PeerTube/shared/extra-utils/videos/videos.ts

925 lines
27 KiB
TypeScript
Raw Normal View History

2020-01-31 16:56:52 +01:00
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
2017-12-29 10:46:27 +01:00
import { expect } from 'chai'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra'
import got, { Response as GotResponse } from 'got/dist/source'
2017-09-07 15:27:35 +02:00
import * as parseTorrent from 'parse-torrent'
import { join } from 'path'
2017-12-28 13:59:22 +01:00
import * as request from 'supertest'
2020-06-16 15:52:05 +02:00
import { v4 as uuidv4 } from 'uuid'
import validator from 'validator'
import { getLowercaseExtension } from '@server/helpers/core-utils'
2021-02-25 13:56:07 +01:00
import { HttpStatusCode } from '@shared/core-utils'
import { VideosCommonQuery } from '@shared/models'
2020-06-16 15:52:05 +02:00
import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
import { VideoDetails, VideoPrivacy } from '../../models/videos'
2021-02-25 13:56:07 +01:00
import {
buildAbsoluteFixturePath,
buildServerDirectory,
dateIsValid,
immutableAssign,
testImage,
wait,
webtorrentAdd
} from '../miscs/miscs'
import { makeGetRequest, makePutBodyRequest, makeRawRequest, makeUploadRequest } from '../requests/requests'
2020-06-16 15:52:05 +02:00
import { waitJobs } from '../server/jobs'
import { ServerInfo } from '../server/servers'
import { getMyUserInformation } from '../users/users'
2017-09-04 21:21:47 +02:00
loadLanguages()
2017-09-04 21:21:47 +02:00
type VideoAttributes = {
name?: string
category?: number
licence?: number
2018-04-23 14:39:52 +02:00
language?: string
2017-09-04 21:21:47 +02:00
nsfw?: boolean
2018-01-03 10:12:36 +01:00
commentsEnabled?: boolean
downloadEnabled?: boolean
waitTranscoding?: boolean
2017-09-04 21:21:47 +02:00
description?: string
2019-02-11 14:41:55 +01:00
originallyPublishedAt?: string
2017-09-04 21:21:47 +02:00
tags?: string[]
2017-10-24 19:41:30 +02:00
channelId?: number
privacy?: VideoPrivacy
2017-09-04 21:21:47 +02:00
fixture?: string
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
support?: string
thumbnailfile?: string
previewfile?: string
scheduleUpdate?: {
updateAt: string
privacy?: VideoPrivacy
}
2017-09-04 21:21:47 +02:00
}
function getVideoCategories (url: string) {
const path = '/api/v1/videos/categories'
2017-12-28 14:29:57 +01:00
return makeGetRequest({
url,
2017-12-28 17:25:10 +01:00
path,
statusCodeExpected: HttpStatusCode.OK_200
2017-12-28 14:29:57 +01:00
})
2017-09-04 21:21:47 +02:00
}
function getVideoLicences (url: string) {
const path = '/api/v1/videos/licences'
2017-12-28 14:29:57 +01:00
return makeGetRequest({
url,
2017-12-28 17:25:10 +01:00
path,
statusCodeExpected: HttpStatusCode.OK_200
2017-12-28 14:29:57 +01:00
})
2017-09-04 21:21:47 +02:00
}
function getVideoLanguages (url: string) {
const path = '/api/v1/videos/languages'
2017-12-28 14:29:57 +01:00
return makeGetRequest({
url,
2017-12-28 17:25:10 +01:00
path,
statusCodeExpected: HttpStatusCode.OK_200
2017-12-28 14:29:57 +01:00
})
2017-09-04 21:21:47 +02:00
}
function getVideoPrivacies (url: string) {
const path = '/api/v1/videos/privacies'
2017-12-28 14:29:57 +01:00
return makeGetRequest({
url,
2017-12-28 17:25:10 +01:00
path,
statusCodeExpected: HttpStatusCode.OK_200
2017-12-28 14:29:57 +01:00
})
}
function getVideo (url: string, id: number | string, expectedStatus = HttpStatusCode.OK_200) {
2017-09-04 21:21:47 +02:00
const path = '/api/v1/videos/' + id
return request(url)
.get(path)
.set('Accept', 'application/json')
.expect(expectedStatus)
}
async function getVideoIdFromUUID (url: string, uuid: string) {
const res = await getVideo(url, uuid)
return res.body.id
}
function getVideoFileMetadataUrl (url: string) {
return request(url)
.get('/')
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
.expect('Content-Type', /json/)
}
function viewVideo (url: string, id: number | string, expectedStatus = HttpStatusCode.NO_CONTENT_204, xForwardedFor?: string) {
2017-11-30 09:21:11 +01:00
const path = '/api/v1/videos/' + id + '/views'
2018-03-29 10:58:24 +02:00
const req = request(url)
2017-11-30 09:21:11 +01:00
.post(path)
.set('Accept', 'application/json')
2018-03-29 10:58:24 +02:00
if (xForwardedFor) {
req.set('X-Forwarded-For', xForwardedFor)
}
return req.expect(expectedStatus)
2017-11-30 09:21:11 +01:00
}
function getVideoWithToken (url: string, token: string, id: number | string, expectedStatus = HttpStatusCode.OK_200) {
const path = '/api/v1/videos/' + id
return request(url)
.get(path)
.set('Authorization', 'Bearer ' + token)
.set('Accept', 'application/json')
.expect(expectedStatus)
2017-09-04 21:21:47 +02:00
}
2017-10-30 10:16:27 +01:00
function getVideoDescription (url: string, descriptionPath: string) {
return request(url)
.get(descriptionPath)
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
2017-10-30 10:16:27 +01:00
.expect('Content-Type', /json/)
}
2017-09-04 21:21:47 +02:00
function getVideosList (url: string) {
const path = '/api/v1/videos'
return request(url)
.get(path)
.query({ sort: 'name' })
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
2017-09-04 21:21:47 +02:00
.expect('Content-Type', /json/)
}
2018-07-20 14:35:18 +02:00
function getVideosListWithToken (url: string, token: string, query: { nsfw?: boolean } = {}) {
const path = '/api/v1/videos'
return request(url)
.get(path)
.set('Authorization', 'Bearer ' + token)
2018-07-20 14:35:18 +02:00
.query(immutableAssign(query, { sort: 'name' }))
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
.expect('Content-Type', /json/)
}
2018-03-13 10:24:28 +01:00
function getLocalVideos (url: string) {
const path = '/api/v1/videos'
return request(url)
.get(path)
.query({ sort: 'name', filter: 'local' })
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
2018-03-13 10:24:28 +01:00
.expect('Content-Type', /json/)
}
2019-12-30 14:31:39 +01:00
function getMyVideos (url: string, accessToken: string, start: number, count: number, sort?: string, search?: string) {
const path = '/api/v1/users/me/videos'
const req = request(url)
.get(path)
.query({ start: start })
.query({ count: count })
2019-12-30 14:31:39 +01:00
.query({ search: search })
if (sort) req.query({ sort })
return req.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + accessToken)
.expect(HttpStatusCode.OK_200)
.expect('Content-Type', /json/)
}
function getMyVideosWithFilter (url: string, accessToken: string, query: { isLive?: boolean }) {
const path = '/api/v1/users/me/videos'
return makeGetRequest({
url,
path,
token: accessToken,
query,
statusCodeExpected: HttpStatusCode.OK_200
})
}
2018-07-20 14:35:18 +02:00
function getAccountVideos (
url: string,
accessToken: string,
accountName: string,
start: number,
count: number,
sort?: string,
query: {
nsfw?: boolean
search?: string
} = {}
2018-07-20 14:35:18 +02:00
) {
2018-05-25 09:57:16 +02:00
const path = '/api/v1/accounts/' + accountName + '/videos'
2018-04-25 10:21:38 +02:00
return makeGetRequest({
url,
path,
2018-07-20 14:35:18 +02:00
query: immutableAssign(query, {
2018-04-25 10:21:38 +02:00
start,
count,
sort
2018-07-20 14:35:18 +02:00
}),
2018-04-25 10:21:38 +02:00
token: accessToken,
statusCodeExpected: HttpStatusCode.OK_200
2018-04-25 10:21:38 +02:00
})
}
function getVideoChannelVideos (
url: string,
accessToken: string,
2018-08-17 15:45:42 +02:00
videoChannelName: string,
2018-04-25 10:21:38 +02:00
start: number,
count: number,
2018-07-20 14:35:18 +02:00
sort?: string,
query: { nsfw?: boolean } = {}
2018-04-25 10:21:38 +02:00
) {
2018-08-17 15:45:42 +02:00
const path = '/api/v1/video-channels/' + videoChannelName + '/videos'
2018-04-25 10:21:38 +02:00
return makeGetRequest({
url,
path,
2018-07-20 14:35:18 +02:00
query: immutableAssign(query, {
2018-04-25 10:21:38 +02:00
start,
count,
sort
2018-07-20 14:35:18 +02:00
}),
2018-04-25 10:21:38 +02:00
token: accessToken,
statusCodeExpected: HttpStatusCode.OK_200
2018-04-25 10:21:38 +02:00
})
}
2019-02-26 10:55:40 +01:00
function getPlaylistVideos (
url: string,
accessToken: string,
playlistId: number | string,
start: number,
count: number,
query: { nsfw?: boolean } = {}
) {
const path = '/api/v1/video-playlists/' + playlistId + '/videos'
return makeGetRequest({
url,
path,
query: immutableAssign(query, {
start,
count
}),
token: accessToken,
statusCodeExpected: HttpStatusCode.OK_200
2019-02-26 10:55:40 +01:00
})
}
2020-01-08 14:15:16 +01:00
function getVideosListPagination (url: string, start: number, count: number, sort?: string, skipCount?: boolean) {
2017-09-04 21:21:47 +02:00
const path = '/api/v1/videos'
const req = request(url)
.get(path)
.query({ start: start })
.query({ count: count })
if (sort) req.query({ sort })
2020-01-08 14:15:16 +01:00
if (skipCount) req.query({ skipCount })
2017-09-04 21:21:47 +02:00
return req.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
2017-09-04 21:21:47 +02:00
.expect('Content-Type', /json/)
}
function getVideosListSort (url: string, sort: string) {
const path = '/api/v1/videos'
return request(url)
.get(path)
.query({ sort: sort })
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
2017-09-04 21:21:47 +02:00
.expect('Content-Type', /json/)
}
function getVideosWithFilters (url: string, query: VideosCommonQuery) {
2017-09-04 21:21:47 +02:00
const path = '/api/v1/videos'
return request(url)
2018-07-19 16:17:54 +02:00
.get(path)
2018-07-20 14:35:18 +02:00
.query(query)
2017-12-05 17:46:33 +01:00
.set('Accept', 'application/json')
.expect(HttpStatusCode.OK_200)
2017-12-05 17:46:33 +01:00
.expect('Content-Type', /json/)
2017-09-04 21:21:47 +02:00
}
function removeVideo (url: string, token: string, id: number | string, expectedStatus = HttpStatusCode.NO_CONTENT_204) {
const path = '/api/v1/videos'
2017-09-04 21:21:47 +02:00
return request(url)
2018-07-20 14:35:18 +02:00
.delete(path + '/' + id)
2017-09-04 21:21:47 +02:00
.set('Accept', 'application/json')
2018-07-20 14:35:18 +02:00
.set('Authorization', 'Bearer ' + token)
.expect(expectedStatus)
2017-09-04 21:21:47 +02:00
}
2020-11-04 14:16:57 +01:00
async function removeAllVideos (server: ServerInfo) {
const resVideos = await getVideosList(server.url)
for (const v of resVideos.body.data) {
await removeVideo(server.url, server.accessToken, v.id)
}
}
2018-10-03 16:43:57 +02:00
async function checkVideoFilesWereRemoved (
videoUUID: string,
serverNumber: number,
2019-01-29 08:37:25 +01:00
directories = [
'redundancy',
'videos',
'thumbnails',
'torrents',
'previews',
'captions',
join('playlists', 'hls'),
join('redundancy', 'hls')
]
2018-10-03 16:43:57 +02:00
) {
for (const directory of directories) {
2020-11-24 15:22:56 +01:00
const directoryPath = buildServerDirectory({ internalServerNumber: serverNumber }, directory)
2019-03-05 10:58:44 +01:00
const directoryExists = await pathExists(directoryPath)
if (directoryExists === false) continue
2018-08-27 16:23:34 +02:00
const files = await readdir(directoryPath)
for (const file of files) {
2021-02-01 11:18:50 +01:00
expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID)
}
}
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
async function uploadVideo (
url: string,
accessToken: string,
videoAttributesArg: VideoAttributes,
specialStatus = HttpStatusCode.OK_200,
mode: 'legacy' | 'resumable' = 'legacy'
) {
2017-10-24 19:41:30 +02:00
let defaultChannelId = '1'
try {
const res = await getMyUserInformation(url, accessToken)
defaultChannelId = res.body.videoChannels[0].id
} catch (e) { /* empty */ }
2017-09-04 21:21:47 +02:00
// Override default attributes
const attributes = Object.assign({
2017-09-04 21:21:47 +02:00
name: 'my super video',
category: 5,
licence: 4,
2018-04-23 14:39:52 +02:00
language: 'zh',
2017-10-24 19:41:30 +02:00
channelId: defaultChannelId,
2017-09-04 21:21:47 +02:00
nsfw: true,
waitTranscoding: false,
2017-09-04 21:21:47 +02:00
description: 'my super description',
support: 'my super support text',
2017-09-04 21:21:47 +02:00
tags: [ 'tag' ],
privacy: VideoPrivacy.PUBLIC,
2018-01-03 10:12:36 +01:00
commentsEnabled: true,
downloadEnabled: true,
2017-09-04 21:21:47 +02:00
fixture: 'video_short.webm'
}, videoAttributesArg)
2017-09-04 21:21:47 +02:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
const res = mode === 'legacy'
? await buildLegacyUpload(url, accessToken, attributes, specialStatus)
: await buildResumeUpload(url, accessToken, attributes, specialStatus)
// Wait torrent generation
if (specialStatus === HttpStatusCode.OK_200) {
let video: VideoDetails
do {
const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid)
video = resVideo.body
await wait(50)
} while (!video.files[0].torrentUrl)
}
return res
}
function checkUploadVideoParam (
url: string,
token: string,
attributes: Partial<VideoAttributes>,
specialStatus = HttpStatusCode.OK_200,
mode: 'legacy' | 'resumable' = 'legacy'
) {
return mode === 'legacy'
? buildLegacyUpload(url, token, attributes, specialStatus)
: buildResumeUpload(url, token, attributes, specialStatus)
}
async function buildLegacyUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
const path = '/api/v1/videos/upload'
2017-09-04 21:21:47 +02:00
const req = request(url)
.post(path)
.set('Accept', 'application/json')
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
.set('Authorization', 'Bearer ' + token)
2019-06-11 10:39:30 +02:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
buildUploadReq(req, attributes)
2017-09-07 17:58:09 +02:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
if (attributes.fixture !== undefined) {
req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
return req.expect(specialStatus)
}
2017-09-04 21:21:47 +02:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
async function buildResumeUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
let size = 0
let videoFilePath: string
let mimetype = 'video/mp4'
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
if (attributes.fixture) {
videoFilePath = buildAbsoluteFixturePath(attributes.fixture)
size = (await stat(videoFilePath)).size
if (videoFilePath.endsWith('.mkv')) {
mimetype = 'video/x-matroska'
} else if (videoFilePath.endsWith('.webm')) {
mimetype = 'video/webm'
}
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
const initializeSessionRes = await prepareResumableUpload({ url, token, attributes, size, mimetype })
const initStatus = initializeSessionRes.status
if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
const locationHeader = initializeSessionRes.header['location']
expect(locationHeader).to.not.be.undefined
const pathUploadId = locationHeader.split('?')[1]
return sendResumableChunks({ url, token, pathUploadId, videoFilePath, size, specialStatus })
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
const expectedInitStatus = specialStatus === HttpStatusCode.OK_200
? HttpStatusCode.CREATED_201
: specialStatus
2021-02-25 13:56:07 +01:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
expect(initStatus).to.equal(expectedInitStatus)
2021-02-25 13:56:07 +01:00
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
return initializeSessionRes
}
async function prepareResumableUpload (options: {
url: string
token: string
attributes: VideoAttributes
size: number
mimetype: string
}) {
const { url, token, attributes, size, mimetype } = options
const path = '/api/v1/videos/upload-resumable'
const req = request(url)
.post(path)
.set('Authorization', 'Bearer ' + token)
.set('X-Upload-Content-Type', mimetype)
.set('X-Upload-Content-Length', size.toString())
buildUploadReq(req, attributes)
if (attributes.fixture) {
req.field('filename', attributes.fixture)
2021-02-25 13:56:07 +01:00
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
return req
}
function sendResumableChunks (options: {
url: string
token: string
pathUploadId: string
videoFilePath: string
size: number
specialStatus?: HttpStatusCode
contentLength?: number
contentRangeBuilder?: (start: number, chunk: any) => string
}) {
const { url, token, pathUploadId, videoFilePath, size, specialStatus, contentLength, contentRangeBuilder } = options
const expectedStatus = specialStatus || HttpStatusCode.OK_200
const path = '/api/v1/videos/upload-resumable'
let start = 0
const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
return new Promise<GotResponse>((resolve, reject) => {
readable.on('data', async function onData (chunk) {
readable.pause()
const headers = {
'Authorization': 'Bearer ' + token,
'Content-Type': 'application/octet-stream',
'Content-Range': contentRangeBuilder
? contentRangeBuilder(start, chunk)
: `bytes ${start}-${start + chunk.length - 1}/${size}`,
'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
}
const res = await got({
url,
method: 'put',
headers,
path: path + '?' + pathUploadId,
body: chunk,
responseType: 'json',
throwHttpErrors: false
})
start += chunk.length
if (res.statusCode === expectedStatus) {
return resolve(res)
}
if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
readable.off('data', onData)
return reject(new Error('Incorrect transient behaviour sending intermediary chunks'))
}
readable.resume()
})
})
2017-09-04 21:21:47 +02:00
}
function updateVideo (
url: string,
accessToken: string,
id: number | string,
attributes: VideoAttributes,
statusCodeExpected = HttpStatusCode.NO_CONTENT_204
) {
2017-09-04 21:21:47 +02:00
const path = '/api/v1/videos/' + id
const body = {}
if (attributes.name) body['name'] = attributes.name
if (attributes.category) body['category'] = attributes.category
if (attributes.licence) body['licence'] = attributes.licence
if (attributes.language) body['language'] = attributes.language
2018-01-03 10:12:36 +01:00
if (attributes.nsfw !== undefined) body['nsfw'] = JSON.stringify(attributes.nsfw)
if (attributes.commentsEnabled !== undefined) body['commentsEnabled'] = JSON.stringify(attributes.commentsEnabled)
if (attributes.downloadEnabled !== undefined) body['downloadEnabled'] = JSON.stringify(attributes.downloadEnabled)
2019-02-11 14:41:55 +01:00
if (attributes.originallyPublishedAt !== undefined) body['originallyPublishedAt'] = attributes.originallyPublishedAt
2017-09-04 21:21:47 +02:00
if (attributes.description) body['description'] = attributes.description
if (attributes.tags) body['tags'] = attributes.tags
if (attributes.privacy) body['privacy'] = attributes.privacy
2018-05-11 15:10:13 +02:00
if (attributes.channelId) body['channelId'] = attributes.channelId
if (attributes.scheduleUpdate) body['scheduleUpdate'] = attributes.scheduleUpdate
2017-09-04 21:21:47 +02:00
// Upload request
if (attributes.thumbnailfile || attributes.previewfile) {
const attaches: any = {}
if (attributes.thumbnailfile) attaches.thumbnailfile = attributes.thumbnailfile
if (attributes.previewfile) attaches.previewfile = attributes.previewfile
return makeUploadRequest({
url,
method: 'PUT',
path,
token: accessToken,
fields: body,
attaches,
statusCodeExpected
})
}
return makePutBodyRequest({
url,
path,
fields: body,
token: accessToken,
statusCodeExpected
})
2017-09-04 21:21:47 +02:00
}
function rateVideo (url: string, accessToken: string, id: number | string, rating: string, specialStatus = HttpStatusCode.NO_CONTENT_204) {
2017-09-04 21:21:47 +02:00
const path = '/api/v1/videos/' + id + '/rate'
return request(url)
.put(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + accessToken)
.send({ rating })
.expect(specialStatus)
}
function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolution: number) {
2017-09-07 15:27:35 +02:00
return new Promise<any>((res, rej) => {
const torrentName = videoUUID + '-' + resolution + '.torrent'
2020-11-24 15:22:56 +01:00
const torrentPath = buildServerDirectory(server, join('torrents', torrentName))
2017-09-07 15:27:35 +02:00
readFile(torrentPath, (err, data) => {
if (err) return rej(err)
return res(parseTorrent(data))
})
})
}
2017-12-29 10:46:27 +01:00
async function completeVideoCheck (
url: string,
video: any,
attributes: {
name: string
category: number
licence: number
2018-04-23 14:39:52 +02:00
language: string
2017-12-29 10:46:27 +01:00
nsfw: boolean
2018-01-03 10:12:36 +01:00
commentsEnabled: boolean
downloadEnabled: boolean
2017-12-29 10:46:27 +01:00
description: string
publishedAt?: string
support: string
2020-01-31 16:56:52 +01:00
originallyPublishedAt?: string
2018-03-12 11:06:15 +01:00
account: {
name: string
host: string
}
isLocal: boolean
tags: string[]
privacy: number
likes?: number
dislikes?: number
duration: number
2017-12-29 10:46:27 +01:00
channel: {
displayName: string
name: string
2017-12-29 11:51:55 +01:00
description
2017-12-29 10:46:27 +01:00
isLocal: boolean
}
fixture: string
2017-12-29 10:46:27 +01:00
files: {
resolution: number
size: number
2020-01-31 16:56:52 +01:00
}[]
thumbnailfile?: string
previewfile?: string
2017-12-29 10:46:27 +01:00
}
) {
2017-12-29 11:51:55 +01:00
if (!attributes.likes) attributes.likes = 0
if (!attributes.dislikes) attributes.dislikes = 0
const host = new URL(url).host
const originHost = attributes.account.host
2017-12-29 10:46:27 +01:00
expect(video.name).to.equal(attributes.name)
expect(video.category.id).to.equal(attributes.category)
2018-04-23 14:39:52 +02:00
expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Misc')
expect(video.licence.id).to.equal(attributes.licence)
2018-04-23 14:39:52 +02:00
expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown')
expect(video.language.id).to.equal(attributes.language)
2018-04-23 14:39:52 +02:00
expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown')
expect(video.privacy.id).to.deep.equal(attributes.privacy)
expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy])
2017-12-29 10:46:27 +01:00
expect(video.nsfw).to.equal(attributes.nsfw)
expect(video.description).to.equal(attributes.description)
2018-04-25 14:32:19 +02:00
expect(video.account.id).to.be.a('number')
2018-03-12 11:06:15 +01:00
expect(video.account.host).to.equal(attributes.account.host)
expect(video.account.name).to.equal(attributes.account.name)
expect(video.channel.displayName).to.equal(attributes.channel.displayName)
expect(video.channel.name).to.equal(attributes.channel.name)
2017-12-29 11:51:55 +01:00
expect(video.likes).to.equal(attributes.likes)
expect(video.dislikes).to.equal(attributes.dislikes)
2017-12-29 10:46:27 +01:00
expect(video.isLocal).to.equal(attributes.isLocal)
2017-12-29 11:51:55 +01:00
expect(video.duration).to.equal(attributes.duration)
2017-12-29 10:46:27 +01:00
expect(dateIsValid(video.createdAt)).to.be.true
2018-04-04 10:21:36 +02:00
expect(dateIsValid(video.publishedAt)).to.be.true
2017-12-29 10:46:27 +01:00
expect(dateIsValid(video.updatedAt)).to.be.true
if (attributes.publishedAt) {
expect(video.publishedAt).to.equal(attributes.publishedAt)
}
2019-02-11 14:41:55 +01:00
if (attributes.originallyPublishedAt) {
expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt)
} else {
expect(video.originallyPublishedAt).to.be.null
}
const res = await getVideo(url, video.uuid)
2018-05-11 15:10:13 +02:00
const videoDetails: VideoDetails = res.body
2017-12-29 10:46:27 +01:00
expect(videoDetails.files).to.have.lengthOf(attributes.files.length)
expect(videoDetails.tags).to.deep.equal(attributes.tags)
2018-03-12 11:29:46 +01:00
expect(videoDetails.account.name).to.equal(attributes.account.name)
expect(videoDetails.account.host).to.equal(attributes.account.host)
expect(video.channel.displayName).to.equal(attributes.channel.displayName)
expect(video.channel.name).to.equal(attributes.channel.name)
2018-05-11 15:10:13 +02:00
expect(videoDetails.channel.host).to.equal(attributes.account.host)
2017-12-29 10:46:27 +01:00
expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal)
2018-05-11 15:10:13 +02:00
expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true
expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true
expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled)
expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled)
2017-12-29 10:46:27 +01:00
for (const attributeFile of attributes.files) {
2018-03-19 12:36:41 +01:00
const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution)
2017-12-29 10:46:27 +01:00
expect(file).not.to.be.undefined
let extension = getLowercaseExtension(attributes.fixture)
2019-04-26 08:50:52 +02:00
// Transcoding enabled: extension will always be .mp4
if (attributes.files.length > 1) extension = '.mp4'
2017-12-29 11:51:55 +01:00
2017-12-29 10:46:27 +01:00
expect(file.magnetUri).to.have.lengthOf.above(2)
expect(file.torrentDownloadUrl).to.equal(`http://${host}/download/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`)
expect(file.torrentUrl).to.equal(`http://${host}/lazy-static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`)
expect(file.fileUrl).to.equal(`http://${originHost}/static/webseed/${videoDetails.uuid}-${file.resolution.id}${extension}`)
expect(file.fileDownloadUrl).to.equal(`http://${originHost}/download/videos/${videoDetails.uuid}-${file.resolution.id}${extension}`)
await Promise.all([
makeRawRequest(file.torrentUrl, 200),
makeRawRequest(file.torrentDownloadUrl, 200),
makeRawRequest(file.metadataUrl, 200),
// Backward compatibility
makeRawRequest(`http://${originHost}/static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`, 200)
])
expect(file.resolution.id).to.equal(attributeFile.resolution)
expect(file.resolution.label).to.equal(attributeFile.resolution + 'p')
2017-12-29 11:51:55 +01:00
const minSize = attributeFile.size - ((10 * attributeFile.size) / 100)
const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100)
2020-01-31 16:56:52 +01:00
expect(
file.size,
'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')'
).to.be.above(minSize).and.below(maxSize)
2017-12-29 10:46:27 +01:00
const torrent = await webtorrentAdd(file.magnetUri, true)
2017-12-29 10:46:27 +01:00
expect(torrent.files).to.be.an('array')
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
}
2020-01-23 14:23:19 +01:00
2021-06-02 10:41:46 +02:00
expect(videoDetails.thumbnailPath).to.exist
2020-01-23 14:23:19 +01:00
await testImage(url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath)
if (attributes.previewfile) {
2021-06-02 10:41:46 +02:00
expect(videoDetails.previewPath).to.exist
2020-01-23 14:23:19 +01:00
await testImage(url, attributes.previewfile, videoDetails.previewPath)
}
2017-12-29 10:46:27 +01:00
}
2019-03-05 10:58:44 +01:00
async function videoUUIDToId (url: string, id: number | string) {
if (validator.isUUID('' + id) === false) return id
const res = await getVideo(url, id)
return res.body.id
}
2020-01-10 10:11:28 +01:00
async function uploadVideoAndGetId (options: {
2020-01-31 16:56:52 +01:00
server: ServerInfo
videoName: string
nsfw?: boolean
privacy?: VideoPrivacy
2020-01-10 10:11:28 +01:00
token?: string
fixture?: string
2020-01-10 10:11:28 +01:00
}) {
2019-03-05 10:58:44 +01:00
const videoAttrs: any = { name: options.videoName }
if (options.nsfw) videoAttrs.nsfw = options.nsfw
2020-01-10 10:11:28 +01:00
if (options.privacy) videoAttrs.privacy = options.privacy
if (options.fixture) videoAttrs.fixture = options.fixture
2019-03-05 10:58:44 +01:00
const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs)
return { id: res.body.video.id, uuid: res.body.video.uuid }
}
2020-01-10 10:11:28 +01:00
async function getLocalIdByUUID (url: string, uuid: string) {
const res = await getVideo(url, uuid)
return res.body.id
}
2020-06-16 15:52:05 +02:00
// serverNumber starts from 1
async function uploadRandomVideoOnServers (servers: ServerInfo[], serverNumber: number, additionalParams: any = {}) {
const server = servers.find(s => s.serverNumber === serverNumber)
const res = await uploadRandomVideo(server, false, additionalParams)
await waitJobs(servers)
return res
}
async function uploadRandomVideo (server: ServerInfo, wait = true, additionalParams: any = {}) {
const prefixName = additionalParams.prefixName || ''
const name = prefixName + uuidv4()
const data = Object.assign({ name }, additionalParams)
const res = await uploadVideo(server.url, server.accessToken, data)
if (wait) await waitJobs([ server ])
return { uuid: res.body.video.uuid, name }
}
2017-09-04 21:21:47 +02:00
// ---------------------------------------------------------------------------
export {
2017-10-30 10:16:27 +01:00
getVideoDescription,
2017-09-04 21:21:47 +02:00
getVideoCategories,
2020-06-16 15:52:05 +02:00
uploadRandomVideo,
2017-09-04 21:21:47 +02:00
getVideoLicences,
2019-03-05 10:58:44 +01:00
videoUUIDToId,
getVideoPrivacies,
2017-09-04 21:21:47 +02:00
getVideoLanguages,
getMyVideos,
2018-04-25 10:21:38 +02:00
getAccountVideos,
getVideoChannelVideos,
2017-09-04 21:21:47 +02:00
getVideo,
getVideoFileMetadataUrl,
getVideoWithToken,
2017-09-04 21:21:47 +02:00
getVideosList,
2020-11-04 14:16:57 +01:00
removeAllVideos,
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
checkUploadVideoParam,
2017-09-04 21:21:47 +02:00
getVideosListPagination,
getVideosListSort,
removeVideo,
getVideosListWithToken,
2017-09-04 21:21:47 +02:00
uploadVideo,
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
sendResumableChunks,
2018-07-20 14:35:18 +02:00
getVideosWithFilters,
2020-06-16 15:52:05 +02:00
uploadRandomVideoOnServers,
2017-09-04 21:21:47 +02:00
updateVideo,
2017-09-07 15:27:35 +02:00
rateVideo,
2017-11-30 09:21:11 +01:00
viewVideo,
2017-12-29 10:46:27 +01:00
parseTorrentVideo,
2018-03-13 10:24:28 +01:00
getLocalVideos,
completeVideoCheck,
2019-02-26 10:55:40 +01:00
checkVideoFilesWereRemoved,
2019-03-05 10:58:44 +01:00
getPlaylistVideos,
getMyVideosWithFilter,
2020-01-10 10:11:28 +01:00
uploadVideoAndGetId,
getLocalIdByUUID,
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
getVideoIdFromUUID,
prepareResumableUpload
}
// ---------------------------------------------------------------------------
function buildUploadReq (req: request.Test, attributes: VideoAttributes) {
for (const key of [ 'name', 'support', 'channelId', 'description', 'originallyPublishedAt' ]) {
if (attributes[key] !== undefined) {
req.field(key, attributes[key])
}
}
for (const key of [ 'nsfw', 'commentsEnabled', 'downloadEnabled', 'waitTranscoding' ]) {
if (attributes[key] !== undefined) {
req.field(key, JSON.stringify(attributes[key]))
}
}
for (const key of [ 'language', 'privacy', 'category', 'licence' ]) {
if (attributes[key] !== undefined) {
req.field(key, attributes[key].toString())
}
}
const tags = attributes.tags || []
for (let i = 0; i < tags.length; i++) {
req.field('tags[' + i + ']', attributes.tags[i])
}
for (const key of [ 'thumbnailfile', 'previewfile' ]) {
if (attributes[key] !== undefined) {
req.attach(key, buildAbsoluteFixturePath(attributes[key]))
}
}
if (attributes.scheduleUpdate) {
if (attributes.scheduleUpdate.updateAt) {
req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt)
}
if (attributes.scheduleUpdate.privacy) {
req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy)
}
}
2017-09-04 21:21:47 +02:00
}