PeerTube/shared/extra-utils/videos/videos.ts

220 lines
8.1 KiB
TypeScript
Raw Normal View History

2020-01-31 16:56:52 +01:00
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
2017-12-29 10:46:27 +01:00
import { expect } from 'chai'
2021-07-15 10:02:54 +02:00
import { pathExists, readdir } from 'fs-extra'
import { join } from 'path'
import { getLowercaseExtension } from '@server/helpers/core-utils'
2021-07-16 10:42:24 +02:00
import { HttpStatusCode } from '@shared/models'
2021-07-15 10:02:54 +02:00
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants'
import { dateIsValid, testImage, webtorrentAdd } from '../miscs'
import { makeRawRequest } from '../requests/requests'
import { waitJobs } from '../server'
2021-07-16 09:47:51 +02:00
import { PeerTubeServer } from '../server/server'
2021-07-15 10:02:54 +02:00
import { VideoEdit } from './videos-command'
2020-11-04 14:16:57 +01:00
2018-10-03 16:43:57 +02:00
async function checkVideoFilesWereRemoved (
videoUUID: string,
2021-07-16 09:47:51 +02:00
server: PeerTubeServer,
2019-01-29 08:37:25 +01:00
directories = [
'redundancy',
'videos',
'thumbnails',
'torrents',
'previews',
'captions',
join('playlists', 'hls'),
join('redundancy', 'hls')
]
2018-10-03 16:43:57 +02:00
) {
for (const directory of directories) {
2021-07-16 09:04:35 +02:00
const directoryPath = server.servers.buildDirectory(directory)
2019-03-05 10:58:44 +01:00
const directoryExists = await pathExists(directoryPath)
if (directoryExists === false) continue
2018-08-27 16:23:34 +02:00
const files = await readdir(directoryPath)
for (const file of files) {
2021-02-01 11:18:50 +01:00
expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID)
}
}
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
function checkUploadVideoParam (
2021-07-16 09:47:51 +02:00
server: PeerTubeServer,
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
token: string,
2021-07-15 10:02:54 +02:00
attributes: Partial<VideoEdit>,
expectedStatus = HttpStatusCode.OK_200,
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
mode: 'legacy' | 'resumable' = 'legacy'
) {
return mode === 'legacy'
2021-07-16 09:04:35 +02:00
? server.videos.buildLegacyUpload({ token, attributes, expectedStatus })
: server.videos.buildResumeUpload({ token, attributes, expectedStatus })
2017-09-07 15:27:35 +02:00
}
2017-12-29 10:46:27 +01:00
async function completeVideoCheck (
2021-07-16 09:47:51 +02:00
server: PeerTubeServer,
2017-12-29 10:46:27 +01:00
video: any,
attributes: {
name: string
category: number
licence: number
2018-04-23 14:39:52 +02:00
language: string
2017-12-29 10:46:27 +01:00
nsfw: boolean
2018-01-03 10:12:36 +01:00
commentsEnabled: boolean
downloadEnabled: boolean
2017-12-29 10:46:27 +01:00
description: string
publishedAt?: string
support: string
2020-01-31 16:56:52 +01:00
originallyPublishedAt?: string
2018-03-12 11:06:15 +01:00
account: {
name: string
host: string
}
isLocal: boolean
tags: string[]
privacy: number
likes?: number
dislikes?: number
duration: number
2017-12-29 10:46:27 +01:00
channel: {
displayName: string
name: string
2017-12-29 11:51:55 +01:00
description
2017-12-29 10:46:27 +01:00
isLocal: boolean
}
fixture: string
2017-12-29 10:46:27 +01:00
files: {
resolution: number
size: number
2020-01-31 16:56:52 +01:00
}[]
thumbnailfile?: string
previewfile?: string
2017-12-29 10:46:27 +01:00
}
) {
2017-12-29 11:51:55 +01:00
if (!attributes.likes) attributes.likes = 0
if (!attributes.dislikes) attributes.dislikes = 0
2021-07-15 10:02:54 +02:00
const host = new URL(server.url).host
const originHost = attributes.account.host
2017-12-29 10:46:27 +01:00
expect(video.name).to.equal(attributes.name)
expect(video.category.id).to.equal(attributes.category)
2018-04-23 14:39:52 +02:00
expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Misc')
expect(video.licence.id).to.equal(attributes.licence)
2018-04-23 14:39:52 +02:00
expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown')
expect(video.language.id).to.equal(attributes.language)
2018-04-23 14:39:52 +02:00
expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown')
expect(video.privacy.id).to.deep.equal(attributes.privacy)
expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy])
2017-12-29 10:46:27 +01:00
expect(video.nsfw).to.equal(attributes.nsfw)
expect(video.description).to.equal(attributes.description)
2018-04-25 14:32:19 +02:00
expect(video.account.id).to.be.a('number')
2018-03-12 11:06:15 +01:00
expect(video.account.host).to.equal(attributes.account.host)
expect(video.account.name).to.equal(attributes.account.name)
expect(video.channel.displayName).to.equal(attributes.channel.displayName)
expect(video.channel.name).to.equal(attributes.channel.name)
2017-12-29 11:51:55 +01:00
expect(video.likes).to.equal(attributes.likes)
expect(video.dislikes).to.equal(attributes.dislikes)
2017-12-29 10:46:27 +01:00
expect(video.isLocal).to.equal(attributes.isLocal)
2017-12-29 11:51:55 +01:00
expect(video.duration).to.equal(attributes.duration)
2017-12-29 10:46:27 +01:00
expect(dateIsValid(video.createdAt)).to.be.true
2018-04-04 10:21:36 +02:00
expect(dateIsValid(video.publishedAt)).to.be.true
2017-12-29 10:46:27 +01:00
expect(dateIsValid(video.updatedAt)).to.be.true
if (attributes.publishedAt) {
expect(video.publishedAt).to.equal(attributes.publishedAt)
}
2019-02-11 14:41:55 +01:00
if (attributes.originallyPublishedAt) {
expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt)
} else {
expect(video.originallyPublishedAt).to.be.null
}
2021-07-16 09:04:35 +02:00
const videoDetails = await server.videos.get({ id: video.uuid })
2017-12-29 10:46:27 +01:00
expect(videoDetails.files).to.have.lengthOf(attributes.files.length)
expect(videoDetails.tags).to.deep.equal(attributes.tags)
2018-03-12 11:29:46 +01:00
expect(videoDetails.account.name).to.equal(attributes.account.name)
expect(videoDetails.account.host).to.equal(attributes.account.host)
expect(video.channel.displayName).to.equal(attributes.channel.displayName)
expect(video.channel.name).to.equal(attributes.channel.name)
2018-05-11 15:10:13 +02:00
expect(videoDetails.channel.host).to.equal(attributes.account.host)
2017-12-29 10:46:27 +01:00
expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal)
2018-05-11 15:10:13 +02:00
expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true
expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true
expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled)
expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled)
2017-12-29 10:46:27 +01:00
for (const attributeFile of attributes.files) {
2018-03-19 12:36:41 +01:00
const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution)
2017-12-29 10:46:27 +01:00
expect(file).not.to.be.undefined
let extension = getLowercaseExtension(attributes.fixture)
2019-04-26 08:50:52 +02:00
// Transcoding enabled: extension will always be .mp4
if (attributes.files.length > 1) extension = '.mp4'
2017-12-29 11:51:55 +01:00
2017-12-29 10:46:27 +01:00
expect(file.magnetUri).to.have.lengthOf.above(2)
expect(file.torrentDownloadUrl).to.equal(`http://${host}/download/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`)
expect(file.torrentUrl).to.equal(`http://${host}/lazy-static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`)
expect(file.fileUrl).to.equal(`http://${originHost}/static/webseed/${videoDetails.uuid}-${file.resolution.id}${extension}`)
expect(file.fileDownloadUrl).to.equal(`http://${originHost}/download/videos/${videoDetails.uuid}-${file.resolution.id}${extension}`)
await Promise.all([
makeRawRequest(file.torrentUrl, 200),
makeRawRequest(file.torrentDownloadUrl, 200),
makeRawRequest(file.metadataUrl, 200),
// Backward compatibility
makeRawRequest(`http://${originHost}/static/torrents/${videoDetails.uuid}-${file.resolution.id}.torrent`, 200)
])
expect(file.resolution.id).to.equal(attributeFile.resolution)
expect(file.resolution.label).to.equal(attributeFile.resolution + 'p')
2017-12-29 11:51:55 +01:00
const minSize = attributeFile.size - ((10 * attributeFile.size) / 100)
const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100)
2020-01-31 16:56:52 +01:00
expect(
file.size,
'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')'
).to.be.above(minSize).and.below(maxSize)
2017-12-29 10:46:27 +01:00
const torrent = await webtorrentAdd(file.magnetUri, true)
2017-12-29 10:46:27 +01:00
expect(torrent.files).to.be.an('array')
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
}
2020-01-23 14:23:19 +01:00
2021-06-02 10:41:46 +02:00
expect(videoDetails.thumbnailPath).to.exist
2021-07-15 10:02:54 +02:00
await testImage(server.url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath)
2020-01-23 14:23:19 +01:00
if (attributes.previewfile) {
2021-06-02 10:41:46 +02:00
expect(videoDetails.previewPath).to.exist
2021-07-15 10:02:54 +02:00
await testImage(server.url, attributes.previewfile, videoDetails.previewPath)
2020-01-23 14:23:19 +01:00
}
2017-12-29 10:46:27 +01:00
}
2020-06-16 15:52:05 +02:00
// serverNumber starts from 1
2021-07-15 10:02:54 +02:00
async function uploadRandomVideoOnServers (
2021-07-16 09:47:51 +02:00
servers: PeerTubeServer[],
2021-07-15 10:02:54 +02:00
serverNumber: number,
additionalParams?: VideoEdit & { prefixName?: string }
) {
2020-06-16 15:52:05 +02:00
const server = servers.find(s => s.serverNumber === serverNumber)
2021-07-16 14:27:30 +02:00
const res = await server.videos.randomUpload({ wait: false, additionalParams })
2020-06-16 15:52:05 +02:00
await waitJobs(servers)
return res
}
2017-09-04 21:21:47 +02:00
// ---------------------------------------------------------------------------
export {
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
checkUploadVideoParam,
completeVideoCheck,
2021-07-15 10:02:54 +02:00
uploadRandomVideoOnServers,
checkVideoFilesWereRemoved
2017-09-04 21:21:47 +02:00
}