diff --git a/.eslintrc.json b/.eslintrc.json index a49a9e71b..c7597cef3 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -102,6 +102,9 @@ "parserOptions": { "project": [ "./tsconfig.json", + "./shared/tsconfig.json", + "./scripts/tsconfig.json", + "./server/tsconfig.json", "./server/tools/tsconfig.json" ] } diff --git a/.gitignore b/.gitignore index 3027b6058..cd7d3f59b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ yarn-error.log /server/tools/import-mediacore.ts /docker-volume/ /init.mp4 + +# TypeScript +*.tsbuildinfo diff --git a/scripts/build/server.sh b/scripts/build/server.sh index b903f8250..865bdd095 100755 --- a/scripts/build/server.sh +++ b/scripts/build/server.sh @@ -4,7 +4,10 @@ set -eu rm -rf ./dist -npm run tsc -cp "./tsconfig.json" "./dist" +npm run tsc -- -b --verbose +cp "./tsconfig.base.json" "./tsconfig.json" "./dist/" +cp "./scripts/tsconfig.json" "./dist/scripts/" +cp "./server/tsconfig.json" "./dist/server/" +cp "./shared/tsconfig.json" "./dist/shared/" cp -r "./server/static" "./server/assets" "./dist/server" cp -r "./server/lib/emails" "./dist/server/lib" diff --git a/scripts/client-build-stats.ts b/scripts/client-build-stats.ts index 70ceda975..91844dfcd 100644 --- a/scripts/client-build-stats.ts +++ b/scripts/client-build-stats.ts @@ -3,7 +3,7 @@ registerTSPaths() import { readdir, stat } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' async function run () { const result = { diff --git a/scripts/setup/cli.sh b/scripts/setup/cli.sh index ea327e5a1..d0ad2ec5f 100755 --- a/scripts/setup/cli.sh +++ b/scripts/setup/cli.sh @@ -11,6 +11,6 @@ rm -rf ./dist/server/tools/ yarn install --pure-lockfile ) -npm run tsc -- --build ./server/tools/tsconfig.json +npm run tsc -- --build --verbose ./server/tools/tsconfig.json cp -r "./server/tools/node_modules" "./dist/server/tools" cp "./tsconfig.json" "./dist" diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json new file mode 100644 index 000000000..0d9716f2d --- /dev/null +++ b/scripts/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "../dist/scripts", + }, + "references": [ + { "path": "../shared" }, + { "path": "../server" } + ] +} diff --git a/server/controllers/api/users/token.ts b/server/controllers/api/users/token.ts index 1d4004ce0..42f4f6096 100644 --- a/server/controllers/api/users/token.ts +++ b/server/controllers/api/users/token.ts @@ -1,7 +1,7 @@ import express from 'express' import RateLimit from 'express-rate-limit' import { logger } from '@server/helpers/logger' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { CONFIG } from '@server/initializers/config' import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' import { handleOAuthToken } from '@server/lib/auth/oauth' diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts index 8b7a76718..f8f07b1c6 100644 --- a/server/controllers/api/video-playlist.ts +++ b/server/controllers/api/video-playlist.ts @@ -1,6 +1,6 @@ import express from 'express' import { join } from 'path' -import { uuidToShort } from '@server/helpers/uuid' +import { uuidToShort } from '@shared/core-utils/uuid' import { scheduleRefreshIfNeeded } from '@server/lib/activitypub/playlists' import { Hooks } from '@server/lib/plugins/hooks' import { getServerActor } from '@server/models/application/application' diff --git a/server/controllers/api/videos/live.ts b/server/controllers/api/videos/live.ts index 3e1480cf2..e466d041b 100644 --- a/server/controllers/api/videos/live.ts +++ b/server/controllers/api/videos/live.ts @@ -1,6 +1,6 @@ import express from 'express' import { createReqFiles } from '@server/helpers/express-utils' -import { buildUUID, uuidToShort } from '@server/helpers/uuid' +import { buildUUID, uuidToShort } from '@shared/core-utils/uuid' import { CONFIG } from '@server/initializers/config' import { ASSETS_PATH, MIMETYPES } from '@server/initializers/constants' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 1be87f746..a4d0f980f 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -1,9 +1,9 @@ import express from 'express' import { move } from 'fs-extra' import { basename } from 'path' -import { getLowercaseExtension } from '@server/helpers/core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { getResumableUploadPath } from '@server/helpers/upload' -import { uuidToShort } from '@server/helpers/uuid' +import { uuidToShort } from '@shared/core-utils/uuid' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' import { generateWebTorrentVideoFilename } from '@server/lib/paths' diff --git a/server/controllers/client.ts b/server/controllers/client.ts index 2157ae533..8a56f2f75 100644 --- a/server/controllers/client.ts +++ b/server/controllers/client.ts @@ -7,7 +7,7 @@ import { CONFIG } from '@server/initializers/config' import { Hooks } from '@server/lib/plugins/hooks' import { buildFileLocale, getCompleteLocale, is18nLocale, LOCALE_FILES } from '@shared/core-utils/i18n' import { HttpStatusCode } from '@shared/models' -import { root } from '../helpers/core-utils' +import { root } from '@shared/core-utils' import { STATIC_MAX_AGE } from '../initializers/constants' import { ClientHtml, sendHTML, serveIndexHTML } from '../lib/client-html' import { asyncMiddleware, embedCSP } from '../middlewares' diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 0d94cac9b..87bceba7a 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -5,7 +5,7 @@ import { serveIndexHTML } from '@server/lib/client-html' import { ServerConfigManager } from '@server/lib/server-config-manager' import { HttpStatusCode } from '@shared/models' import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo/nodeinfo.model' -import { root } from '../helpers/core-utils' +import { root } from '@shared/core-utils' import { CONFIG, isEmailEnabled } from '../initializers/config' import { CONSTRAINTS_FIELDS, diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 2cbf0f8fe..531ccfba9 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -6,9 +6,8 @@ */ import { exec, ExecOptions } from 'child_process' -import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' +import { randomBytes } from 'crypto' import { truncate } from 'lodash' -import { basename, extname, isAbsolute, join, resolve } from 'path' import { createPrivateKey as createPrivateKey_1, getPublicKey as getPublicKey_1 } from 'pem' import { pipeline } from 'stream' import { URL } from 'url' @@ -159,34 +158,6 @@ function getAppNumber () { // --------------------------------------------------------------------------- -let rootPath: string - -function root () { - if (rootPath) return rootPath - - rootPath = __dirname - - if (basename(rootPath) === 'helpers') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'server') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..') - - return rootPath -} - -function buildPath (path: string) { - if (isAbsolute(path)) return path - - return join(root(), path) -} - -function getLowercaseExtension (filename: string) { - const ext = extname(filename) || '' - - return ext.toLowerCase() -} - -// --------------------------------------------------------------------------- - // Consistent with .length, lodash truncate function is not function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { const truncatedStr = truncate(str, options) @@ -221,16 +192,6 @@ function parseSemVersion (s: string) { // --------------------------------------------------------------------------- -function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { - return createHash('sha256').update(str).digest(encoding) -} - -function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { - return createHash('sha1').update(str).digest(encoding) -} - -// --------------------------------------------------------------------------- - function execShell (command: string, options?: ExecOptions) { return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { exec(command, options, (err, stdout, stderr) => { @@ -298,9 +259,6 @@ export { objectConverter, mapToJSON, - root, - buildPath, - getLowercaseExtension, sanitizeUrl, sanitizeHost, @@ -309,9 +267,6 @@ export { pageToStartAndCount, peertubeTruncate, - sha256, - sha1, - promisify0, promisify1, promisify2, diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index c19a3e5eb..eaabdbbea 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -2,7 +2,7 @@ import 'multer' import { UploadFilesForCheck } from 'express' import { sep } from 'path' import validator from 'validator' -import { isShortUUID, shortToUUID } from '../uuid' +import { isShortUUID, shortToUUID } from '@shared/core-utils/uuid' function exists (value: any) { return value !== undefined && value !== null diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index 7b81ed71b..780fd6345 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -1,9 +1,9 @@ -import express from 'express' +import express, { RequestHandler } from 'express' import multer, { diskStorage } from 'multer' import { HttpStatusCode } from '../../shared/models/http/http-error-codes' import { CONFIG } from '../initializers/config' import { REMOTE_SCHEME } from '../initializers/constants' -import { getLowercaseExtension } from './core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { isArray } from './custom-validators/misc' import { logger } from './logger' import { deleteFileAndCatch, generateRandomString } from './utils' @@ -69,7 +69,7 @@ function createReqFiles ( fieldNames: string[], mimeTypes: { [id: string]: string | string[] }, destinations: { [fieldName: string]: string } -) { +): RequestHandler { const storage = diskStorage({ destination: (req, file, cb) => { cb(null, destinations[file.fieldname]) diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts index e15628e2a..595112bce 100644 --- a/server/helpers/ffprobe-utils.ts +++ b/server/helpers/ffprobe-utils.ts @@ -1,9 +1,22 @@ -import { ffprobe, FfprobeData } from 'fluent-ffmpeg' +import { FfprobeData } from 'fluent-ffmpeg' import { getMaxBitrate } from '@shared/core-utils' -import { VideoFileMetadata, VideoResolution, VideoTranscodingFPS } from '../../shared/models/videos' +import { VideoResolution, VideoTranscodingFPS } from '../../shared/models/videos' import { CONFIG } from '../initializers/config' import { VIDEO_TRANSCODING_FPS } from '../initializers/constants' import { logger } from './logger' +import { + canDoQuickAudioTranscode, + ffprobePromise, + getDurationFromVideoFile, + getAudioStream, + getMaxAudioBitrate, + getMetadataFromFile, + getVideoFileBitrate, + getVideoFileFPS, + getVideoFileResolution, + getVideoStreamFromFile, + getVideoStreamSize +} from '@shared/extra-utils/ffprobe' /** * @@ -11,79 +24,6 @@ import { logger } from './logger' * */ -function ffprobePromise (path: string) { - return new Promise((res, rej) => { - ffprobe(path, (err, data) => { - if (err) return rej(err) - - return res(data) - }) - }) -} - -async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) { - // without position, ffprobe considers the last input only - // we make it consider the first input only - // if you pass a file path to pos, then ffprobe acts on that file directly - const data = existingProbe || await ffprobePromise(videoPath) - - if (Array.isArray(data.streams)) { - const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio') - - if (audioStream) { - return { - absolutePath: data.format.filename, - audioStream, - bitrate: parseInt(audioStream['bit_rate'] + '', 10) - } - } - } - - return { absolutePath: data.format.filename } -} - -function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) { - const maxKBitrate = 384 - const kToBits = (kbits: number) => kbits * 1000 - - // If we did not manage to get the bitrate, use an average value - if (!bitrate) return 256 - - if (type === 'aac') { - switch (true) { - case bitrate > kToBits(maxKBitrate): - return maxKBitrate - - default: - return -1 // we interpret it as a signal to copy the audio stream as is - } - } - - /* - a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. - That's why, when using aac, we can go to lower kbit/sec. The equivalences - made here are not made to be accurate, especially with good mp3 encoders. - */ - switch (true) { - case bitrate <= kToBits(192): - return 128 - - case bitrate <= kToBits(384): - return 256 - - default: - return maxKBitrate - } -} - -async function getVideoStreamSize (path: string, existingProbe?: FfprobeData): Promise<{ width: number, height: number }> { - const videoStream = await getVideoStreamFromFile(path, existingProbe) - - return videoStream === null - ? { width: 0, height: 0 } - : { width: videoStream.width, height: videoStream.height } -} - async function getVideoStreamCodec (path: string) { const videoStream = await getVideoStreamFromFile(path) @@ -143,69 +83,6 @@ async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { return 'mp4a.40.2' // Fallback } -async function getVideoFileResolution (path: string, existingProbe?: FfprobeData) { - const size = await getVideoStreamSize(path, existingProbe) - - return { - width: size.width, - height: size.height, - ratio: Math.max(size.height, size.width) / Math.min(size.height, size.width), - resolution: Math.min(size.height, size.width), - isPortraitMode: size.height > size.width - } -} - -async function getVideoFileFPS (path: string, existingProbe?: FfprobeData) { - const videoStream = await getVideoStreamFromFile(path, existingProbe) - if (videoStream === null) return 0 - - for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { - const valuesText: string = videoStream[key] - if (!valuesText) continue - - const [ frames, seconds ] = valuesText.split('/') - if (!frames || !seconds) continue - - const result = parseInt(frames, 10) / parseInt(seconds, 10) - if (result > 0) return Math.round(result) - } - - return 0 -} - -async function getMetadataFromFile (path: string, existingProbe?: FfprobeData) { - const metadata = existingProbe || await ffprobePromise(path) - - return new VideoFileMetadata(metadata) -} - -async function getVideoFileBitrate (path: string, existingProbe?: FfprobeData): Promise { - const metadata = await getMetadataFromFile(path, existingProbe) - - let bitrate = metadata.format.bit_rate as number - if (bitrate && !isNaN(bitrate)) return bitrate - - const videoStream = await getVideoStreamFromFile(path, existingProbe) - if (!videoStream) return undefined - - bitrate = videoStream?.bit_rate - if (bitrate && !isNaN(bitrate)) return bitrate - - return undefined -} - -async function getDurationFromVideoFile (path: string, existingProbe?: FfprobeData) { - const metadata = await getMetadataFromFile(path, existingProbe) - - return Math.round(metadata.format.duration) -} - -async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData) { - const metadata = await getMetadataFromFile(path, existingProbe) - - return metadata.streams.find(s => s.codec_type === 'video') || null -} - function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') { const configResolutions = type === 'vod' ? CONFIG.TRANSCODING.RESOLUTIONS @@ -263,26 +140,6 @@ async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Pro return true } -async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { - const parsedAudio = await getAudioStream(path, probe) - - if (!parsedAudio.audioStream) return true - - if (parsedAudio.audioStream['codec_name'] !== 'aac') return false - - const audioBitrate = parsedAudio.bitrate - if (!audioBitrate) return false - - const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) - if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false - - const channelLayout = parsedAudio.audioStream['channel_layout'] - // Causes playback issues with Chrome - if (!channelLayout || channelLayout === 'unknown') return false - - return true -} - function getClosestFramerateStandard > (fps: number, type: K) { return VIDEO_TRANSCODING_FPS[type].slice(0) .sort((a, b) => fps % a - fps % b)[0] diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index 4305584d5..ced288045 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -1,9 +1,9 @@ import { copy, readFile, remove, rename } from 'fs-extra' import Jimp, { read } from 'jimp' -import { getLowercaseExtension } from './core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { convertWebPToJPG, processGIF } from './ffmpeg-utils' import { logger } from './logger' -import { buildUUID } from './uuid' +import { buildUUID } from '@shared/core-utils/uuid' function generateImageFilename (extension = '.jpg') { return buildUUID() + extension diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 44d90d9f1..31705e7fa 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts @@ -4,7 +4,8 @@ import { Request } from 'express' import { cloneDeep } from 'lodash' import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' import { MActor } from '../types/models' -import { createPrivateKey, getPublicKey, promisify1, promisify2, sha256 } from './core-utils' +import { sha256 } from '@shared/core-utils/crypto' +import { createPrivateKey, getPublicKey, promisify1, promisify2 } from './core-utils' import { jsonld } from './custom-jsonld-signature' import { logger } from './logger' diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 6c95a43b6..882f808ab 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -3,7 +3,8 @@ import { Instance as ParseTorrent } from 'parse-torrent' import { join } from 'path' import { ResultList } from '../../shared' import { CONFIG } from '../initializers/config' -import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' +import { sha256 } from '@shared/core-utils/crypto' +import { execPromise, execPromise2, randomBytesPromise } from './core-utils' import { logger } from './logger' function deleteFileAndCatch (path: string) { diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index ecc703646..67cb3971d 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -14,7 +14,8 @@ import { MVideo } from '@server/types/models/video/video' import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' import { CONFIG } from '../initializers/config' -import { promisify2, sha1 } from './core-utils' +import { promisify2 } from './core-utils' +import { sha1 } from '@shared/core-utils/crypto' import { logger } from './logger' import { generateVideoImportTmpPath } from './utils' import { extractVideo } from './video' diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 70179d25c..e3e8c426e 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -6,7 +6,8 @@ import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-red import { BroadcastMessageLevel } from '@shared/models/server' import { VideoPrivacy, VideosRedundancyStrategy } from '../../shared/models' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' -import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' +import { buildPath, root } from '../../shared/core-utils' +import { parseBytes, parseDurationToMs } from '../helpers/core-utils' // Use a variable to reload the configuration if we need let config: IConfig = require('config') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 70b8e3d27..026c715c2 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -18,8 +18,9 @@ import { FollowState } from '../../shared/models/actors' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model' import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model' +import { root } from '../../shared/core-utils' // Do not use barrels, remain constants as independent as possible -import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' +import { isTestInstance, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0080-video-channels.ts b/server/initializers/migrations/0080-video-channels.ts index 0e6952350..82971c9f5 100644 --- a/server/initializers/migrations/0080-video-channels.ts +++ b/server/initializers/migrations/0080-video-channels.ts @@ -1,4 +1,4 @@ -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import * as Sequelize from 'sequelize' async function up (utils: { diff --git a/server/initializers/migrations/0345-video-playlists.ts b/server/initializers/migrations/0345-video-playlists.ts index 8dd631dff..5cc52e7ee 100644 --- a/server/initializers/migrations/0345-video-playlists.ts +++ b/server/initializers/migrations/0345-video-playlists.ts @@ -1,5 +1,5 @@ import * as Sequelize from 'sequelize' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { VideoPlaylistPrivacy, VideoPlaylistType } from '../../../shared/models/videos' import { WEBSERVER } from '../constants' diff --git a/server/initializers/migrations/0560-user-feed-token.ts b/server/initializers/migrations/0560-user-feed-token.ts index 042301352..961777e35 100644 --- a/server/initializers/migrations/0560-user-feed-token.ts +++ b/server/initializers/migrations/0560-user-feed-token.ts @@ -1,5 +1,5 @@ import * as Sequelize from 'sequelize' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' async function up (utils: { transaction: Sequelize.Transaction diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts index 1612b3ad0..1ad89ac56 100644 --- a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts @@ -1,6 +1,6 @@ -import { getLowercaseExtension } from '@server/helpers/core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { MIMETYPES } from '@server/initializers/constants' import { ActorModel } from '@server/models/actor/actor' import { FilteredModelAttributes } from '@server/types' diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts index 497773536..47bc8c055 100644 --- a/server/lib/auth/oauth.ts +++ b/server/lib/auth/oauth.ts @@ -8,7 +8,8 @@ import { UnauthorizedClientError, UnsupportedGrantTypeError } from 'oauth2-server' -import { randomBytesPromise, sha1 } from '@server/helpers/core-utils' +import { sha1 } from '@shared/core-utils/crypto' +import { randomBytesPromise } from '@server/helpers/core-utils' import { MOAuthClient } from '@server/types/models' import { OAUTH_LIFETIME } from '../../initializers/constants' import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index adc3d712e..dee7ca8ed 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts @@ -8,7 +8,8 @@ import { HTMLServerConfig } from '@shared/models' import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' import { HttpStatusCode } from '../../shared/models/http/http-error-codes' import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' -import { isTestInstance, sha256 } from '../helpers/core-utils' +import { isTestInstance } from '../helpers/core-utils' +import { sha256 } from '@shared/core-utils/crypto' import { logger } from '../helpers/logger' import { mdToPlainText } from '../helpers/markdown' import { CONFIG } from '../initializers/config' diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index 60284ea28..ebad43650 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -4,7 +4,8 @@ import { createTransport, Transporter } from 'nodemailer' import { join } from 'path' import { EmailPayload } from '@shared/models' import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model' -import { isTestInstance, root } from '../helpers/core-utils' +import { isTestInstance } from '../helpers/core-utils' +import { root } from '@shared/core-utils' import { bunyanLogger, logger } from '../helpers/logger' import { CONFIG, isEmailEnabled } from '../initializers/config' import { WEBSERVER } from '../initializers/constants' diff --git a/server/lib/hls.ts b/server/lib/hls.ts index f2fe893a9..220b7733b 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -2,7 +2,7 @@ import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, import { flatten, uniq } from 'lodash' import { basename, dirname, join } from 'path' import { MStreamingPlaylistFilesVideo, MVideo, MVideoUUID } from '@server/types/models' -import { sha256 } from '../helpers/core-utils' +import { sha256 } from '@shared/core-utils/crypto' import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils' import { logger } from '../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index a91c2ef80..0d9e80cb8 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -1,6 +1,6 @@ import { Job } from 'bull' import { copy, stat } from 'fs-extra' -import { getLowercaseExtension } from '@server/helpers/core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { CONFIG } from '@server/initializers/config' import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 4ce1a6c30..e5730e746 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -1,6 +1,6 @@ import { Job } from 'bull' import { move, remove, stat } from 'fs-extra' -import { getLowercaseExtension } from '@server/helpers/core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { retryTransactionWrapper } from '@server/helpers/database-utils' import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' import { isPostImportVideoAccepted } from '@server/lib/moderation' diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts index 821a92b91..572696f2a 100644 --- a/server/lib/local-actor.ts +++ b/server/lib/local-actor.ts @@ -2,8 +2,8 @@ import 'multer' import { queue } from 'async' import LRUCache from 'lru-cache' import { join } from 'path' -import { getLowercaseExtension } from '@server/helpers/core-utils' -import { buildUUID } from '@server/helpers/uuid' +import { getLowercaseExtension } from '@shared/core-utils' +import { buildUUID } from '@shared/core-utils/uuid' import { ActorModel } from '@server/models/actor/actor' import { ActivityPubActorType, ActorImageType } from '@shared/models' import { retryTransactionWrapper } from '../helpers/database-utils' diff --git a/server/lib/paths.ts b/server/lib/paths.ts index 434e637c6..d8cf812e3 100644 --- a/server/lib/paths.ts +++ b/server/lib/paths.ts @@ -1,5 +1,5 @@ import { join } from 'path' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { CONFIG } from '@server/initializers/config' import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' diff --git a/server/lib/user.ts b/server/lib/user.ts index 936403692..230bf37d0 100644 --- a/server/lib/user.ts +++ b/server/lib/user.ts @@ -1,5 +1,5 @@ import { Transaction } from 'sequelize/types' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { UserModel } from '@server/models/user/user' import { MActorDefault } from '@server/types/models/actor' import { ActivityPubActorType } from '../../shared/models/activitypub' diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts index 27058005c..429b36df9 100644 --- a/server/lib/video-path-manager.ts +++ b/server/lib/video-path-manager.ts @@ -1,6 +1,6 @@ import { remove } from 'fs-extra' import { extname, join } from 'path' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { extractVideo } from '@server/helpers/video' import { CONFIG } from '@server/initializers/config' import { diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts index 8df49951d..cbd3f0e4a 100644 --- a/server/models/actor/actor.ts +++ b/server/models/actor/actor.ts @@ -16,9 +16,8 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { getLowercaseExtension } from '@server/helpers/core-utils' import { ModelCache } from '@server/models/model-cache' -import { AttributesOnly } from '@shared/core-utils' +import { getLowercaseExtension, AttributesOnly } from '@shared/core-utils' import { ActivityIconObject, ActivityPubActorType } from '../../../shared/models/activitypub' import { ActorImage } from '../../../shared/models/actors/actor-image.model' import { activityPubContextify } from '../../helpers/activitypub' diff --git a/server/models/user/user-notification.ts b/server/models/user/user-notification.ts index 04c5513a9..55d65d6b2 100644 --- a/server/models/user/user-notification.ts +++ b/server/models/user/user-notification.ts @@ -1,6 +1,6 @@ import { FindOptions, ModelIndexesOptions, Op, WhereOptions } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' -import { uuidToShort } from '@server/helpers/uuid' +import { uuidToShort } from '@shared/core-utils/uuid' import { UserNotificationIncludes, UserNotificationModelForApi } from '@server/types/models/user' import { AttributesOnly } from '@shared/core-utils' import { UserNotification, UserNotificationType } from '../../../shared' diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index fd4da68ed..f6c750ccf 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -1,4 +1,4 @@ -import { uuidToShort } from '@server/helpers/uuid' +import { uuidToShort } from '@shared/core-utils/uuid' import { generateMagnetUri } from '@server/helpers/webtorrent' import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' import { VideoViews } from '@server/lib/video-views' diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index d24be56c3..590e72e52 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -15,7 +15,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionVideo } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 630684a88..d0c73cbd1 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -17,7 +17,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { buildUUID, uuidToShort } from '@server/helpers/uuid' +import { buildUUID, uuidToShort } from '@shared/core-utils/uuid' import { MAccountId, MChannelId } from '@server/types/models' import { AttributesOnly, buildPlaylistEmbedPath, buildPlaylistWatchPath, pick } from '@shared/core-utils' import { ActivityIconObject } from '../../../shared/models/activitypub/objects' diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index e36852cad..23b3fbcbe 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -21,7 +21,7 @@ import { MStreamingPlaylist, MVideo } from '@server/types/models' import { AttributesOnly } from '@shared/core-utils' import { VideoStorage } from '@shared/models' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' -import { sha1 } from '../../helpers/core-utils' +import { sha1 } from '@shared/core-utils/crypto' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { isArrayOf } from '../../helpers/custom-validators/misc' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 1050463d2..efd4d8462 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -25,7 +25,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { buildNSFWFilter } from '@server/helpers/express-utils' -import { uuidToShort } from '@server/helpers/uuid' +import { uuidToShort } from '@shared/core-utils/uuid' import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' import { LiveManager } from '@server/lib/live/live-manager' import { removeHLSObjectStorage, removeWebTorrentObjectStorage } from '@server/lib/object-storage' diff --git a/server/tests/api/activitypub/security.ts b/server/tests/api/activitypub/security.ts index 94d946563..2700cff13 100644 --- a/server/tests/api/activitypub/security.ts +++ b/server/tests/api/activitypub/security.ts @@ -7,7 +7,7 @@ import { buildDigest } from '@server/helpers/peertube-crypto' import { HTTP_SIGNATURE } from '@server/initializers/constants' import { buildGlobalHeaders } from '@server/lib/job-queue/handlers/utils/activitypub-http-utils' import { buildAbsoluteFixturePath, cleanupTests, createMultipleServers, killallServers, PeerTubeServer, wait } from '@shared/extra-utils' -import { makeFollowRequest, makePOSTAPRequest } from '@shared/extra-utils/requests/activitypub' +import { makeFollowRequest, makePOSTAPRequest } from '@server/tests/shared' import { HttpStatusCode } from '@shared/models' const expect = chai.expect diff --git a/server/tests/api/notifications/moderation-notifications.ts b/server/tests/api/notifications/moderation-notifications.ts index 81ce8061b..6d8e5359b 100644 --- a/server/tests/api/notifications/moderation-notifications.ts +++ b/server/tests/api/notifications/moderation-notifications.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { checkAbuseStateChange, checkAutoInstanceFollowing, diff --git a/server/tests/api/notifications/user-notifications.ts b/server/tests/api/notifications/user-notifications.ts index 9af20843e..6db0347cc 100644 --- a/server/tests/api/notifications/user-notifications.ts +++ b/server/tests/api/notifications/user-notifications.ts @@ -2,7 +2,7 @@ import 'mocha' import * as chai from 'chai' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { CheckerBaseParams, checkMyVideoImportIsFinished, diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts index 832ba561a..748f4cd35 100644 --- a/server/tests/api/server/follows.ts +++ b/server/tests/api/server/follows.ts @@ -4,7 +4,6 @@ import 'mocha' import * as chai from 'chai' import { cleanupTests, - completeVideoCheck, createMultipleServers, dateIsValid, expectAccountFollows, @@ -15,6 +14,7 @@ import { waitJobs } from '@shared/extra-utils' import { VideoCreateResult, VideoPrivacy } from '@shared/models' +import { completeVideoCheck } from '@server/tests/shared/video' const expect = chai.expect diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index fa1da8fe0..2d059c0ed 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -5,7 +5,6 @@ import * as chai from 'chai' import { cleanupTests, CommentsCommand, - completeVideoCheck, createMultipleServers, killallServers, PeerTubeServer, @@ -14,6 +13,7 @@ import { waitJobs } from '@shared/extra-utils' import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models' +import { completeVideoCheck } from '@server/tests/shared/video' const expect = chai.expect diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index c6c279064..9e7b39cfa 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -8,7 +8,6 @@ import { checkTmpIsEmpty, checkVideoFilesWereRemoved, cleanupTests, - completeVideoCheck, createMultipleServers, dateIsValid, doubleFollow, @@ -21,6 +20,7 @@ import { webtorrentAdd } from '@shared/extra-utils' import { HttpStatusCode, VideoCommentThreadTree, VideoPrivacy } from '@shared/models' +import { completeVideoCheck } from '@server/tests/shared/video' const expect = chai.expect diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index a0e4a156c..100067f18 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -5,7 +5,6 @@ import * as chai from 'chai' import { checkVideoFilesWereRemoved, cleanupTests, - completeVideoCheck, createSingleServer, PeerTubeServer, setAccessTokensToServers, @@ -13,6 +12,7 @@ import { wait } from '@shared/extra-utils' import { Video, VideoPrivacy } from '@shared/models' +import { completeVideoCheck } from '@server/tests/shared/video' const expect = chai.expect diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 1c0282da9..4b4f75527 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts @@ -4,7 +4,7 @@ import 'mocha' import * as chai from 'chai' import { createFile, readdir } from 'fs-extra' import { join } from 'path' -import { buildUUID } from '@server/helpers/uuid' +import { buildUUID } from '@shared/core-utils/uuid' import { cleanupTests, CLICommand, diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts new file mode 100644 index 000000000..938817268 --- /dev/null +++ b/server/tests/shared/index.ts @@ -0,0 +1,2 @@ +export * from './requests' +export * from './video' diff --git a/shared/extra-utils/requests/activitypub.ts b/server/tests/shared/requests.ts similarity index 54% rename from shared/extra-utils/requests/activitypub.ts rename to server/tests/shared/requests.ts index 4ae878384..9eb596029 100644 --- a/shared/extra-utils/requests/activitypub.ts +++ b/server/tests/shared/requests.ts @@ -1,9 +1,9 @@ -import { activityPubContextify } from '../../../server/helpers/activitypub' -import { doRequest } from '../../../server/helpers/requests' -import { HTTP_SIGNATURE } from '../../../server/initializers/constants' -import { buildGlobalHeaders } from '../../../server/lib/job-queue/handlers/utils/activitypub-http-utils' +import { doRequest } from '@server/helpers/requests' +import { activityPubContextify } from '@server/helpers/activitypub' +import { HTTP_SIGNATURE } from '@server/initializers/constants' +import { buildGlobalHeaders } from '@server/lib/job-queue/handlers/utils/activitypub-http-utils' -function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) { +export function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) { const options = { method: 'POST' as 'POST', json: body, @@ -14,7 +14,7 @@ function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: return doRequest(url, options) } -async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) { +export async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) { const follow = { type: 'Follow', id: by.url + '/' + new Date().getTime(), @@ -35,8 +35,3 @@ async function makeFollowRequest (to: { url: string }, by: { url: string, privat return makePOSTAPRequest(to.url + '/inbox', body, httpSignature, headers) } - -export { - makePOSTAPRequest, - makeFollowRequest -} diff --git a/server/tests/shared/video.ts b/server/tests/shared/video.ts new file mode 100644 index 000000000..0e6a00f5c --- /dev/null +++ b/server/tests/shared/video.ts @@ -0,0 +1,148 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions */ +import { dateIsValid, makeRawRequest, PeerTubeServer, testImage, webtorrentAdd } from '@shared/extra-utils' +import { expect } from 'chai' +import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '@server/initializers/constants' +import { getLowercaseExtension, uuidRegex } from '@shared/core-utils' + +export async function completeVideoCheck ( + server: PeerTubeServer, + video: any, + attributes: { + name: string + category: number + licence: number + language: string + nsfw: boolean + commentsEnabled: boolean + downloadEnabled: boolean + description: string + publishedAt?: string + support: string + originallyPublishedAt?: string + account: { + name: string + host: string + } + isLocal: boolean + tags: string[] + privacy: number + likes?: number + dislikes?: number + duration: number + channel: { + displayName: string + name: string + description: string + isLocal: boolean + } + fixture: string + files: { + resolution: number + size: number + }[] + thumbnailfile?: string + previewfile?: string + } +) { + if (!attributes.likes) attributes.likes = 0 + if (!attributes.dislikes) attributes.dislikes = 0 + + const host = new URL(server.url).host + const originHost = attributes.account.host + + expect(video.name).to.equal(attributes.name) + expect(video.category.id).to.equal(attributes.category) + expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Misc') + expect(video.licence.id).to.equal(attributes.licence) + expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown') + expect(video.language.id).to.equal(attributes.language) + expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown') + expect(video.privacy.id).to.deep.equal(attributes.privacy) + expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy]) + expect(video.nsfw).to.equal(attributes.nsfw) + expect(video.description).to.equal(attributes.description) + expect(video.account.id).to.be.a('number') + expect(video.account.host).to.equal(attributes.account.host) + expect(video.account.name).to.equal(attributes.account.name) + expect(video.channel.displayName).to.equal(attributes.channel.displayName) + expect(video.channel.name).to.equal(attributes.channel.name) + expect(video.likes).to.equal(attributes.likes) + expect(video.dislikes).to.equal(attributes.dislikes) + expect(video.isLocal).to.equal(attributes.isLocal) + expect(video.duration).to.equal(attributes.duration) + expect(video.url).to.contain(originHost) + expect(dateIsValid(video.createdAt)).to.be.true + expect(dateIsValid(video.publishedAt)).to.be.true + expect(dateIsValid(video.updatedAt)).to.be.true + + if (attributes.publishedAt) { + expect(video.publishedAt).to.equal(attributes.publishedAt) + } + + if (attributes.originallyPublishedAt) { + expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt) + } else { + expect(video.originallyPublishedAt).to.be.null + } + + const videoDetails = await server.videos.get({ id: video.uuid }) + + expect(videoDetails.files).to.have.lengthOf(attributes.files.length) + expect(videoDetails.tags).to.deep.equal(attributes.tags) + expect(videoDetails.account.name).to.equal(attributes.account.name) + expect(videoDetails.account.host).to.equal(attributes.account.host) + expect(video.channel.displayName).to.equal(attributes.channel.displayName) + expect(video.channel.name).to.equal(attributes.channel.name) + expect(videoDetails.channel.host).to.equal(attributes.account.host) + expect(videoDetails.channel.isLocal).to.equal(attributes.channel.isLocal) + expect(dateIsValid(videoDetails.channel.createdAt.toString())).to.be.true + expect(dateIsValid(videoDetails.channel.updatedAt.toString())).to.be.true + expect(videoDetails.commentsEnabled).to.equal(attributes.commentsEnabled) + expect(videoDetails.downloadEnabled).to.equal(attributes.downloadEnabled) + + for (const attributeFile of attributes.files) { + const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution) + expect(file).not.to.be.undefined + + let extension = getLowercaseExtension(attributes.fixture) + // Transcoding enabled: extension will always be .mp4 + if (attributes.files.length > 1) extension = '.mp4' + + expect(file.magnetUri).to.have.lengthOf.above(2) + + expect(file.torrentDownloadUrl).to.match(new RegExp(`http://${host}/download/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) + expect(file.torrentUrl).to.match(new RegExp(`http://${host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) + + expect(file.fileUrl).to.match(new RegExp(`http://${originHost}/static/webseed/${uuidRegex}-${file.resolution.id}${extension}`)) + expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) + + await Promise.all([ + makeRawRequest(file.torrentUrl, 200), + makeRawRequest(file.torrentDownloadUrl, 200), + makeRawRequest(file.metadataUrl, 200) + ]) + + expect(file.resolution.id).to.equal(attributeFile.resolution) + expect(file.resolution.label).to.equal(attributeFile.resolution + 'p') + + const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) + const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) + expect( + file.size, + 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')' + ).to.be.above(minSize).and.below(maxSize) + + const torrent = await webtorrentAdd(file.magnetUri, true) + expect(torrent.files).to.be.an('array') + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + } + + expect(videoDetails.thumbnailPath).to.exist + await testImage(server.url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) + + if (attributes.previewfile) { + expect(videoDetails.previewPath).to.exist + await testImage(server.url, attributes.previewfile, videoDetails.previewPath) + } +} diff --git a/server/tools/cli.ts b/server/tools/cli.ts index 52e6ea593..7c763734f 100644 --- a/server/tools/cli.ts +++ b/server/tools/cli.ts @@ -5,7 +5,9 @@ import { createLogger, format, transports } from 'winston' import { PeerTubeServer } from '@shared/extra-utils' import { UserRole } from '@shared/models' import { VideoPrivacy } from '../../shared/models/videos' -import { getAppNumber, isTestInstance, root } from '../helpers/core-utils' +import { getAppNumber, isTestInstance } from '../helpers/core-utils' +import { root } from '@shared/core-utils' +import { loadLanguages } from '@server/initializers/constants' let configName = 'PeerTube/CLI' if (isTestInstance()) configName += `-${getAppNumber()}` @@ -180,6 +182,7 @@ function getServerCredentials (program: Command) { } function buildServer (url: string) { + loadLanguages() return new PeerTubeServer({ url }) } diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts index a758beef9..223bf7f1b 100644 --- a/server/tools/peertube-import-videos.ts +++ b/server/tools/peertube-import-videos.ts @@ -5,7 +5,7 @@ import { program } from 'commander' import { accessSync, constants } from 'fs' import { remove } from 'fs-extra' import { join } from 'path' -import { sha256 } from '../helpers/core-utils' +import { sha256 } from '@shared/core-utils/crypto' import { doRequestAndSaveToFile } from '../helpers/requests' import { assignToken, diff --git a/server/tools/tsconfig.json b/server/tools/tsconfig.json index 156a8ed22..575133ec8 100644 --- a/server/tools/tsconfig.json +++ b/server/tools/tsconfig.json @@ -1,5 +1,11 @@ { "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/server/tools" + }, "include": [ ".", "../typings" ], + "references": [ + { "path": "../" }, + ], "exclude": [ ] // Overwrite exclude property } diff --git a/server/tsconfig.json b/server/tsconfig.json new file mode 100644 index 000000000..4be7ae2f4 --- /dev/null +++ b/server/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "../dist/server" + }, + "references": [ + { "path": "../shared" } + ], + "exclude": [ + "tools/" + ] +} diff --git a/shared/core-utils/crypto.ts b/shared/core-utils/crypto.ts new file mode 100644 index 000000000..d6d1150d0 --- /dev/null +++ b/shared/core-utils/crypto.ts @@ -0,0 +1,14 @@ +import { BinaryToTextEncoding, createHash } from 'crypto' + +function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { + return createHash('sha256').update(str).digest(encoding) +} + +function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { + return createHash('sha1').update(str).digest(encoding) +} + +export { + sha256, + sha1 +} diff --git a/shared/core-utils/index.ts b/shared/core-utils/index.ts index e0a6a8087..ee5cd4412 100644 --- a/shared/core-utils/index.ts +++ b/shared/core-utils/index.ts @@ -1,8 +1,10 @@ export * from './abuse' export * from './common' export * from './i18n' +export * from './path' export * from './plugins' export * from './renderer' export * from './users' export * from './utils' export * from './videos' +export * from './uuid' diff --git a/shared/core-utils/path.ts b/shared/core-utils/path.ts new file mode 100644 index 000000000..b1a45d69b --- /dev/null +++ b/shared/core-utils/path.ts @@ -0,0 +1,34 @@ +import { basename, extname, isAbsolute, join, resolve } from 'path' + +let rootPath: string + +function root () { + if (rootPath) return rootPath + + rootPath = __dirname + + if (basename(rootPath) === 'core-utils') rootPath = resolve(rootPath, '..') + if (basename(rootPath) === 'shared') rootPath = resolve(rootPath, '..') + if (basename(rootPath) === 'server') rootPath = resolve(rootPath, '..') + if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..') + + return rootPath +} + +function buildPath (path: string) { + if (isAbsolute(path)) return path + + return join(root(), path) +} + +function getLowercaseExtension (filename: string) { + const ext = extname(filename) || '' + + return ext.toLowerCase() +} + +export { + root, + buildPath, + getLowercaseExtension +} diff --git a/server/helpers/uuid.ts b/shared/core-utils/uuid.ts similarity index 100% rename from server/helpers/uuid.ts rename to shared/core-utils/uuid.ts diff --git a/shared/extra-utils/ffprobe.ts b/shared/extra-utils/ffprobe.ts new file mode 100644 index 000000000..9257bbd5f --- /dev/null +++ b/shared/extra-utils/ffprobe.ts @@ -0,0 +1,180 @@ +import { ffprobe, FfprobeData } from 'fluent-ffmpeg' +import { VideoFileMetadata } from '@shared/models/videos' + +/** + * + * Helpers to run ffprobe and extract data from the JSON output + * + */ + +function ffprobePromise (path: string) { + return new Promise((res, rej) => { + ffprobe(path, (err, data) => { + if (err) return rej(err) + + return res(data) + }) + }) +} + +async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) { + // without position, ffprobe considers the last input only + // we make it consider the first input only + // if you pass a file path to pos, then ffprobe acts on that file directly + const data = existingProbe || await ffprobePromise(videoPath) + + if (Array.isArray(data.streams)) { + const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio') + + if (audioStream) { + return { + absolutePath: data.format.filename, + audioStream, + bitrate: parseInt(audioStream['bit_rate'] + '', 10) + } + } + } + + return { absolutePath: data.format.filename } +} + +function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) { + const maxKBitrate = 384 + const kToBits = (kbits: number) => kbits * 1000 + + // If we did not manage to get the bitrate, use an average value + if (!bitrate) return 256 + + if (type === 'aac') { + switch (true) { + case bitrate > kToBits(maxKBitrate): + return maxKBitrate + + default: + return -1 // we interpret it as a signal to copy the audio stream as is + } + } + + /* + a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. + That's why, when using aac, we can go to lower kbit/sec. The equivalences + made here are not made to be accurate, especially with good mp3 encoders. + */ + switch (true) { + case bitrate <= kToBits(192): + return 128 + + case bitrate <= kToBits(384): + return 256 + + default: + return maxKBitrate + } +} + +async function getVideoStreamSize (path: string, existingProbe?: FfprobeData): Promise<{ width: number, height: number }> { + const videoStream = await getVideoStreamFromFile(path, existingProbe) + + return videoStream === null + ? { width: 0, height: 0 } + : { width: videoStream.width, height: videoStream.height } +} + +async function getVideoFileResolution (path: string, existingProbe?: FfprobeData) { + const size = await getVideoStreamSize(path, existingProbe) + + return { + width: size.width, + height: size.height, + ratio: Math.max(size.height, size.width) / Math.min(size.height, size.width), + resolution: Math.min(size.height, size.width), + isPortraitMode: size.height > size.width + } +} + +async function getVideoFileFPS (path: string, existingProbe?: FfprobeData) { + const videoStream = await getVideoStreamFromFile(path, existingProbe) + if (videoStream === null) return 0 + + for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { + const valuesText: string = videoStream[key] + if (!valuesText) continue + + const [ frames, seconds ] = valuesText.split('/') + if (!frames || !seconds) continue + + const result = parseInt(frames, 10) / parseInt(seconds, 10) + if (result > 0) return Math.round(result) + } + + return 0 +} + +async function getMetadataFromFile (path: string, existingProbe?: FfprobeData) { + const metadata = existingProbe || await ffprobePromise(path) + + return new VideoFileMetadata(metadata) +} + +async function getVideoFileBitrate (path: string, existingProbe?: FfprobeData): Promise { + const metadata = await getMetadataFromFile(path, existingProbe) + + let bitrate = metadata.format.bit_rate as number + if (bitrate && !isNaN(bitrate)) return bitrate + + const videoStream = await getVideoStreamFromFile(path, existingProbe) + if (!videoStream) return undefined + + bitrate = videoStream?.bit_rate + if (bitrate && !isNaN(bitrate)) return bitrate + + return undefined +} + +async function getDurationFromVideoFile (path: string, existingProbe?: FfprobeData) { + const metadata = await getMetadataFromFile(path, existingProbe) + + return Math.round(metadata.format.duration) +} + +async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData) { + const metadata = await getMetadataFromFile(path, existingProbe) + + return metadata.streams.find(s => s.codec_type === 'video') || null +} + +async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { + const parsedAudio = await getAudioStream(path, probe) + + if (!parsedAudio.audioStream) return true + + if (parsedAudio.audioStream['codec_name'] !== 'aac') return false + + const audioBitrate = parsedAudio.bitrate + if (!audioBitrate) return false + + const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) + if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false + + const channelLayout = parsedAudio.audioStream['channel_layout'] + // Causes playback issues with Chrome + if (!channelLayout || channelLayout === 'unknown') return false + + return true +} + +// --------------------------------------------------------------------------- + +export { + getVideoStreamSize, + getVideoFileResolution, + getMetadataFromFile, + getMaxAudioBitrate, + getVideoStreamFromFile, + getDurationFromVideoFile, + getAudioStream, + getVideoFileFPS, + ffprobePromise, + getVideoFileBitrate, + canDoQuickAudioTranscode +} diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index b1be214b1..589928997 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { pathExists, readFile } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { HttpStatusCode } from '@shared/models' import { makeGetRequest } from '../requests' import { PeerTubeServer } from '../server' diff --git a/shared/extra-utils/miscs/generate.ts b/shared/extra-utils/miscs/generate.ts index 3b29c0ad4..93673a063 100644 --- a/shared/extra-utils/miscs/generate.ts +++ b/shared/extra-utils/miscs/generate.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import ffmpeg from 'fluent-ffmpeg' import { ensureDir, pathExists } from 'fs-extra' import { dirname } from 'path' -import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' +import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@shared/extra-utils/ffprobe' import { getMaxBitrate } from '@shared/core-utils' import { buildAbsoluteFixturePath } from './tests' diff --git a/shared/extra-utils/server/directories.ts b/shared/extra-utils/server/directories.ts index b6465cbf4..e6f72d6fc 100644 --- a/shared/extra-utils/server/directories.ts +++ b/shared/extra-utils/server/directories.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { PeerTubeServer } from './server' async function checkTmpIsEmpty (server: PeerTubeServer) { diff --git a/shared/extra-utils/server/plugins-command.ts b/shared/extra-utils/server/plugins-command.ts index 9bf24afff..1c44711da 100644 --- a/shared/extra-utils/server/plugins-command.ts +++ b/shared/extra-utils/server/plugins-command.ts @@ -2,7 +2,7 @@ import { readJSON, writeJSON } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { HttpStatusCode, PeerTubePlugin, diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 9da293877..339b9cabb 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts @@ -1,8 +1,7 @@ import { ChildProcess, fork } from 'child_process' import { copy } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' -import { randomInt } from '@shared/core-utils' +import { root, randomInt } from '@shared/core-utils' import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '../../models/videos' import { BulkCommand } from '../bulk' import { CLICommand } from '../cli' diff --git a/shared/extra-utils/server/servers-command.ts b/shared/extra-utils/server/servers-command.ts index 776d2123c..47420c95f 100644 --- a/shared/extra-utils/server/servers-command.ts +++ b/shared/extra-utils/server/servers-command.ts @@ -1,7 +1,7 @@ import { exec } from 'child_process' import { copy, ensureDir, readFile, remove } from 'fs-extra' import { basename, join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { HttpStatusCode } from '@shared/models' import { getFileSize, isGithubCI, wait } from '../miscs' import { AbstractCommand, OverrideCommandOptions } from '../shared' diff --git a/shared/extra-utils/server/tracker.ts b/shared/extra-utils/server/tracker.ts index f04e8f8a1..ed43a5924 100644 --- a/shared/extra-utils/server/tracker.ts +++ b/shared/extra-utils/server/tracker.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { sha1 } from '@server/helpers/core-utils' +import { sha1 } from '@shared/core-utils/crypto' import { makeGetRequest } from '../requests' async function hlsInfohashExist (serverUrl: string, masterPlaylistUrl: string, fileNumber: number) { diff --git a/shared/extra-utils/users/actors.ts b/shared/extra-utils/users/actors.ts index cfcc7d0a7..12c3e078a 100644 --- a/shared/extra-utils/users/actors.ts +++ b/shared/extra-utils/users/actors.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { Account, VideoChannel } from '@shared/models' import { PeerTubeServer } from '../server' diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index 6671e3fa6..0451c0efe 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts @@ -1,6 +1,6 @@ import { expect } from 'chai' import { basename } from 'path' -import { sha256 } from '@server/helpers/core-utils' +import { sha256 } from '@shared/core-utils/crypto' import { removeFragmentedMP4Ext } from '@shared/core-utils' import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models' import { PeerTubeServer } from '../server' diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 7ec9c3647..8ea828b40 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts @@ -5,8 +5,7 @@ import { createReadStream, stat } from 'fs-extra' import got, { Response as GotResponse } from 'got' import { omit } from 'lodash' import validator from 'validator' -import { buildUUID } from '@server/helpers/uuid' -import { loadLanguages } from '@server/initializers/constants' +import { buildUUID } from '@shared/core-utils/uuid' import { pick } from '@shared/core-utils' import { HttpStatusCode, @@ -23,7 +22,7 @@ import { } from '@shared/models' import { buildAbsoluteFixturePath, wait } from '../miscs' import { unwrapBody } from '../requests' -import { PeerTubeServer, waitJobs } from '../server' +import { waitJobs } from '../server' import { AbstractCommand, OverrideCommandOptions } from '../shared' export type VideoEdit = Partial> & { @@ -33,13 +32,6 @@ export type VideoEdit = Partial f.resolution.id === attributeFile.resolution) - expect(file).not.to.be.undefined - - let extension = getLowercaseExtension(attributes.fixture) - // Transcoding enabled: extension will always be .mp4 - if (attributes.files.length > 1) extension = '.mp4' - - expect(file.magnetUri).to.have.lengthOf.above(2) - - expect(file.torrentDownloadUrl).to.match(new RegExp(`http://${host}/download/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) - expect(file.torrentUrl).to.match(new RegExp(`http://${host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) - - expect(file.fileUrl).to.match(new RegExp(`http://${originHost}/static/webseed/${uuidRegex}-${file.resolution.id}${extension}`)) - expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) - - await Promise.all([ - makeRawRequest(file.torrentUrl, 200), - makeRawRequest(file.torrentDownloadUrl, 200), - makeRawRequest(file.metadataUrl, 200) - ]) - - expect(file.resolution.id).to.equal(attributeFile.resolution) - expect(file.resolution.label).to.equal(attributeFile.resolution + 'p') - - const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) - const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) - expect( - file.size, - 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')' - ).to.be.above(minSize).and.below(maxSize) - - const torrent = await webtorrentAdd(file.magnetUri, true) - expect(torrent.files).to.be.an('array') - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - expect(torrent.files[0].name).to.equal(`${videoDetails.name} ${file.resolution.id}p${extension}`) - } - - expect(videoDetails.thumbnailPath).to.exist - await testImage(server.url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) - - if (attributes.previewfile) { - expect(videoDetails.previewPath).to.exist - await testImage(server.url, attributes.previewfile, videoDetails.previewPath) - } -} - // serverNumber starts from 1 async function uploadRandomVideoOnServers ( servers: PeerTubeServer[], @@ -247,7 +98,6 @@ async function uploadRandomVideoOnServers ( export { checkUploadVideoParam, - completeVideoCheck, uploadRandomVideoOnServers, checkVideoFilesWereRemoved, saveVideoInServers diff --git a/shared/tsconfig.json b/shared/tsconfig.json new file mode 100644 index 000000000..95892077b --- /dev/null +++ b/shared/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "../dist/shared" + } +} diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 000000000..ef86b9797 --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es2015", + "noImplicitAny": false, + "sourceMap": false, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "importHelpers": true, + "removeComments": true, + "strictBindCallApply": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "lib": [ + "es2015", + "es2016", + "es2017", + "es2018", + "es2019" + ], + "typeRoots": [ + "node_modules/@types", + ], + "baseUrl": "./", + "outDir": "./dist/", + "paths": { + "@server/*": [ "server/*" ], + "@shared/*": [ "shared/*" ] + }, + "resolveJsonModule": true, + "strict": false, + "skipLibCheck": true, + "composite": true + } +} diff --git a/tsconfig.json b/tsconfig.json index 075a3d86e..a14a97dfb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,46 +1,9 @@ { - "compilerOptions": { - "module": "commonjs", - "target": "es2015", - "noImplicitAny": false, - "sourceMap": false, - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "importHelpers": true, - "removeComments": true, - "strictBindCallApply": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "outDir": "./dist", - "lib": [ - "dom", - "es2015", - "es2016", - "es2017", - "es2018", - "es2019" - ], - "typeRoots": [ - "node_modules/@types", - "server/typings" - ], - "baseUrl": "./", - "paths": { - "@server/*": [ "server/*" ], - "@shared/*": [ "shared/*" ] - } - }, - "exclude": [ - "server/tools/", - "node_modules", - "dist", - "storage", - "client", - "test1", - "test2", - "test3", - "test4", - "test5", - "test6" - ] + "extends": "./tsconfig.base.json", + "references": [ + { "path": "./shared" }, + { "path": "./server" }, + { "path": "./scripts" } + ], + "files": ["server.ts"] } diff --git a/tsconfig.types.json b/tsconfig.types.json new file mode 100644 index 000000000..c9447d86d --- /dev/null +++ b/tsconfig.types.json @@ -0,0 +1,19 @@ +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "incremental": true, + "sourceMap": true, + "stripInternal": true, + "removeComments": false, + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true + }, + "references": [ + { "path": "./shared/tsconfig.types.json" }, + { "path": "./server/tsconfig.types.json" }, + { "path": "./scripts/tsconfig.types.json" } + ], + "files": [] +} +