From ea54cd04c1ff0e55651cd5fb1a83672acde68604 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 8 Jun 2021 09:33:03 +0200 Subject: [PATCH] Fix video upload with a capitalized ext --- server/controllers/api/videos/upload.ts | 4 +- server/helpers/core-utils.ts | 90 ++++++++++++------- server/helpers/express-utils.ts | 4 +- server/helpers/image-utils.ts | 7 +- .../shared/object-to-model-attributes.ts | 4 +- .../job-queue/handlers/video-file-import.ts | 3 +- server/lib/job-queue/handlers/video-import.ts | 3 +- server/lib/local-actor.ts | 5 +- server/models/actor/actor.ts | 6 +- shared/extra-utils/videos/videos.ts | 5 +- 10 files changed, 82 insertions(+), 49 deletions(-) diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts index 783cc329a..e767492bc 100644 --- a/server/controllers/api/videos/upload.ts +++ b/server/controllers/api/videos/upload.ts @@ -1,6 +1,6 @@ import * as express from 'express' import { move } from 'fs-extra' -import { extname } from 'path' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' @@ -225,7 +225,7 @@ async function addVideo (options: { async function buildNewFile (video: MVideo, videoPhysicalFile: express.VideoUploadFile) { const videoFile = new VideoFileModel({ - extname: extname(videoPhysicalFile.filename), + extname: getLowercaseExtension(videoPhysicalFile.filename), size: videoPhysicalFile.size, videoStreamingPlaylistId: null, metadata: await getMetadataFromFile(videoPhysicalFile.path) diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index b93868c12..b5bf2c92c 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -8,7 +8,7 @@ import { exec, ExecOptions } from 'child_process' import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' import { truncate } from 'lodash' -import { basename, isAbsolute, join, resolve } from 'path' +import { basename, extname, isAbsolute, join, resolve } from 'path' import * as pem from 'pem' import { pipeline } from 'stream' import { URL } from 'url' @@ -32,6 +32,18 @@ const objectConverter = (oldObject: any, keyConverter: (e: string) => string, va return newObject } +function mapToJSON (map: Map) { + const obj: any = {} + + for (const [ k, v ] of map) { + obj[k] = v + } + + return obj +} + +// --------------------------------------------------------------------------- + const timeTable = { ms: 1, second: 1000, @@ -110,6 +122,8 @@ export function parseBytes (value: string | number): number { } } +// --------------------------------------------------------------------------- + function sanitizeUrl (url: string) { const urlObject = new URL(url) @@ -129,6 +143,8 @@ function sanitizeHost (host: string, remoteScheme: string) { return host.replace(new RegExp(`:${toRemove}$`), '') } +// --------------------------------------------------------------------------- + function isTestInstance () { return process.env.NODE_ENV === 'test' } @@ -141,6 +157,8 @@ function getAppNumber () { return process.env.NODE_APP_INSTANCE } +// --------------------------------------------------------------------------- + let rootPath: string function root () { @@ -154,28 +172,20 @@ function root () { return rootPath } -function pageToStartAndCount (page: number, itemsPerPage: number) { - const start = (page - 1) * itemsPerPage - - return { start, count: itemsPerPage } -} - -function mapToJSON (map: Map) { - const obj: any = {} - - for (const [ k, v ] of map) { - obj[k] = v - } - - return obj -} - function buildPath (path: string) { if (isAbsolute(path)) return path return join(root(), path) } +function getLowercaseExtension (filename: string) { + const ext = extname(filename) || '' + + return ext.toLowerCase() +} + +// --------------------------------------------------------------------------- + // Consistent with .length, lodash truncate function is not function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { const truncatedStr = truncate(str, options) @@ -189,6 +199,27 @@ function peertubeTruncate (str: string, options: { length: number, separator?: R return truncate(str, options) } +function pageToStartAndCount (page: number, itemsPerPage: number) { + const start = (page - 1) * itemsPerPage + + return { start, count: itemsPerPage } +} + +// --------------------------------------------------------------------------- + +type SemVersion = { major: number, minor: number, patch: number } +function parseSemVersion (s: string) { + const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i) + + return { + major: parseInt(parsed[1]), + minor: parseInt(parsed[2]), + patch: parseInt(parsed[3]) + } as SemVersion +} + +// --------------------------------------------------------------------------- + function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { return createHash('sha256').update(str).digest(encoding) } @@ -197,6 +228,8 @@ function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { return createHash('sha1').update(str).digest(encoding) } +// --------------------------------------------------------------------------- + function execShell (command: string, options?: ExecOptions) { return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { exec(command, options, (err, stdout, stderr) => { @@ -208,6 +241,8 @@ function execShell (command: string, options?: ExecOptions) { }) } +// --------------------------------------------------------------------------- + function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { return function promisified (): Promise { return new Promise((resolve: (arg: A) => void, reject: (err: any) => void) => { @@ -233,17 +268,6 @@ function promisify2 (func: (arg1: T, arg2: U, cb: (err: any, result: A) } } -type SemVersion = { major: number, minor: number, patch: number } -function parseSemVersion (s: string) { - const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i) - - return { - major: parseInt(parsed[1]), - minor: parseInt(parsed[2]), - patch: parseInt(parsed[3]) - } as SemVersion -} - const randomBytesPromise = promisify1(randomBytes) const createPrivateKey = promisify1(pem.createPrivateKey) const getPublicKey = promisify1(pem.getPublicKey) @@ -259,17 +283,21 @@ export { getAppNumber, objectConverter, + mapToJSON, + root, - pageToStartAndCount, + buildPath, + getLowercaseExtension, sanitizeUrl, sanitizeHost, - buildPath, + execShell, + + pageToStartAndCount, peertubeTruncate, sha256, sha1, - mapToJSON, promisify0, promisify1, diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index 010c6961a..003e02818 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -1,9 +1,9 @@ import * as express from 'express' import * as multer from 'multer' -import { extname } from 'path' import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' import { CONFIG } from '../initializers/config' import { REMOTE_SCHEME } from '../initializers/constants' +import { getLowercaseExtension } from './core-utils' import { isArray } from './custom-validators/misc' import { logger } from './logger' import { deleteFileAndCatch, generateRandomString } from './utils' @@ -79,7 +79,7 @@ function createReqFiles ( filename: async (req, file, cb) => { let extension: string - const fileExtension = extname(file.originalname) + const fileExtension = getLowercaseExtension(file.originalname) const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype) // Take the file extension if we don't understand the mime type diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index 6f6f8d4da..122fb009d 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -1,7 +1,7 @@ import { copy, readFile, remove, rename } from 'fs-extra' import * as Jimp from 'jimp' -import { extname } from 'path' import { v4 as uuidv4 } from 'uuid' +import { getLowercaseExtension } from './core-utils' import { convertWebPToJPG, processGIF } from './ffmpeg-utils' import { logger } from './logger' @@ -15,7 +15,7 @@ async function processImage ( newSize: { width: number, height: number }, keepOriginal = false ) { - const extension = extname(path) + const extension = getLowercaseExtension(path) if (path === destination) { throw new Error('Jimp/FFmpeg needs an input path different that the output path.') @@ -61,7 +61,8 @@ async function jimpProcessor (path: string, destination: string, newSize: { widt await remove(destination) // Optimization if the source file has the appropriate size - if (await skipProcessing({ jimpInstance, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt: extname(destination) })) { + const outputExt = getLowercaseExtension(destination) + if (skipProcessing({ jimpInstance, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) { return copy(path, destination) } diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts index 66b22c952..f53b98448 100644 --- a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts @@ -1,5 +1,5 @@ -import { extname } from 'path' import { v4 as uuidv4 } from 'uuid' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { MIMETYPES } from '@server/initializers/constants' import { ActorModel } from '@server/models/actor/actor' @@ -43,7 +43,7 @@ function getImageInfoFromObject (actorObject: ActivityPubActor, type: ActorImage if (icon.mediaType) { extension = mimetypes.MIMETYPE_EXT[icon.mediaType] } else { - const tmp = extname(icon.url) + const tmp = getLowercaseExtension(icon.url) if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp } diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 8297a1571..048963033 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts @@ -1,6 +1,7 @@ import * as Bull from 'bull' import { copy, stat } from 'fs-extra' import { extname } from 'path' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { UserModel } from '@server/models/user/user' @@ -55,7 +56,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { const { size } = await stat(inputFilePath) const fps = await getVideoFileFPS(inputFilePath) - const fileExt = extname(inputFilePath) + const fileExt = getLowercaseExtension(inputFilePath) const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === videoFileResolution) diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index d71053e87..937f586c9 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -1,6 +1,7 @@ import * as Bull from 'bull' import { move, remove, stat } from 'fs-extra' import { extname } from 'path' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { retryTransactionWrapper } from '@server/helpers/database-utils' import { YoutubeDL } from '@server/helpers/youtube-dl' import { isPostImportVideoAccepted } from '@server/lib/moderation' @@ -119,7 +120,7 @@ async function processFile (downloader: () => Promise, videoImport: MVid const duration = await getDurationFromVideoFile(tempVideoPath) // Prepare video file object for creation in database - const fileExt = extname(tempVideoPath) + const fileExt = getLowercaseExtension(tempVideoPath) const videoFileData = { extname: fileExt, resolution: videoFileResolution, diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts index 55e77dd04..2d2bd43a1 100644 --- a/server/lib/local-actor.ts +++ b/server/lib/local-actor.ts @@ -1,8 +1,9 @@ import 'multer' import { queue } from 'async' import * as LRUCache from 'lru-cache' -import { extname, join } from 'path' +import { join } from 'path' import { v4 as uuidv4 } from 'uuid' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { ActorModel } from '@server/models/actor/actor' import { ActivityPubActorType, ActorImageType } from '@shared/models' import { retryTransactionWrapper } from '../helpers/database-utils' @@ -41,7 +42,7 @@ async function updateLocalActorImageFile ( ? ACTOR_IMAGES_SIZE.AVATARS : ACTOR_IMAGES_SIZE.BANNERS - const extension = extname(imagePhysicalFile.filename) + const extension = getLowercaseExtension(imagePhysicalFile.filename) const imageName = uuidv4() + extension const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts index 65c53f8f8..0cd30f545 100644 --- a/server/models/actor/actor.ts +++ b/server/models/actor/actor.ts @@ -1,5 +1,4 @@ import { values } from 'lodash' -import { extname } from 'path' import { literal, Op, Transaction } from 'sequelize' import { AllowNull, @@ -17,6 +16,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { ModelCache } from '@server/models/model-cache' import { AttributesOnly } from '@shared/core-utils' import { ActivityIconObject, ActivityPubActorType } from '../../../shared/models/activitypub' @@ -567,7 +567,7 @@ export class ActorModel extends Model>> { let image: ActivityIconObject if (this.avatarId) { - const extension = extname(this.Avatar.filename) + const extension = getLowercaseExtension(this.Avatar.filename) icon = { type: 'Image', @@ -580,7 +580,7 @@ export class ActorModel extends Model>> { if (this.bannerId) { const banner = (this as MActorAPChannel).Banner - const extension = extname(banner.filename) + const extension = getLowercaseExtension(banner.filename) image = { type: 'Image', diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 98a568a02..a3a276188 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -4,10 +4,11 @@ import { expect } from 'chai' import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra' import got, { Response as GotResponse } from 'got/dist/source' import * as parseTorrent from 'parse-torrent' -import { extname, join } from 'path' +import { join } from 'path' import * as request from 'supertest' import { v4 as uuidv4 } from 'uuid' import validator from 'validator' +import { getLowercaseExtension } from '@server/helpers/core-utils' import { HttpStatusCode } from '@shared/core-utils' import { VideosCommonQuery } from '@shared/models' import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' @@ -738,7 +739,7 @@ async function completeVideoCheck ( const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution) expect(file).not.to.be.undefined - let extension = extname(attributes.fixture) + let extension = getLowercaseExtension(attributes.fixture) // Transcoding enabled: extension will always be .mp4 if (attributes.files.length > 1) extension = '.mp4'