From e11f68a3562d2468480c396f47f1bdd2a306e17a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 8 Dec 2017 10:08:36 +0100 Subject: [PATCH] Optimise transaction for video upload --- server/controllers/api/videos/index.ts | 95 +++++++++++++------------- server/tests/api/single-server.ts | 13 ++-- server/tests/utils/videos.ts | 2 +- 3 files changed, 53 insertions(+), 57 deletions(-) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index f427a25c0..0f71a7f7f 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -164,73 +164,72 @@ async function addVideoRetryWrapper (req: express.Request, res: express.Response }).end() } -function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) { +async function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) { const videoInfo: VideoCreate = req.body + // Prepare data so we don't block the transaction + const videoData = { + name: videoInfo.name, + remote: false, + extname: extname(videoPhysicalFile.filename), + category: videoInfo.category, + licence: videoInfo.licence, + language: videoInfo.language, + nsfw: videoInfo.nsfw, + description: videoInfo.description, + privacy: videoInfo.privacy, + duration: videoPhysicalFile['duration'], // duration was added by a previous middleware + channelId: res.locals.videoChannel.id + } + const video = db.Video.build(videoData) + video.url = getVideoActivityPubUrl(video) + + const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename) + const videoFileHeight = await getVideoFileHeight(videoFilePath) + + const videoFileData = { + extname: extname(videoPhysicalFile.filename), + resolution: videoFileHeight, + size: videoPhysicalFile.size + } + const videoFile = db.VideoFile.build(videoFileData) + const videoDir = CONFIG.STORAGE.VIDEOS_DIR + const source = join(videoDir, videoPhysicalFile.filename) + const destination = join(videoDir, video.getVideoFilename(videoFile)) + + await renamePromise(source, destination) + // This is important in case if there is another attempt in the retry process + videoPhysicalFile.filename = video.getVideoFilename(videoFile) + + const tasks = [] + + tasks.push( + video.createTorrentAndSetInfoHash(videoFile), + video.createThumbnail(videoFile), + video.createPreview(videoFile) + ) + await Promise.all(tasks) + return db.sequelize.transaction(async t => { const sequelizeOptions = { transaction: t } - const videoData = { - name: videoInfo.name, - remote: false, - extname: extname(videoPhysicalFile.filename), - category: videoInfo.category, - licence: videoInfo.licence, - language: videoInfo.language, - nsfw: videoInfo.nsfw, - description: videoInfo.description, - privacy: videoInfo.privacy, - duration: videoPhysicalFile['duration'], // duration was added by a previous middleware - channelId: res.locals.videoChannel.id - } - const video = db.Video.build(videoData) - video.url = getVideoActivityPubUrl(video) - - const videoFilePath = join(CONFIG.STORAGE.VIDEOS_DIR, videoPhysicalFile.filename) - const videoFileHeight = await getVideoFileHeight(videoFilePath) - - const videoFileData = { - extname: extname(videoPhysicalFile.filename), - resolution: videoFileHeight, - size: videoPhysicalFile.size - } - const videoFile = db.VideoFile.build(videoFileData) - const videoDir = CONFIG.STORAGE.VIDEOS_DIR - const source = join(videoDir, videoPhysicalFile.filename) - const destination = join(videoDir, video.getVideoFilename(videoFile)) - - await renamePromise(source, destination) - // This is important in case if there is another attempt in the retry process - videoPhysicalFile.filename = video.getVideoFilename(videoFile) - - const tasks = [] - - tasks.push( - video.createTorrentAndSetInfoHash(videoFile), - video.createThumbnail(videoFile), - video.createPreview(videoFile) - ) - if (CONFIG.TRANSCODING.ENABLED === true) { // Put uuid because we don't have id auto incremented for now const dataInput = { videoUUID: video.uuid } - tasks.push( - transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput) - ) + await transcodingJobScheduler.createJob(t, 'videoFileOptimizer', dataInput) } - await Promise.all(tasks) const videoCreated = await video.save(sequelizeOptions) // Do not forget to add video channel information to the created video videoCreated.VideoChannel = res.locals.videoChannel videoFile.videoId = video.id - await videoFile.save(sequelizeOptions) - video.VideoFiles = [videoFile] + + video.VideoFiles = [ videoFile ] if (videoInfo.tags) { const tagInstances = await db.Tag.findOrCreateTags(videoInfo.tags, t) diff --git a/server/tests/api/single-server.ts b/server/tests/api/single-server.ts index ed79f9e1c..e99955ef4 100644 --- a/server/tests/api/single-server.ts +++ b/server/tests/api/single-server.ts @@ -364,7 +364,7 @@ describe('Test a single server', function () { 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' ] - // const tasks: Promise[] = [] + const tasks: Promise[] = [] for (const video of videos) { const videoAttributes = { name: video + ' name', @@ -378,13 +378,10 @@ describe('Test a single server', function () { } const p = uploadVideo(server.url, server.accessToken, videoAttributes) - await p + tasks.push(p) } - // FIXME: concurrent uploads does not work :( - // tasks.push(p) - // } - // - // await Promise.all(tasks) + + await Promise.all(tasks) }) it('Should have the correct durations', async function () { @@ -712,7 +709,7 @@ describe('Test a single server', function () { const filePath = join(__dirname, '..', 'api', 'fixtures', 'video_short.webm') await req.attach('videofile', filePath) - .expect(204) + .expect(200) const res = await getVideosList(server.url) const video = res.body.data.find(v => v.name === 'minimum parameters') diff --git a/server/tests/utils/videos.ts b/server/tests/utils/videos.ts index bdf3368ac..fb758cf29 100644 --- a/server/tests/utils/videos.ts +++ b/server/tests/utils/videos.ts @@ -201,7 +201,7 @@ async function testVideoImage (url: string, imageName: string, imagePath: string } } -async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = 201) { +async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = 200) { const path = '/api/v1/videos/upload' let defaultChannelId = '1'