From 66a7fc947321b923fa1e7182202e03d0e21ffb3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 17 Aug 2021 10:33:36 +0200 Subject: [PATCH] Remove optimize old videos script It is not compatible with HLS and I don't have time to maintain it --- package.json | 1 - scripts/help.sh | 1 - scripts/optimize-old-videos.ts | 92 ------------------------ server/lib/object-storage/keys.ts | 2 +- server/tests/cli/index.ts | 1 - server/tests/cli/optimize-old-videos.ts | 96 ------------------------- support/doc/tools.md | 18 ----- 7 files changed, 1 insertion(+), 210 deletions(-) delete mode 100644 scripts/optimize-old-videos.ts delete mode 100644 server/tests/cli/optimize-old-videos.ts diff --git a/package.json b/package.json index d2b1e1245..32f4e7c31 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,6 @@ "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", "parse-log": "node ./dist/scripts/parse-log.js", "prune-storage": "node ./dist/scripts/prune-storage.js", - "optimize-old-videos": "node ./dist/scripts/optimize-old-videos.js", "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)", "tsc": "tsc", "commander": "commander", diff --git a/scripts/help.sh b/scripts/help.sh index da127092d..b58f52fa0 100755 --- a/scripts/help.sh +++ b/scripts/help.sh @@ -13,7 +13,6 @@ printf " reset-password -- -u [user] -> Reset the password of user [user]\n" printf " create-transcoding-job -- -v [video UUID] \n" printf " -> Create a transcoding job for a particular video\n" printf " prune-storage -> Delete (after confirmation) unknown video files/thumbnails/previews... (due to a bad video deletion, transcoding job not finished...)\n" -printf " optimize-old-videos -> Re-transcode videos that have a high bitrate, to make them suitable for streaming over slow connections" printf " dev -> Watch, run the livereload and run the server so that you can develop the application\n" printf " start -> Run the server\n" printf " update-host -> Upgrade scheme/host in torrent files according to the webserver configuration (config/ folder)\n" diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts deleted file mode 100644 index 245e4cf28..000000000 --- a/scripts/optimize-old-videos.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { registerTSPaths } from '../server/helpers/register-ts-paths' -registerTSPaths() - -import { copy, move, remove } from 'fs-extra' -import { basename, dirname } from 'path' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { CONFIG } from '@server/initializers/config' -import { processMoveToObjectStorage } from '@server/lib/job-queue/handlers/move-to-object-storage' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { getMaxBitrate } from '@shared/core-utils' -import { MoveObjectStoragePayload } from '@shared/models' -import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffprobe-utils' -import { initDatabaseModels } from '../server/initializers/database' -import { optimizeOriginalVideofile } from '../server/lib/transcoding/video-transcoding' -import { VideoModel } from '../server/models/video/video' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -let currentVideoId: string -let currentFilePath: string - -process.on('SIGINT', async function () { - console.log('Cleaning up temp files') - await remove(`${currentFilePath}_backup`) - await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`) - process.exit(0) -}) - -async function run () { - await initDatabaseModels(true) - - const localVideos = await VideoModel.listLocal() - - for (const localVideo of localVideos) { - const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(localVideo.id) - - currentVideoId = video.id - - for (const file of video.VideoFiles) { - await VideoPathManager.Instance.makeAvailableVideoFile(video, file, async path => { - currentFilePath = path - - const [ videoBitrate, fps, dataResolution ] = await Promise.all([ - getVideoFileBitrate(currentFilePath), - getVideoFileFPS(currentFilePath), - getVideoFileResolution(currentFilePath) - ]) - - const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) - const isMaxBitrateExceeded = videoBitrate > maxBitrate - if (isMaxBitrateExceeded) { - console.log( - 'Optimizing video file %s with bitrate %s kbps (max: %s kbps)', - basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000 - ) - - const backupFile = `${currentFilePath}_backup` - await copy(currentFilePath, backupFile) - - await optimizeOriginalVideofile(video, file) - // Update file path, the video filename changed - currentFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) - - const originalDuration = await getDurationFromVideoFile(backupFile) - const newDuration = await getDurationFromVideoFile(currentFilePath) - - if (originalDuration === newDuration) { - console.log('Finished optimizing %s', basename(currentFilePath)) - await remove(backupFile) - return - } - - console.log('Failed to optimize %s, restoring original', basename(currentFilePath)) - await move(backupFile, currentFilePath, { overwrite: true }) - await createTorrentAndSetInfoHash(video, file) - await file.save() - } - }) - } - - if (CONFIG.OBJECT_STORAGE.ENABLED === true) { - await processMoveToObjectStorage({ data: { videoUUID: video.uuid } as MoveObjectStoragePayload } as any) - } - } - - console.log('Finished optimizing videos') -} diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts index 519474775..12acb3aec 100644 --- a/server/lib/object-storage/keys.ts +++ b/server/lib/object-storage/keys.ts @@ -6,7 +6,7 @@ function generateHLSObjectStorageKey (playlist: MStreamingPlaylist, video: MVide } function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylist, video: MVideoUUID) { - return playlist.getStringType() + '_' + video.uuid + return join(playlist.getStringType(), video.uuid) } function generateWebTorrentObjectStorageKey (filename: string) { diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts index 7e6eebd17..c6dd0581a 100644 --- a/server/tests/cli/index.ts +++ b/server/tests/cli/index.ts @@ -1,7 +1,6 @@ // Order of the tests we want to execute import './create-import-video-file-job' import './create-transcoding-job' -import './optimize-old-videos' import './peertube' import './plugins' import './print-transcode-command' diff --git a/server/tests/cli/optimize-old-videos.ts b/server/tests/cli/optimize-old-videos.ts deleted file mode 100644 index 9b75ae164..000000000 --- a/server/tests/cli/optimize-old-videos.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import 'mocha' -import * as chai from 'chai' -import { getMaxBitrate } from '@shared/core-utils' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - generateHighBitrateVideo, - PeerTubeServer, - setAccessTokensToServers, - wait, - waitJobs -} from '@shared/extra-utils' -import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../../helpers/ffprobe-utils' - -const expect = chai.expect - -describe('Test optimize old videos', function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(200000) - - // Run server 2 to have transcoding enabled - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - const tempFixturePath = await generateHighBitrateVideo() - - // Upload two videos for our needs - await servers[0].videos.upload({ attributes: { name: 'video1', fixture: tempFixturePath } }) - await servers[0].videos.upload({ attributes: { name: 'video2', fixture: tempFixturePath } }) - - await waitJobs(servers) - }) - - it('Should have two video files on each server', async function () { - this.timeout(30000) - - for (const server of servers) { - const { data } = await server.videos.list() - expect(data).to.have.lengthOf(2) - - for (const video of data) { - const videoDetails = await server.videos.get({ id: video.uuid }) - expect(videoDetails.files).to.have.lengthOf(1) - } - } - }) - - it('Should run optimize script', async function () { - this.timeout(200000) - - await servers[0].cli.execWithEnv('npm run optimize-old-videos') - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - expect(data).to.have.lengthOf(2) - - for (const video of data) { - await server.videos.view({ id: video.uuid }) - - // Refresh video - await waitJobs(servers) - await wait(5000) - await waitJobs(servers) - - const videoDetails = await server.videos.get({ id: video.uuid }) - - expect(videoDetails.files).to.have.lengthOf(1) - const file = videoDetails.files[0] - - expect(file.size).to.be.below(8000000) - - const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl) - const bitrate = await getVideoFileBitrate(path) - const fps = await getVideoFileFPS(path) - const data = await getVideoFileResolution(path) - - expect(data.resolution).to.equal(file.resolution.id) - - const maxBitrate = getMaxBitrate({ ...data, fps }) - expect(bitrate).to.be.below(maxBitrate) - } - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/support/doc/tools.md b/support/doc/tools.md index 1956dc572..433aaeff7 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md @@ -19,7 +19,6 @@ - [create-transcoding-job.js](#create-transcoding-jobjs) - [create-import-video-file-job.js](#create-import-video-file-jobjs) - [prune-storage.js](#prune-storagejs) - - [optimize-old-videos.js](#optimize-old-videosjs) - [update-host.js](#update-hostjs) - [reset-password.js](#reset-passwordjs) - [plugin install/uninstall](#plugin-installuninstall) @@ -344,23 +343,6 @@ $ cd /var/www/peertube/peertube-latest $ sudo systemctl stop peertube && sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage ``` -### optimize-old-videos.js - -Before version v1.0.0-beta.16, Peertube did not specify a bitrate for the -transcoding of uploaded videos. This means that videos might be encoded into -very large files that are too large for streaming. This script re-transcodes -these videos so that they can be watched properly, even on slow connections. - -```bash -$ # Basic installation -$ cd /var/www/peertube/peertube-latest -$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run optimize-old-videos - -$ # Docker installation -$ cd /var/www/peertube-docker -$ docker-compose exec -u peertube peertube npm run optimize-old-videos -``` - ### update-host.js