Optimize torrent URL update

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
           - 10389:10389
 
       s3ninja:
-        image: scireum/s3-ninja
+        image: chocobozzz/s3-ninja
         ports:
           - 9444:9000
 

package.json

@@ -77,6 +77,7 @@
     "async": "^3.0.1",
     "async-lru": "^1.1.1",
     "bcrypt": "5.0.1",
+    "bencode": "^2.0.2",
     "bittorrent-tracker": "^9.0.0",
     "bluebird": "^3.5.0",
     "bull": "^3.4.2",

scripts/update-host.ts

@@ -17,7 +17,7 @@ import { VideoCommentModel } from '../server/models/video/video-comment'
 import { AccountModel } from '../server/models/account/account'
 import { VideoChannelModel } from '../server/models/video/video-channel'
 import { initDatabaseModels } from '../server/initializers/database'
-import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
+import { updateTorrentUrls } from '@server/helpers/webtorrent'
 import { getServerActor } from '@server/models/application/application'
 
 run()
@@ -126,7 +126,7 @@ async function run () {
 
     for (const file of video.VideoFiles) {
       console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
-      await createTorrentAndSetInfoHash(video, file)
+      await updateTorrentUrls(video, file)
 
       await file.save()
     }
@@ -135,7 +135,7 @@ async function run () {
     for (const file of (playlist?.VideoFiles || [])) {
       console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
 
-      await createTorrentAndSetInfoHash(video, file)
+      await updateTorrentUrls(video, file)
 
       await file.save()
     }

server/helpers/webtorrent.ts

@@ -1,5 +1,6 @@
+import * as bencode from 'bencode'
 import * as createTorrent from 'create-torrent'
-import { createWriteStream, ensureDir, remove, writeFile } from 'fs-extra'
+import { createWriteStream, ensureDir, readFile, remove, writeFile } from 'fs-extra'
 import * as magnetUtil from 'magnet-uri'
 import * as parseTorrent from 'parse-torrent'
 import { dirname, join } from 'path'
@@ -79,43 +80,65 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
   })
 }
 
-function createTorrentAndSetInfoHash (
+function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
-  videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
-  videoFile: MVideoFile
-) {
   const video = extractVideo(videoOrPlaylist)
 
   const options = {
     // Keep the extname, it's used by the client to stream the file inside a web browser
     name: `${video.name} ${videoFile.resolution}p${videoFile.extname}`,
     createdBy: 'PeerTube',
-    announceList: [
+    announceList: buildAnnounceList(),
-      [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
+    urlList: buildUrlList(video, videoFile)
-      [ WEBSERVER.URL + '/tracker/announce' ]
-    ],
-    urlList: [ videoFile.getFileUrl(video) ]
   }
 
   return VideoPathManager.Instance.makeAvailableVideoFile(videoOrPlaylist, videoFile, async videoPath => {
-    const torrent = await createTorrentPromise(videoPath, options)
+    const torrentContent = await createTorrentPromise(videoPath, options)
 
     const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
     const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
     logger.info('Creating torrent %s.', torrentPath)
 
-    await writeFile(torrentPath, torrent)
+    await writeFile(torrentPath, torrentContent)
 
     // Remove old torrent file if it existed
     if (videoFile.hasTorrent()) {
       await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
     }
 
-    const parsedTorrent = parseTorrent(torrent)
+    const parsedTorrent = parseTorrent(torrentContent)
     videoFile.infoHash = parsedTorrent.infoHash
     videoFile.torrentFilename = torrentFilename
   })
 }
 
+async function updateTorrentUrls (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
+  const video = extractVideo(videoOrPlaylist)
+
+  const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
+
+  const torrentContent = await readFile(oldTorrentPath)
+  const decoded = bencode.decode(torrentContent)
+
+  decoded['announce-list'] = buildAnnounceList()
+  decoded.announce = decoded['announce-list'][0][0]
+
+  decoded['url-list'] = buildUrlList(video, videoFile)
+
+  const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
+  const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename)
+
+  logger.info('Updating torrent URLs %s.', newTorrentPath)
+
+  await writeFile(newTorrentPath, bencode.encode(decoded))
+
+  // Remove old torrent file if it existed
+  if (videoFile.hasTorrent()) {
+    await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
+  }
+
+  videoFile.torrentFilename = newTorrentFilename
+}
+
 function generateMagnetUri (
   video: MVideo,
   videoFile: MVideoFileRedundanciesOpt,
@@ -143,6 +166,7 @@ function generateMagnetUri (
 
 export {
   createTorrentPromise,
+  updateTorrentUrls,
   createTorrentAndSetInfoHash,
   generateMagnetUri,
   downloadWebTorrentVideo
@@ -186,3 +210,14 @@ function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath
   remove(toRemovePath)
     .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err }))
 }
+
+function buildAnnounceList () {
+  return [
+    [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
+    [ WEBSERVER.URL + '/tracker/announce' ]
+  ]
+}
+
+function buildUrlList (video: MVideo, videoFile: MVideoFile) {
+  return [ videoFile.getFileUrl(video) ]
+}

server/lib/job-queue/handlers/move-to-object-storage.ts

@@ -2,7 +2,7 @@ import * as Bull from 'bull'
 import { remove } from 'fs-extra'
 import { join } from 'path'
 import { logger } from '@server/helpers/logger'
-import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
+import { updateTorrentUrls } from '@server/helpers/webtorrent'
 import { CONFIG } from '@server/initializers/config'
 import { storeHLSFile, storeWebTorrentFile } from '@server/lib/object-storage'
 import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
@@ -106,7 +106,7 @@ async function onFileMoved (options: {
   file.fileUrl = fileUrl
   file.storage = VideoStorage.OBJECT_STORAGE
 
-  await createTorrentAndSetInfoHash(videoOrPlaylist, file)
+  await updateTorrentUrls(videoOrPlaylist, file)
   await file.save()
 
   logger.debug('Removing %s because it\'s now on object storage', oldPath)

server/tests/api/object-storage/videos.ts

@@ -10,6 +10,7 @@ import {
   createMultipleServers,
   createSingleServer,
   doubleFollow,
+  expectLogDoesNotContain,
   expectStartWith,
   killallServers,
   makeRawRequest,
@@ -235,6 +236,12 @@ function runTestSuite (options: {
     }
   })
 
+  it('Should not have downloaded files from object storage', async function () {
+    for (const server of servers) {
+      await expectLogDoesNotContain(server, 'from object storage')
+    }
+  })
+
   after(async function () {
     mockObjectStorage.terminate()
 

shared/extra-utils/miscs/checks.ts

@@ -20,6 +20,12 @@ function expectStartWith (str: string, start: string) {
   expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
 }
 
+async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
+  const content = await server.servers.getLogContent()
+
+  expect(content.toString()).to.not.contain(str)
+}
+
 async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
   const res = await makeGetRequest({
     url,
@@ -46,6 +52,7 @@ async function testFileExistsOrNot (server: PeerTubeServer, directory: string, f
 export {
   dateIsValid,
   testImage,
+  expectLogDoesNotContain,
   testFileExistsOrNot,
   expectStartWith
 }

shared/extra-utils/server/servers-command.ts

@@ -55,7 +55,7 @@ export class ServersCommand extends AbstractCommand {
   }
 
   async waitUntilLog (str: string, count = 1, strictCount = true) {
-    const logfile = this.server.servers.buildDirectory('logs/peertube.log')
+    const logfile = this.buildDirectory('logs/peertube.log')
 
     while (true) {
       const buf = await readFile(logfile)
@@ -80,6 +80,10 @@ export class ServersCommand extends AbstractCommand {
     return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl)))
   }
 
+  getLogContent () {
+    return readFile(this.buildDirectory('logs/peertube.log'))
+  }
+
   async getServerFileSize (subPath: string) {
     const path = this.server.servers.buildDirectory(subPath)