From e9fc9e03c120fb048ed00b38157d15144770ec23 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Wed, 7 Sep 2022 17:30:21 +0200
Subject: [PATCH] Optimize fetching playlist urls

---
 server/helpers/youtube-dl/youtube-dl-cli.ts   |  5 +++
 .../helpers/youtube-dl/youtube-dl-wrapper.ts  |  9 ++--
 server/lib/sync-channel.ts                    | 42 ++++++++++++-------
 server/tests/api/videos/multiple-servers.ts   |  2 +-
 4 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
index 508055b85..fc4c40787 100644
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ b/server/helpers/youtube-dl/youtube-dl-cli.ts
@@ -142,6 +142,11 @@ export class YoutubeDLCLI {
   }): Promise<{ upload_date: string, webpage_url: string }[]> {
     const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
 
+    if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
+      // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
+      additionalYoutubeDLArgs.push('--flat-playlist')
+    }
+
     if (options.latestVideosCount !== undefined) {
       additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
     }
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
index 2c3ba2feb..966b8df78 100644
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -1,5 +1,6 @@
 import { move, pathExists, readdir, remove } from 'fs-extra'
 import { dirname, join } from 'path'
+import { inspect } from 'util'
 import { CONFIG } from '@server/initializers/config'
 import { isVideoFileExtnameValid } from '../custom-validators/videos'
 import { logger, loggerTagsFactory } from '../logger'
@@ -59,13 +60,9 @@ class YoutubeDLWrapper {
       processOptions
     })
 
-    if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}`)
+    if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
 
-    return list.map(info => {
-      const infoBuilder = new YoutubeDLInfoBuilder(info)
-
-      return infoBuilder.getInfo()
-    })
+    return list.map(info => info.webpage_url)
   }
 
   async getSubtitles (): Promise<YoutubeDLSubs> {
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index eb5ca1703..3a81daac0 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config'
 import { buildYoutubeDLImport } from '@server/lib/video-import'
 import { UserModel } from '@server/models/user/user'
 import { VideoImportModel } from '@server/models/video/video-import'
-import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
+import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models'
 import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
 import { CreateJobArgument, JobQueue } from './job-queue'
 import { ServerConfigManager } from './server-config-manager'
@@ -31,15 +31,7 @@ export async function synchronizeChannel (options: {
     CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
   )
 
-  const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
-
-  const targetUrls = infoList
-    .filter(videoInfo => {
-      if (!onlyAfter) return true
-
-      return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
-    })
-    .map(videoInfo => videoInfo.webpageUrl)
+  const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
 
   logger.info(
     'Fetched %d candidate URLs for sync channel %s.',
@@ -58,10 +50,7 @@ export async function synchronizeChannel (options: {
   const children: CreateJobArgument[] = []
 
   for (const targetUrl of targetUrls) {
-    if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
-      logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
-      continue
-    }
+    if (await skipImport(channel, targetUrl, onlyAfter)) continue
 
     const { job } = await buildYoutubeDLImport({
       user,
@@ -86,3 +75,28 @@ export async function synchronizeChannel (options: {
 
   await JobQueue.Instance.createJobWithChildren(parent, children)
 }
+
+// ---------------------------------------------------------------------------
+
+async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
+  if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
+    logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
+    return true
+  }
+
+  if (onlyAfter) {
+    const youtubeDL = new YoutubeDLWrapper(
+      targetUrl,
+      ServerConfigManager.Instance.getEnabledResolutions('vod'),
+      CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
+    )
+
+    const videoInfo = await youtubeDL.getInfoForDownload()
+
+    if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
+      return true
+    }
+  }
+
+  return false
+}
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index 23790ba65..d47807a79 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -1021,7 +1021,7 @@ describe('Test multiple servers', function () {
 
   describe('With minimum parameters', function () {
     it('Should upload and propagate the video', async function () {
-      this.timeout(60000)
+      this.timeout(120000)
 
       const path = '/api/v1/videos/upload'