From 48f1d4b18606f80dcdf61f2e357b57bde8117044 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 16 Feb 2024 10:14:12 +0100 Subject: [PATCH] More robust channel sync --- .../shared-main/video/video-import.service.ts | 2 +- .../tests/src/api/videos/video-imports.ts | 21 +++++++--- .../handlers/video-channel-import.ts | 2 +- server/core/lib/sync-channel.ts | 38 +++++++++++-------- server/core/models/video/video-import.ts | 18 ++++++--- 5 files changed, 54 insertions(+), 27 deletions(-) diff --git a/client/src/app/shared/shared-main/video/video-import.service.ts b/client/src/app/shared/shared-main/video/video-import.service.ts index bb9052401..7aa330bc4 100644 --- a/client/src/app/shared/shared-main/video/video-import.service.ts +++ b/client/src/app/shared/shared-main/video/video-import.service.ts @@ -61,7 +61,7 @@ export class VideoImportService { } return this.authHttp - .get>(UserService.BASE_USERS_URL + '/me/videos/imports', { params }) + .get>(UserService.BASE_USERS_URL + 'me/videos/imports', { params }) .pipe( switchMap(res => this.extractVideoImports(res)), catchError(err => this.restExtractor.handleError(err)) diff --git a/packages/tests/src/api/videos/video-imports.ts b/packages/tests/src/api/videos/video-imports.ts index 6c874d713..3ffaa225c 100644 --- a/packages/tests/src/api/videos/video-imports.ts +++ b/packages/tests/src/api/videos/video-imports.ts @@ -232,12 +232,23 @@ describe('Test video imports', function () { }) it('Should search in my imports', async function () { - const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' }) - expect(total).to.equal(1) - expect(videoImports).to.have.lengthOf(1) + { + const { total, data } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) - expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet) - expect(videoImports[0].video.name).to.equal('super peertube2 video') + expect(data[0].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(data[0].video.name).to.equal('super peertube2 video') + } + + { + const { total, data } = await servers[0].imports.getMyVideoImports({ search: FIXTURE_URLS.magnet }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + expect(data[0].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(data[0].video.name).to.equal('super peertube2 video') + } }) it('Should have the video listed on the two instances', async function () { diff --git a/server/core/lib/job-queue/handlers/video-channel-import.ts b/server/core/lib/job-queue/handlers/video-channel-import.ts index b5f87e11f..a92e1b6f1 100644 --- a/server/core/lib/job-queue/handlers/video-channel-import.ts +++ b/server/core/lib/job-queue/handlers/video-channel-import.ts @@ -26,7 +26,7 @@ export async function processVideoChannelImport (job: Job) { channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) if (!channelSync) { - throw new Error('Unlnown channel sync specified in videos channel import') + throw new Error('Unknown channel sync specified in videos channel import') } } diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.ts index b0cbf43ba..5a346f84a 100644 --- a/server/core/lib/sync-channel.ts +++ b/server/core/lib/sync-channel.ts @@ -1,4 +1,4 @@ -import { logger } from '@server/helpers/logger.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js' import { CONFIG } from '@server/initializers/config.js' import { buildYoutubeDLImport } from '@server/lib/video-pre-import.js' @@ -9,6 +9,8 @@ import { VideoChannelSyncState, VideoPrivacy } from '@peertube/peertube-models' import { CreateJobArgument, JobQueue } from './job-queue/index.js' import { ServerConfigManager } from './server-config-manager.js' +const lTags = loggerTagsFactory('channel-synchronization') + export async function synchronizeChannel (options: { channel: MChannelAccountDefault externalChannelUrl: string @@ -36,7 +38,7 @@ export async function synchronizeChannel (options: { logger.info( 'Fetched %d candidate URLs for sync channel %s.', - targetUrls.length, channel.Actor.preferredUsername, { targetUrls } + targetUrls.length, channel.Actor.preferredUsername, { targetUrls, ...lTags() } ) if (targetUrls.length === 0) { @@ -51,19 +53,25 @@ export async function synchronizeChannel (options: { const children: CreateJobArgument[] = [] for (const targetUrl of targetUrls) { - if (await skipImport(channel, targetUrl, onlyAfter)) continue + logger.debug(`Import candidate: ${targetUrl}`, lTags()) - const { job } = await buildYoutubeDLImport({ - user, - channel, - targetUrl, - channelSync, - importDataOverride: { - privacy: VideoPrivacy.PUBLIC - } - }) + try { + if (await skipImport(channel, targetUrl, onlyAfter)) continue - children.push(job) + const { job } = await buildYoutubeDLImport({ + user, + channel, + targetUrl, + channelSync, + importDataOverride: { + privacy: VideoPrivacy.PUBLIC + } + }) + + children.push(job) + } catch (err) { + logger.error(`Cannot build import for ${targetUrl} in channel ${channel.name}`, { err, ...lTags() }) + } } // Will update the channel sync status @@ -76,7 +84,7 @@ export async function synchronizeChannel (options: { await JobQueue.Instance.createJobWithChildren(parent, children) } catch (err) { - logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err }) + logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err, ...lTags() }) channelSync.state = VideoChannelSyncState.FAILED await channelSync.save() } @@ -86,7 +94,7 @@ export async function synchronizeChannel (options: { async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { - logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name) + logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name, lTags()) return true } diff --git a/server/core/models/video/video-import.ts b/server/core/models/video/video-import.ts index 4de0cb104..3be691c00 100644 --- a/server/core/models/video/video-import.ts +++ b/server/core/models/video/video-import.ts @@ -159,7 +159,7 @@ export class VideoImportModel extends Model