From 3204f4d17dec79413a94eae8a71b72e0c7ec7ba5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 16 Sep 2022 09:37:07 +0200 Subject: [PATCH] Fix sync import of latest videos --- .../youtube-dl/youtube-dl-info-builder.ts | 4 +- .../handlers/video-channel-import.ts | 4 +- server/lib/sync-channel.ts | 5 +- server/lib/video-import.ts | 2 +- .../tests/api/videos/video-channel-syncs.ts | 138 ++++++++++++------ 5 files changed, 106 insertions(+), 47 deletions(-) diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts index 303e4051f..a74904e43 100644 --- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts +++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts @@ -12,7 +12,7 @@ type YoutubeDLInfo = { tags?: string[] thumbnailUrl?: string ext?: string - originallyPublishedAt?: Date + originallyPublishedAtWithoutTime?: Date webpageUrl?: string urls?: string[] @@ -81,7 +81,7 @@ class YoutubeDLInfoBuilder { tags: this.getTags(obj.tags), thumbnailUrl: obj.thumbnail || undefined, urls: this.buildAvailableUrl(obj), - originallyPublishedAt: this.buildOriginallyPublishedAt(obj), + originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj), ext: obj.ext, webpageUrl: obj.webpage_url } diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts index 9aaad659e..600292844 100644 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ b/server/lib/job-queue/handlers/video-channel-import.ts @@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel' import { VideoChannelModel } from '@server/models/video/video-channel' import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' import { MChannelSync } from '@server/types/models' -import { VideoChannelImportPayload } from '@shared/models' +import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models' export async function processVideoChannelImport (job: Job) { const payload = job.data as VideoChannelImportPayload @@ -42,5 +42,7 @@ export async function processVideoChannelImport (job: Job) { }) } catch (err) { logger.error(`Failed to import channel ${videoChannel.name}`, { err }) + channelSync.state = VideoChannelSyncState.FAILED + await channelSync.save() } } diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index 3a81daac0..6cd1fc26d 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts @@ -93,7 +93,10 @@ async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Dat const videoInfo = await youtubeDL.getInfoForDownload() - if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) { + const onlyAfterWithoutTime = new Date(onlyAfter) + onlyAfterWithoutTime.setHours(0, 0, 0) + + if (videoInfo.originallyPublishedAtWithoutTime.getTime() >= onlyAfterWithoutTime.getTime()) { return true } } diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts index de95116aa..d9da09040 100644 --- a/server/lib/video-import.ts +++ b/server/lib/video-import.ts @@ -125,7 +125,7 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride channelId, originallyPublishedAt: importDataOverride?.originallyPublishedAt ? new Date(importDataOverride?.originallyPublishedAt) - : importData.originallyPublishedAt + : importData.originallyPublishedAtWithoutTime } videoData = await Hooks.wrapObject( diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts index 4e18ddfc7..65b143a4f 100644 --- a/server/tests/api/videos/video-channel-syncs.ts +++ b/server/tests/api/videos/video-channel-syncs.ts @@ -3,9 +3,9 @@ import { FIXTURE_URLS } from '@server/tests/shared' import { areHttpImportTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models' import { - ChannelSyncsCommand, - createSingleServer, + createMultipleServers, getServerImportConfig, + killallServers, PeerTubeServer, setAccessTokensToServers, setDefaultAccountAvatar, @@ -20,8 +20,7 @@ describe('Test channel synchronizations', function () { function runSuite (mode: 'youtube-dl' | 'yt-dlp') { describe('Sync using ' + mode, function () { - let server: PeerTubeServer - let command: ChannelSyncsCommand + let servers: PeerTubeServer[] let startTestDate: Date @@ -35,7 +34,7 @@ describe('Test channel synchronizations', function () { } async function changeDateForSync (channelSyncId: number, newDate: string) { - await server.sql.updateQuery( + await servers[0].sql.updateQuery( `UPDATE "videoChannelSync" ` + `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + `WHERE id=${channelSyncId}` @@ -43,25 +42,23 @@ describe('Test channel synchronizations', function () { } before(async function () { - this.timeout(120_000) + this.timeout(240_000) startTestDate = new Date() - server = await createSingleServer(1, getServerImportConfig(mode)) + servers = await createMultipleServers(2, getServerImportConfig(mode)) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - await setDefaultChannelAvatar([ server ]) - await setDefaultAccountAvatar([ server ]) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultChannelAvatar(servers) + await setDefaultAccountAvatar(servers) - await server.config.enableChannelSync() - - command = server.channelSyncs + await servers[0].config.enableChannelSync() { - userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) + userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username) - const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken }) + const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken }) userInfo.channelId = videoChannels[0].id } }) @@ -70,9 +67,9 @@ describe('Test channel synchronizations', function () { this.timeout(120_000) { - const { video } = await server.imports.importVideo({ + const { video } = await servers[0].imports.importVideo({ attributes: { - channelId: server.store.channel.id, + channelId: servers[0].store.channel.id, privacy: VideoPrivacy.PUBLIC, targetUrl: FIXTURE_URLS.youtube } @@ -80,16 +77,16 @@ describe('Test channel synchronizations', function () { expect(video.name).to.equal('small video - youtube') - const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) + const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) expect(total).to.equal(1) } - const { videoChannelSync } = await command.create({ + const { videoChannelSync } = await servers[0].channelSyncs.create({ attributes: { externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: server.store.channel.id + videoChannelId: servers[0].store.channel.id }, - token: server.accessToken, + token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) rootChannelSyncId = videoChannelSync.id @@ -97,16 +94,19 @@ describe('Test channel synchronizations', function () { // Ensure any missing video not already fetched will be considered as new await changeDateForSync(videoChannelSync.id, '1970-01-01') - await server.debug.sendCommand({ + await servers[0].debug.sendCommand({ body: { command: 'process-video-channel-sync-latest' } }) { - await waitJobs(server) + await waitJobs(servers) - const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) + const { total, data } = await servers[0].videos.listByChannel({ + handle: 'root_channel', + include: VideoInclude.NOT_PUBLISHED_STATE + }) expect(total).to.equal(2) expect(data[0].name).to.equal('test') } @@ -115,18 +115,18 @@ describe('Test channel synchronizations', function () { it('Should add another synchronization', async function () { const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' - const { videoChannelSync } = await command.create({ + const { videoChannelSync } = await servers[0].channelSyncs.create({ attributes: { externalChannelUrl, - videoChannelId: server.store.channel.id + videoChannelId: servers[0].store.channel.id }, - token: server.accessToken, + token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) expect(videoChannelSync.channel).to.include({ - id: server.store.channel.id, + id: servers[0].store.channel.id, name: 'root_channel' }) expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) @@ -134,7 +134,7 @@ describe('Test channel synchronizations', function () { }) it('Should add a synchronization for another user', async function () { - const { videoChannelSync } = await command.create({ + const { videoChannelSync } = await servers[0].channelSyncs.create({ attributes: { externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', videoChannelId: userInfo.channelId @@ -145,9 +145,9 @@ describe('Test channel synchronizations', function () { }) it('Should not import a channel if not asked', async function () { - await waitJobs(server) + await waitJobs(servers) - const { data } = await command.listByAccount({ accountName: userInfo.username }) + const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) expect(data[0].state).to.contain({ id: VideoChannelSyncState.WAITING_FIRST_RUN, @@ -160,15 +160,15 @@ describe('Test channel synchronizations', function () { await changeDateForSync(userInfo.syncId, '2019-03-01') - await server.debug.sendCommand({ + await servers[0].debug.sendCommand({ body: { command: 'process-video-channel-sync-latest' } }) - await waitJobs(server) + await waitJobs(servers) - const { data, total } = await server.videos.listByChannel({ + const { data, total } = await servers[0].videos.listByChannel({ handle: userInfo.channelName, include: VideoInclude.NOT_PUBLISHED_STATE }) @@ -180,7 +180,7 @@ describe('Test channel synchronizations', function () { it('Should list channel synchronizations', async function () { // Root { - const { total, data } = await command.listByAccount({ accountName: 'root' }) + const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' }) expect(total).to.equal(2) expect(data[0]).to.deep.contain({ @@ -193,13 +193,13 @@ describe('Test channel synchronizations', function () { expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) - expect(data[0].channel).to.contain({ id: server.store.channel.id }) + expect(data[0].channel).to.contain({ id: servers[0].store.channel.id }) expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) } // User { - const { total, data } = await command.listByAccount({ accountName: userInfo.username }) + const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) expect(total).to.equal(1) expect(data[0]).to.deep.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', @@ -212,7 +212,7 @@ describe('Test channel synchronizations', function () { }) it('Should list imports of a channel synchronization', async function () { - const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) + const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) expect(total).to.equal(1) expect(data).to.have.lengthOf(1) @@ -220,14 +220,68 @@ describe('Test channel synchronizations', function () { }) it('Should remove user\'s channel synchronizations', async function () { - await command.delete({ channelSyncId: userInfo.syncId }) + await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId }) - const { total } = await command.listByAccount({ accountName: userInfo.username }) + const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) expect(total).to.equal(0) }) + // FIXME: youtube-dl doesn't work when speicifying a port after the hostname + // it('Should import a remote PeerTube channel', async function () { + // this.timeout(240_000) + + // await servers[1].videos.quickUpload({ name: 'remote 1' }) + // await waitJobs(servers) + + // const { videoChannelSync } = await servers[0].channelSyncs.create({ + // attributes: { + // externalChannelUrl: servers[1].url + '/c/root_channel', + // videoChannelId: userInfo.channelId + // }, + // token: userInfo.accessToken + // }) + // await servers[0].channels.importVideos({ + // channelName: userInfo.channelName, + // externalChannelUrl: servers[1].url + '/c/root_channel', + // videoChannelSyncId: videoChannelSync.id, + // token: userInfo.accessToken + // }) + + // await waitJobs(servers) + + // const { data, total } = await servers[0].videos.listByChannel({ + // handle: userInfo.channelName, + // include: VideoInclude.NOT_PUBLISHED_STATE + // }) + + // expect(total).to.equal(2) + // expect(data[0].name).to.equal('remote 1') + // }) + + // it('Should keep synced a remote PeerTube channel', async function () { + // this.timeout(240_000) + + // await servers[1].videos.quickUpload({ name: 'remote 2' }) + // await waitJobs(servers) + + // await servers[0].debug.sendCommand({ + // body: { + // command: 'process-video-channel-sync-latest' + // } + // }) + + // await waitJobs(servers) + + // const { data, total } = await servers[0].videos.listByChannel({ + // handle: userInfo.channelName, + // include: VideoInclude.NOT_PUBLISHED_STATE + // }) + // expect(total).to.equal(2) + // expect(data[0].name).to.equal('remote 2') + // }) + after(async function () { - await server?.kill() + await killallServers(servers) }) }) }