/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' import { cleanupTests, makeRawRequest, ObjectStorageCommand, PeerTubeServer, waitJobs } from '@peertube/peertube-server-commands' import { HttpStatusCode, LiveVideoLatencyMode, UserImportState, UserNotificationSettingValue, VideoCreateResult, VideoPlaylistPrivacy, VideoPlaylistType, VideoPrivacy } from '@peertube/peertube-models' import { prepareImportExportTests } from '@tests/shared/import-export.js' import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { writeFile } from 'fs/promises' import { join } from 'path' import { expect } from 'chai' import { testImage, testImageSize } from '@tests/shared/checks.js' import { completeVideoCheck } from '@tests/shared/videos.js' import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js' function runTest (withObjectStorage: boolean) { let server: PeerTubeServer let remoteServer: PeerTubeServer let blockedServer: PeerTubeServer let noahToken: string let noahId: number const emails: object[] = [] let externalVideo: VideoCreateResult let noahVideo: VideoCreateResult let mouskaVideo: VideoCreateResult let remoteNoahToken: string let remoteNoahId: number let archivePath: string let objectStorage: ObjectStorageCommand let latestImportId: number before(async function () { this.timeout(240000) objectStorage = withObjectStorage ? new ObjectStorageCommand() : undefined; ({ noahId, externalVideo, noahVideo, noahToken, server, remoteNoahId, remoteNoahToken, remoteServer, mouskaVideo, blockedServer } = await prepareImportExportTests({ emails, objectStorage, withBlockedServer: true })) await blockedServer.videos.quickUpload({ name: 'blocked video' }) await waitJobs([ blockedServer ]) // Also add some blocks const blocks = [ { account: 'mouska' }, { account: 'root@' + blockedServer.host }, { server: blockedServer.host } ] for (const toBlock of blocks) { await server.blocklist.addToMyBlocklist({ token: noahToken, ...toBlock }) } // Add avatars await server.users.updateMyAvatar({ token: noahToken, fixture: 'avatar.gif' }) // Add password protected video await server.videos.upload({ token: noahToken, attributes: { name: 'noah password video', privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords: [ 'password1', 'password2' ] } }) // Add a video in watch later playlist const { data: playlists } = await server.playlists.listByAccount({ token: noahToken, handle: 'noah', playlistType: VideoPlaylistType.WATCH_LATER }) await server.playlists.addElement({ playlistId: playlists[0].id, attributes: { videoId: noahVideo.uuid } }) await waitJobs([ server, remoteServer, blockedServer ]) // --------------------------------------------------------------------------- await server.userExports.request({ userId: noahId, withVideoFiles: true }) await server.userExports.waitForCreation({ userId: noahId }) const { data } = await server.userExports.list({ userId: noahId }) const res = await makeRawRequest({ url: data[0].privateDownloadUrl, responseType: 'arraybuffer', redirects: 1, expectedStatus: HttpStatusCode.OK_200 }) archivePath = join(server.getDirectoryPath('tmp'), 'archive.zip') await writeFile(archivePath, res.body) }) it('Should import an archive with video files', async function () { this.timeout(240000) const { userImport } = await remoteServer.userImports.importArchive({ fixture: archivePath, userId: remoteNoahId }) latestImportId = userImport.id await waitJobs([ server, remoteServer ]) }) it('Should have a valid import status', async function () { const userImport = await remoteServer.userImports.getLatestImport({ userId: remoteNoahId, token: remoteNoahToken }) expect(userImport.id).to.equal(latestImportId) expect(userImport.state.id).to.equal(UserImportState.COMPLETED) expect(userImport.state.label).to.equal('Completed') }) it('Should have correctly imported blocklist', async function () { { const { data } = await remoteServer.blocklist.listMyAccountBlocklist({ start: 0, count: 5, token: remoteNoahToken }) expect(data).to.have.lengthOf(2) expect(data.find(a => a.blockedAccount.host === server.host && a.blockedAccount.name === 'mouska')).to.exist expect(data.find(a => a.blockedAccount.host === blockedServer.host && a.blockedAccount.name === 'root')).to.exist } { const { data } = await remoteServer.blocklist.listMyServerBlocklist({ start: 0, count: 5, token: remoteNoahToken }) expect(data).to.have.lengthOf(1) expect(data.find(a => a.blockedServer.host === blockedServer.host)).to.exist } }) it('Should have correctly imported account', async function () { const me = await remoteServer.users.getMyInfo({ token: remoteNoahToken }) expect(me.account.displayName).to.equal('noah') expect(me.username).to.equal('noah_remote') expect(me.account.description).to.equal('super noah description') for (const avatar of me.account.avatars) { await testImageSize(remoteServer.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, '.gif') } }) it('Should have correctly imported user settings', async function () { { const me = await remoteServer.users.getMyInfo({ token: remoteNoahToken }) expect(me.p2pEnabled).to.be.false const settings = me.notificationSettings expect(settings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL) expect(settings.myVideoPublished).to.equal(UserNotificationSettingValue.NONE) expect(settings.commentMention).to.equal(UserNotificationSettingValue.EMAIL) } }) it('Should have correctly imported channels', async function () { const { data: channels } = await remoteServer.channels.listByAccount({ token: remoteNoahToken, accountName: 'noah_remote' }) // One default + 2 imported expect(channels).to.have.lengthOf(3) await remoteServer.channels.get({ token: remoteNoahToken, channelName: 'noah_remote_channel' }) const importedMain = await remoteServer.channels.get({ token: remoteNoahToken, channelName: 'noah_channel' }) expect(importedMain.displayName).to.equal('Main noah channel') expect(importedMain.avatars).to.have.lengthOf(0) expect(importedMain.banners).to.have.lengthOf(0) const importedSecond = await remoteServer.channels.get({ token: remoteNoahToken, channelName: 'noah_second_channel' }) expect(importedSecond.displayName).to.equal('noah display name') expect(importedSecond.description).to.equal('noah description') expect(importedSecond.support).to.equal('noah support') await testImage(remoteServer.url, 'banner-resized', importedSecond.banners[0].path) for (const avatar of importedSecond.avatars) { await testImage(remoteServer.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') } { // Also check the correct count on origin server const { data: channels } = await server.channels.listByAccount({ accountName: 'noah_remote@' + remoteServer.host }) expect(channels).to.have.lengthOf(2) // noah_remote_channel doesn't have videos so it has not been federated } }) it('Should have correctly imported following', async function () { const { data } = await remoteServer.subscriptions.list({ token: remoteNoahToken }) expect(data).to.have.lengthOf(2) expect(data.find(f => f.name === 'mouska_channel' && f.host === server.host)).to.exist expect(data.find(f => f.name === 'root_channel' && f.host === remoteServer.host)).to.exist }) it('Should not have reimported followers (it is not a migration)', async function () { for (const checkServer of [ server, remoteServer ]) { const { data } = await checkServer.channels.listFollowers({ channelName: 'noah_channel@' + remoteServer.host }) expect(data).to.have.lengthOf(0) } }) it('Should not have imported comments (it is not a migration)', async function () { for (const checkServer of [ server, remoteServer ]) { { const threads = await checkServer.comments.listThreads({ videoId: noahVideo.uuid }) expect(threads.total).to.equal(2) } { const threads = await checkServer.comments.listThreads({ videoId: mouskaVideo.uuid }) expect(threads.total).to.equal(1) } } }) it('Should have correctly imported likes/dislikes', async function () { { const { rating } = await remoteServer.users.getMyRating({ videoId: mouskaVideo.uuid, token: remoteNoahToken }) expect(rating).to.equal('like') for (const checkServer of [ server, remoteServer ]) { const video = await checkServer.videos.get({ id: mouskaVideo.uuid }) expect(video.likes).to.equal(2) // Old account + new account rates expect(video.dislikes).to.equal(0) } } { const { rating } = await remoteServer.users.getMyRating({ videoId: noahVideo.uuid, token: remoteNoahToken }) expect(rating).to.equal('like') } { const { rating } = await remoteServer.users.getMyRating({ videoId: externalVideo.uuid, token: remoteNoahToken }) expect(rating).to.equal('dislike') } }) it('Should have correctly imported user video playlists', async function () { const { data } = await remoteServer.playlists.listByAccount({ handle: 'noah_remote', token: remoteNoahToken }) // Should merge the watch later playlists expect(data).to.have.lengthOf(3) { const watchLater = data.find(p => p.type.id === VideoPlaylistType.WATCH_LATER) expect(watchLater).to.exist expect(watchLater.privacy.id).to.equal(VideoPlaylistPrivacy.PRIVATE) // Playlists were merged expect(watchLater.videosLength).to.equal(1) const { data: videos } = await remoteServer.playlists.listVideos({ playlistId: watchLater.id, token: remoteNoahToken }) expect(videos[0].position).to.equal(1) expect(videos[0].video.uuid).to.equal(noahVideo.uuid) // Not federated await server.playlists.get({ playlistId: watchLater.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) } { const playlist1 = data.find(p => p.displayName === 'noah playlist 1') expect(playlist1).to.exist expect(playlist1.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) expect(playlist1.videosLength).to.equal(2) // 1 private video could not be imported const { data: videos } = await remoteServer.playlists.listVideos({ playlistId: playlist1.id, token: remoteNoahToken }) expect(videos[0].position).to.equal(1) expect(videos[0].startTimestamp).to.equal(2) expect(videos[0].stopTimestamp).to.equal(3) expect(videos[0].video).to.not.exist // Mouska is blocked expect(videos[1].position).to.equal(2) expect(videos[1].video.uuid).to.equal(noahVideo.uuid) // Federated await server.playlists.get({ playlistId: playlist1.uuid }) } { const playlist2 = data.find(p => p.displayName === 'noah playlist 2') expect(playlist2).to.exist expect(playlist2.privacy.id).to.equal(VideoPlaylistPrivacy.PRIVATE) expect(playlist2.videosLength).to.equal(0) // Federated await server.playlists.get({ playlistId: playlist2.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) } }) it('Should have correctly imported user videos', async function () { const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken }) expect(data).to.have.lengthOf(5) { const privateVideo = data.find(v => v.name === 'noah private video') expect(privateVideo).to.exist expect(privateVideo.privacy.id).to.equal(VideoPrivacy.PRIVATE) // Not federated await server.videos.get({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) } { const publicVideo = data.find(v => v.name === 'noah public video') expect(publicVideo).to.exist expect(publicVideo.privacy.id).to.equal(VideoPrivacy.PUBLIC) // Federated await server.videos.get({ id: publicVideo.uuid }) } { const passwordVideo = data.find(v => v.name === 'noah password video') expect(passwordVideo).to.exist expect(passwordVideo.privacy.id).to.equal(VideoPrivacy.PASSWORD_PROTECTED) const { data: passwords } = await remoteServer.videoPasswords.list({ videoId: passwordVideo.uuid }) expect(passwords.map(p => p.password).sort()).to.deep.equal([ 'password1', 'password2' ]) // Not federated await server.videos.get({ id: passwordVideo.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) } { const otherVideo = data.find(v => v.name === 'noah public video second channel') expect(otherVideo).to.exist for (const checkServer of [ server, remoteServer ]) { await completeVideoCheck({ server: checkServer, originServer: remoteServer, videoUUID: otherVideo.uuid, objectStorageBaseUrl: objectStorage?.getMockWebVideosBaseUrl(), attributes: { name: 'noah public video second channel', privacy: (VideoPrivacy.PUBLIC), category: (12), tags: [ 'tag1', 'tag2' ], commentsEnabled: false, downloadEnabled: false, nsfw: false, description: ('video description'), support: ('video support'), language: 'fr', licence: 1, originallyPublishedAt: new Date(0).toISOString(), account: { name: 'noah_remote', host: remoteServer.host }, isLocal: checkServer === remoteServer, likes: 0, dislikes: 0, duration: 5, channel: { displayName: 'noah display name', name: 'noah_second_channel', description: 'noah description', isLocal: checkServer === remoteServer }, fixture: 'video_short.webm', files: [ { resolution: 720, size: 61000 }, { resolution: 240, size: 23000 } ], thumbnailfile: 'custom-thumbnail-from-preview', previewfile: 'custom-preview' } }) } await completeCheckHlsPlaylist({ hlsOnly: false, servers: [ remoteServer, server ], videoUUID: otherVideo.uuid, objectStorageBaseUrl: objectStorage?.getMockPlaylistBaseUrl(), resolutions: [ 720, 240 ] }) const source = await remoteServer.videos.getSource({ id: otherVideo.uuid }) expect(source.filename).to.equal('video_short.webm') } { const liveVideo = data.find(v => v.name === 'noah live video') expect(liveVideo).to.exist await remoteServer.videos.get({ id: liveVideo.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) const video = await remoteServer.videos.getWithPassword({ id: liveVideo.uuid, password: 'password1' }) const live = await remoteServer.live.get({ videoId: liveVideo.uuid, token: remoteNoahToken }) expect(video.isLive).to.be.true expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY) expect(live.saveReplay).to.be.true expect(live.permanentLive).to.be.true expect(live.streamKey).to.exist expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) expect(video.channel.name).to.equal('noah_second_channel') expect(video.privacy.id).to.equal(VideoPrivacy.PASSWORD_PROTECTED) expect(video.duration).to.equal(0) expect(video.files).to.have.lengthOf(0) expect(video.streamingPlaylists).to.have.lengthOf(0) } }) it('Should re-import the same file', async function () { this.timeout(240000) const { userImport } = await remoteServer.userImports.importArchive({ fixture: archivePath, userId: remoteNoahId }) await waitJobs([ remoteServer ]) latestImportId = userImport.id }) it('Should have the status of this new reimport', async function () { const userImport = await remoteServer.userImports.getLatestImport({ userId: remoteNoahId, token: remoteNoahToken }) expect(userImport.id).to.equal(latestImportId) expect(userImport.state.id).to.equal(UserImportState.COMPLETED) expect(userImport.state.label).to.equal('Completed') }) it('Should not have duplicated data', async function () { // Blocklist { { const { data } = await remoteServer.blocklist.listMyAccountBlocklist({ start: 0, count: 5, token: remoteNoahToken }) expect(data).to.have.lengthOf(2) } { const { data } = await remoteServer.blocklist.listMyServerBlocklist({ start: 0, count: 5, token: remoteNoahToken }) expect(data).to.have.lengthOf(1) } } // My avatars { const me = await remoteServer.users.getMyInfo({ token: remoteNoahToken }) expect(me.account.avatars).to.have.lengthOf(2) } // Channels { const { data: channels } = await remoteServer.channels.listByAccount({ token: remoteNoahToken, accountName: 'noah_remote' }) expect(channels).to.have.lengthOf(3) } // Following { const { data } = await remoteServer.subscriptions.list({ token: remoteNoahToken }) expect(data).to.have.lengthOf(2) } // Likes/dislikes { const video = await remoteServer.videos.get({ id: mouskaVideo.uuid }) expect(video.likes).to.equal(2) expect(video.dislikes).to.equal(0) const { rating } = await remoteServer.users.getMyRating({ videoId: mouskaVideo.uuid, token: remoteNoahToken }) expect(rating).to.equal('like') } // Playlists { const { data } = await remoteServer.playlists.listByAccount({ handle: 'noah_remote', token: remoteNoahToken }) expect(data).to.have.lengthOf(3) } // Videos { const { data } = await remoteServer.videos.listMyVideos({ token: remoteNoahToken }) expect(data).to.have.lengthOf(5) } }) it('Should have received an email on finished import', async function () { const email = emails.reverse().find(e => { return e['to'][0]['address'] === 'noah_remote@example.com' && e['subject'].includes('archive import has finished') }) expect(email).to.exist expect(email['text']).to.contain('as considered duplicate: 5') // 5 videos are considered as duplicates }) it('Should auto blacklist imported videos if enabled by the administrator', async function () { this.timeout(240000) await blockedServer.config.enableAutoBlacklist() const { token, userId } = await blockedServer.users.generate('blocked_user') await blockedServer.userImports.importArchive({ fixture: archivePath, userId, token }) await waitJobs([ blockedServer ]) { const { data } = await blockedServer.videos.listMyVideos({ token }) expect(data).to.have.lengthOf(5) for (const video of data) { expect(video.blacklisted).to.be.true } } }) after(async function () { MockSmtpServer.Instance.kill() await cleanupTests([ server, remoteServer, blockedServer ]) }) } describe('Test user import', function () { describe('From filesystem', function () { runTest(false) }) describe('From object storage', function () { if (areMockObjectStorageTestsDisabled()) return runTest(true) }) })