From b488ba1e26b803ac6c637e8b11bdd444ca4c803f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 30 Jul 2020 09:43:12 +0200 Subject: [PATCH] Don't rely on youtube for tests Use another import URL when possible, and disable import tests when we want to do a youtube import test --- .github/workflows/test.yml | 10 +++++--- scripts/ci.sh | 2 +- server/tests/api/check-params/users.ts | 13 +++++----- .../tests/api/check-params/video-imports.ts | 24 +++++++++---------- .../api/notifications/user-notifications.ts | 10 ++++---- server/tests/api/videos/video-blacklist.ts | 10 ++++---- server/tests/api/videos/video-imports.ts | 14 ++++++----- server/tests/plugins/filter-hooks.ts | 8 +++---- shared/extra-utils/miscs/miscs.ts | 15 ++++++++---- shared/extra-utils/videos/video-imports.ts | 7 +++++- 10 files changed, 64 insertions(+), 49 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index feb885378..a2f93ce3f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,15 @@ jobs: POSTGRES_HOST_AUTH_METHOD: trust strategy: - fail-fast: true + fail-fast: false matrix: test_suite: [ misc, api-1, api-2, api-3, api-4, cli, lint ] env: PGUSER: peertube PGHOST: localhost + DISABLE_HTTP_IMPORT_TESTS: true + NODE_PENDING_JOB_WAIT: 2000 steps: - uses: actions/checkout@v2 @@ -50,7 +52,9 @@ jobs: - name: Cache Node.js modules uses: actions/cache@v2 with: - path: ~/.cache/yarn + path: | + **/node_modules + ~/fixtures key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.OS }}-node- @@ -69,5 +73,5 @@ jobs: uses: actions/upload-artifact@v2 if: failure() with: - name: test-storages + name: test-storages-${{ matrix.test_suite }} path: test*/logs diff --git a/scripts/ci.sh b/scripts/ci.sh index f5bd0bdc9..cef276da3 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -21,7 +21,7 @@ if [ "$1" = "misc" ]; then server/tests/plugins/index.ts elif [ "$1" = "cli" ]; then npm run build:server - CC=gcc-4.9 CXX=g++-4.9 npm run setup:cli + npm run setup:cli mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail server/tests/cli/index.ts elif [ "$1" = "api-1" ]; then npm run build:server diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index 2fea2cbd5..3e53c445d 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { omit } from 'lodash' import 'mocha' +import { expect } from 'chai' +import { omit } from 'lodash' import { join } from 'path' import { User, UserRole, VideoImport, VideoImportState } from '../../../../shared' - import { addVideoChannel, blockUser, @@ -31,17 +31,16 @@ import { uploadVideo, userLogin } from '../../../../shared/extra-utils' +import { MockSmtpServer } from '../../../../shared/extra-utils/miscs/email' import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' -import { VideoPrivacy } from '../../../../shared/models/videos' import { waitJobs } from '../../../../shared/extra-utils/server/jobs' -import { expect } from 'chai' +import { getMagnetURI, getMyVideoImports, getGoodVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model' -import { MockSmtpServer } from '../../../../shared/extra-utils/miscs/email' +import { VideoPrivacy } from '../../../../shared/models/videos' describe('Test users API validators', function () { const path = '/api/v1/users/' @@ -1047,7 +1046,7 @@ describe('Test users API validators', function () { channelId: 1, privacy: VideoPrivacy.PUBLIC } - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index 3759e47c4..f954ba089 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts @@ -1,9 +1,8 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { omit } from 'lodash' import 'mocha' +import { omit } from 'lodash' import { join } from 'path' -import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' import { cleanupTests, createUser, @@ -23,7 +22,8 @@ import { checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { getMagnetURI, getYoutubeVideoUrl } from '../../../../shared/extra-utils/videos/video-imports' +import { getMagnetURI, getGoodVideoUrl } from '../../../../shared/extra-utils/videos/video-imports' +import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' describe('Test video imports API validator', function () { const path = '/api/v1/videos/imports' @@ -76,7 +76,7 @@ describe('Test video imports API validator', function () { before(function () { baseCorrectParams = { - targetUrl: getYoutubeVideoUrl(), + targetUrl: getGoodVideoUrl(), name: 'my super name', category: 5, licence: 1, @@ -246,15 +246,13 @@ describe('Test video imports API validator', function () { it('Should succeed with the correct parameters', async function () { this.timeout(30000) - { - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - statusCodeExpected: 200 - }) - } + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + statusCodeExpected: 200 + }) }) it('Should forbid to import http videos', async function () { diff --git a/server/tests/api/notifications/user-notifications.ts b/server/tests/api/notifications/user-notifications.ts index e5f6aa864..af4ff42b0 100644 --- a/server/tests/api/notifications/user-notifications.ts +++ b/server/tests/api/notifications/user-notifications.ts @@ -24,7 +24,7 @@ import { prepareNotificationsTest } from '../../../../shared/extra-utils/users/user-notifications' import { addUserSubscription, removeUserSubscription } from '../../../../shared/extra-utils/users/user-subscriptions' -import { getBadVideoUrl, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' +import { getBadVideoUrl, getGoodVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' import { UserNotification, UserNotificationType } from '../../../../shared/models/users' import { VideoPrivacy } from '../../../../shared/models/videos' @@ -213,7 +213,7 @@ describe('Test user notifications', function () { name, channelId, privacy: VideoPrivacy.PUBLIC, - targetUrl: getYoutubeVideoUrl() + targetUrl: getGoodVideoUrl() } const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) const uuid = res.body.video.uuid @@ -284,7 +284,7 @@ describe('Test user notifications', function () { name, channelId, privacy: VideoPrivacy.PUBLIC, - targetUrl: getYoutubeVideoUrl(), + targetUrl: getGoodVideoUrl(), waitTranscoding: true } const res = await importVideo(servers[1].url, servers[1].accessToken, attributes) @@ -371,13 +371,13 @@ describe('Test user notifications', function () { name, channelId, privacy: VideoPrivacy.PRIVATE, - targetUrl: getYoutubeVideoUrl() + targetUrl: getGoodVideoUrl() } const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) const uuid = res.body.video.uuid await waitJobs(servers) - await checkMyVideoImportIsFinished(baseParams, name, uuid, getYoutubeVideoUrl(), true, 'presence') + await checkMyVideoImportIsFinished(baseParams, name, uuid, getGoodVideoUrl(), true, 'presence') }) }) diff --git a/server/tests/api/videos/video-blacklist.ts b/server/tests/api/videos/video-blacklist.ts index a8500627b..52cac20d9 100644 --- a/server/tests/api/videos/video-blacklist.ts +++ b/server/tests/api/videos/video-blacklist.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { orderBy } from 'lodash' -import 'mocha' import { addVideoToBlacklist, cleanupTests, @@ -25,10 +25,10 @@ import { } from '../../../../shared/extra-utils/index' import { doubleFollow } from '../../../../shared/extra-utils/server/follows' import { waitJobs } from '../../../../shared/extra-utils/server/jobs' -import { VideoBlacklist, VideoBlacklistType } from '../../../../shared/models/videos' -import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model' +import { getGoodVideoUrl, getMagnetURI, importVideo } from '../../../../shared/extra-utils/videos/video-imports' import { User, UserRole } from '../../../../shared/models/users' -import { getMagnetURI, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' +import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model' +import { VideoBlacklist, VideoBlacklistType } from '../../../../shared/models/videos' const expect = chai.expect @@ -424,7 +424,7 @@ describe('Test video blacklist', function () { this.timeout(15000) const attributes = { - targetUrl: getYoutubeVideoUrl(), + targetUrl: getGoodVideoUrl(), name: 'URL import', channelId: channelOfUserWithoutFlag } diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index 05ee36b27..efda4fc7b 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -1,8 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import * as chai from 'chai' import 'mocha' -import { VideoDetails, VideoImport, VideoPrivacy, VideoCaption } from '../../../../shared/models/videos' +import * as chai from 'chai' import { cleanupTests, doubleFollow, @@ -11,15 +10,16 @@ import { getMyVideos, getVideo, getVideosList, - listVideoCaptions, - testCaptionFile, immutableAssign, + listVideoCaptions, ServerInfo, - setAccessTokensToServers + setAccessTokensToServers, + testCaptionFile } from '../../../../shared/extra-utils' +import { areHttpImportTestsDisabled, testImage } from '../../../../shared/extra-utils/miscs/miscs' import { waitJobs } from '../../../../shared/extra-utils/server/jobs' import { getMagnetURI, getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../../shared/extra-utils/videos/video-imports' -import { testImage } from '../../../../shared/extra-utils/miscs/miscs' +import { VideoCaption, VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' const expect = chai.expect @@ -28,6 +28,8 @@ describe('Test video imports', function () { let channelIdServer1: number let channelIdServer2: number + if (areHttpImportTestsDisabled()) return + async function checkVideosServer1 (url: string, idHttp: string, idMagnet: string, idTorrent: string) { const resHttp = await getVideo(url, idHttp) const videoHttp: VideoDetails = resHttp.body diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts index 41242318e..4d354b68e 100644 --- a/server/tests/plugins/filter-hooks.ts +++ b/server/tests/plugins/filter-hooks.ts @@ -24,7 +24,7 @@ import { waitJobs } from '../../../shared/extra-utils' import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers' -import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports' +import { getGoodVideoUrl, getMyVideoImports, importVideo } from '../../../shared/extra-utils/videos/video-imports' import { VideoDetails, VideoImport, VideoImportState, VideoPrivacy } from '../../../shared/models/videos' import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model' @@ -92,7 +92,7 @@ describe('Test plugin filter hooks', function () { name: 'normal title', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].videoChannel.id, - targetUrl: getYoutubeVideoUrl() + 'bad' + targetUrl: getGoodVideoUrl() + 'bad' } await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403) }) @@ -117,7 +117,7 @@ describe('Test plugin filter hooks', function () { name: 'title with bad word', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].videoChannel.id, - targetUrl: getYoutubeVideoUrl() + targetUrl: getGoodVideoUrl() } const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes) videoImportId = res.body.id @@ -220,7 +220,7 @@ describe('Test plugin filter hooks', function () { const attributes = { name: 'video please blacklist me', - targetUrl: getYoutubeVideoUrl(), + targetUrl: getGoodVideoUrl(), channelId: servers[0].videoChannel.id } const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) diff --git a/shared/extra-utils/miscs/miscs.ts b/shared/extra-utils/miscs/miscs.ts index 2239d5bcc..caf09c01f 100644 --- a/shared/extra-utils/miscs/miscs.ts +++ b/shared/extra-utils/miscs/miscs.ts @@ -64,19 +64,25 @@ async function testImage (url: string, imageName: string, imagePath: string, ext } function buildAbsoluteFixturePath (path: string, customCIPath = false) { - if (isAbsolute(path)) { - return path - } + if (isAbsolute(path)) return path if (customCIPath) { if (process.env.GITLAB_CI) return join(root(), 'cached-fixtures', path) - if (process.env.TRAVIS) return join(process.env.HOME, 'fixtures', path) + return join(process.env.HOME, 'fixtures', path) } return join(root(), 'server', 'tests', 'fixtures', path) } +function areHttpImportTestsDisabled () { + const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' + + if (disabled) console.log('Import tests are disabled') + + return disabled +} + async function generateHighBitrateVideo () { const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) @@ -131,6 +137,7 @@ async function generateVideoWithFramerate (fps = 60) { export { dateIsValid, wait, + areHttpImportTestsDisabled, buildServerDirectory, webtorrentAdd, immutableAssign, diff --git a/shared/extra-utils/videos/video-imports.ts b/shared/extra-utils/videos/video-imports.ts index d235181b0..6249e8a94 100644 --- a/shared/extra-utils/videos/video-imports.ts +++ b/shared/extra-utils/videos/video-imports.ts @@ -15,6 +15,10 @@ function getBadVideoUrl () { return 'https://download.cpy.re/peertube/bad_video.mp4' } +function getGoodVideoUrl () { + return 'https://download.cpy.re/peertube/good_video.mp4' +} + function importVideo (url: string, token: string, attributes: VideoImportCreate & { torrentfile?: string }, statusCodeExpected = 200) { const path = '/api/v1/videos/imports' @@ -53,5 +57,6 @@ export { getYoutubeVideoUrl, importVideo, getMagnetURI, - getMyVideoImports + getMyVideoImports, + getGoodVideoUrl }