From ca3d5912e16b39697bdeeda35d10b44ed8f711aa Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 9 May 2022 11:49:25 +0200 Subject: [PATCH] Add use proxy for s3 (#4973) * Fix object storage to be accessible via proxy * fix lint * Use hpagent * Fix lint * Fix PR Co-authored-by: noellabo --- package.json | 1 + server/helpers/requests.ts | 1 + server/lib/object-storage/shared/client.ts | 17 ++++++++- server/tests/api/server/proxy.ts | 42 +++++++++++++++++++++- server/tests/shared/checks.ts | 5 +++ yarn.lock | 41 +++++++++++++++++++++ 6 files changed, 105 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 360bd781f..ec9a2a632 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.23.0", "@aws-sdk/lib-storage": "^3.72.0", + "@aws-sdk/node-http-handler": "^3.82.0", "@babel/parser": "7.17.8", "@peertube/feed": "^5.0.1", "@peertube/http-signature": "^1.4.0", diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 327610558..a9869e987 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -212,6 +212,7 @@ export { doRequestAndSaveToFile, isBinaryResponse, downloadImage, + getAgent, findLatestRedirection, peertubeGot } diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts index c9a614593..d5cb074df 100644 --- a/server/lib/object-storage/shared/client.ts +++ b/server/lib/object-storage/shared/client.ts @@ -1,8 +1,22 @@ import { S3Client } from '@aws-sdk/client-s3' +import { NodeHttpHandler } from '@aws-sdk/node-http-handler' import { logger } from '@server/helpers/logger' +import { isProxyEnabled } from '@server/helpers/proxy' +import { getAgent } from '@server/helpers/requests' import { CONFIG } from '@server/initializers/config' import { lTags } from './logger' +function getProxyRequestHandler () { + if (!isProxyEnabled()) return null + + const { agent } = getAgent() + + return new NodeHttpHandler({ + httpAgent: agent.http, + httpsAgent: agent.https + }) +} + let endpointParsed: URL function getEndpointParsed () { if (endpointParsed) return endpointParsed @@ -26,7 +40,8 @@ function getClient () { accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY } - : undefined + : undefined, + requestHandler: getProxyRequestHandler() }) logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts index 2a8ff56d2..e238edaf4 100644 --- a/server/tests/api/server/proxy.ts +++ b/server/tests/api/server/proxy.ts @@ -2,12 +2,14 @@ import 'mocha' import * as chai from 'chai' -import { FIXTURE_URLS, MockProxy } from '@server/tests/shared' +import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' +import { areObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoPrivacy } from '@shared/models' import { cleanupTests, createMultipleServers, doubleFollow, + ObjectStorageCommand, PeerTubeServer, setAccessTokensToServers, setDefaultVideoChannel, @@ -120,6 +122,44 @@ describe('Test proxy', function () { }) }) + describe('Object storage', function () { + if (areObjectStorageTestsDisabled()) return + + before(async function () { + this.timeout(30000) + + await ObjectStorageCommand.prepareDefaultBuckets() + }) + + it('Should succeed to upload to object storage with the appropriate proxy config', async function () { + this.timeout(120000) + + await servers[0].kill() + await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + + expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + }) + + it('Should fail to upload to object storage with a wrong proxy config', async function () { + this.timeout(120000) + + await servers[0].kill() + await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + + expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + }) + }) + after(async function () { await proxy.terminate() diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts index dcc16d7ea..33b917f31 100644 --- a/server/tests/shared/checks.ts +++ b/server/tests/shared/checks.ts @@ -19,6 +19,10 @@ function expectStartWith (str: string, start: string) { expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true } +function expectNotStartWith (str: string, start: string) { + expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false +} + async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { const content = await server.servers.getLogContent() @@ -92,6 +96,7 @@ export { expectLogDoesNotContain, testFileExistsOrNot, expectStartWith, + expectNotStartWith, checkBadStartPagination, checkBadCountPagination, checkBadSortPagination diff --git a/yarn.lock b/yarn.lock index 104f4c241..bfc4102a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -142,6 +142,14 @@ "@aws-sdk/types" "3.55.0" tslib "^2.3.1" +"@aws-sdk/abort-controller@3.78.0": + version "3.78.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/abort-controller/-/abort-controller-3.78.0.tgz#f2b0f8d63954afe51136254f389a18dd24a8f6f3" + integrity sha512-iz1YLwM2feJUj/y97yO4XmDeTxs+yZ1XJwQgoawKuc8IDBKUutnJNCHL5jL04WUKU7Nrlq+Hr2fCTScFh2z9zg== + dependencies: + "@aws-sdk/types" "3.78.0" + tslib "^2.3.1" + "@aws-sdk/chunked-blob-reader-native@3.58.0": version "3.58.0" resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.58.0.tgz#1db413c5c80b32e24f1b62b22e15e9ad74d75cda" @@ -678,6 +686,17 @@ "@aws-sdk/types" "3.55.0" tslib "^2.3.1" +"@aws-sdk/node-http-handler@^3.82.0": + version "3.82.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/node-http-handler/-/node-http-handler-3.82.0.tgz#e28064815c6c6caf22a16bb7fee4e9e7e73ef3bb" + integrity sha512-yyq/DA/IMzL4fLJhV7zVfP7aUQWPHfOKTCJjWB3KeV5YPiviJtSKb/KyzNi+gQyO7SmsL/8vQbQrf3/s7N/2OA== + dependencies: + "@aws-sdk/abort-controller" "3.78.0" + "@aws-sdk/protocol-http" "3.78.0" + "@aws-sdk/querystring-builder" "3.78.0" + "@aws-sdk/types" "3.78.0" + tslib "^2.3.1" + "@aws-sdk/property-provider@3.55.0": version "3.55.0" resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.55.0.tgz#0eabe5e84d9258c85c2c5e44bcb09379ae9429d2" @@ -694,6 +713,14 @@ "@aws-sdk/types" "3.55.0" tslib "^2.3.1" +"@aws-sdk/protocol-http@3.78.0": + version "3.78.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/protocol-http/-/protocol-http-3.78.0.tgz#8a30db90e3373fe94e2b0007c3cba47b5c9e08bd" + integrity sha512-SQB26MhEK96yDxyXd3UAaxLz1Y/ZvgE4pzv7V3wZiokdEedM0kawHKEn1UQJlqJLEZcQI9QYyysh3rTvHZ3fyg== + dependencies: + "@aws-sdk/types" "3.78.0" + tslib "^2.3.1" + "@aws-sdk/querystring-builder@3.55.0": version "3.55.0" resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.55.0.tgz#7d6d4e2c597eb3d636bd3a368b494dac175ba329" @@ -703,6 +730,15 @@ "@aws-sdk/util-uri-escape" "3.55.0" tslib "^2.3.1" +"@aws-sdk/querystring-builder@3.78.0": + version "3.78.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.78.0.tgz#29068c4d1fad056e26f848779a31335469cb0038" + integrity sha512-aib6RW1WAaTQDqVgRU1Ku9idkhm90gJKbCxVaGId+as6QHNUqMChEfK2v+0afuKiPNOs5uWmqvOXI9+Gt+UGDg== + dependencies: + "@aws-sdk/types" "3.78.0" + "@aws-sdk/util-uri-escape" "3.55.0" + tslib "^2.3.1" + "@aws-sdk/querystring-parser@3.55.0": version "3.55.0" resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.55.0.tgz#ea35642c1b8324dd896d45185f99ad9d6c3af6d2" @@ -760,6 +796,11 @@ resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.55.0.tgz#d524d567e2b2722f2d6be83e2417dd6d46ce1490" integrity sha512-wrDZjuy1CVAYxDCbm3bWQIKMGfNs7XXmG0eG4858Ixgqmq2avsIn5TORy8ynBxcXn9aekV/+tGEQ7BBSYzIVNQ== +"@aws-sdk/types@3.78.0": + version "3.78.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.78.0.tgz#51dc80b2142ee20821fb9f476bdca6e541021443" + integrity sha512-I9PTlVNSbwhIgMfmDM5as1tqRIkVZunjVmfogb2WVVPp4CaX0Ll01S0FSMSLL9k6tcQLXqh45pFRjrxCl9WKdQ== + "@aws-sdk/url-parser@3.55.0": version "3.55.0" resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.55.0.tgz#03b47a45c591d52c9d00dc40c630b91094991fe7"