feat: config option object_storage.max_request_attempts (#6418)

* feat: config option object_storage.max_attempts

Backblaze recommends to have a high amount of attempts since they've
designed their architecture so that it will return 5xx errors to
indicate that the client should do a new attempt.

https://www.backblaze.com/blog/b2-503-500-server-error/

closes #6415

* Rephrase comment

---------

Co-authored-by: Chocobozzz <me@florianbigard.com>

config/default.yaml

@@ -214,6 +214,11 @@ object_storage:
   # Maximum amount to upload in one request to object storage
   max_upload_part: 100MB
 
+  # Maximum number of attempts to make a request to object storage
+  # Some object storage providers (for instance Backblaze) expects the client to retry upload upon 5xx errors
+  # If you're using such a provider then you can increase this value
+  max_request_attempts: 3
+
   streaming_playlists:
     bucket_name: 'streaming-playlists'
 

config/production.yaml.example

@@ -212,6 +212,11 @@ object_storage:
   # Maximum amount to upload in one request to object storage
   max_upload_part: 100MB
 
+  # Maximum number of attempts to make a request to object storage
+  # Some object storage providers (for instance Backblaze) expects the client to retry upload upon 5xx errors
+  # If you're using such a provider then you can increase this value
+  max_request_attempts: 3
+
   streaming_playlists:
     bucket_name: 'streaming-playlists'
 

server/core/initializers/checker-before-init.ts

@@ -67,7 +67,7 @@ function checkMissedConfig () {
     'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name',
     'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name',
     'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', 'object_storage.original_video_files.bucket_name',
-    'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url',
+    'object_storage.original_video_files.prefix', 'object_storage.original_video_files.base_url', 'object_storage.max_request_attempts',
     'theme.default',
     'feeds.videos.count', 'feeds.comments.count',
     'geo_ip.enabled', 'geo_ip.country.database_url', 'geo_ip.city.database_url',

server/core/initializers/config.ts

@@ -133,6 +133,7 @@ const CONFIG = {
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),
     MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
+    MAX_REQUEST_ATTEMPTS: config.get<number>('object_storage.max_request_attempts'),
     ENDPOINT: config.get<string>('object_storage.endpoint'),
     REGION: config.get<string>('object_storage.region'),
     UPLOAD_ACL: {

server/core/lib/object-storage/shared/client.ts

@@ -45,7 +45,8 @@ function getClient () {
           secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
         }
         : undefined,
-      requestHandler: await getProxyRequestHandler()
+      requestHandler: await getProxyRequestHandler(),
+      maxAttempts: CONFIG.OBJECT_STORAGE.MAX_REQUEST_ATTEMPTS
     })
 
     logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())