Fix redundancy timeout

pull/4597/head
Chocobozzz 2021-11-29 15:45:02 +01:00
parent c5e53d0e39
commit 4c99953acd
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
4 changed files with 14 additions and 8 deletions

View File

@ -3,7 +3,7 @@ import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, Reque
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
import { join } from 'path'
import { CONFIG } from '../initializers/config'
import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
import { pipelinePromise } from './core-utils'
import { processImage } from './image-utils'
import { logger, loggerTagsFactory } from './logger'
@ -20,6 +20,7 @@ export interface PeerTubeRequestError extends Error {
}
type PeerTubeRequestOptions = {
timeout?: number
activityPub?: boolean
bodyKBLimit?: number // 1MB
httpSignature?: {
@ -129,7 +130,7 @@ async function doRequestAndSaveToFile (
destPath: string,
options: PeerTubeRequestOptions = {}
) {
const gotOptions = buildGotOptions(options)
const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
const outFile = createWriteStream(destPath)
@ -235,7 +236,7 @@ function buildGotOptions (options: PeerTubeRequestOptions) {
return {
method: options.method,
dnsCache: true,
timeout: REQUEST_TIMEOUT,
timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
json: options.json,
searchParams: options.searchParams,
retry: 2,

View File

@ -202,7 +202,12 @@ const JOB_PRIORITY = {
const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
const AP_CLEANER_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-cleaner job
const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
const REQUEST_TIMEOUT = 7000 // 7 seconds
const REQUEST_TIMEOUTS = {
DEFAULT: 7000, // 7 seconds
FILE: 30000, // 30 seconds
REDUNDANCY: JOB_TTL['video-redundancy']
}
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
@ -896,7 +901,7 @@ export {
FFMPEG_NICE,
ABUSE_STATES,
LRU_CACHE,
REQUEST_TIMEOUT,
REQUEST_TIMEOUTS,
USER_PASSWORD_RESET_LIFETIME,
USER_PASSWORD_CREATE_LIFETIME,
MEMOIZE_TTL,

View File

@ -1,7 +1,7 @@
import WebFinger from 'webfinger.js'
import { isProdInstance } from '@server/helpers/core-utils'
import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
import { REQUEST_TIMEOUT, WEBSERVER } from '@server/initializers/constants'
import { REQUEST_TIMEOUTS, WEBSERVER } from '@server/initializers/constants'
import { ActorModel } from '@server/models/actor/actor'
import { MActorFull } from '@server/types/models'
import { WebFingerData } from '@shared/models'
@ -10,7 +10,7 @@ const webfinger = new WebFinger({
webfist_fallback: false,
tls_only: isProdInstance(),
uri_fallback: false,
request_timeout: REQUEST_TIMEOUT
request_timeout: REQUEST_TIMEOUTS.DEFAULT
})
async function loadActorUrlOrGetFromWebfinger (uriArg: string) {

View File

@ -130,7 +130,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
for (const fileUrl of fileUrls) {
const destPath = join(tmpDirectory, basename(fileUrl))
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit })
await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.FILE })
const { size } = await stat(destPath)
remainingBodyKBLimit -= (size / 1000)