OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Add job ttl

pull/904/merge
Chocobozzz 2018-08-03 10:19:51 +02:00
parent 516df59b3b
commit 2b86fe7274
2 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
server/initializers/constants.ts
View File

@ -100,9 +100,18 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
'video-import': 1,
'email': 5
}
const JOB_TTL: { [ id in JobType ]: number } = {
'activitypub-http-broadcast': 60000 * 10, // 10 minutes
'activitypub-http-unicast': 60000 * 10, // 10 minutes
'activitypub-http-fetcher': 60000 * 10, // 10 minutes
'activitypub-follow': 60000 * 10, // 10 minutes
'video-file-import': 1000 * 3600, // 1 hour
'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
'video-import': 1000 * 3600 * 5, // 5 hours
'email': 60000 * 10 // 10 minutes
}
const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
const JOB_REQUEST_TTL = 60000 * 10 // 10 minutes
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
// 1 hour
@ -576,6 +585,7 @@ export {
ROUTE_CACHE_LIFETIME,
SORTABLE_COLUMNS,
FEEDS,
JOB_TTL,
NSFW_POLICY_TYPES,
STATIC_MAX_AGE,
STATIC_PATHS,
@ -592,7 +602,6 @@ export {
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
JOB_REQUEST_TIMEOUT,
JOB_REQUEST_TTL,
USER_PASSWORD_RESET_LIFETIME,
IMAGE_MIMETYPE_EXT,
SCHEDULER_INTERVALS_MS,

10
server/lib/job-queue/job-queue.ts
View File

@ -2,7 +2,7 @@ import * as Bull from 'bull'
import { JobState, JobType } from '../../../shared/models'
import { logger } from '../../helpers/logger'
import { Redis } from '../redis'
import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_REQUEST_TTL } from '../../initializers'
import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers'
import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
@ -79,6 +79,7 @@ class JobQueue {
const handler = handlers[handlerName]
queue.process(JOB_CONCURRENCY[handlerName], handler)
.catch(err => logger.error('Error in job queue processor %s.', handlerName, { err }))
queue.on('failed', (job, err) => {
logger.error('Cannot execute job %d in queue %s.', job.id, handlerName, { payload: job.data, err })
@ -109,11 +110,8 @@ class JobQueue {
const jobArgs: Bull.JobOptions = {
backoff: { delay: 60 * 1000, type: 'exponential' },
attempts: JOB_ATTEMPTS[obj.type]
}
if (jobsWithRequestTimeout[obj.type] === true) {
jobArgs.timeout = JOB_REQUEST_TTL
attempts: JOB_ATTEMPTS[obj.type],
timeout: JOB_TTL[obj.type]
}
return queue.add(obj.payload, jobArgs)