diff --git a/server/core/lib/opentelemetry/metric-helpers/worker-threads-observers.ts b/server/core/lib/opentelemetry/metric-helpers/worker-threads-observers.ts index 608010cef..3761b8aa0 100644 --- a/server/core/lib/opentelemetry/metric-helpers/worker-threads-observers.ts +++ b/server/core/lib/opentelemetry/metric-helpers/worker-threads-observers.ts @@ -1,5 +1,5 @@ import { Meter } from '@opentelemetry/api' -import { getWorkersQueueSize } from '@server/lib/worker/parent-process.js' +import { getWorkersStats } from '@server/lib/worker/parent-process.js' export class WorkerThreadsObserversBuilder { @@ -11,12 +11,22 @@ export class WorkerThreadsObserversBuilder { this.meter.createObservableGauge('peertube_worker_thread_queue_total', { description: 'Total tasks waiting for a PeerTube worker thread' }).addCallback(observableResult => { - const stats = getWorkersQueueSize() + const stats = getWorkersStats() for (const stat of stats) { observableResult.observe(stat.queueSize, { state: 'waiting', workerThread: stat.label }) } }) + + this.meter.createObservableGauge('peertube_worker_thread_completed_total', { + description: 'Total tasks completed in PeerTube worker threads' + }).addCallback(observableResult => { + const stats = getWorkersStats() + + for (const stat of stats) { + observableResult.observe(stat.completed, { workerThread: stat.label }) + } + }) } } diff --git a/server/core/lib/worker/parent-process.ts b/server/core/lib/worker/parent-process.ts index a4bbed06d..7aa003fe9 100644 --- a/server/core/lib/worker/parent-process.ts +++ b/server/core/lib/worker/parent-process.ts @@ -149,39 +149,47 @@ export function buildDigestFromWorker ( // --------------------------------------------------------------------------- -export function getWorkersQueueSize () { +export function getWorkersStats () { return [ { label: 'downloadImage', - queueSize: downloadImageWorker?.queueSize || 0 + queueSize: downloadImageWorker?.queueSize || 0, + completed: downloadImageWorker?.completed || 0 }, { label: 'processImageWorker', - queueSize: processImageWorker?.queueSize || 0 + queueSize: processImageWorker?.queueSize || 0, + completed: processImageWorker?.completed || 0 }, { label: 'getImageSizeWorker', - queueSize: getImageSizeWorker?.queueSize || 0 + queueSize: getImageSizeWorker?.queueSize || 0, + completed: getImageSizeWorker?.completed || 0 }, { label: 'parallelHTTPBroadcastWorker', - queueSize: parallelHTTPBroadcastWorker?.queueSize || 0 + queueSize: parallelHTTPBroadcastWorker?.queueSize || 0, + completed: parallelHTTPBroadcastWorker?.completed || 0 }, { label: 'sequentialHTTPBroadcastWorker', - queueSize: sequentialHTTPBroadcastWorker?.queueSize || 0 + queueSize: sequentialHTTPBroadcastWorker?.queueSize || 0, + completed: sequentialHTTPBroadcastWorker?.completed || 0 }, { label: 'httpUnicastWorker', - queueSize: httpUnicastWorker?.queueSize || 0 + queueSize: httpUnicastWorker?.queueSize || 0, + completed: httpUnicastWorker?.completed || 0 }, { label: 'signJsonLDObjectWorker', - queueSize: signJsonLDObjectWorker?.queueSize || 0 + queueSize: signJsonLDObjectWorker?.queueSize || 0, + completed: signJsonLDObjectWorker?.completed || 0 }, { label: 'buildDigestWorker', - queueSize: buildDigestWorker?.queueSize || 0 + queueSize: buildDigestWorker?.queueSize || 0, + completed: buildDigestWorker?.completed || 0 } ] }