chore: refactor test

.github/workflows/transcription.yml

@@ -0,0 +1,30 @@
+name: Benchmark
+
+on:
+  push:
+    branches:
+      - transcription-backend-workbench
+
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: './.github/actions/reusable-prepare-peertube-build'
+        with:
+          node-version: '18.x'
+
+      - uses: './.github/actions/reusable-prepare-peertube-run'
+
+      - name: Install Python libraries
+        run: |
+          pip install openai-whisper
+          pip install whisper-ctranslate2
+          pip install whisper-timestamped
+
+      - name: Run transcription tests
+        run: |
+          npm run mocha -- --exit --bail packages/tests/src/transcription/whisper/transcriber/openai-transcriber.spec.ts

packages/tests/src/transcription/benchmark.spec.ts

@@ -0,0 +1,122 @@
+import { createLogger } from 'winston'
+import { join } from 'path'
+import { expect } from 'chai'
+import { existsSync } from 'node:fs'
+import { rm, mkdir, readFile } from 'node:fs/promises'
+import { buildAbsoluteFixturePath, root } from '@peertube/peertube-node-utils'
+import { toHumanReadable, transcriberFactory } from '@peertube/peertube-transcription'
+import { performance, PerformanceObserver } from 'node:perf_hooks'
+
+// const WER_TOLERANCE = 1
+// const CER_TOLERANCE = 1
+//
+// interface TestResult {
+//   WER: number
+//   CER: number
+//   duration: number
+//   engine: TranscriptionEngine
+//   dataThroughput: number // relevant ?
+//   cpus: CpuInfo[]
+//   cpuUsages: CpuUsage[]
+//   memoryUsages: Record<number, MemoryUsage>
+//   // Prints:
+//   // {
+//   //  rss: 4935680,
+//   //  heapTotal: 1826816,
+//   //  heapUsed: 650472,
+//   //  external: 49879,
+//   //  arrayBuffers: 9386
+//   // }
+//
+//   // heapTotal and heapUsed refer to V8's memory usage.
+//   // external refers to the memory usage of C++ objects bound to JavaScript objects managed by V8.
+//   // rss, Resident Set Size, is the amount of space occupied in the main memory device (that is a subset of the total allocated memory) for the process, including all C++ and JavaScript objects and code.
+//   // arrayBuffers refers to memory allocated for ArrayBuffers and SharedArrayBuffers, including all Node.js Buffers. This is also included in the external value. When Node.js is used as an embedded library, this value may be 0 because allocations for ArrayBuffers may not be tracked in that case.
+//   //
+//   // When using Worker threads, rss will be a value that is valid for the entire process, while the other fields will only refer to the current thread.
+//   //
+//   // The process.memoryUsage() method iterates over each page to gather information about memory usage which might be slow depending on the program memory allocations.
+// }
+//
+// // var os = require('os');
+// //
+// console.log(cpus())
+// // console.log(os.totalmem());
+// // console.log(os.freemem())
+//
+// const testsResults: Record<string, TestResult> = {
+//   cpus: []
+// }
+//
+// async function testTranscriptGeneration (transformerBackend: string, model: string, mediaFilePath: string) {
+//   const testResults = {
+//     WER: 3,
+//     CER: 3,
+//     duration: 3
+//   }
+//
+//   return testResults
+// }
+
+describe('Transcribers', function () {
+  const transcriptDirectory = join(root(), 'test-transcript')
+  const expectedVttTranscriptPath = join(transcriptDirectory, 'video_short.vtt')
+  const mediaFilePath = buildAbsoluteFixturePath('video_short.mp4')
+  const transcribers = [
+    'openai-whisper',
+    'whisper-ctranslate2',
+    'whisper-timestamped'
+  ]
+
+  before(async function () {
+    await mkdir(transcriptDirectory, { recursive: true })
+
+    const performanceObserver = new PerformanceObserver((items) => {
+      items
+        .getEntries()
+        .forEach((entry) => console.log(`Transcription ${entry.name} took ${toHumanReadable(entry.duration)}`))
+    })
+    performanceObserver.observe({ type: 'measure' })
+
+    // console.table
+  })
+
+  transcribers.forEach(function (transcriberName) {
+    describe(`${transcriberName}`, function () {
+      it(`Should instanciate`, function () {
+        transcriberFactory.createFromEngineName(transcriberName)
+      })
+
+      it('Should run transcription on a media file without raising any errors', async function () {
+        const transcriber = transcriberFactory.createFromEngineName(
+          transcriberName,
+          createLogger(),
+          transcriptDirectory
+        )
+        const transcript = await transcriber.transcribe(
+          mediaFilePath,
+          { name: 'tiny' },
+          'fr',
+          'vtt'
+        )
+        expect(transcript).to.deep.equals({
+          path: expectedVttTranscriptPath,
+          language: 'fr',
+          format: 'vtt'
+        })
+        expect(transcript.path).to.equals(expectedVttTranscriptPath)
+
+        // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+        expect(existsSync(transcript.path), `Transcript file ${transcript.path} doesn't exist.`).to.be.true
+        expect(await readFile(transcript.path, 'utf8'), `Transcript file ${transcript.path} doesn't exist.`).to.equal('...')
+
+        await rm(transcript.path)
+      })
+    })
+  })
+
+  after(async function () {
+    await rm(transcriptDirectory, { recursive: true, force: true })
+    performance.clearMarks()
+  })
+})

packages/tests/src/transcription/benchmark.ts

@@ -1,34 +0,0 @@
-import { CpuInfo, cpus } from 'os'
-import { TranscriptionEngine } from '@peertube/transcription'
-
-const WER_TOLERANCE = 1
-const CER_TOLERANCE = 1
-
-interface TestResult {
-  WER: number
-  CER: number
-  duration: number
-  engine: TranscriptionEngine
-  dataThroughput: number // relevant ?
-  cpus: CpuInfo[]
-}
-
-// var os = require('os');
-//
-console.log(cpus())
-// console.log(os.totalmem());
-// console.log(os.freemem())
-
-const testsResults: Record<string, TestResult> = {
-  cpus: []
-}
-
-async function testTranscriptGeneration (transformerBackend: string, model: string, mediaFilePath: string) {
-  const testResults = {
-    WER: 3,
-    CER: 3,
-    duration: 3
-  }
-
-  return testResults
-}

packages/tests/src/transcription/transcriber-factory.spec.ts

@@ -0,0 +1,17 @@
+import { transcriberFactory } from '@peertube/peertube-transcription'
+
+describe('Transcriber factory', function () {
+  const transcribers = [
+    'openai-whisper',
+    'whisper-ctranslate2',
+    'whisper-timestamped'
+  ]
+
+  describe('Should be able to create a transcriber for each available transcription engine', function () {
+    transcribers.forEach(function (transcriberName) {
+      it(`Should be able to create a(n) ${transcriberName} transcriber`, function () {
+        transcriberFactory.createFromEngineName(transcriberName)
+      })
+    })
+  })
+})

packages/tests/src/transcription/transcribers.spec.ts

@@ -1,68 +0,0 @@
-import { createLogger } from 'winston'
-import { join } from 'path'
-import { expect } from 'chai'
-import { existsSync } from 'node:fs'
-import { rm, mkdir, readFile } from 'node:fs/promises'
-import { buildAbsoluteFixturePath, root } from '@peertube/peertube-node-utils'
-import { toHumanReadable, transcriberFactory } from '@peertube/peertube-transcription'
-import { performance, PerformanceObserver } from 'node:perf_hooks'
-
-describe('Transcribers', function () {
-  const transcriptDirectory = join(root(), 'test-transcript')
-  const vttTranscriptPath = join(transcriptDirectory, 'video_short.vtt')
-  const transcribers = [
-    'openai-whisper',
-    'whisper-ctranslate2',
-    'whisper-timestamped'
-  ]
-
-  before(async function () {
-    await mkdir(transcriptDirectory, { recursive: true })
-
-    const performanceObserver = new PerformanceObserver((items) => {
-      items
-        .getEntries()
-        .forEach((entry) => console.log(`Transcription ${entry.name} took ${toHumanReadable(entry.duration)}`))
-    })
-    performanceObserver.observe({ type: 'measure' })
-  })
-
-  transcribers.forEach(function (transcriberName) {
-    describe(`${transcriberName}`, function () {
-      it(`Should instanciate`, function () {
-        transcriberFactory.createFromEngineName(transcriberName)
-      })
-
-      it('Should run transcription on a media file without raising any errors', async function () {
-        const transcriber = transcriberFactory.createFromEngineName(
-          transcriberName,
-          createLogger(),
-          transcriptDirectory
-        )
-        const mediaFilePath = buildAbsoluteFixturePath('video_short.mp4')
-        const transcript = await transcriber.transcribe(
-          mediaFilePath,
-          { name: 'tiny' },
-          'fr',
-          'vtt'
-        )
-        expect(transcript).to.deep.equals({
-          path: vttTranscriptPath,
-          language: 'fr',
-          format: 'vtt'
-        })
-        expect(transcript.path).to.equals(vttTranscriptPath)
-
-        expect(existsSync(transcript.path), `Transcript file ${transcript.path} doesn't exist.`).to.be.true
-
-        console.log(await readFile(transcript.path, 'utf8'))
-        await rm(transcript.path)
-      })
-    })
-  })
-
-  after(async function () {
-    await rm(transcriptDirectory, { recursive: true, force: true })
-    performance.clearMarks()
-  })
-})

packages/tests/src/transcription/whisper/transcriber/openai-transcriber.spec.ts

@@ -0,0 +1,60 @@
+import { createLogger } from 'winston'
+import { join } from 'path'
+import { expect, config } from 'chai'
+import { existsSync } from 'node:fs'
+import { mkdir, readFile, rm } from 'node:fs/promises'
+import { buildAbsoluteFixturePath, root } from '@peertube/peertube-node-utils'
+import { OpenaiTranscriber } from '@peertube/peertube-transcription'
+
+config.truncateThreshold = 0
+
+describe('Open AI transcriber', function () {
+  const transcriptDirectory = join(root(), 'test-transcript')
+  const expectedVttTranscriptPath = join(transcriptDirectory, 'video_short.vtt')
+
+  before(async function () {
+    await mkdir(transcriptDirectory, { recursive: true })
+  })
+
+  it('Should transcribe a media file', async function () {
+    const transcriber = new OpenaiTranscriber(
+      {
+        name: 'openai-whisper',
+        requirements: [],
+        language: '',
+        type: 'binary',
+        license: '',
+        supportedModelFormats: [ 'PyTorch' ]
+      },
+      createLogger(),
+      transcriptDirectory
+    )
+    const transcript = await transcriber.transcribe(
+      buildAbsoluteFixturePath('video_short.mp4'),
+      { name: 'tiny' },
+      'fr',
+      'vtt'
+    )
+
+    expect(transcript).to.deep.equals({
+      path: expectedVttTranscriptPath,
+      language: 'fr',
+      format: 'vtt'
+    })
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+    expect(existsSync(transcript.path), `Transcript file ${transcript.path} doesn't exist.`).to.be.true
+    expect(await readFile(transcript.path, 'utf8')).to.equal(
+      `WEBVTT
+
+00:00.000 --> 00:02.000
+You
+
+`
+    )
+  })
+
+  after(async function () {
+    await rm(transcriptDirectory, { recursive: true, force: true })
+  })
+})

packages/tests/src/transcription/whisper/transcriber/whisper-ctranslate2.spec.ts

@@ -0,0 +1,60 @@
+import { createLogger } from 'winston'
+import { join } from 'path'
+import { expect, config } from 'chai'
+import { existsSync } from 'node:fs'
+import { mkdir, readFile, rm } from 'node:fs/promises'
+import { buildAbsoluteFixturePath, root } from '@peertube/peertube-node-utils'
+import { OpenaiTranscriber } from '@peertube/peertube-transcription'
+
+config.truncateThreshold = 0
+
+describe('Open AI transcriber', function () {
+  const transcriptDirectory = join(root(), 'test-transcript')
+  const expectedVttTranscriptPath = join(transcriptDirectory, 'video_short.vtt')
+
+  before(async function () {
+    await mkdir(transcriptDirectory, { recursive: true })
+  })
+
+  it('Should transcribe a media file', async function () {
+    const transcriber = new OpenaiTranscriber(
+      {
+        name: 'openai-whisper',
+        requirements: [],
+        language: '',
+        type: 'binary',
+        license: '',
+        supportedModelFormats: [ 'PyTorch' ]
+      },
+      createLogger(),
+      transcriptDirectory
+    )
+    const transcript = await transcriber.transcribe(
+      buildAbsoluteFixturePath('video_short.mp4'),
+      { name: 'tiny' },
+      'fr',
+      'vtt'
+    )
+
+    expect(transcript).to.deep.equals({
+      path: expectedVttTranscriptPath,
+      language: 'fr',
+      format: 'vtt'
+    })
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+    expect(existsSync(transcript.path), `Transcript file ${transcript.path} doesn't exist.`).to.be.true
+    expect(await readFile(transcript.path, 'utf8')).to.equal(
+      `WEBVTT
+
+00:00.000 --> 00:02.000
+You
+
+`
+    )
+  })
+
+  after(async function () {
+    await rm(transcriptDirectory, { recursive: true, force: true })
+  })
+})

packages/transcription/src/abstract-transcriber.ts

@@ -1,13 +1,13 @@
-import { Logger } from 'winston'
-import { join } from 'path'
+import { join } from 'node:path'
+import { existsSync } from 'node:fs'
+import { PerformanceObserver } from 'node:perf_hooks'
+import assert from 'node:assert'
+import { createLogger, Logger } from 'winston'
+import short from 'short-uuid'
 import { root } from '@peertube/peertube-node-utils'
 import { TranscriptionEngine } from './transcription-engine.js'
 import { TranscriptionModel } from './transcription-model.js'
 import { Transcript, TranscriptFormat } from './transcript.js'
-import { existsSync } from 'fs'
-import { PerformanceObserver } from 'node:perf_hooks'
-import short from 'short-uuid'
-import assert from 'node:assert'
 
 export abstract class AbstractTranscriber {
   public static DEFAULT_TRANSCRIPT_DIRECTORY = join(root(), 'dist', 'transcripts')
@@ -20,7 +20,7 @@ export abstract class AbstractTranscriber {
 
   constructor (
     engine: TranscriptionEngine,
-    logger: Logger,
+    logger: Logger = createLogger(),
     transcriptDirectory: string = AbstractTranscriber.DEFAULT_TRANSCRIPT_DIRECTORY,
     performanceObserver?: PerformanceObserver
   ) {

packages/transcription/src/index.ts

@@ -6,5 +6,6 @@ export * from './duration.js'
 export * from './transcription-engine.js'
 export * from './transcription-model.js'
 export * from './transcript.js'
+export * from './whisper/index.js'
 
 export const transcriberFactory = new TranscriberFactory(engines)

packages/transcription/src/transcription-engine.ts

@@ -5,13 +5,13 @@ import { ModelFormat } from './transcription-model.js'
  */
 export interface TranscriptionEngine {
   name: string
-  description: string
+  description?: string
   language: string
   requirements: string[]
   type: 'binary' | 'bindings' | 'ws'
   binary?: string
   license: string
-  forgeURL: string
+  forgeURL?: string
   supportedModelFormats: ModelFormat[]
 
   // There could be a default models.