/** * Local voice transcription for the staff-chat module (whisper.cpp + ffmpeg). * * Chat-owned on purpose (isolation rule): it mirrors the pipeline in * server/api/mobile/transcribe.post.ts but is self-contained and RETURNS the * transcript instead of writing it onto an iDempiere request — the chat caller * decides what to do with it. Fully fail-soft: any missing binary / model / * ffmpeg / decode error resolves to '' so a voice message still sends, just * without a transcript. Uses async exec so it never blocks the event loop. */ import { exec } from 'child_process' import { promisify } from 'util' import { writeFile, unlink, mkdir } from 'fs/promises' import { existsSync } from 'fs' import { join } from 'path' const pexec = promisify(exec) // Whisper language. IMPORTANT: `-l` is a HARD override, not a hint — forcing the // wrong language makes whisper decode (effectively translate) the audio into that // language. The spoken language is NOT the UI language, so we default to whisper's // own auto-detection and only pin a language when a caller explicitly asks for one. const langArg = (lang?: string): string => { const l = (lang || '').toLowerCase().slice(0, 2) if (l === 'de' || l === 'en' || l === 'es') return l return 'auto' } // Light, dictionary-free cleanup of a raw whisper transcript before it lands in // chat. Safe for any language — only fixes whitespace, punctuation spacing, // sentence capitalization and a missing final stop. It does NOT correct // misspelled words (that needs a real spell/grammar engine — see notes). const cleanupTranscript = (raw: string): string => { let s = (raw || '').replace(/\s+/g, ' ').trim() if (!s) return '' s = s.replace(/\s+([,.;:!?])/g, '$1') // no space before punctuation s = s.replace(/([.!?])([A-Za-zÄÖÜäöü])/g, '$1 $2') // space after . ! ? if glued to a word s = s.replace(/\s{2,}/g, ' ').trim() s = s.charAt(0).toUpperCase() + s.slice(1) // capitalize the first word s = s.replace(/([.!?]\s+)(\p{Ll})/gu, (_m, p, c) => p + c.toUpperCase()) // …and each new sentence if (!/[.!?…]$/.test(s)) s += '.' // ensure a terminal stop return s } const resolveWhisper = () => { // Persistent install (survives npm install / deploys) with a node_modules fallback for dev. const persistentDir = '/opt/whisper-cpp' const nodeModulesDir = join(process.cwd(), 'node_modules/whisper-node/lib/whisper.cpp') const bin = existsSync(join(persistentDir, 'main')) ? join(persistentDir, 'main') : join(nodeModulesDir, 'main') const model = existsSync(join(persistentDir, 'ggml-base.bin')) ? join(persistentDir, 'ggml-base.bin') : join(nodeModulesDir, 'models/ggml-base.bin') return { bin, model } } export interface TranscriptSegment { start: number; end: number; text: string } export interface TranscribeOptions { /** whisper + ffmpeg timeouts (ms). Chat voice memos are short; call recordings are not. */ timeoutMs?: number /** Extra whisper.cpp args (e.g. ['-t', '4']). */ extraArgs?: string[] } const parseStamp = (s: string): number => { const m = /(\d{2}):(\d{2}):(\d{2})\.(\d{3})/.exec(s) return m ? Number(m[1]) * 3600 + Number(m[2]) * 60 + Number(m[3]) + Number(m[4]) / 1000 : 0 } /** * Transcribe an audio buffer and return the text PLUS the timestamped segments * whisper produced. Returns { text: '', segments: [] } on any failure (fail-soft, * never throws). `transcribeAudio()` below is the plain-text wrapper the chat * and ticket flows use; the call-recording transcription uses the segments. * @param buffer raw audio (webm/wav/mp3 … anything ffmpeg decodes) * @param filename original filename (only used for the temp extension) * @param lang 'de' | 'en' | 'es' | undefined (→ auto) */ export const transcribeAudioDetailed = async (buffer: Buffer, filename = 'audio.webm', lang?: string, opts: TranscribeOptions = {}): Promise<{ text: string; segments: TranscriptSegment[] }> => { const { bin, model } = resolveWhisper() if (!existsSync(bin) || !existsSync(model)) { console.warn('[ChatTranscribe] whisper binary/model not found — skipping transcription') return { text: '', segments: [] } } const tmpDir = join(process.cwd(), '.tmp-chat-voice') const safeName = filename.replace(/[^\w.\-]/g, '_') const stamp = `${Date.now()}-${Math.floor(Math.random() * 1e9)}` const srcPath = join(tmpDir, `${stamp}-${safeName}`) // Distinct name even when the source already IS a .wav (ffmpeg refuses in-place edits). const wavPath = srcPath.replace(/\.\w+$/, '') + '.16k.wav' const timeout = Math.max(30000, Number(opts.timeoutMs) || 120000) const extra = (opts.extraArgs || []).map((a) => `"${String(a).replace(/"/g, '')}"`).join(' ') try { if (!existsSync(tmpDir)) await mkdir(tmpDir, { recursive: true }) await writeFile(srcPath, buffer) // 1) normalise to 16 kHz mono WAV await pexec(`ffmpeg -i "${srcPath}" -ar 16000 -ac 1 -y "${wavPath}"`, { timeout }) // 2) run whisper.cpp const { stdout } = await pexec( `"${bin}" -m "${model}" -l ${langArg(lang)} ${extra} -f "${wavPath}"`, { timeout, maxBuffer: 10 * 1024 * 1024 } ) // whisper prints "[00:00:00.000 --> 00:00:05.000] text" const segments: TranscriptSegment[] = [] for (const line of (stdout || '').split('\n')) { const m = /^\[([\d:.]+)\s*-->\s*([\d:.]+)\]\s*(.*)$/.exec(line.trim()) if (!m) continue const text = (m[3] || '').trim() if (text) segments.push({ start: parseStamp(m[1] || ''), end: parseStamp(m[2] || ''), text }) } const text = cleanupTranscript(segments.map((s) => s.text).join(' ').trim()) return { text, segments } } catch (err: any) { console.error('[ChatTranscribe] failed:', err?.message || err) return { text: '', segments: [] } } finally { try { await unlink(srcPath) } catch {} try { await unlink(wavPath) } catch {} } } /** * Transcribe an audio buffer. Returns the recognised text, or '' on any failure. * @param buffer raw audio (typically audio/webm from MediaRecorder) * @param filename original filename (only used for the temp extension) * @param lang 'de' | 'en' | 'es' | undefined (→ auto) */ export const transcribeAudio = async (buffer: Buffer, filename = 'audio.webm', lang?: string): Promise => { const res = await transcribeAudioDetailed(buffer, filename, lang) return res.text }