From 432a691e381c9cd5f6ef71ee556cbe6720291135 Mon Sep 17 00:00:00 2001 From: henrique Date: Mon, 2 Nov 2026 18:39:48 -0300 Subject: [PATCH] ASR local com faster-whisper (proxy LiteLLM nao aceita audio) + integracao judge/transcribe --- .env.example | 6 +- server/.env.asr | 5 + server/src/lib/asr.ts | 169 ++++++++++++++++++++++++++++++++ server/src/routes/judge.ts | 4 +- server/src/routes/transcribe.ts | 61 ++++-------- tools/asr_whisper.py | 55 +++++++++++ tools/probe-audio.mjs | 24 +++++ 7 files changed, 278 insertions(+), 46 deletions(-) create mode 100644 server/.env.asr create mode 100644 server/src/lib/asr.ts create mode 100644 tools/asr_whisper.py create mode 100644 tools/probe-audio.mjs diff --git a/.env.example b/.env.example index 624f6c9..967d481 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,11 @@ LITELLM_API_KEY=coloqu…aqui # Modelos CHAT_MODEL=qwen3.8-max # juiz de feedback / geracao de missões FAST_MODEL=qwen3.8-flash # drills e tarefas leves -OMNI_MODEL=qwen3-omni-flash # transcricao de audio do aluno (ASR tolerante a sotaque) +OMNI_MODEL=qwen3-omni-flash # transcricao de audio do aluno (NOTA: proxy LiteLLM nao aceita audio, ver ASR abaixo) + +# ASR local via faster-whisper (proxy LiteLLM NAO aceita partes de audio) +ASR_PYTHON=C:\Users\henrique\AppData\Local\Programs\Python\Python312\python.exe # caminho do Python com faster-whisper instalado +ASR_MODEL=small # modelo whisper: tiny, base, small, medium, large-v2 # Porta do server PORT=8787 diff --git a/server/.env.asr b/server/.env.asr new file mode 100644 index 0000000..34d027a --- /dev/null +++ b/server/.env.asr @@ -0,0 +1,5 @@ +# ASR local via faster-whisper (Python 3.12 — onde o pacote esta instalado). +# O proxy LiteLLM nao aceita partes de audio (omni devolve 400), entao a transcricao +# roda na maquina e o LLM so julga o texto. Ver tools/probe-audio.mjs. +ASR_PYTHON=C:\Users\henrique\AppData\Local\Programs\Python\Python312\python.exe +ASR_MODEL=small diff --git a/server/src/lib/asr.ts b/server/src/lib/asr.ts new file mode 100644 index 0000000..0f06410 --- /dev/null +++ b/server/src/lib/asr.ts @@ -0,0 +1,169 @@ +/** + * ASR local via faster-whisper (Python). + * O proxy LiteLLM NÃO aceita partes de áudio (qwen omni devolve 400), + * então a transcrição roda local com faster-whisper. + */ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +// lib/asr.ts -> server/src/lib -> repo root = ../../.. +const REPO_ROOT = path.resolve(__dirname, "..", "..", ".."); + +export interface TranscriptionResult { + text: string; + language: string; + probability: number; +} + +/** + * Detecta formato do áudio pelos magic bytes iniciais. + * Retorna extensão adequada para o arquivo temporário. + */ +function detectAudioExtension(buffer: Buffer, mimeHint?: string): string { + // Magic bytes detection + if (buffer.length >= 4) { + const b0 = buffer[0]!; + const b1 = buffer[1]!; + const b2 = buffer[2]!; + const b3 = buffer[3]!; + + // EBML (WebM/MKV): 1A 45 DF A3 + if (b0 === 0x1a && b1 === 0x45 && b2 === 0xdf && b3 === 0xa3) { + return ".webm"; + } + // ID3 (MP3 com tag): 49 44 33 + if (b0 === 0x49 && b1 === 0x44 && b2 === 0x33) { + return ".mp3"; + } + // MP3 frame sync: FF FB ou FF F3 ou FF F2 + if (b0 === 0xff && (b1 & 0xe0) === 0xe0) { + return ".mp3"; + } + // RIFF (WAV): 52 49 46 46 ... 57 41 56 45 + if (b0 === 0x52 && b1 === 0x49 && b2 === 0x46 && b3 === 0x46) { + if (buffer.length >= 12 && buffer[8] === 0x57 && buffer[9] === 0x41 && buffer[10] === 0x56 && buffer[11] === 0x45) { + return ".wav"; + } + } + // OGG: 4F 67 67 53 + if (b0 === 0x4f && b1 === 0x67 && b2 === 0x67 && b3 === 0x53) { + return ".ogg"; + } + } + + // Fallback para mime hint + if (mimeHint) { + const mimeMap: Record = { + "audio/webm": ".webm", + "audio/mp3": ".mp3", + "audio/mpeg": ".mp3", + "audio/wav": ".wav", + "audio/ogg": ".ogg", + "audio/mp4": ".mp4", + }; + const extFromMime = mimeMap[mimeHint]; + if (extFromMime) { + return extFromMime; + } + } + + // Default seguro + return ".webm"; +} + +/** + * Transcreve áudio usando faster-whisper local. + * @param buffer - Buffer do áudio + * @param mimeHint - MIME type opcional para ajudar na detecção de formato + */ +export async function transcribeAudio( + buffer: Buffer, + mimeHint?: string, +): Promise { + const asrPython = process.env.ASR_PYTHON || "python"; + const asrModel = process.env.ASR_MODEL || "small"; + const scriptPath = path.join(REPO_ROOT, "tools", "asr_whisper.py"); + + // Verificar se o script existe + try { + await fs.access(scriptPath); + } catch { + throw new Error(`Script ASR não encontrado: ${scriptPath}`); + } + + // Criar arquivo temporário com extensão adequada + const ext = detectAudioExtension(buffer, mimeHint); + const tmpDir = os.tmpdir(); + const tmpFile = path.join(tmpDir, `asr-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`); + + try { + await fs.writeFile(tmpFile, buffer); + + // Executar o script Python + const result = await new Promise((resolve, reject) => { + const proc = spawn(asrPython, [scriptPath, tmpFile, "--model", asrModel, "--json"], { + timeout: 60000, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + + proc.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + + proc.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + proc.on("error", (err) => { + reject(new Error(`Falha ao executar ASR: ${err.message}`)); + }); + + proc.on("close", (code) => { + if (code !== 0) { + reject(new Error(`ASR falhou (código ${code}): ${stderr.trim() || "sem detalhes"}`)); + return; + } + resolve(stdout); + }); + }); + + // Parsear a última linha como JSON (o script imprime logs antes) + const lines = result.trim().split("\n"); + const lastLine = lines[lines.length - 1]; + + if (!lastLine) { + throw new Error("ASR não retornou saída"); + } + + try { + const parsed = JSON.parse(lastLine) as { + text?: string; + language?: string; + language_probability?: number; + }; + + return { + text: (parsed.text ?? "").trim(), + language: parsed.language ?? "unknown", + probability: parsed.language_probability ?? 0, + }; + } catch (parseErr) { + throw new Error(`Saída ASR inválida: ${(parseErr as Error).message}. Raw: ${lastLine.slice(0, 200)}`); + } + } finally { + // Limpar arquivo temporário + try { + await fs.unlink(tmpFile); + } catch { + // Ignorar erros de limpeza + } + } +} diff --git a/server/src/routes/judge.ts b/server/src/routes/judge.ts index 0c6f685..3ced482 100644 --- a/server/src/routes/judge.ts +++ b/server/src/routes/judge.ts @@ -126,7 +126,9 @@ judgeRouter.post("/", async (req, res, next) => { try { const input = JudgeInputSchema.parse(req.body); - // Se veio áudio sem transcrição, transcrever primeiro via OMNI + // Se veio áudio sem transcrição, transcrever primeiro via ASR local (faster-whisper). + // NOTA: O proxy LiteLLM NÃO aceita partes de áudio (qwen omni devolve 400), + // então usamos faster-whisper local em vez do modelo omni. let transcript = input.transcriptRaw ?? input.utteranceText ?? ""; if (!transcript && input.audioBase64) { diff --git a/server/src/routes/transcribe.ts b/server/src/routes/transcribe.ts index 4f58be4..e2a5d96 100644 --- a/server/src/routes/transcribe.ts +++ b/server/src/routes/transcribe.ts @@ -1,11 +1,11 @@ /** - * POST /api/transcribe — ASR via OMNI_MODEL (qwen3-omni-flash). - * Formato OpenAI-compatible com input_audio. + * POST /api/transcribe — ASR local via faster-whisper. + * NOTA: O proxy LiteLLM NÃO aceita partes de áudio (qwen omni devolve 400), + * então a transcrição roda local com faster-whisper em vez do modelo omni. */ import { Router } from "express"; import { z } from "zod"; -import type { ChatCompletionContentPartInputAudio, ChatCompletionContentPartText } from "openai/resources/chat/completions"; -import { getClient, MODELS } from "../lib/llm.js"; +import { transcribeAudio as asrTranscribe } from "../lib/asr.js"; export const transcribeRouter = Router(); @@ -17,58 +17,31 @@ const TranscribeInputSchema = z.object({ const TranscribeOutputSchema = z.object({ text: z.string(), words: z.array(z.string()).optional(), + language: z.string().optional(), + probability: z.number().optional(), }); /** - * Transcreve áudio base64 usando o modelo omni. + * Transcreve áudio base64 usando faster-whisper local. * Exportada para uso interno pelo /judge quando recebe áudio sem transcrição. */ export async function transcribeAudio( audioBase64: string, mime: string, -): Promise<{ text: string; words?: string[] }> { - const client = getClient(); +): Promise<{ text: string; words?: string[]; language?: string; probability?: number }> { + // Converter base64 para buffer + const buffer = Buffer.from(audioBase64, "base64"); - // Determinar formato do áudio para a API - // O SDK openai tipa format como 'wav' | 'mp3', mas o modelo omni via LiteLLM - // aceita outros formatos em runtime. Mapeamos para os tipos do SDK. - const formatMap: Record = { - "audio/webm": "mp3", // fallback: webm não é suportado pelo SDK, usa mp3 como proxy - "audio/mp3": "mp3", - "audio/mpeg": "mp3", - "audio/wav": "wav", - "audio/ogg": "mp3", // fallback - "audio/mp4": "mp3", // fallback - }; - const format = formatMap[mime] ?? "mp3"; - - const audioPart: ChatCompletionContentPartInputAudio = { - type: "input_audio", - input_audio: { - data: audioBase64, - format, - }, - }; - const textPart: ChatCompletionContentPartText = { - type: "text", - text: "Trascrivi esattamente quello che senti in italiano. Se non riesci a capire qualcosa, scrivi [incomprensibile]. Rispondi SOLO con la trascrizione, niente altro.", - }; - - const response = await client.chat.completions.create({ - model: MODELS.omni, - messages: [ - { - role: "user", - content: [audioPart, textPart], - }, - ], - temperature: 0.1, - }); + if (buffer.length === 0) { + throw new Error("Áudio vazio"); + } - const rawText = response.choices[0]?.message?.content ?? ""; + const result = await asrTranscribe(buffer, mime); return TranscribeOutputSchema.parse({ - text: rawText.trim(), + text: result.text, + language: result.language, + probability: result.probability, }); } diff --git a/tools/asr_whisper.py b/tools/asr_whisper.py new file mode 100644 index 0000000..c55c1e8 --- /dev/null +++ b/tools/asr_whisper.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""ASR local para o Italiano Vivo via faster-whisper. + +O proxy LiteLLM (10.8.0.1:4000) nao aceita partes de audio no chat/completions +(qwen omni devolve 400 "Invalid video file" / "provided URL does not appear to be valid"), +entao a transcricao roda aqui e o server Node repassa so o texto para o juiz LLM. + +Uso: + python asr_whisper.py [--model small] [--json] + +Imprime o texto transcrito (ou JSON com segments/timing com --json). +Idempotente: baixa o modelo uma vez e usa cache local do HuggingFace. +""" +import argparse +import json +import sys + + +def main() -> int: + ap = argparse.ArgumentParser(description="Transcricao IT/PT com faster-whisper") + ap.add_argument("audio", help="caminho do arquivo de audio (mp3/wav/webm)") + ap.add_argument("--model", default="small", help="tiny|base|small|medium (default small)") + ap.add_argument("--lang", default=None, help="forcar idioma, ex.: it") + ap.add_argument("--json", action="store_true", dest="as_json") + args = ap.parse_args() + + from faster_whisper import WhisperModel + + model = WhisperModel(args.model, device="cpu", compute_type="int8") + segments, info = model.transcribe( + args.audio, language=args.lang, beam_size=5, vad_filter=True + ) + + parts = [] + seg_list = [] + for s in segments: + parts.append(s.text.strip()) + seg_list.append({"start": round(s.start, 2), "end": round(s.end, 2), "text": s.text.strip()}) + + text = " ".join(p for p in parts if p).strip() + + if args.as_json: + print(json.dumps({ + "text": text, + "language": info.language, + "language_probability": round(float(info.language_probability), 3), + "segments": seg_list, + }, ensure_ascii=False)) + else: + print(text) + return 0 if text else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/probe-audio.mjs b/tools/probe-audio.mjs new file mode 100644 index 0000000..6e339dd --- /dev/null +++ b/tools/probe-audio.mjs @@ -0,0 +1,24 @@ +// Probe: qual formato de audio o proxy LiteLLM aceita para transcricao (omni) +import fs from 'node:fs'; + +const key = process.env.K; +const b64 = fs.readFileSync(new URL('../audio/b1-chunk-01.mp3', import.meta.url)).toString('base64'); + +async function tryOne(model, part) { + const r = await fetch('http://10.8.0.1:4000/v1/chat/completions', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: [{ type: 'text', text: 'Transcreva exatamente a fala em italiano, sem comentarios.' }, part] }], + max_tokens: 120, + }), + }); + const t = await r.text(); + console.log('===', model, JSON.stringify(Object.keys(part)), r.status); + console.log(t.slice(0, 300).replace(/\n/g, ' ')); +} + +await tryOne('qwen3.5-omni-flash', { type: 'video_url', video_url: { url: 'data:audio/mpeg;base64,' + b64 } }); +await tryOne('qwen3-omni-flash', { type: 'input_audio', input_audio: { data: b64, format: 'mp3' } }); +await tryOne('qwen3.5-plus', { type: 'input_audio', input_audio: { data: b64, format: 'mp3' } });