Преглед изворни кода

perf(asr): streaming partial ASR during speech + reduce VAD silence to 0.4s

- Background _partial_asr loop transcribes new audio every 1s during speech
- Partial results sent immediately via 'partial' messages
- On speech end, full final transcription still runs for accuracy
- VAD MIN_SILENCE_S reduced from 0.8s -> 0.4s (xiaozhi uses 200ms)
- Partial task cancelled on speech restart (prevents stale transcription)
wenhongquan пре 3 недеља
родитељ
комит
b3de0309a2
1 измењених фајлова са 40 додато и 4 уклоњено
  1. 40 4
      asr_agent/conversation_worker.py

+ 40 - 4
asr_agent/conversation_worker.py

@@ -52,7 +52,7 @@ LLM_TEMPERATURE = 0.7
 LLM_TIMEOUT     = 25
 
 SYSTEM_PROMPT = (
-    "你是友好的中文语音助手,名字叫小智。"
+    "你是友好的中文语音助手,名字叫狄诺尼试验员。"
     "回答简洁自然,2-3句即可,用口语化中文。"
     "不要使用 Markdown、代码块、表格或特殊符号。"
     "不要输出括号注释、不要使用英文缩写。"
@@ -203,7 +203,7 @@ VAD_THRESHOLD_LOW  = float(os.environ.get("VAD_THRESHOLD_LOW", "0.2"))
 VAD_WINDOW_SIZE    = int(os.environ.get("VAD_WINDOW_SIZE", "5"))
 VAD_VOICE_RATIO    = float(os.environ.get("VAD_VOICE_RATIO", "0.5"))
 MIN_SPEECH_S       = float(os.environ.get("MIN_SPEECH_S", "0.3"))
-MIN_SILENCE_S      = float(os.environ.get("MIN_SILENCE_S", "0.8"))
+MIN_SILENCE_S      = float(os.environ.get("MIN_SILENCE_S", "0.4"))
 MAX_SPEECH_S       = float(os.environ.get("MAX_SPEECH_S", "8.0"))
 
 # Global ONNX session (shared across participants)
@@ -365,9 +365,36 @@ class Worker:
         vad = _SileroVAD()
         busy = False
         play_tasks: list[asyncio.Task] = []
+        partial_task: asyncio.Task | None = None
+
+        async def _partial_asr():
+            """Streaming partial ASR during speech (xiaozhi-style interim)."""
+            last_pos = 0
+            try:
+                while vad.in_speech:
+                    await asyncio.sleep(1.0)
+                    if busy or not vad.in_speech:
+                        continue
+                    total = len(buf.buffer)
+                    if total - last_pos < 4800:  # 0.3s new audio minimum
+                        continue
+                    audio = buf.buffer[last_pos:]
+                    if len(audio) < 4800:
+                        continue
+                    loop = asyncio.get_running_loop()
+                    result = await loop.run_in_executor(None, self.asr.transcribe_array, audio)
+                    txt = result.get("text", "").strip()
+                    if txt:
+                        logger.info("[%s] partial ASR: %s", sid, txt[:60])
+                        await self._send({"type": "partial", "text": txt, "seq": 0})
+                    last_pos = total
+            except asyncio.CancelledError:
+                pass
+            except Exception:
+                logger.exception("[%s] partial_asr failed", sid)
 
         async def _acc():
-            nonlocal busy, play_tasks
+            nonlocal busy, play_tasks, partial_task
             stream = rtc.AudioStream(track, sample_rate=sr, num_channels=1)
             fc = 0
             async for ev in stream:
@@ -380,6 +407,12 @@ class Worker:
                 was_speech = vad.in_speech
                 vad.process(arr)
 
+                # Start partial ASR when speech begins
+                if vad.in_speech and not was_speech:
+                    if partial_task and not partial_task.done():
+                        partial_task.cancel()
+                    partial_task = asyncio.create_task(_partial_asr())
+
                 if fc % 100 == 0:
                     logger.info("[%s] af#%d e=%.4f vad=%s blen=%d",
                                 sid, fc, float(np.sqrt(np.mean(arr ** 2))),
@@ -387,7 +420,8 @@ class Worker:
 
                 if was_speech and vad.should_end() and vad.min_speech_met():
                     logger.info("[%s] VAD END blen=%d", sid, len(buf.buffer))
-                    # Cancel any ongoing TTS (interruption)
+                    if partial_task and not partial_task.done():
+                        partial_task.cancel()
                     for pt in play_tasks:
                         if not pt.done():
                             pt.cancel()
@@ -403,6 +437,8 @@ class Worker:
 
                 if vad.in_speech and vad.total_samples >= int(MAX_SPEECH_S * sr):
                     logger.info("[%s] VAD MAX blen=%d", sid, len(buf.buffer))
+                    if partial_task and not partial_task.done():
+                        partial_task.cancel()
                     for pt in play_tasks:
                         if not pt.done():
                             pt.cancel()