Просмотр исходного кода

fix(tts): ordered TTS generation - collect segments first, generate sequentially

Previously concurrent asyncio tasks could finish out of order,
causing later segments to play before earlier ones. Now collects
all segments, then generates TTS in sequence for correct order.
wenhongquan 3 недель назад
Родитель
Сommit
34cb936a5e
1 измененных файлов с 21 добавлено и 38 удалено
  1. 21 38
      asr_agent/conversation_worker.py

+ 21 - 38
asr_agent/conversation_worker.py

@@ -241,9 +241,9 @@ async def _asr_mimo_stream(audio_data: np.ndarray):
 # ═══════════════════════════════════════════════════════════════
 
 async def _tts_stream(text: str, voice: str = "Chloe"):
-    """Mimo v2.5 streaming TTS. Yields (pcm_chunk: np.ndarray, sr: int, is_final: bool)."""
+    """Mimo TTS. Non-streaming to avoid chunk-boundary artifacts."""
     if not MIMO_KEY:
-        yield np.array([], dtype=np.float32), 24000, True
+        yield np.array([], dtype=np.int16), 24000, True
         return
     if voice.startswith("data:"):
         model = "mimo-v2.5-tts-voiceclone"
@@ -258,7 +258,6 @@ async def _tts_stream(text: str, voice: str = "Chloe"):
             {"role": "assistant", "content": text},
         ],
         "audio": {"format": "wav", "voice": voice},
-        "stream": True,
     }
 
     async with aiohttp.ClientSession() as s:
@@ -268,28 +267,15 @@ async def _tts_stream(text: str, voice: str = "Chloe"):
                 json=body, headers=h,
                 timeout=aiohttp.ClientTimeout(total=30),
             ) as r:
-                async for line in r.content:
-                    line = line.decode().strip()
-                    if not line.startswith("data: "):
-                        continue
-                    data = line[6:]
-                    if data == "[DONE]":
-                        break
-                    try:
-                        chunk = json.loads(data)
-                        delta = chunk.get("choices", [{}])[0].get("delta", {})
-                        audio = delta.get("audio")
-                        if audio and isinstance(audio, dict) and "data" in audio:
-                            pcm = np.frombuffer(
-                                base64.b64decode(audio["data"]), dtype=np.int16
-                            )
-                            if len(pcm) > 0:
-                                yield pcm, 24000, False
-                    except Exception:
-                        continue
+                result = await r.json()
+                b64 = result.get("choices", [{}])[0].get("message", {}).get("audio", {}).get("data", "")
+                if b64:
+                    pcm = np.frombuffer(base64.b64decode(b64), dtype=np.int16)
+                    if len(pcm) > 0:
+                        yield pcm, 24000, False
         except Exception:
-            logger.exception("TTS stream failed")
-    yield np.array([], dtype=np.float32), 24000, True
+            logger.exception("TTS failed")
+    yield np.array([], dtype=np.int16), 24000, True
 
 
 def _clean_tts_text(text: str) -> str:
@@ -616,30 +602,32 @@ class Worker:
         logger.info("ASR: %s", txt[:80])
         await self._send({"type": "utterance", "text": txt, "seq": 0})
 
-        # ── Streaming LLM + concurrent TTS generation (sequential playback) ──
-        # TTS is generated concurrently per segment, but the player consumes the
-        # queue in order on a single track, so segments never overlap.
+        # ── Streaming LLM, collect segments for ordered TTS ──
         reply_full = ""
         seg = _TTSSegmenter()
+        tts_segments: list[str] = []
 
         try:
             async for delta, is_final in _llm_stream(txt, self.hist):
                 if delta:
                     reply_full += delta
                     await self._send({"type": "reply_partial", "text": reply_full, "seq": 0})
-                    for s in seg.feed(delta):
-                        logger.info("TTS seg(%d): %s", len(s), s[:40])
-                        asyncio.create_task(self._tts_enqueue(s, play_q, gen))
+                    tts_segments.extend(seg.feed(delta))
 
                 if is_final:
                     remaining = seg.flush()
                     if remaining:
-                        asyncio.create_task(self._tts_enqueue(remaining, play_q, gen))
+                        tts_segments.append(remaining)
                     break
         except Exception:
             logger.exception("LLM stream failed")
             reply_full = "抱歉,我暂时无法回答。"
 
+        # Ordered TTS: generate and enqueue in sequence
+        for s in tts_segments:
+            logger.info("TTS seg(%d): %s", len(s), s[:40])
+            await self._tts_enqueue(s, play_q, gen)
+
         if reply_full:
             await self._send({"type": "reply", "text": reply_full})
             self.hist.extend([
@@ -649,18 +637,13 @@ class Worker:
             self.hist[:] = self.hist[-20:]
 
     async def _tts_enqueue(self, text: str, play_q, gen: int):
-        """Stream TTS audio for one segment, then enqueue it for playback."""
+        """Generate TTS audio and enqueue for playback."""
         try:
-            chunks: list[np.ndarray] = []
             async for pcm, sr, is_final in _tts_stream(text):
                 if len(pcm) > 0:
-                    chunks.append(pcm)
+                    await play_q.put((gen, pcm))
                 if is_final:
                     break
-            if chunks:
-                audio = np.concatenate(chunks)
-                if len(audio) > 0:
-                    await play_q.put((gen, audio))
         except asyncio.CancelledError:
             pass
         except Exception: