Преглед на файлове

fix(tts): ordered TTS generation - collect segments first, generate sequentially

Previously concurrent asyncio tasks could finish out of order,
causing later segments to play before earlier ones. Now collects
all segments, then generates TTS in sequence for correct order.
wenhongquan преди 3 седмици
родител
ревизия
34cb936a5e
променени са 1 файла, в които са добавени 21 реда и са изтрити 38 реда
  1. 21 38
      asr_agent/conversation_worker.py

+ 21 - 38
asr_agent/conversation_worker.py

@@ -241,9 +241,9 @@ async def _asr_mimo_stream(audio_data: np.ndarray):
 # ═══════════════════════════════════════════════════════════════
 # ═══════════════════════════════════════════════════════════════
 
 
 async def _tts_stream(text: str, voice: str = "Chloe"):
 async def _tts_stream(text: str, voice: str = "Chloe"):
-    """Mimo v2.5 streaming TTS. Yields (pcm_chunk: np.ndarray, sr: int, is_final: bool)."""
+    """Mimo TTS. Non-streaming to avoid chunk-boundary artifacts."""
     if not MIMO_KEY:
     if not MIMO_KEY:
-        yield np.array([], dtype=np.float32), 24000, True
+        yield np.array([], dtype=np.int16), 24000, True
         return
         return
     if voice.startswith("data:"):
     if voice.startswith("data:"):
         model = "mimo-v2.5-tts-voiceclone"
         model = "mimo-v2.5-tts-voiceclone"
@@ -258,7 +258,6 @@ async def _tts_stream(text: str, voice: str = "Chloe"):
             {"role": "assistant", "content": text},
             {"role": "assistant", "content": text},
         ],
         ],
         "audio": {"format": "wav", "voice": voice},
         "audio": {"format": "wav", "voice": voice},
-        "stream": True,
     }
     }
 
 
     async with aiohttp.ClientSession() as s:
     async with aiohttp.ClientSession() as s:
@@ -268,28 +267,15 @@ async def _tts_stream(text: str, voice: str = "Chloe"):
                 json=body, headers=h,
                 json=body, headers=h,
                 timeout=aiohttp.ClientTimeout(total=30),
                 timeout=aiohttp.ClientTimeout(total=30),
             ) as r:
             ) as r:
-                async for line in r.content:
-                    line = line.decode().strip()
-                    if not line.startswith("data: "):
-                        continue
-                    data = line[6:]
-                    if data == "[DONE]":
-                        break
-                    try:
-                        chunk = json.loads(data)
-                        delta = chunk.get("choices", [{}])[0].get("delta", {})
-                        audio = delta.get("audio")
-                        if audio and isinstance(audio, dict) and "data" in audio:
-                            pcm = np.frombuffer(
-                                base64.b64decode(audio["data"]), dtype=np.int16
-                            )
-                            if len(pcm) > 0:
-                                yield pcm, 24000, False
-                    except Exception:
-                        continue
+                result = await r.json()
+                b64 = result.get("choices", [{}])[0].get("message", {}).get("audio", {}).get("data", "")
+                if b64:
+                    pcm = np.frombuffer(base64.b64decode(b64), dtype=np.int16)
+                    if len(pcm) > 0:
+                        yield pcm, 24000, False
         except Exception:
         except Exception:
-            logger.exception("TTS stream failed")
-    yield np.array([], dtype=np.float32), 24000, True
+            logger.exception("TTS failed")
+    yield np.array([], dtype=np.int16), 24000, True
 
 
 
 
 def _clean_tts_text(text: str) -> str:
 def _clean_tts_text(text: str) -> str:
@@ -616,30 +602,32 @@ class Worker:
         logger.info("ASR: %s", txt[:80])
         logger.info("ASR: %s", txt[:80])
         await self._send({"type": "utterance", "text": txt, "seq": 0})
         await self._send({"type": "utterance", "text": txt, "seq": 0})
 
 
-        # ── Streaming LLM + concurrent TTS generation (sequential playback) ──
-        # TTS is generated concurrently per segment, but the player consumes the
-        # queue in order on a single track, so segments never overlap.
+        # ── Streaming LLM, collect segments for ordered TTS ──
         reply_full = ""
         reply_full = ""
         seg = _TTSSegmenter()
         seg = _TTSSegmenter()
+        tts_segments: list[str] = []
 
 
         try:
         try:
             async for delta, is_final in _llm_stream(txt, self.hist):
             async for delta, is_final in _llm_stream(txt, self.hist):
                 if delta:
                 if delta:
                     reply_full += delta
                     reply_full += delta
                     await self._send({"type": "reply_partial", "text": reply_full, "seq": 0})
                     await self._send({"type": "reply_partial", "text": reply_full, "seq": 0})
-                    for s in seg.feed(delta):
-                        logger.info("TTS seg(%d): %s", len(s), s[:40])
-                        asyncio.create_task(self._tts_enqueue(s, play_q, gen))
+                    tts_segments.extend(seg.feed(delta))
 
 
                 if is_final:
                 if is_final:
                     remaining = seg.flush()
                     remaining = seg.flush()
                     if remaining:
                     if remaining:
-                        asyncio.create_task(self._tts_enqueue(remaining, play_q, gen))
+                        tts_segments.append(remaining)
                     break
                     break
         except Exception:
         except Exception:
             logger.exception("LLM stream failed")
             logger.exception("LLM stream failed")
             reply_full = "抱歉,我暂时无法回答。"
             reply_full = "抱歉,我暂时无法回答。"
 
 
+        # Ordered TTS: generate and enqueue in sequence
+        for s in tts_segments:
+            logger.info("TTS seg(%d): %s", len(s), s[:40])
+            await self._tts_enqueue(s, play_q, gen)
+
         if reply_full:
         if reply_full:
             await self._send({"type": "reply", "text": reply_full})
             await self._send({"type": "reply", "text": reply_full})
             self.hist.extend([
             self.hist.extend([
@@ -649,18 +637,13 @@ class Worker:
             self.hist[:] = self.hist[-20:]
             self.hist[:] = self.hist[-20:]
 
 
     async def _tts_enqueue(self, text: str, play_q, gen: int):
     async def _tts_enqueue(self, text: str, play_q, gen: int):
-        """Stream TTS audio for one segment, then enqueue it for playback."""
+        """Generate TTS audio and enqueue for playback."""
         try:
         try:
-            chunks: list[np.ndarray] = []
             async for pcm, sr, is_final in _tts_stream(text):
             async for pcm, sr, is_final in _tts_stream(text):
                 if len(pcm) > 0:
                 if len(pcm) > 0:
-                    chunks.append(pcm)
+                    await play_q.put((gen, pcm))
                 if is_final:
                 if is_final:
                     break
                     break
-            if chunks:
-                audio = np.concatenate(chunks)
-                if len(audio) > 0:
-                    await play_q.put((gen, audio))
         except asyncio.CancelledError:
         except asyncio.CancelledError:
             pass
             pass
         except Exception:
         except Exception: