修复网页对话并接入 SearXNG 搜索

services/bot_server.py

@@ -33,9 +33,11 @@ import os
 import sys
 import time
 from collections import deque
+from datetime import UTC, datetime
 from pathlib import Path
 
 import numpy as np
+import pipecat.processors.frameworks.rtvi.models as RTVI
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
@@ -46,13 +48,24 @@ from pipecat.frames.frames import (
     EndFrame,
     InputAudioRawFrame,
     InterruptionFrame,
+    LLMConfigureOutputFrame,
+    LLMContextFrame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMMessagesAppendFrame,
+    LLMTextFrame,
     OutputAudioRawFrame,
     StartFrame,
-    TextFrame,
+    TranscriptionFrame,
 )
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.worker import PipelineParams, PipelineWorker
+from pipecat.processors.aggregators.llm_context import LLMContext
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import (
+    RTVIObserverParams,
+    RTVIProcessor,
+)
 from pipecat.transports.base_transport import TransportParams
 from pipecat.transports.smallwebrtc.request_handler import (
     SmallWebRTCRequest,
@@ -156,6 +169,24 @@ class SimpleVAD:
 
 # ---------------------------------------------------------------- 引擎处理器
 
+def extract_user_text(messages: list) -> str:
+    """从 RTVI/OpenAI 形式的消息列表中取最后一条用户文本。"""
+    for message in reversed(messages):
+        if not isinstance(message, dict) or message.get("role") != "user":
+            continue
+        content = message.get("content", "")
+        if isinstance(content, str):
+            return content.strip()
+        if isinstance(content, list):
+            parts = [
+                item.get("text", "")
+                for item in content
+                if isinstance(item, dict) and isinstance(item.get("text"), str)
+            ]
+            return " ".join(part.strip() for part in parts if part.strip())
+    return ""
+
+
 class EngineProcessor(FrameProcessor):
     """把 ConversationEngine 包成一个 Pipecat 处理器。
 
@@ -163,37 +194,50 @@ class EngineProcessor(FrameProcessor):
     Pipecat 只负责音频进出和帧路由。**
     """
 
-    def __init__(self, cfg: AppConfig, transport: SmallWebRTCTransport):
+    def __init__(
+        self,
+        cfg: AppConfig,
+        transport: SmallWebRTCTransport,
+        rtvi: RTVIProcessor,
+    ):
         super().__init__()
         self.cfg = cfg
         self.transport = transport
+        self.rtvi = rtvi
         self._vad = SimpleVAD()
         self._pre_roll: deque[bytes] = deque(maxlen=50)  # 约 1 秒
         self._buf = bytearray()          # 当前话语的 PCM16
         self._turn: asyncio.Task | None = None
+        self._turn_kind: str | None = None
         self._engine: ConversationEngine | None = None
+        self._engine_lock = asyncio.Lock()
         self._memory: SQLiteMemoryStore | None = None
         self._llm: FallbackProvider | None = None
         self._session = ""
         self._bot_speaking = False
+        self._skip_tts = False
         # 计时：用于拿真实延迟数字
         self._t_speech_end = 0.0
         self._t_first_audio = 0.0
 
     # ------------------------------------------------------------ 生命周期
 
-    async def _ensure_engine(self) -> ConversationEngine:
-        if self._engine is None:
-            self._memory = SQLiteMemoryStore(
-                Path.home() / ".local/share/ai_assistant/memory.db")
-            await self._memory.open()
-            self._llm = FallbackProvider(create_provider(self.cfg.llm),
-                                         create_provider(self.cfg.llm_fallback))
-            self._engine = ConversationEngine(
-                self.cfg, self._llm, self._memory, ManualIdentityProvider(),
-                create_tts(self.cfg.voice), StreamingSTT(self.cfg.voice))
-            self._session = await self._engine.start_session(user_id="u_ylhan")
-        return self._engine
+    async def _ensure_engine(self, *, require_stt: bool = False) -> ConversationEngine:
+        async with self._engine_lock:
+            if self._engine is None:
+                self._memory = SQLiteMemoryStore(
+                    Path.home() / ".local/share/ai_assistant/memory.db")
+                await self._memory.open()
+                self._llm = FallbackProvider(create_provider(self.cfg.llm),
+                                             create_provider(self.cfg.llm_fallback))
+                # 文字首轮不应等待 STT 模型加载；第一次语音输入时再初始化。
+                self._engine = ConversationEngine(
+                    self.cfg, self._llm, self._memory, ManualIdentityProvider(),
+                    create_tts(self.cfg.voice), stt=None)
+                self._session = await self._engine.start_session(user_id="u_ylhan")
+            if require_stt and self._engine.stt is None:
+                self._engine.stt = StreamingSTT(self.cfg.voice)
+            return self._engine
 
     async def setup(self, setup):
         await super().setup(setup)
@@ -208,6 +252,26 @@ class EngineProcessor(FrameProcessor):
         elif isinstance(frame, InputAudioRawFrame):
             await self._on_audio(frame)
 
+        elif isinstance(frame, LLMConfigureOutputFrame):
+            # RTVI 在 send-text 前后用这个帧表达 audio_response 选项。
+            self._skip_tts = frame.skip_tts
+            await self.push_frame(frame, direction)
+
+        elif isinstance(frame, LLMMessagesAppendFrame):
+            text = extract_user_text(frame.messages)
+            if text and frame.run_llm is not False:
+                log.info("收到网页文字输入: %d 字", len(text))
+                await self._cancel_turn()
+                self._t_speech_end = time.perf_counter()
+                self._t_first_audio = 0.0
+                self._turn = self.create_task(
+                    self._run_turn(text=text, audio_response=not self._skip_tts),
+                    "conversation-text-turn",
+                )
+                self._turn_kind = "text"
+            else:
+                await self.push_frame(frame, direction)
+
         elif isinstance(frame, InterruptionFrame):
             # 客户端判定打断后发来的信号（或 Pipecat 内部产生）
             log.info("收到打断信号，取消当前生成")
@@ -264,28 +328,94 @@ class EngineProcessor(FrameProcessor):
             audio = np.frombuffer(bytes(self._buf), dtype=np.int16).astype(np.float32) / 32768.0
             self._buf.clear()
             if len(audio) > IN_RATE * 0.3:      # 太短的不处理
+                # Playground 始终开着麦克风。文字轮次期间的环境声或扬声器回采
+                # 不能抢占文字请求；真正的设备端打断将在 P2 由客户端判定。
+                if (
+                    self._turn_kind == "text"
+                    and self._turn is not None
+                    and not self._turn.done()
+                ):
+                    log.info("文字轮次进行中，忽略并发麦克风话段")
+                    return
                 self._t_speech_end = time.perf_counter()
                 self._t_first_audio = 0.0
                 await self._cancel_turn()
-                self._turn = self.create_task(self._run_turn(audio), "conversation-turn")
+                self._turn = self.create_task(
+                    self._run_turn(audio=audio), "conversation-audio-turn")
+                self._turn_kind = "audio"
 
     async def _cancel_turn(self) -> None:
         if self._turn and not self._turn.done():
             await self.cancel_task(self._turn, timeout=5.0)
         self._turn = None
+        self._turn_kind = None
 
     # ------------------------------------------------------------ 跑一轮
 
-    async def _run_turn(self, audio: np.ndarray) -> None:
+    async def _run_turn(
+        self,
+        *,
+        audio: np.ndarray | None = None,
+        text: str | None = None,
+        audio_response: bool = True,
+    ) -> None:
+        llm_started = False
+        reply_parts: list[str] = []
         try:
-            engine = await self._ensure_engine()
-            async for ev in engine.respond_to_audio(audio, session_id=self._session):
-                if ev.kind == EventKind.RESPONSE_DELTA and ev.text:
-                    await self.push_frame(TextFrame(text=ev.text))
-                elif ev.kind == EventKind.SPEAK and ev.audio is not None and len(ev.audio):
+            engine = await self._ensure_engine(require_stt=audio is not None)
+            if text is not None:
+                # RTVIObserver 从标准上下文帧生成 user-llm-text，供网页
+                # Conversation 区显示。实际会话历史仍由自有引擎管理。
+                await self.push_frame(LLMContextFrame(
+                    context=LLMContext(messages=[
+                        {"role": "user", "content": text},
+                    ]),
+                ))
+                await self.rtvi.push_transport_message(
+                    RTVI.UserLLMTextMessage(data=RTVI.TextMessageData(text=text))
+                )
+            events = (
+                engine.respond_to_text(text, session_id=self._session)
+                if text is not None
+                else engine.respond_to_audio(audio, session_id=self._session)
+            )
+            async for ev in events:
+                if ev.kind == EventKind.TRANSCRIPT and ev.text:
+                    await self.push_frame(TranscriptionFrame(
+                        text=ev.text,
+                        user_id="u_ylhan",
+                        timestamp=datetime.now(UTC).isoformat(),
+                        finalized=True,
+                    ))
+                    await self.rtvi.push_transport_message(
+                        RTVI.UserTranscriptionMessage(
+                            data=RTVI.UserTranscriptionMessageData(
+                                text=ev.text,
+                                user_id="u_ylhan",
+                                timestamp=datetime.now(UTC).isoformat(),
+                                final=True,
+                            )
+                        )
+                    )
+                elif ev.kind == EventKind.RESPONSE_DELTA and ev.text:
+                    reply_parts.append(ev.text)
+                    if not llm_started:
+                        await self.push_frame(LLMFullResponseStartFrame())
+                        await self.rtvi.push_transport_message(RTVI.BotLLMStartedMessage())
+                        llm_started = True
+                    await self.push_frame(LLMTextFrame(text=ev.text))
+                    await self.rtvi.push_transport_message(
+                        RTVI.BotLLMTextMessage(data=RTVI.TextMessageData(text=ev.text))
+                    )
+                elif (
+                    ev.kind == EventKind.SPEAK
+                    and audio_response
+                    and ev.audio is not None
+                    and len(ev.audio)
+                ):
                     if self._t_first_audio == 0.0:
                         self._t_first_audio = time.perf_counter()
-                        log.info("首音延迟（说完→首段音频）: %.0f ms",
+                        log.info("首音延迟（输入完成→首段音频）: %.0f ms",
                                  (self._t_first_audio - self._t_speech_end) * 1000)
                     await self._push_audio(ev.audio, ev.sample_rate)
                 elif ev.kind == EventKind.METRICS and ev.metrics:
@@ -295,6 +425,28 @@ class EngineProcessor(FrameProcessor):
             raise
         except Exception as e:  # noqa: BLE001
             log.exception("跑轮失败: %s", e)
+        finally:
+            if llm_started:
+                await self.push_frame(LLMFullResponseEndFrame())
+                await self.rtvi.push_transport_message(RTVI.BotLLMStoppedMessage())
+            reply = "".join(reply_parts).strip()
+            if reply:
+                # Playground 1.7 的 Conversation 视图主要消费完整转写/
+                # bot-output；仅发送 token 级 bot-llm-text 不会稳定渲染。
+                await self.rtvi.push_transport_message(
+                    RTVI.BotTranscriptionMessage(
+                        data=RTVI.TextMessageData(text=reply)
+                    )
+                )
+                await self.rtvi.push_transport_message(
+                    RTVI.BotOutputMessage(
+                        data=RTVI.BotOutputMessageData(
+                            text=reply,
+                            aggregated_by="sentence",
+                            spoken=audio_response,
+                        )
+                    )
+                )
 
     async def _push_audio(self, samples: np.ndarray, sample_rate: int) -> None:
         """把 float32 音频推给传输层。必要时重采样到 OUT_RATE。"""
@@ -352,7 +504,8 @@ async def offer(request: SmallWebRTCRequest):
                 audio_out_channels=1,
             ),
         )
-        engine_proc = EngineProcessor(_cfg, transport)
+        rtvi = RTVIProcessor()
+        engine_proc = EngineProcessor(_cfg, transport, rtvi)
         pipeline = Pipeline([transport.input(), engine_proc, transport.output()])
         # 注意：这个版本里 PipelineParams 没有 allow_interruptions——
         # 打断在 1.10 已移到 pipecat/turns/ 框架。实验阶段我们不用它的
@@ -370,6 +523,14 @@ async def offer(request: SmallWebRTCRequest):
             pipeline,
             params=PIPELINE_PARAMS,
             enable_rtvi=True,
+            rtvi_processor=rtvi,
+            # EngineProcessor 显式发送这些事件，避免自有异步引擎产生的帧在
+            # observer 来源判定中丢失或重复。
+            rtvi_observer_params=RTVIObserverParams(
+                bot_llm_enabled=False,
+                user_llm_enabled=False,
+                user_transcription_enabled=False,
+            ),
             idle_timeout_secs=None,
         )
         _tasks.append(task)