ylhan/ai-companion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复 Pipecat 实时链路并补齐网页测试

Browse Source
This commit is contained in:
ylhan
2026-09-16 09:50:38 +08:00
parent 74a30b11d7
commit c3d2c7f3a8
6 changed files with 296 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/bot_server.py
+113 -41
View File
@@ -32,7 +32,7 @@ import logging
import os
import sys
import time
import uuid
from collections import deque
from pathlib import Path
import numpy as np
@@ -51,7 +51,7 @@ from pipecat.frames.frames import (
TextFrame,
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.smallwebrtc.request_handler import (
@@ -73,20 +73,25 @@ logging.basicConfig(level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("bot")
# Pipecat 会在每个 worker 启动时预热 NLTK 分句器。固定到私有、可预装的目录，
# 避免使用权限过宽的 ~/nltk_data，也避免首个会话临时联网下载。
DEFAULT_NLTK_DATA = Path.home() / ".local/share/ai-companion/nltk_data"
os.environ.setdefault("NLTK_DATA", str(DEFAULT_NLTK_DATA))
#: 输入音频格式（WebRTC 侧协商后的格式）
IN_RATE = 16000
IN_CHANNELS = 1
#: 输出音频格式（Qwen3-TTS 输出 24kHz）
OUT_RATE = 24000
#: 管线参数。给 PipelineTask 构造用。
#: 管线参数。给 PipelineWorker 构造用。
PIPELINE_PARAMS = PipelineParams(
audio_in_sample_rate=IN_RATE,
audio_out_sample_rate=OUT_RATE,
enable_metrics=True,
)
#: 注意：PipelineTask.run() 要的是 WorkerParams（内含 task_manager），
#: 注意：PipelineWorker.run() 要的是 WorkerParams（内含 task_manager），
#: **不是** PipelineParams。传错会依次报两个错，且都发生在后台任务里，
#: 主日志里什么都看不到：
#: 1. 不传 → "PipelineWorker.run() missing 1 required positional argument: 'params'"
@@ -121,8 +126,13 @@ class SimpleVAD:
self._silence = 0.0
self._speech = 0.0
@property
def active(self) -> bool:
"""是否已经进入一段候选语音。"""
return self._speech > 0.0
def feed(self, pcm16: np.ndarray, chunk_ms: float) -> str | None:
"""喂一块音频，返回 'start' / 'end' / None。"""
"""喂一块音频，返回 start / end / discard / None。"""
if pcm16.size == 0:
return None
rms = float(np.sqrt(np.mean(pcm16.astype(np.float32) ** 2)))
@@ -138,6 +148,8 @@ class SimpleVAD:
if self._silence >= self.silence_ms:
if self._speech >= self.min_speech_ms:
event = "end"
else:
event = "discard"
self.reset()
return event
@@ -153,13 +165,15 @@ class EngineProcessor(FrameProcessor):
def __init__(self, cfg: AppConfig, transport: SmallWebRTCTransport):
super().__init__()
log.info("[诊断] EngineProcessor 已创建")
self.cfg = cfg
self.transport = transport
self._vad = SimpleVAD()
self._pre_roll: deque[bytes] = deque(maxlen=50) # 约 1 秒
self._buf = bytearray() # 当前话语的 PCM16
self._turn: asyncio.Task | None = None
self._engine: ConversationEngine | None = None
self._memory: SQLiteMemoryStore | None = None
self._llm: FallbackProvider | None = None
self._session = ""
self._bot_speaking = False
# 计时：用于拿真实延迟数字
@@ -170,32 +184,24 @@ class EngineProcessor(FrameProcessor):
async def _ensure_engine(self) -> ConversationEngine:
if self._engine is None:
mem = SQLiteMemoryStore(Path.home() / ".local/share/ai_assistant/memory.db")
await mem.open()
llm = FallbackProvider(create_provider(self.cfg.llm),
create_provider(self.cfg.llm_fallback))
self._memory = SQLiteMemoryStore(
Path.home() / ".local/share/ai_assistant/memory.db")
await self._memory.open()
self._llm = FallbackProvider(create_provider(self.cfg.llm),
create_provider(self.cfg.llm_fallback))
self._engine = ConversationEngine(
self.cfg, llm, mem, ManualIdentityProvider(),
self.cfg, self._llm, self._memory, ManualIdentityProvider(),
create_tts(self.cfg.voice), StreamingSTT(self.cfg.voice))
self._session = await self._engine.start_session(user_id="u_ylhan")
return self._engine
async def setup(self, setup):
log.info("[诊断] EngineProcessor.setup() 被调用")
await super().setup(setup)
log.info("[诊断] EngineProcessor 初始化完成")
log.info("会话处理器已就绪")
async def process_frame(self, frame, direction: FrameDirection):
log.info("[诊断·顶] 进入 process_frame: %s", type(frame).__name__)
await super().process_frame(frame, direction)
# [诊断] 记录到达的帧类型分布
self._seen = getattr(self, "_seen", {})
k = type(frame).__name__
self._seen[k] = self._seen.get(k, 0) + 1
if sum(self._seen.values()) <= 40 or self._seen[k] == 1:
log.info("[诊断] 帧: %s (累计 %d)", k, sum(self._seen.values()))
if isinstance(frame, StartFrame):
await self.push_frame(frame, direction)
@@ -233,9 +239,27 @@ class EngineProcessor(FrameProcessor):
if self._bot_speaking and float(np.sqrt(np.mean(pcm16.astype(np.float32) ** 2))) > self._vad.threshold:
log.info("检测到用户插话（助手正在播放）")
self._buf.extend(frame.audio)
was_active = self._vad.active
self._pre_roll.append(frame.audio)
event = self._vad.feed(pcm16, chunk_ms)
if event == "start":
# 带上说话前约 1 秒，避免 VAD 切掉首字。
self._buf = bytearray().join(self._pre_roll)
elif was_active:
self._buf.extend(frame.audio)
# 防止噪声或异常客户端让常驻进程无限积累音频。
max_bytes = IN_RATE * 2 * 60
if len(self._buf) > max_bytes:
log.warning("单次话语超过 60 秒，强制截断处理")
event = "end"
self._vad.reset()
if event == "discard":
self._buf.clear()
return
if event == "end":
audio = np.frombuffer(bytes(self._buf), dtype=np.int16).astype(np.float32) / 32768.0
self._buf.clear()
@@ -243,15 +267,11 @@ class EngineProcessor(FrameProcessor):
self._t_speech_end = time.perf_counter()
self._t_first_audio = 0.0
await self._cancel_turn()
self._turn = asyncio.create_task(self._run_turn(audio))
self._turn = self.create_task(self._run_turn(audio), "conversation-turn")
async def _cancel_turn(self) -> None:
if self._turn and not self._turn.done():
self._turn.cancel()
try:
await self._turn
except (asyncio.CancelledError, Exception): # noqa: BLE001
pass
await self.cancel_task(self._turn, timeout=5.0)
self._turn = None
# ------------------------------------------------------------ 跑一轮
@@ -295,6 +315,15 @@ class EngineProcessor(FrameProcessor):
async def cleanup(self):
await self._cancel_turn()
if self._engine is not None:
await self._engine.end_session()
if self._llm is not None:
await self._llm.aclose()
if self._memory is not None:
await self._memory.close()
self._engine = None
self._llm = None
self._memory = None
await super().cleanup()
@@ -303,7 +332,7 @@ class EngineProcessor(FrameProcessor):
app = FastAPI(title="AI Companion · Pipecat experimental server")
handler = SmallWebRTCRequestHandler()
_cfg: AppConfig | None = None
_tasks: list[PipelineTask] = []
_tasks: list[PipelineWorker] = []
_bg_tasks: set[asyncio.Task] = set() # 持有后台 task 引用，避免被 GC
@@ -329,24 +358,39 @@ async def offer(request: SmallWebRTCRequest):
# 打断在 1.10 已移到 pipecat/turns/ 框架。实验阶段我们不用它的
# 轮次框架（方案 §5 的边界决定是"打断由客户端判定"），
# 所以这里只开指标。
# 注意：这个版本的 PipelineTask.run(params) **要求单独传 params**，
# 只传给构造函数不够。漏传的报错是
# 注意：PipelineWorker 的音频参数传构造函数；run() 另收
# WorkerParams（主要提供 task_manager）。漏传 run 参数会报：
# "PipelineWorker.run() missing 1 required positional argument: 'params'"
# 而这个异常发生在后台任务里，会被静默吞掉——
# 现象是"连接正常但什么反应都没有"，而且日志里一条错误都看不到。
# 所以下面的 _run_task 里必须 log.exception，不能让异常无声消失。
task = PipelineTask(pipeline)
# 保留 RTVI：网页 prebuilt 客户端用它显示状态、转写和指标；
# N5095 的精简客户端可以忽略不认识的 data-channel 消息。
task = PipelineWorker(
pipeline,
params=PIPELINE_PARAMS,
enable_rtvi=True,
idle_timeout_secs=None,
)
_tasks.append(task)
log.info("新连接已建立，管线启动")
@task.event_handler("on_pipeline_started")
async def _on_pipeline_started(worker, frame):
log.info("管线已就绪")
@task.event_handler("on_pipeline_timeout")
async def _on_pipeline_timeout(worker, frame):
log.error("管线等待 %s 超时", type(frame).__name__)
@task.event_handler("on_pipeline_error")
async def _on_pipeline_error(worker, frame):
log.error("管线错误: %s", frame)
@transport.event_handler("on_client_connected")
async def _on_conn(t, client):
# **这一步不能省。** 传输层的音频推流由 StartFrame 驱动
# （base_input.start()），而 PipelineTask.run() 自己不会补发。
# 不喂 StartFrame 的话：ICE 连通、信令全对、但一帧音频都收不到，
# 表现为"连上了但助手永远不说话"。
log.info("客户端已连接，发送 StartFrame 启动音频流")
await task.queue_frames([StartFrame()])
# PipelineWorker.run() 会自动发送且只应发送一次 StartFrame。
log.info("客户端已连接")
@transport.event_handler("on_client_disconnected")
async def _on_disc(t, client):
@@ -360,13 +404,13 @@ async def offer(request: SmallWebRTCRequest):
# 于是 on_client_connected 永远不触发、StartFrame 不发、一帧音频都收不到。
# 官方 runner 用的是 background_tasks.add_task(bot, args)，同样的道理。
async def _run_task():
log.info("[诊断] PipelineTask 开始运行")
log.info("PipelineWorker 开始运行")
try:
# TaskManager 必须在这里建——模块级没有运行中的事件循环
await task.run(WorkerParams(task_manager=TaskManager()))
except Exception as e: # noqa: BLE001
log.exception("[诊断] PipelineTask 异常退出: %s", e)
log.info("[诊断] PipelineTask 已结束")
log.exception("PipelineWorker 异常退出: %s", e)
log.info("PipelineWorker 已结束")
bg = asyncio.create_task(_run_task())
_bg_tasks.add(bg)
@@ -378,6 +422,20 @@ async def offer(request: SmallWebRTCRequest):
return answer
@app.post("/start")
async def start_web_client():
"""给 Pipecat Playground 返回本服务的 WebRTC 信令入口。
prebuilt 网页客户端会先 POST /start，再用响应里的 endpoint 发 SDP offer；
自写 aiortc/N5095 客户端则可以继续直接调用 /api/offer。
"""
return {
"webrtcRequestParams": {
"endpoint": "/api/offer",
},
}
@app.get("/health")
async def health():
return {"status": "ok", "connections": len([t for t in _tasks if not t.has_finished()])}
@@ -399,6 +457,19 @@ def mount_client() -> bool:
return False
def check_runtime_data() -> None:
"""启动前验证运行时数据，避免首个会话里偷偷联网下载。"""
import nltk
try:
nltk.data.find("tokenizers/punkt_tab")
except LookupError as e:
raise RuntimeError(
"缺少 NLTK punkt_tab。请在部署阶段运行 "
"`python -m nltk.downloader punkt_tab`，并设置 NLTK_DATA。"
) from e
def main() -> int:
global _cfg
p = argparse.ArgumentParser()
@@ -406,6 +477,7 @@ def main() -> int:
p.add_argument("--port", type=int, default=7860)
a = p.parse_args()
check_runtime_data()
_cfg = AppConfig.load()
log.info("LLM=%s TTS=%s/%s STT=%d线程",
_cfg.llm.model, _cfg.voice.tts_backend, _cfg.voice.tts_speaker,