ylhan
98bbba93d0
按 P1-Pipecat Flows Agent化与记忆改造交接.md 的 Phase 4 完成。 新增 p1/phase4_acceptance.py：跑真实服务端验 7 条，每条之间查数据库。 实测结果（2026-09-16） 1. "记住我不吃香菜" → confirmed 落库 ✅ 2. **新连接** "我不吃什么？" → "你不吃香菜。" ✅ 连接开始时加载档案生效 3. "我今天吃了香菜" → 不新增 confirmed，模型还追问了是否口味变了 ✅ 4. "其实我现在吃香菜了" → 同键仍只一条 confirmed，值已更新 ✅ 5. "忘掉我的饮食偏好" → 已删除 ✅ 6/7. 访客与跨用户隔离 → 由单测覆盖（客户端不支持切身份） 实现：会话开始时加载稳定档案 - load_stable_memories()：取已确认、未过期记忆（身份未确认返回空） - _conversation_node 改用 build_system_prompt，与 legacy 共用同一套 人格与记忆注入格式；记忆进**同一条** system 消息，仍不用 role_message 修掉一个真实缺陷：Flows 路径误用了语音的 max_tokens - 症状：一句"忘掉我的饮食偏好"，服务端日志 "Failed to parse function call arguments: {"query": "饮食"（JSON 被截断）， 回复为空、记忆没删掉，且**没有任何报错** - 根因：Flows 路径沿用了语音的 max_tokens=200（为"回复要短"而设）。 原始 API 对照实测：max_tokens=200 时模型 finish=stop、根本不调工具； 600 时正常产生 tool_calls 且参数 JSON 完整 - 修法：新增 AgentConfig.max_tokens（AI_AGENT_MAX_TOKENS，默认 600）， Flows 路径用它，语音路径仍用 200 新增 client/test_client.py 能力（为验收所需） - RTVIReplies：收服务端 RTVI 文本消息（bot-output / bot-transcription） - 打印 BOT_REPLY 供验收脚本断言 回归：test_agent_tools 45/45、test_flows_runtime 24/24、 test_memory 30/30、test_identity 19/19 服务器已改回 AI_AGENT_RUNTIME=legacy（改默认是 Phase 7 的事）
462 lines
18 KiB
Python
462 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""最小 aiortc 客户端（实验用）。
|
||
|
||
**为什么需要它**：Pipecat 的官方客户端 SDK 覆盖浏览器 / 移动端 / C++ / ESP32，
|
||
唯独没有适配"跑 Linux 的 x86 盒子接 USB 麦克风阵列"——也就是 N5095 的形态。
|
||
所以客户端必须自己写，这个文件是雏形。
|
||
|
||
它同时是验证工具：不依赖浏览器就能测服务端。用法：
|
||
|
||
# 发一段 wav，收回应答音频
|
||
python client/test_client.py --server http://<服务器>:7860 \
|
||
--wav p0/stt_tts_bench/wavs/stt_1.wav --out /tmp/reply.wav
|
||
|
||
# 只做连接测试，不发音频
|
||
python client/test_client.py --server http://<服务器>:7860 --connect-only
|
||
|
||
**它还不是什么**：不做 AEC、不做播客参考、不做打断（那些要真实声卡和硬件），
|
||
只用文件音频验证信令与媒体通路。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import fractions
|
||
import json
|
||
import logging
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
import wave
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
from aiortc import (
|
||
RTCConfiguration,
|
||
RTCPeerConnection,
|
||
RTCSessionDescription,
|
||
)
|
||
from aiortc.mediastreams import AudioStreamTrack, MediaStreamError
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||
log = logging.getLogger("client")
|
||
|
||
|
||
class SilentTrack(AudioStreamTrack):
|
||
"""静音音轨。只为触发 ICE 候选收集，不发真实音频。"""
|
||
|
||
kind = "audio"
|
||
|
||
def __init__(self, sample_rate: int = 16000):
|
||
super().__init__()
|
||
self.sample_rate = sample_rate
|
||
self._n = 0
|
||
|
||
async def recv(self):
|
||
import fractions as _fr
|
||
from av import AudioFrame as _AF
|
||
await asyncio.sleep(0.02)
|
||
pcm = np.zeros(self.sample_rate // 50, dtype=np.int16)
|
||
f = _AF(format="s16", layout="mono", samples=len(pcm))
|
||
f.planes[0].update(pcm.tobytes())
|
||
f.sample_rate = self.sample_rate
|
||
f.pts = self._n
|
||
f.time_base = _fr.Fraction(1, self.sample_rate)
|
||
self._n += len(pcm)
|
||
return f
|
||
|
||
|
||
class FileAudioTrack(AudioStreamTrack):
|
||
"""把 wav 文件当作麦克风音源，按实时速率吐出来。
|
||
|
||
实时速率很重要——如果一次性灌完，服务端的 VAD 会把它当成
|
||
一段超长的连续语音，测出来的延迟也不真实。
|
||
"""
|
||
|
||
kind = "audio"
|
||
|
||
def __init__(self, wav_path: Path, *, sample_rate: int = 16000,
|
||
tail_silence_s: float = 1.2):
|
||
super().__init__()
|
||
with wave.open(str(wav_path)) as w:
|
||
assert w.getnchannels() == 1, "需要单声道"
|
||
self.src_rate = w.getframerate()
|
||
pcm = np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16)
|
||
self.samples = pcm.astype(np.float32) / 32768.0
|
||
self.sample_rate = sample_rate
|
||
if self.src_rate != sample_rate:
|
||
n = int(len(self.samples) * sample_rate / self.src_rate)
|
||
self.samples = np.interp(np.linspace(0, len(self.samples) - 1, n),
|
||
np.arange(len(self.samples)), self.samples)
|
||
self._source_pos = 0
|
||
self._timestamp_samples = 0
|
||
self._tail_remaining = int(tail_silence_s * sample_rate)
|
||
self.source_finished_at: float | None = None
|
||
self._t0: float | None = None
|
||
self._frame_samples = sample_rate // 100 # 10ms
|
||
|
||
async def recv(self):
|
||
"""按 10ms 一块、以真实时间节奏产出。"""
|
||
if self._t0 is None:
|
||
self._t0 = time.perf_counter()
|
||
self._source_pos = 0
|
||
|
||
frame_pts = self._timestamp_samples
|
||
if self._source_pos >= len(self.samples):
|
||
if self.source_finished_at is None:
|
||
self.source_finished_at = time.perf_counter()
|
||
audio = np.zeros(self._frame_samples, dtype=np.float32)
|
||
self._tail_remaining = max(0, self._tail_remaining - self._frame_samples)
|
||
else:
|
||
audio = self.samples[self._source_pos:self._source_pos + self._frame_samples]
|
||
if len(audio) < self._frame_samples:
|
||
audio = np.pad(audio, (0, self._frame_samples - len(audio)))
|
||
self._source_pos += self._frame_samples
|
||
|
||
# 即使源文件已经结束，静音帧也必须持续推进时钟和 PTS。
|
||
self._timestamp_samples += self._frame_samples
|
||
|
||
# 实时节奏
|
||
target = self._t0 + self._timestamp_samples / self.sample_rate
|
||
delay = target - time.perf_counter()
|
||
if delay > 0:
|
||
await asyncio.sleep(delay)
|
||
|
||
pcm = (np.clip(audio, -1, 1) * 32767).astype(np.int16)
|
||
from av import AudioFrame
|
||
f = AudioFrame(format="s16", layout="mono", samples=len(pcm))
|
||
f.planes[0].update(pcm.tobytes())
|
||
f.sample_rate = self.sample_rate
|
||
f.pts = int(frame_pts * 90000 / self.sample_rate)
|
||
f.time_base = fractions.Fraction(1, 90000)
|
||
return f
|
||
|
||
@property
|
||
def finished(self) -> bool:
|
||
return self._source_pos >= len(self.samples) and self._tail_remaining <= 0
|
||
|
||
|
||
class ReplyRecorder:
|
||
"""收服务端音频，写 wav，并记录首音延迟。"""
|
||
|
||
def __init__(self, out_path: Path, sample_rate: int = 24000,
|
||
speech_rms_threshold: float = 80.0):
|
||
self.out_path = out_path
|
||
self.sample_rate = sample_rate
|
||
self.chunks: list[np.ndarray] = []
|
||
self.first_non_silent_at: float | None = None
|
||
self.speech_rms_threshold = speech_rms_threshold
|
||
|
||
def feed(self, frame) -> None:
|
||
arr = frame.to_ndarray()
|
||
channels = len(frame.layout.channels)
|
||
if frame.format.is_planar:
|
||
raw = arr.astype(np.float32).mean(axis=0)
|
||
else:
|
||
raw = arr.astype(np.float32).reshape(-1, channels).mean(axis=1)
|
||
# WebRTC/Opus 通常以 48kHz 交付，即使服务端原始输出是 24kHz。
|
||
# WAV 必须记录接收帧的实际采样率，否则时长会被算错。
|
||
if not self.chunks:
|
||
self.sample_rate = frame.sample_rate
|
||
elif frame.sample_rate != self.sample_rate:
|
||
raise ValueError(
|
||
f"回复音轨采样率发生变化: {self.sample_rate} -> {frame.sample_rate}")
|
||
raw = raw.reshape(-1)
|
||
rms = float(np.sqrt(np.mean(raw ** 2))) if raw.size else 0.0
|
||
if self.first_non_silent_at is None and rms >= self.speech_rms_threshold:
|
||
self.first_non_silent_at = time.perf_counter()
|
||
self.chunks.append(raw / 32768.0)
|
||
|
||
def save(self) -> float:
|
||
if not self.chunks:
|
||
return 0.0
|
||
audio = np.concatenate(self.chunks)
|
||
pcm = (np.clip(audio, -1, 1) * 32767).astype(np.int16)
|
||
self.out_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with wave.open(str(self.out_path), "wb") as w:
|
||
w.setnchannels(1)
|
||
w.setsampwidth(2)
|
||
w.setframerate(self.sample_rate)
|
||
w.writeframes(pcm.tobytes())
|
||
return len(audio) / self.sample_rate
|
||
|
||
@property
|
||
def has_speech(self) -> bool:
|
||
return self.first_non_silent_at is not None
|
||
|
||
def first_audio_ms_from(self, started_at: float | None) -> float | None:
|
||
if started_at is None or self.first_non_silent_at is None:
|
||
return None
|
||
return (self.first_non_silent_at - started_at) * 1000
|
||
|
||
|
||
class RTVIReplies:
|
||
"""收集服务端发来的 RTVI 文本消息，供验收脚本断言。
|
||
|
||
网页用的 bot-output / bot-transcription 是完整文本，
|
||
bot-llm-text 是逐 token 增量。验收看完整文本。
|
||
"""
|
||
|
||
def __init__(self):
|
||
self.bot_texts: list[str] = []
|
||
self.user_texts: list[str] = []
|
||
self.other: list[str] = []
|
||
|
||
def handle(self, raw: str) -> None:
|
||
try:
|
||
m = json.loads(raw)
|
||
except (json.JSONDecodeError, TypeError):
|
||
return
|
||
if not isinstance(m, dict):
|
||
return
|
||
t = m.get("type")
|
||
data = m.get("data") or {}
|
||
text = data.get("text") if isinstance(data, dict) else None
|
||
if t == "bot-output" and text:
|
||
self.bot_texts.append(text)
|
||
elif t == "bot-transcription" and text:
|
||
self.bot_texts.append(text)
|
||
elif t == "user-llm-text" and text:
|
||
self.user_texts.append(text)
|
||
elif t:
|
||
self.other.append(t)
|
||
|
||
@property
|
||
def last_bot(self) -> str:
|
||
return self.bot_texts[-1] if self.bot_texts else ""
|
||
|
||
|
||
def rtvi_send_text(channel, text: str, *, audio_response: bool = True) -> None:
|
||
"""通过 data channel 发一条 RTVI send-text。
|
||
|
||
信封格式来自 pipecat 的 RTVI 模型（实测校正过两次）：
|
||
label='rtvi-ai'，**字段是 id 不是 msg_id**，type='send-text'，
|
||
data.content（**是 content 不是 text**）。
|
||
写错任何一个，服务端都会以 "Invalid RTVI transport message: Field required" 拒掉，
|
||
而客户端看不到任何反馈——表现为"发了消息但助手毫无反应"。
|
||
"""
|
||
import uuid
|
||
msg = {
|
||
"label": "rtvi-ai",
|
||
"id": uuid.uuid4().hex,
|
||
"type": "send-text",
|
||
"data": {"content": text,
|
||
"options": {"audio_response": audio_response}},
|
||
}
|
||
channel.send(json.dumps(msg))
|
||
|
||
|
||
def post_offer(server: str, sdp: str, type_: str) -> dict:
|
||
"""把 offer 发给服务端，取回 answer。"""
|
||
payload = json.dumps({"sdp": sdp, "type": type_, "pc_id": None,
|
||
"restart_pc": False, "request_data": None}).encode()
|
||
req = urllib.request.Request(f"{server.rstrip('/')}/api/offer", data=payload,
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST")
|
||
with urllib.request.urlopen(req, timeout=30) as r:
|
||
return json.loads(r.read().decode())
|
||
|
||
|
||
async def run(server: str, wav: Path | None, out: Path, *, connect_only: bool,
|
||
timeout_s: float, texts: list[str] | None = None) -> int:
|
||
pc = RTCPeerConnection(RTCConfiguration(iceServers=[]))
|
||
recorder = ReplyRecorder(out)
|
||
track = None
|
||
got_audio = asyncio.Event()
|
||
|
||
@pc.on("connectionstatechange")
|
||
async def _state():
|
||
log.info("连接状态: %s", pc.connectionState)
|
||
if pc.connectionState in ("failed", "closed"):
|
||
got_audio.set()
|
||
|
||
@pc.on("track")
|
||
def _track(t):
|
||
log.info("收到服务端音轨: %s", t.kind)
|
||
|
||
async def _pump():
|
||
try:
|
||
while True:
|
||
frame = await t.recv()
|
||
recorder.feed(frame)
|
||
except (MediaStreamError, asyncio.CancelledError):
|
||
pass
|
||
finally:
|
||
got_audio.set()
|
||
|
||
asyncio.ensure_future(_pump())
|
||
|
||
if wav and not connect_only:
|
||
track = FileAudioTrack(wav)
|
||
pc.addTrack(track)
|
||
log.info("音源: %s（%d 采样点）", wav.name, len(track.samples))
|
||
elif texts:
|
||
# 文字模式：仍要一条音轨，否则 aiortc 不收集 ICE 候选
|
||
pc.addTrack(SilentTrack())
|
||
log.info("文字模式：将发送 %d 条文本", len(texts))
|
||
else:
|
||
# --connect-only 也要加一条（静音）音轨。
|
||
# **不能省**：aiortc 的 offer 里没有媒体时完全不收集 ICE 候选，
|
||
# 现象是"信令成功、SDP 交换正常、但 ICE 永远停在 new"。
|
||
pc.addTrack(SilentTrack())
|
||
log.info("连接测试模式：已加静音音轨（仅为触发 ICE 收集）")
|
||
|
||
# **必须建 data channel 并定期发 "ping"。**
|
||
# Pipecat 的 SmallWebRTC 用 data channel 上的消息来判断"客户端还活着"：
|
||
# if isinstance(message, str) and message.startswith("ping"):
|
||
# self._last_received_time = time.time()
|
||
# 而 connection 的 connected 事件、进而 on_client_connected 回调、
|
||
# 进而 StartFrame（音频推流的开关）都依赖它。
|
||
# 不发 ping 的现象是：ICE 连通、音轨协商成功、但**服务端一帧音频都不推**、
|
||
# 助手永远不说话，而且日志里只有一句 "Data channel not established within 10s"。
|
||
channel = pc.createDataChannel("pipecat")
|
||
dc_state = {"ok": False}
|
||
|
||
replies = RTVIReplies()
|
||
|
||
@channel.on("open")
|
||
def _dc_open():
|
||
dc_state["ok"] = True
|
||
log.info("data channel 已打开，开始发心跳")
|
||
|
||
@channel.on("message")
|
||
def _dc_message(msg):
|
||
if isinstance(msg, str):
|
||
replies.handle(msg)
|
||
|
||
async def _keepalive():
|
||
while pc.connectionState not in ("closed", "failed"):
|
||
if dc_state["ok"]:
|
||
try:
|
||
channel.send("ping")
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
await asyncio.sleep(1.0)
|
||
|
||
asyncio.ensure_future(_keepalive())
|
||
|
||
offer = await pc.createOffer()
|
||
await pc.setLocalDescription(offer)
|
||
|
||
# 等 ICE 收集完成再发 offer——**这步不能省**。
|
||
# 不等的话 SDP 里没有任何候选地址，服务端拿到的是一份"空地址"的 offer，
|
||
# 信令看着成功，但 ICE 永远连不上（状态停在 new）。
|
||
# 这叫非 trickle ICE：把候选都塞进 SDP 一次发完。
|
||
# 真实产品会用 trickle（边收集边发候选），但那样需要额外的信令端点。
|
||
for _ in range(100):
|
||
if pc.iceGatheringState == "complete":
|
||
break
|
||
await asyncio.sleep(0.1)
|
||
n_cand = pc.localDescription.sdp.count("candidate:")
|
||
log.info("ICE 收集: %s，SDP 含 %d 个候选", pc.iceGatheringState, n_cand)
|
||
if n_cand == 0:
|
||
log.warning("SDP 里没有候选地址，ICE 很可能连不上")
|
||
|
||
log.info("本地 SDP 已生成，发给服务端…")
|
||
|
||
t_send = time.perf_counter()
|
||
answer = post_offer(server, pc.localDescription.sdp, pc.localDescription.type)
|
||
if answer.get("type") not in ("answer", None):
|
||
log.error("服务端返回异常: %s", answer)
|
||
return 1
|
||
if answer.get("sdp"):
|
||
await pc.setRemoteDescription(RTCSessionDescription(
|
||
sdp=answer["sdp"], type=answer["type"]))
|
||
log.info("远端 SDP 已设置（往返 %.0f ms）", (time.perf_counter() - t_send) * 1000)
|
||
else:
|
||
log.info("服务端未返回 SDP（可能是 pc_id 复用）: %s", answer)
|
||
|
||
# 等 ICE 连通
|
||
for _ in range(100):
|
||
if pc.connectionState in ("connected", "completed"):
|
||
break
|
||
if pc.connectionState in ("failed", "closed"):
|
||
log.error("连接失败")
|
||
await pc.close()
|
||
return 1
|
||
await asyncio.sleep(0.1)
|
||
log.info("✅ ICE 已连通（%s）", pc.connectionState)
|
||
print(f" ✅ 连接建立，状态 {pc.connectionState}")
|
||
|
||
if texts:
|
||
# 等 data channel 就绪（服务端靠它判断客户端存活）
|
||
for _ in range(60):
|
||
if dc_state["ok"]:
|
||
break
|
||
await asyncio.sleep(0.1)
|
||
log.info("data channel: %s", "已就绪" if dc_state["ok"] else "未就绪")
|
||
for t in texts:
|
||
await asyncio.sleep(0.5)
|
||
rtvi_send_text(channel, t)
|
||
log.info("已发送文本: %r", t)
|
||
# 等**非静音**音频出现。
|
||
# 不能用"收到多少块"来判断——WebRTC 输出轨空闲时也在推静音，
|
||
# 块数会立刻涨上去，导致客户端在服务端还在生成时就断开
|
||
# （服务端日志表现为"本轮被取消"，客户端表现为"没反应"）。
|
||
recorder.first_non_silent_at = None
|
||
for _ in range(int(timeout_s * 10)):
|
||
if recorder.has_speech:
|
||
break
|
||
await asyncio.sleep(0.1)
|
||
log.info("本轮 %s", "收到语音回复" if recorder.has_speech else "未收到语音")
|
||
# 让它把话说完
|
||
await asyncio.sleep(4)
|
||
# 打印助手回复全文，供验收脚本断言
|
||
if replies.bot_texts:
|
||
print(f" BOT_REPLY: {replies.bot_texts[-1]}", flush=True)
|
||
replies.bot_texts.clear()
|
||
await asyncio.sleep(2)
|
||
elif track:
|
||
# 等音频放完 + 留出响应时间
|
||
t0 = time.perf_counter()
|
||
while not track.finished and time.perf_counter() - t0 < timeout_s:
|
||
await asyncio.sleep(0.2)
|
||
log.info("音频已发完，等待回复…")
|
||
# 等首个非静音回复；SmallWebRTC 空闲时也会持续发送静音。
|
||
t_wait = time.perf_counter()
|
||
while time.perf_counter() - t_wait < timeout_s:
|
||
if recorder.has_speech:
|
||
break
|
||
await asyncio.sleep(0.1)
|
||
# 再收一会儿，让回复说完
|
||
await asyncio.sleep(6)
|
||
|
||
dur = recorder.save()
|
||
await pc.close()
|
||
got_audio.set()
|
||
|
||
print()
|
||
print("=== 结果 ===")
|
||
print(f" 回复: {'✅ 收到非静音语音' if recorder.has_speech else '❌ 只有静音'}")
|
||
if dur:
|
||
print(f" 录制音轨: {dur:.2f} 秒 → {out}")
|
||
fa = recorder.first_audio_ms_from(track.source_finished_at if track else None)
|
||
if fa is not None:
|
||
print(f" 首音延迟（源音频结束→首个非静音输出）: {fa:.0f} ms")
|
||
return 0 if (connect_only or recorder.has_speech) else 1
|
||
|
||
|
||
def main() -> int:
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--server", default="http://cadadmin-server.taild0f91d.ts.net:7860",
|
||
help="服务端地址。默认用 tailscale 域名——LAN 地址会被重新编址"
|
||
"（2026-09-15 实测从 192.168.0.x 变成 172.16.1.x）")
|
||
p.add_argument("--wav", type=Path)
|
||
p.add_argument("--out", type=Path, default=Path("/tmp/client_reply.wav"))
|
||
p.add_argument("--connect-only", action="store_true")
|
||
p.add_argument("--text", action="append", default=None,
|
||
help="通过 RTVI 发送文本（可重复），用于测 flows 文字路径")
|
||
p.add_argument("--timeout", type=float, default=30.0)
|
||
a = p.parse_args()
|
||
if not a.connect_only and not a.wav and not a.text:
|
||
print("需要 --wav / --text / --connect-only", file=sys.stderr)
|
||
return 2
|
||
return asyncio.run(run(a.server, a.wav, a.out,
|
||
connect_only=a.connect_only, timeout_s=a.timeout,
|
||
texts=a.text))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|