ylhan/ai-companion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复 Pipecat 实时链路并补齐网页测试

Browse Source
This commit is contained in:
ylhan
2026-09-16 09:50:38 +08:00
parent 74a30b11d7
commit c3d2c7f3a8
6 changed files with 296 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/test_client.py
+56 -31
View File
@@ -76,7 +76,8 @@ class FileAudioTrack(AudioStreamTrack):
kind = "audio"
def __init__(self, wav_path: Path, *, sample_rate: int = 16000):
def __init__(self, wav_path: Path, *, sample_rate: int = 16000,
tail_silence_s: float = 1.2):
super().__init__()
with wave.open(str(wav_path)) as w:
assert w.getnchannels() == 1, "需要单声道"
@@ -88,7 +89,10 @@ class FileAudioTrack(AudioStreamTrack):
n = int(len(self.samples) * sample_rate / self.src_rate)
self.samples = np.interp(np.linspace(0, len(self.samples) - 1, n),
np.arange(len(self.samples)), self.samples)
self._pos = 0
self._source_pos = 0
self._timestamp_samples = 0
self._tail_remaining = int(tail_silence_s * sample_rate)
self.source_finished_at: float | None = None
self._t0: float | None = None
self._frame_samples = sample_rate // 100 # 10ms
@@ -96,19 +100,25 @@ class FileAudioTrack(AudioStreamTrack):
"""按 10ms 一块、以真实时间节奏产出。"""
if self._t0 is None:
self._t0 = time.perf_counter()
self._pos = 0
self._source_pos = 0
# 音频放完后持续吐静音，让服务端的 VAD 能判到"说完"
if self._pos >= len(self.samples):
frame_pts = self._timestamp_samples
if self._source_pos >= len(self.samples):
if self.source_finished_at is None:
self.source_finished_at = time.perf_counter()
audio = np.zeros(self._frame_samples, dtype=np.float32)
self._tail_remaining = max(0, self._tail_remaining - self._frame_samples)
else:
audio = self.samples[self._pos:self._pos + self._frame_samples]
audio = self.samples[self._source_pos:self._source_pos + self._frame_samples]
if len(audio) < self._frame_samples:
audio = np.pad(audio, (0, self._frame_samples - len(audio)))
self._pos += self._frame_samples
self._source_pos += self._frame_samples
# 即使源文件已经结束，静音帧也必须持续推进时钟和 PTS。
self._timestamp_samples += self._frame_samples
# 实时节奏
target = self._t0 + self._pos / self.sample_rate
target = self._t0 + self._timestamp_samples / self.sample_rate
delay = target - time.perf_counter()
if delay > 0:
await asyncio.sleep(delay)
@@ -118,33 +128,45 @@ class FileAudioTrack(AudioStreamTrack):
f = AudioFrame(format="s16", layout="mono", samples=len(pcm))
f.planes[0].update(pcm.tobytes())
f.sample_rate = self.sample_rate
f.pts = int(self._pos * 90000 / self.sample_rate)
f.pts = int(frame_pts * 90000 / self.sample_rate)
f.time_base = fractions.Fraction(1, 90000)
return f
@property
def finished(self) -> bool:
return self._pos >= len(self.samples)
return self._source_pos >= len(self.samples) and self._tail_remaining <= 0
class ReplyRecorder:
"""收服务端音频，写 wav，并记录首音延迟。"""
def __init__(self, out_path: Path, sample_rate: int = 24000):
def __init__(self, out_path: Path, sample_rate: int = 24000,
speech_rms_threshold: float = 80.0):
self.out_path = out_path
self.sample_rate = sample_rate
self.chunks: list[np.ndarray] = []
self.first_at: float | None = None
self.t0 = time.perf_counter()
self.first_non_silent_at: float | None = None
self.speech_rms_threshold = speech_rms_threshold
def feed(self, frame) -> None:
if self.first_at is None:
self.first_at = time.perf_counter()
arr = frame.to_ndarray()
# 多声道取第一路
if arr.shape[0] > 1:
arr = arr[0:1]
self.chunks.append(arr.astype(np.float32).reshape(-1) / 32768.0)
channels = len(frame.layout.channels)
if frame.format.is_planar:
raw = arr.astype(np.float32).mean(axis=0)
else:
raw = arr.astype(np.float32).reshape(-1, channels).mean(axis=1)
# WebRTC/Opus 通常以 48kHz 交付，即使服务端原始输出是 24kHz。
# WAV 必须记录接收帧的实际采样率，否则时长会被算错。
if not self.chunks:
self.sample_rate = frame.sample_rate
elif frame.sample_rate != self.sample_rate:
raise ValueError(
f"回复音轨采样率发生变化: {self.sample_rate} -> {frame.sample_rate}")
raw = raw.reshape(-1)
rms = float(np.sqrt(np.mean(raw ** 2))) if raw.size else 0.0
if self.first_non_silent_at is None and rms >= self.speech_rms_threshold:
self.first_non_silent_at = time.perf_counter()
self.chunks.append(raw / 32768.0)
def save(self) -> float:
if not self.chunks:
@@ -160,8 +182,13 @@ class ReplyRecorder:
return len(audio) / self.sample_rate
@property
def first_audio_ms(self) -> float | None:
return None if self.first_at is None else (self.first_at - self.t0) * 1000
def has_speech(self) -> bool:
return self.first_non_silent_at is not None
def first_audio_ms_from(self, started_at: float | None) -> float | None:
if started_at is None or self.first_non_silent_at is None:
return None
return (self.first_non_silent_at - started_at) * 1000
def post_offer(server: str, sdp: str, type_: str) -> dict:
@@ -197,8 +224,6 @@ async def run(server: str, wav: Path | None, out: Path, *, connect_only: bool,
while True:
frame = await t.recv()
recorder.feed(frame)
if not got_audio.is_set() and recorder.chunks:
pass
except (MediaStreamError, asyncio.CancelledError):
pass
finally:
@@ -293,10 +318,10 @@ async def run(server: str, wav: Path | None, out: Path, *, connect_only: bool,
while not track.finished and time.perf_counter() - t0 < timeout_s:
await asyncio.sleep(0.2)
log.info("音频已发完，等待回复…")
# 等首个回复音频
# 等首个非静音回复；SmallWebRTC 空闲时也会持续发送静音。
t_wait = time.perf_counter()
while time.perf_counter() - t_wait < timeout_s:
if recorder.chunks:
if recorder.has_speech:
break
await asyncio.sleep(0.1)
# 再收一会儿，让回复说完
@@ -308,13 +333,13 @@ async def run(server: str, wav: Path | None, out: Path, *, connect_only: bool,
print()
print("=== 结果 ===")
print(f" 连接: {'✅ 成功' if dur > 0 else '⚠️ 无音频返回'}")
print(f" 回复: {'✅ 收到非静音语音' if recorder.has_speech else '❌ 只有静音'}")
if dur:
print(f" 收到音频: {dur:.2f} 秒 → {out}")
fa = recorder.first_audio_ms
if fa:
print(f" 首音延迟（发完到收到首段）: {fa:.0f} ms")
return 0 if dur > 0 else 1
print(f" 录制音轨: {dur:.2f} 秒 → {out}")
fa = recorder.first_audio_ms_from(track.source_finished_at if track else None)
if fa is not None:
print(f" 首音延迟（源音频结束→首个非静音输出）: {fa:.0f} ms")
return 0 if (connect_only or recorder.has_speech) else 1
def main() -> int: