助手本体：会话引擎、LLM 路由、记忆、身份、语音适配
从 Obsidian 笔记库剥离为独立仓库。规划文档与 P0 报告留在笔记库， 代码（助手本体、TTS 服务、P0 基准脚本、ESP32 固件）在此。 架构：服务器是助手，本地是客户端 - 助手本体跑在 cadadmin-server，N5095 只做 USB 外设桥与网络桥接 - 起因：实测 N5095 无 AVX2，跑不动可用的语音模型 当前技术栈 - LLM：DeepSeek deepseek-flash（云端，必须关思考） - TTS：Qwen3-TTS 1.7B（服务器 3090，独立 HTTP 服务） - STT：Zipformer 中英双语 int8（CPU，中文识别率未测） 实测：端到端 16/16 通过，首音延迟 1545ms （对比：本地 LLM + vits-zh-ll 是 179ms，但音色不满意——音质与延迟 在此硬件上此消彼长） 设计要点 - MemoryStore 接口按"可能远程"设计（全 async、可超时、可失败） - user_id / session_id / track_id 三者分开，人脸只作候选不自动切身份 - 记忆查询强制带 owner_user_id，无按 id 单独取的口子 - 自动抽取的记忆是"候选"，默认不参与检索 - 异步写记忆绑定创建时的身份，不读"当前用户" - 引擎与传输解耦，只用文件音频即可端到端测试 模型与音频产物不进版本管理（1.1GB），由 download_models.sh 恢复
This commit is contained in:
ylhan
+217
@@ -0,0 +1,217 @@
|
||||
"""配置加载。
|
||||
|
||||
原则（来自方案 §10）：
|
||||
- 密钥保存在本机配置，不写日志，不进版本管理
|
||||
- 设备标识用稳定名字，不用会变的序号
|
||||
- 依赖版本固定，板卡/固件/通道/引脚都记录
|
||||
|
||||
密钥文件是 ~/.config/ai_assistant/env（权限 600），格式是 shell 变量。
|
||||
本模块只读它，不写。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
ENV_FILE = Path.home() / ".config" / "ai_assistant" / "env"
|
||||
|
||||
# 本项目根目录（app/ 的上一级）
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
_lock = threading.Lock()
|
||||
_loaded = False
|
||||
|
||||
|
||||
def load_env(path: Path | None = None) -> dict[str, str]:
|
||||
"""读取密钥文件到 os.environ（已存在的变量不覆盖）。
|
||||
|
||||
只做极简解析：KEY=VALUE，忽略空行和 # 注释。
|
||||
不用 os.system/source，避免把 shell 引进来。
|
||||
"""
|
||||
global _loaded
|
||||
p = path or ENV_FILE
|
||||
with _lock:
|
||||
if _loaded:
|
||||
return dict(os.environ)
|
||||
if p.is_file():
|
||||
for line in p.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
k, v = k.strip(), v.strip().strip('"').strip("'")
|
||||
os.environ.setdefault(k, v)
|
||||
_loaded = True
|
||||
return dict(os.environ)
|
||||
|
||||
|
||||
def _env(key: str, default: str = "") -> str:
|
||||
load_env()
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
def _env_int(key: str, default: int) -> int:
|
||||
try:
|
||||
return int(_env(key, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _env_float(key: str, default: float) -> float:
|
||||
try:
|
||||
return float(_env(key, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- LLM
|
||||
|
||||
@dataclass
|
||||
class LLMConfig:
|
||||
"""一个 LLM 后端的配置。
|
||||
|
||||
thinking 的处理方式各后端不同，所以由后端自己解释这个开关：
|
||||
- 本机 vLLM： chat_template_kwargs={"enable_thinking": false}
|
||||
- DeepSeek： thinking={"type": "disabled"}
|
||||
两者都实测有效，见 P0-本地LLM验证报告.md 与 P0-DeepSeek验证报告.md。
|
||||
"""
|
||||
name: str
|
||||
base_url: str
|
||||
api_key: str
|
||||
model: str
|
||||
thinking: bool = False # 语音场景默认关思考
|
||||
max_tokens: int = 200 # 语音回复要短，方案要求"默认简短自然回复"
|
||||
timeout_s: float = 30.0
|
||||
first_token_timeout_s: float = 8.0
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.base_url and self.api_key and self.model)
|
||||
|
||||
|
||||
def local_llm_config() -> LLMConfig:
|
||||
"""本机/内网 LLM（vLLM 等）。
|
||||
|
||||
默认走回环：服务与助手同机时 127.0.0.1 最省一跳。
|
||||
部署到别的机器时必须用 LOCAL_LLM_BASE_URL 显式覆盖——
|
||||
**不要把具体的内网地址写进默认值**，那会把网络拓扑带进版本历史，
|
||||
而且换环境就得改代码。
|
||||
"""
|
||||
return LLMConfig(
|
||||
name="local",
|
||||
base_url=_env("LOCAL_LLM_BASE_URL", "http://127.0.0.1:18020/v1"),
|
||||
api_key=_env("LOCAL_LLM_API_KEY"),
|
||||
model=_env("LOCAL_LLM_MODEL", "qwen3.8-27b"),
|
||||
)
|
||||
|
||||
|
||||
def deepseek_config() -> LLMConfig:
|
||||
return LLMConfig(
|
||||
name="deepseek",
|
||||
base_url=_env("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"),
|
||||
api_key=_env("DEEPSEEK_API_KEY"),
|
||||
model=_env("DEEPSEEK_MODEL", "deepseek-flash"),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 上下文
|
||||
|
||||
@dataclass
|
||||
class ContextConfig:
|
||||
"""上下文预算。
|
||||
|
||||
实测：真实语音 prompt（人设+记忆+3轮对话）约 406 tokens，
|
||||
本机 LLM 首 token 367ms。上下文越长，首 token 越慢——直接伤"回应及时"。
|
||||
所以这里设的是硬上限，超出就丢最旧的轮次，不是软建议。
|
||||
"""
|
||||
max_prompt_tokens: int = 4096 # 总上限
|
||||
max_turns: int = 12 # 最多保留的对话轮次
|
||||
max_memories: int = 5 # 单轮最多注入的记忆条数
|
||||
memory_timeout_s: float = 0.3 # 检索超时，方案要求"记忆失败不阻塞对话"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 语音
|
||||
|
||||
def _default_model_dir() -> Path:
|
||||
"""模型目录。可用环境变量 AI_MODEL_DIR 覆盖。
|
||||
|
||||
为什么要有这个覆盖：开发在 N5095，运行在服务器，两边模型放的
|
||||
位置不一样（服务器上模型在 ~/ai_companion_bench/models，
|
||||
不在代码目录里）。没有覆盖的话，部署就得改代码——
|
||||
而"改代码才能部署"是部署流程里最容易出错的一环。
|
||||
"""
|
||||
override = _env("AI_MODEL_DIR")
|
||||
if override:
|
||||
return Path(override).expanduser()
|
||||
return PROJECT_ROOT / "p0" / "stt_tts_bench" / "models"
|
||||
|
||||
|
||||
@dataclass
|
||||
class VoiceConfig:
|
||||
"""语音输入输出。
|
||||
|
||||
线程数是实测出来的，不能按"核多给多"猜：
|
||||
- N5095（4核，无AVX2）：STT 超过 2 线程急剧退化，4线程慢 3.9 倍
|
||||
- 服务器（i9-14900K，有AVX2）：同样配置下 RTF 0.03
|
||||
所以线程数按部署机器配置，不写死。服务器上可以给足。
|
||||
"""
|
||||
model_dir: Path = field(default_factory=_default_model_dir)
|
||||
sample_rate: int = 16000
|
||||
stt_threads: int = field(default_factory=lambda: _env_int("AI_STT_THREADS", 2))
|
||||
tts_threads: int = field(default_factory=lambda: _env_int("AI_TTS_THREADS", 4))
|
||||
|
||||
# ---- TTS 后端选择 ----
|
||||
# "local" = sherpa-onnx vits-zh-ll：RTF 0.09，很快，但音色用户不满意
|
||||
# "remote" = Qwen3-TTS 独立服务：RTF 0.55–0.94，音质是另一档，用户试听选定
|
||||
tts_backend: str = field(default_factory=lambda: _env("AI_TTS_BACKEND", "remote"))
|
||||
tts_service_url: str = field(
|
||||
default_factory=lambda: _env("TTS_SERVICE_URL", "http://127.0.0.1:18040"))
|
||||
# 用户试听选定的组合（out/tts_qwen/instruct_7849.wav）
|
||||
tts_speaker: str = field(default_factory=lambda: _env("AI_TTS_SPEAKER", "Serena"))
|
||||
tts_instructions: str = field(
|
||||
default_factory=lambda: _env("AI_TTS_INSTRUCTIONS", "用轻快活泼的语气说"))
|
||||
tts_language: str = field(default_factory=lambda: _env("AI_TTS_LANGUAGE", "Chinese"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 记忆
|
||||
|
||||
@dataclass
|
||||
class MemoryConfig:
|
||||
db_path: Path = field(default_factory=lambda: Path.home() / ".local" / "share" / "ai_assistant" / "memory.db")
|
||||
# 自动抽取的记忆默认是"候选"，不参与检索；只有用户显式确认的才生效。
|
||||
# 对应方案 §8："模型推测不当作确认事实"
|
||||
auto_extract_enabled: bool = True
|
||||
auto_extract_confirm_status: str = "candidate"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 汇总
|
||||
|
||||
@dataclass
|
||||
class AppConfig:
|
||||
#: 主 LLM 由 AI_LLM_PRIMARY 选择：
|
||||
#: "local" = 本机 vLLM（首 token 192ms，但占 21.5GB 显存）
|
||||
#: "deepseek" = 云端（首 token 786ms，网络往返约 700ms）
|
||||
#: 2026-09-15 起用 deepseek：本地 27B 已停，GPU 让给 Qwen3-TTS。
|
||||
#: 这是个可逆的权衡——音质提升换首音延迟增加约 600ms。
|
||||
llm: LLMConfig = field(default_factory=local_llm_config)
|
||||
llm_fallback: LLMConfig = field(default_factory=deepseek_config)
|
||||
context: ContextConfig = field(default_factory=ContextConfig)
|
||||
voice: VoiceConfig = field(default_factory=VoiceConfig)
|
||||
memory: MemoryConfig = field(default_factory=MemoryConfig)
|
||||
|
||||
@classmethod
|
||||
def load(cls) -> "AppConfig":
|
||||
load_env()
|
||||
primary = _env("AI_LLM_PRIMARY", "local").strip().lower()
|
||||
if primary == "deepseek":
|
||||
return cls(llm=deepseek_config(), llm_fallback=local_llm_config())
|
||||
return cls()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppConfig", "LLMConfig", "ContextConfig", "VoiceConfig", "MemoryConfig",
|
||||
"load_env", "local_llm_config", "deepseek_config", "PROJECT_ROOT", "ENV_FILE",
|
||||
]
|
||||
+389
@@ -0,0 +1,389 @@
|
||||
"""会话引擎：身份 → 记忆 → LLM → TTS 的核心回路。
|
||||
|
||||
这一层不依赖 Pipecat、不依赖任何传输方式，只依赖四个接口
|
||||
（LLMProvider / MemoryStore / IdentityProvider / LocalTTS）。
|
||||
好处有两个：
|
||||
|
||||
1. **可以用文件音频端到端测试**，不需要麦克风、音箱、WebRTC。
|
||||
2. 接 Pipecat 时只需写薄适配层；换传输方式（WebRTC / 本地声卡）不动这里。
|
||||
|
||||
一轮的完整流程：
|
||||
|
||||
用户音频
|
||||
└→ STT 转写
|
||||
└→ 身份裁决（决定用谁的记忆）
|
||||
└→ 记忆检索（带超时，失败不阻塞）
|
||||
└→ 拼 system（每轮重建，单条）
|
||||
└→ LLM 流式
|
||||
└→ 按句切分 → 逐句 TTS → 边合成边播
|
||||
└→ 回复结束，异步写记忆（绑定创建时的身份）
|
||||
|
||||
延迟的关键在最后两步：**必须按句切分后立刻合成**。
|
||||
VITS 是非自回归的，整段合成完才出声；只有切成短句才能提前开口。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import AsyncIterator, Sequence
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .config import AppConfig
|
||||
from .identity.base import IdentityDecision, IdentityProvider
|
||||
from .llm.base import LLMProvider, ToolCall
|
||||
from .memory.base import (
|
||||
ConfirmStatus, Memory, MemoryKind, MemoryStore, Scope, Turn,
|
||||
)
|
||||
from .prompts import build_system_prompt, trim_history
|
||||
from .voice.stt import StreamingSTT
|
||||
from .voice.tts import LocalTTS, split_sentences
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EventKind(str, Enum):
|
||||
IDENTITY = "identity" # 本轮的身份裁决结果
|
||||
MEMORY_USED = "memory_used" # 检索到的记忆
|
||||
TRANSCRIPT = "transcript" # STT 结果
|
||||
RESPONSE_DELTA = "response_delta" # LLM 增量文本（未播，仅供显示）
|
||||
SPEAK = "speak" # 一段可播放的音频
|
||||
TOOL_CALL = "tool_call" # 模型请求调用工具
|
||||
METRICS = "metrics" # 本轮计时
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
kind: EventKind
|
||||
text: str = ""
|
||||
audio: np.ndarray | None = None
|
||||
sample_rate: int = 16000
|
||||
identity: IdentityDecision | None = None
|
||||
memories: list[Memory] = field(default_factory=list)
|
||||
tool_call: ToolCall | None = None
|
||||
metrics: "TurnMetrics | None" = None
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurnMetrics:
|
||||
"""按方案 §10 要求记录耗时，便于定位瓶颈。"""
|
||||
stt_ms: float = 0.0
|
||||
identity_ms: float = 0.0
|
||||
retrieve_ms: float = 0.0
|
||||
llm_first_content_ms: float = 0.0
|
||||
first_audio_ms: float = 0.0 # 从开始处理到第一段音频就绪
|
||||
total_ms: float = 0.0
|
||||
llm_backend: str = ""
|
||||
memories_used: int = 0
|
||||
reply_chars: int = 0
|
||||
speaks: int = 0
|
||||
|
||||
def summary(self) -> str:
|
||||
return (f"STT {self.stt_ms:.0f}ms | 身份 {self.identity_ms:.0f}ms | "
|
||||
f"检索 {self.retrieve_ms:.0f}ms | LLM首token {self.llm_first_content_ms:.0f}ms | "
|
||||
f"首音 {self.first_audio_ms:.0f}ms | 总 {self.total_ms:.0f}ms | "
|
||||
f"{self.llm_backend} | 记忆{self.memories_used}条 | {self.speaks}段音频")
|
||||
|
||||
|
||||
class ConversationEngine:
|
||||
def __init__(
|
||||
self,
|
||||
cfg: AppConfig,
|
||||
llm: LLMProvider,
|
||||
memory: MemoryStore,
|
||||
identity: IdentityProvider,
|
||||
tts: LocalTTS,
|
||||
stt: StreamingSTT | None = None,
|
||||
):
|
||||
self.cfg = cfg
|
||||
self.llm = llm
|
||||
self.memory = memory
|
||||
self.identity = identity
|
||||
self.tts = tts
|
||||
self.stt = stt
|
||||
#: 会话历史。切身份时会清空——方案 §8 要求切换身份
|
||||
#: 清理上一人的私有检索上下文。
|
||||
self._history: list[dict] = []
|
||||
self._history_identity_version: int = -1
|
||||
self._session_id: str = ""
|
||||
self._current_turn: asyncio.Task | None = None
|
||||
|
||||
# ------------------------------------------------------------ 会话
|
||||
|
||||
async def start_session(self, session_id: str | None = None,
|
||||
user_id: str | None = None) -> str:
|
||||
sid = session_id or f"s_{uuid.uuid4().hex[:12]}"
|
||||
if user_id:
|
||||
self.identity.switch(user_id, reason="会话指定")
|
||||
d = self.identity.current()
|
||||
await self.memory.start_session(sid, d.user_id)
|
||||
self._session_id = sid
|
||||
self._history_identity_version = self._identity_version()
|
||||
log.info("会话开始 %s，身份 %s", sid, d.user_id)
|
||||
return sid
|
||||
|
||||
async def end_session(self) -> None:
|
||||
if self._session_id:
|
||||
await self.memory.end_session(self._session_id)
|
||||
|
||||
def _identity_version(self) -> int:
|
||||
# ManualIdentityProvider 有 version；其他实现没有时退化为 0
|
||||
return getattr(self.identity, "version", 0)
|
||||
|
||||
def _sync_identity_scope(self) -> None:
|
||||
"""身份变了就丢弃历史。
|
||||
|
||||
不做这件事的后果：切到新用户后，模型仍看得到上一人的对话，
|
||||
第一条回复就可能带出别人的信息。
|
||||
"""
|
||||
v = self._identity_version()
|
||||
if v != self._history_identity_version:
|
||||
if self._history:
|
||||
log.info("身份变更，丢弃 %d 条历史消息", len(self._history))
|
||||
self._history = []
|
||||
self._history_identity_version = v
|
||||
|
||||
# ------------------------------------------------------------ 主回路
|
||||
|
||||
async def respond_to_text(self, user_text: str, *,
|
||||
session_id: str | None = None) -> AsyncIterator[Event]:
|
||||
"""处理一句用户输入，产出事件流。"""
|
||||
t_start = time.perf_counter()
|
||||
metrics = TurnMetrics()
|
||||
sid = session_id or self._session_id or await self.start_session()
|
||||
|
||||
self._sync_identity_scope()
|
||||
identity = self.identity.current()
|
||||
metrics.identity_ms = (time.perf_counter() - t_start) * 1000
|
||||
yield Event(kind=EventKind.IDENTITY, identity=identity)
|
||||
|
||||
if not user_text.strip():
|
||||
yield Event(kind=EventKind.METRICS, metrics=metrics)
|
||||
return
|
||||
|
||||
# ---- 记忆检索：带超时，失败不阻塞（方案 §8）
|
||||
t0 = time.perf_counter()
|
||||
memories = await self._retrieve_memories(identity, user_text)
|
||||
metrics.retrieve_ms = (time.perf_counter() - t0) * 1000
|
||||
metrics.memories_used = len(memories)
|
||||
if memories:
|
||||
yield Event(kind=EventKind.MEMORY_USED, memories=memories)
|
||||
|
||||
# ---- 拼 system（每轮重建，单条）
|
||||
system = build_system_prompt(identity, memories)
|
||||
history = trim_history(self._history,
|
||||
max_turns=self.cfg.context.max_turns)
|
||||
messages = [{"role": "system", "content": system}] + history + \
|
||||
[{"role": "user", "content": user_text}]
|
||||
|
||||
# ---- LLM 流式 + 逐句 TTS
|
||||
reply_parts: list[str] = []
|
||||
buffer = ""
|
||||
speaks = 0
|
||||
async for ev in self._stream_and_speak(messages, metrics):
|
||||
if ev.kind == EventKind.RESPONSE_DELTA:
|
||||
reply_parts.append(ev.text)
|
||||
buffer += ev.text
|
||||
# 增量文本必须**原样透传**给调用方（显示、字幕、日志都要用）。
|
||||
# 这里漏掉 yield 的后果是：音频正常出来了，但上层拿到的是空回复——
|
||||
# 因为文本只进了内部累加器，从未离开这个生成器。
|
||||
yield ev
|
||||
# 攒到句末就送去合成——这是提前出声的唯一手段
|
||||
segs = split_sentences(buffer)
|
||||
if len(segs) > 1:
|
||||
*ready, buffer = segs
|
||||
for seg in ready:
|
||||
async for speak_ev in self._speak(seg, metrics):
|
||||
speaks += 1
|
||||
yield speak_ev
|
||||
else:
|
||||
yield ev
|
||||
|
||||
# 收尾：把剩下的半句也合成掉
|
||||
if buffer.strip():
|
||||
async for speak_ev in self._speak(buffer, metrics):
|
||||
speaks += 1
|
||||
yield speak_ev
|
||||
|
||||
reply = "".join(reply_parts).strip()
|
||||
metrics.reply_chars = len(reply)
|
||||
metrics.speaks = speaks
|
||||
metrics.total_ms = (time.perf_counter() - t_start) * 1000
|
||||
|
||||
# ---- 落历史
|
||||
self._history.append({"role": "user", "content": user_text})
|
||||
if reply:
|
||||
self._history.append({"role": "assistant", "content": reply})
|
||||
|
||||
# ---- 异步写记忆：**绑定创建时的身份**，不读"当前用户"
|
||||
if self.cfg.memory.auto_extract_enabled and reply:
|
||||
self._spawn_memory_write(
|
||||
user_text=user_text, reply=reply,
|
||||
user_id=identity.user_id,
|
||||
session_id=sid,
|
||||
turn_id=f"t_{uuid.uuid4().hex[:10]}")
|
||||
|
||||
await self.memory.add_turn(Turn(
|
||||
turn_id=f"t_{uuid.uuid4().hex[:10]}", session_id=sid,
|
||||
user_id=identity.user_id, role="user", text=user_text))
|
||||
|
||||
yield Event(kind=EventKind.METRICS, metrics=metrics)
|
||||
|
||||
async def respond_to_audio(self, samples: np.ndarray, *,
|
||||
session_id: str | None = None) -> AsyncIterator[Event]:
|
||||
"""从音频走完整条链路（需要 STT）。"""
|
||||
if self.stt is None:
|
||||
yield Event(kind=EventKind.ERROR, error="未配置 STT，无法处理音频")
|
||||
return
|
||||
t0 = time.perf_counter()
|
||||
stream = self.stt.new_stream()
|
||||
stream.accept(samples)
|
||||
text = stream.finish()
|
||||
stt_ms = (time.perf_counter() - t0) * 1000
|
||||
yield Event(kind=EventKind.TRANSCRIPT, text=text)
|
||||
async for ev in self.respond_to_text(text, session_id=session_id):
|
||||
if ev.kind == EventKind.METRICS and ev.metrics:
|
||||
ev.metrics.stt_ms = stt_ms
|
||||
yield ev
|
||||
|
||||
# ------------------------------------------------------------ 内部
|
||||
|
||||
async def _retrieve_memories(self, identity: IdentityDecision,
|
||||
query: str) -> list[Memory]:
|
||||
"""带超时的检索。超时/出错一律返回空，绝不让对话卡住。"""
|
||||
if not identity.confirmed:
|
||||
# 访客不做个性化检索（方案 §6："未知身份只用访客临时上下文"）
|
||||
return []
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
self.memory.retrieve(
|
||||
identity.user_id, query,
|
||||
limit=self.cfg.context.max_memories,
|
||||
scopes=[Scope.PRIVATE, Scope.SHARED]),
|
||||
timeout=self.cfg.context.memory_timeout_s)
|
||||
except asyncio.TimeoutError:
|
||||
log.warning("记忆检索超时 %.2fs，本轮不带记忆继续",
|
||||
self.cfg.context.memory_timeout_s)
|
||||
return []
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("记忆检索失败（%s），本轮不带记忆继续", e)
|
||||
return []
|
||||
|
||||
async def _stream_and_speak(self, messages: list[dict],
|
||||
metrics: TurnMetrics) -> AsyncIterator[Event]:
|
||||
"""只负责 LLM 流式，不做 TTS。TTS 由调用方按句驱动。"""
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
async for chunk in self.llm.stream(messages):
|
||||
if chunk.kind.value == "content" and chunk.text:
|
||||
if not metrics.llm_first_content_ms:
|
||||
metrics.llm_first_content_ms = (time.perf_counter() - t0) * 1000
|
||||
yield Event(kind=EventKind.RESPONSE_DELTA, text=chunk.text)
|
||||
elif chunk.kind.value == "tool_call" and chunk.tool_name:
|
||||
# 骨架阶段只上报，不执行。工具执行要配合
|
||||
# 方案 §9 的"执行 ID + 状态确认 + 不盲目重试"，
|
||||
# 留到接米家时再做。
|
||||
yield Event(kind=EventKind.TOOL_CALL, tool_call=ToolCall(
|
||||
id=chunk.tool_id, name=chunk.tool_name,
|
||||
arguments=chunk.tool_arguments))
|
||||
except asyncio.CancelledError:
|
||||
log.info("本轮被取消（打断）")
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("LLM 流失败：%s", e)
|
||||
yield Event(kind=EventKind.ERROR, error=f"{type(e).__name__}: {e}")
|
||||
metrics.llm_backend = getattr(self.llm, "last_backend", self.llm.name)
|
||||
|
||||
async def _speak(self, text: str, metrics: TurnMetrics) -> AsyncIterator[Event]:
|
||||
"""合成一段并产出可播放音频。"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
# TTS 是 CPU 密集的同步调用，放线程里跑，别阻塞事件循环
|
||||
audio = await asyncio.to_thread(self.tts.synthesize, text)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("TTS 失败：%s", e)
|
||||
yield Event(kind=EventKind.ERROR, error=f"TTS 失败：{e}")
|
||||
return
|
||||
if audio is None or len(audio) == 0:
|
||||
return
|
||||
if not metrics.first_audio_ms:
|
||||
metrics.first_audio_ms = (time.perf_counter() - t0) * 1000
|
||||
yield Event(kind=EventKind.SPEAK, text=text, audio=audio,
|
||||
sample_rate=self.tts.sample_rate)
|
||||
|
||||
def _spawn_memory_write(self, *, user_text: str, reply: str,
|
||||
user_id: str, session_id: str, turn_id: str) -> None:
|
||||
"""起一个后台任务写记忆。
|
||||
|
||||
**身份是参数传进来的，不在这里读 self.identity.current()。**
|
||||
方案 §8 明确要求："异步任务绑定创建时的 user_id/session_id，
|
||||
避免迟到结果写入新用户。" 如果这里去读"当前身份"，
|
||||
用户在这几百毫秒内切了身份，这条记忆就会写到别人名下。
|
||||
"""
|
||||
async def _write():
|
||||
try:
|
||||
for mem in extract_memories(user_text, reply):
|
||||
mem.owner_user_id = user_id # 绑定创建时的身份
|
||||
mem.source_session_id = session_id
|
||||
mem.source_turn_id = turn_id
|
||||
mem.confirm_status = ConfirmStatus(
|
||||
self.cfg.memory.auto_extract_confirm_status)
|
||||
await self.memory.remember(mem)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("异步写记忆失败（不影响对话）：%s", e)
|
||||
|
||||
task = asyncio.create_task(_write())
|
||||
# 持有引用，避免被 GC 提前回收
|
||||
if not hasattr(self, "_bg_tasks"):
|
||||
self._bg_tasks: set[asyncio.Task] = set()
|
||||
self._bg_tasks.add(task)
|
||||
task.add_done_callback(self._bg_tasks.discard)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 记忆抽取
|
||||
|
||||
#: 显式"记住"的触发词。首版用规则，不额外调一次 LLM——
|
||||
#: 那会多一次往返，而语音场景对延迟很敏感。
|
||||
_REMEMBER_PATTERNS = (
|
||||
("记住", ""), ("记一下", ""), ("别忘了", ""), ("帮我记", ""),
|
||||
("我喜欢", "偏好"), ("我不喜欢", "偏好"), ("我习惯", "偏好"),
|
||||
)
|
||||
|
||||
|
||||
def extract_memories(user_text: str, reply: str) -> list[Memory]:
|
||||
"""从一轮对话里抽取候选记忆。
|
||||
|
||||
首版只认显式表达。**抽取结果默认是 candidate，不参与检索**——
|
||||
方案 §8："模型推测不当作确认事实"。要变成 confirmed 需要用户确认
|
||||
或走工具调用。
|
||||
"""
|
||||
out: list[Memory] = []
|
||||
text = (user_text or "").strip()
|
||||
for trigger, kind_hint in _REMEMBER_PATTERNS:
|
||||
if trigger in text:
|
||||
# 取触发词之后的正文
|
||||
body = text.split(trigger, 1)[1].strip(" ：:，,。.！!？?")
|
||||
if len(body) < 2:
|
||||
continue
|
||||
out.append(Memory(
|
||||
kind=MemoryKind.PREFERENCE if kind_hint else MemoryKind.FACT,
|
||||
key=kind_hint or "",
|
||||
value=body[:200],
|
||||
))
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConversationEngine", "Event", "EventKind", "TurnMetrics", "extract_memories",
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
"""身份层。
|
||||
|
||||
对外暴露 `IdentityProvider` 接口与首版 `ManualIdentityProvider` 实现。
|
||||
将来接人脸/声纹只需新增实现，记忆表结构与调用方都不动
|
||||
（这正是方案 §6 要求"预留 IdentityProvider 接口"的原因）。
|
||||
"""
|
||||
|
||||
from .base import (
|
||||
GUEST_USER_ID, TRUST_GUEST, TRUST_NORMAL,
|
||||
IdentityClaim, IdentityDecision, IdentityProvider,
|
||||
)
|
||||
from .manual import SUGGEST_MIN_CONFIDENCE, ManualIdentityProvider
|
||||
|
||||
__all__ = [
|
||||
"IdentityProvider", "ManualIdentityProvider",
|
||||
"IdentityClaim", "IdentityDecision",
|
||||
"GUEST_USER_ID", "TRUST_NORMAL", "TRUST_GUEST", "SUGGEST_MIN_CONFIDENCE",
|
||||
]
|
||||
@@ -0,0 +1,110 @@
|
||||
"""身份抽象层。
|
||||
|
||||
方案 §6 的核心约束，逐条落进接口：
|
||||
|
||||
* "首版可靠身份入口：手动/对话确认切换到已建档的 user_id；
|
||||
无法确认时使用 unknown/guest。"
|
||||
|
||||
* **"看到熟人不等于确认他正在说话"** —— 人脸只提供候选，
|
||||
不自动切换身份。这条最容易做错：摄像头看到谁就切谁的记忆，
|
||||
结果是"路人经过就把你的私人记忆念出来"。
|
||||
|
||||
* "显式报名字用于个性化档案选择，**不是身份认证**，不能据此解锁敏感信息"
|
||||
—— 所以 trust_level 与 user_id 分开存，将来加认证时不用改偏好存储。
|
||||
|
||||
* "预留 IdentityProvider 接口：输入时间戳、track_id、候选身份、置信度与来源；
|
||||
输出确认的 user_id 或 unknown。后续声纹模块可接入此接口，不改记忆表结构。"
|
||||
|
||||
* "视觉 track_id、会话 session_id、永久 user_id 分开" ——
|
||||
所以 IdentityClaim 里 track_id 是线索，不是身份。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Sequence
|
||||
|
||||
#: 无法确认身份时使用的固定 id。永远不落私人记忆。
|
||||
GUEST_USER_ID = "guest"
|
||||
|
||||
#: 信任等级。与 user_id 分开，将来加认证不改这里。
|
||||
TRUST_NORMAL = "normal"
|
||||
TRUST_GUEST = "guest"
|
||||
|
||||
|
||||
@dataclass
|
||||
class IdentityClaim:
|
||||
"""一条身份线索。
|
||||
|
||||
source 取值（首版只用 manual，其余留给后续）：
|
||||
"manual" 用户显式切换/确认——**唯一可单独决定身份的来源**
|
||||
"face" 人脸识别候选（本地特征，需用户主动录入）
|
||||
"client_login" 客户端登录态（手机 App / 浏览器）
|
||||
"voiceprint" 声纹（后续阶段）
|
||||
"""
|
||||
source: str
|
||||
candidate_user_id: str | None = None
|
||||
confidence: float = 0.0
|
||||
track_id: str | None = None
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
note: str = ""
|
||||
|
||||
#: 只有这些来源可以单独把身份定下来。人脸不在其中——
|
||||
#: 方案明确要求人脸仅作候选、经确认后才启用对应记忆。
|
||||
AUTHORITATIVE_SOURCES = ("manual", "client_login")
|
||||
|
||||
@property
|
||||
def is_authoritative(self) -> bool:
|
||||
return self.source in self.AUTHORITATIVE_SOURCES
|
||||
|
||||
|
||||
@dataclass
|
||||
class IdentityDecision:
|
||||
user_id: str
|
||||
trust_level: str
|
||||
confirmed: bool # True = 经确认；False = 默认访客
|
||||
reason: str
|
||||
#: 未被采纳的候选，保留下来供上层提示"是否要切换成 X"
|
||||
suggestions: list[IdentityClaim] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def is_guest(self) -> bool:
|
||||
return not self.confirmed
|
||||
|
||||
|
||||
class IdentityProvider(ABC):
|
||||
"""身份裁决者。
|
||||
|
||||
实现要点：**宁可拒判，不要猜**。判错的代价是把 A 的私人记忆
|
||||
用在 B 身上，比"这次不认识"严重得多。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def current(self) -> IdentityDecision:
|
||||
"""当前身份。任何时候都必须能回答。"""
|
||||
|
||||
@abstractmethod
|
||||
def switch(self, user_id: str, *, display_name: str = "",
|
||||
reason: str = "显式切换") -> IdentityDecision:
|
||||
"""显式切换身份——首版唯一可靠的入口。"""
|
||||
|
||||
@abstractmethod
|
||||
def resolve(self, claims: Sequence[IdentityClaim]) -> IdentityDecision:
|
||||
"""根据一组线索裁决身份。"""
|
||||
|
||||
@abstractmethod
|
||||
def reset(self, reason: str = "回到待机") -> IdentityDecision:
|
||||
"""回到访客身份，并清掉上一人的身份状态。
|
||||
|
||||
方案 §8 要求"切换身份清理上一人的私有检索上下文"——
|
||||
调用方在收到身份变化后必须重新构造 system 消息（含记忆），
|
||||
不能沿用上一轮的。
|
||||
"""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"IdentityProvider", "IdentityClaim", "IdentityDecision",
|
||||
"GUEST_USER_ID", "TRUST_NORMAL", "TRUST_GUEST",
|
||||
]
|
||||
@@ -0,0 +1,123 @@
|
||||
"""首版身份实现：显式切换 + 访客兜底。
|
||||
|
||||
对应方案 §6："首版可靠身份入口：手动/对话确认切换到已建档的 user_id；
|
||||
无法确认时使用 unknown/guest。"
|
||||
|
||||
行为规则：
|
||||
|
||||
1. 显式切换（`switch`）立即生效，这是唯一能单独定身份的方式。
|
||||
2. 人脸/声纹线索只生成**建议**，不自动改身份。上层可以选择性地
|
||||
问一句"要切换成 X 吗"，但不能替用户决定。
|
||||
3. 没有任何线索时是访客，且访客不能落私人记忆（由记忆层用
|
||||
owner_user_id 隔离保证）。
|
||||
4. 切换身份返回一个"版本号"，上层据此判断要不要丢弃上一人的上下文。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Sequence
|
||||
|
||||
from .base import (
|
||||
GUEST_USER_ID, TRUST_GUEST, TRUST_NORMAL,
|
||||
IdentityClaim, IdentityDecision, IdentityProvider,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: 人脸候选被采纳为"建议"的最低置信度。
|
||||
#: 低于这个值连建议都不给——减少无意义的打扰。
|
||||
SUGGEST_MIN_CONFIDENCE = 0.6
|
||||
|
||||
|
||||
class ManualIdentityProvider(IdentityProvider):
|
||||
def __init__(self, initial_user: str | None = None):
|
||||
self._decision = IdentityDecision(
|
||||
user_id=initial_user or GUEST_USER_ID,
|
||||
trust_level=TRUST_NORMAL if initial_user else TRUST_GUEST,
|
||||
confirmed=bool(initial_user),
|
||||
reason="初始身份" if initial_user else "初始为访客",
|
||||
)
|
||||
self._version = 0
|
||||
|
||||
# ---------------------------------------------------------- 查询
|
||||
|
||||
def current(self) -> IdentityDecision:
|
||||
return self._decision
|
||||
|
||||
@property
|
||||
def version(self) -> int:
|
||||
"""身份版本号。每次真正变更时 +1。
|
||||
|
||||
上层用它判断"上一轮的私有检索上下文是否过期"——
|
||||
方案 §8 要求切换身份时清理上一人的上下文。
|
||||
"""
|
||||
return self._version
|
||||
|
||||
# ---------------------------------------------------------- 变更
|
||||
|
||||
def switch(self, user_id: str, *, display_name: str = "",
|
||||
reason: str = "显式切换") -> IdentityDecision:
|
||||
if not user_id:
|
||||
raise ValueError("user_id 不能为空")
|
||||
prev = self._decision.user_id
|
||||
self._decision = IdentityDecision(
|
||||
user_id=user_id,
|
||||
trust_level=TRUST_NORMAL,
|
||||
confirmed=True,
|
||||
reason=reason,
|
||||
)
|
||||
if prev != user_id:
|
||||
self._version += 1
|
||||
log.info("身份切换 %s → %s（%s）", prev, user_id, reason)
|
||||
return self._decision
|
||||
|
||||
def reset(self, reason: str = "回到待机") -> IdentityDecision:
|
||||
if self._decision.user_id != GUEST_USER_ID:
|
||||
self._version += 1
|
||||
self._decision = IdentityDecision(
|
||||
user_id=GUEST_USER_ID,
|
||||
trust_level=TRUST_GUEST,
|
||||
confirmed=False,
|
||||
reason=reason,
|
||||
)
|
||||
return self._decision
|
||||
|
||||
def resolve(self, claims: Sequence[IdentityClaim]) -> IdentityDecision:
|
||||
"""裁决。
|
||||
|
||||
顺序很重要：先看有没有权威来源（显式切换 / 客户端登录态），
|
||||
有就采用；人脸之类的只作为建议附在后面。
|
||||
"""
|
||||
authoritative = [c for c in claims if c.is_authoritative and c.candidate_user_id]
|
||||
suggestions = [
|
||||
c for c in claims
|
||||
if not c.is_authoritative
|
||||
and c.candidate_user_id
|
||||
and c.confidence >= SUGGEST_MIN_CONFIDENCE
|
||||
]
|
||||
|
||||
if authoritative:
|
||||
# 同一批里有多个权威来源且互相矛盾——宁可不切，报出来让人处理
|
||||
ids = {c.candidate_user_id for c in authoritative}
|
||||
if len(ids) > 1:
|
||||
log.warning("权威身份线索互相矛盾 %s，保持当前身份", ids)
|
||||
d = self._decision
|
||||
d.reason = f"线索矛盾 {ids}，保持 {d.user_id}"
|
||||
d.suggestions = suggestions
|
||||
return d
|
||||
c = max(authoritative, key=lambda x: x.confidence)
|
||||
assert c.candidate_user_id is not None
|
||||
self.switch(c.candidate_user_id, reason=f"来源 {c.source}")
|
||||
else:
|
||||
# 没有权威来源：即便人脸认出来了也不切，只给建议
|
||||
self._decision.suggestions = suggestions
|
||||
if suggestions:
|
||||
self._decision.reason = (
|
||||
f"仅有人脸候选 {suggestions[0].candidate_user_id}"
|
||||
f"（置信度 {suggestions[0].confidence:.2f}），未自动切换"
|
||||
)
|
||||
return self._decision
|
||||
|
||||
|
||||
__all__ = ["ManualIdentityProvider", "SUGGEST_MIN_CONFIDENCE"]
|
||||
@@ -0,0 +1,116 @@
|
||||
"""LLM 后端选择与降级。
|
||||
|
||||
设计要点：**降级只在"还没吐出任何内容"时发生。**
|
||||
|
||||
一旦已经向调用方 yield 过 chunk，再切后端就会把两段不同来源的文本接在一起，
|
||||
产出一句读不通的回复。那种情况下宁可让本轮失败——方案 §5 要求
|
||||
"网络失败给简短提示，不反复播旧回复"，而不是拼接残句。
|
||||
|
||||
用法：
|
||||
provider = create_provider(AppConfig.load().llm)
|
||||
router = FallbackProvider(primary, fallback)
|
||||
async for chunk in router.stream(messages): ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ..config import LLMConfig
|
||||
from .base import Chunk, ChunkKind, LLMError, LLMProvider, LLMResult, ToolCall
|
||||
from .deepseek import DeepSeekProvider
|
||||
from .local_vllm import LocalVLLMProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_REGISTRY = {
|
||||
"local": LocalVLLMProvider,
|
||||
"deepseek": DeepSeekProvider,
|
||||
}
|
||||
|
||||
|
||||
def create_provider(cfg: LLMConfig) -> LLMProvider:
|
||||
cls = _REGISTRY.get(cfg.name)
|
||||
if cls is None:
|
||||
raise ValueError(f"未知的 LLM 后端 {cfg.name!r}，可选：{sorted(_REGISTRY)}")
|
||||
return cls(cfg)
|
||||
|
||||
|
||||
class FallbackProvider(LLMProvider):
|
||||
"""主后端失败时切备用后端。
|
||||
|
||||
只对**连接层失败**降级（超时、拒连、5xx、模型不可用），
|
||||
不对 4xx 这类"请求本身有问题"降级——那种情况换个后端也一样失败，
|
||||
降级只会掩盖真正的 bug。
|
||||
"""
|
||||
|
||||
supports_vision = True # 只要备用支持就算支持，实际由 _choose 时判断
|
||||
|
||||
def __init__(self, primary: LLMProvider, fallback: LLMProvider | None = None):
|
||||
self.primary = primary
|
||||
self.fallback = fallback
|
||||
self.cfg = primary.cfg
|
||||
self.name = f"{primary.name}+{fallback.name if fallback else 'none'}"
|
||||
self.last_backend: str = primary.name
|
||||
self._session = None
|
||||
self._owns_session = False
|
||||
|
||||
def _build_body(self, *a, **kw): # 实际不会走到，路由在 stream 里
|
||||
raise NotImplementedError
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.primary.aclose()
|
||||
if self.fallback:
|
||||
await self.fallback.aclose()
|
||||
|
||||
async def stream(self, messages, *, tools=None, max_tokens=None):
|
||||
emitted = False
|
||||
try:
|
||||
async for ch in self.primary.stream(messages, tools=tools, max_tokens=max_tokens):
|
||||
emitted = True
|
||||
self.last_backend = self.primary.name
|
||||
yield ch
|
||||
return
|
||||
except Exception as e: # noqa: BLE001 — 要按类型决定是否降级
|
||||
if emitted:
|
||||
log.error("主后端 %s 在已输出后失败，不再降级：%s", self.primary.name, e)
|
||||
raise
|
||||
if self.fallback is None:
|
||||
raise
|
||||
if not self._should_fallback(e):
|
||||
log.error("主后端 %s 失败且不适合降级：%s", self.primary.name, e)
|
||||
raise
|
||||
log.warning("主后端 %s 失败（%s），切备用 %s",
|
||||
self.primary.name, type(e).__name__, self.fallback.name)
|
||||
|
||||
async for ch in self.fallback.stream(messages, tools=tools, max_tokens=max_tokens):
|
||||
self.last_backend = self.fallback.name
|
||||
yield ch
|
||||
|
||||
@staticmethod
|
||||
def _should_fallback(err: Exception) -> bool:
|
||||
"""连接/超时/服务端错误才降级；请求本身的问题不降级。"""
|
||||
if isinstance(err, LLMError):
|
||||
# LLMError 里带的是 HTTP 状态码，只有 5xx / 429 值得重试到别的后端
|
||||
msg = str(err)
|
||||
for code in ("500", "502", "503", "504", "429"):
|
||||
if f"HTTP {code}" in msg:
|
||||
return True
|
||||
return False
|
||||
# 超时、连接失败、DNS 等
|
||||
return isinstance(err, (TimeoutError, ConnectionError, OSError))
|
||||
|
||||
async def health(self) -> tuple[bool, str]:
|
||||
ok, detail = await self.primary.health()
|
||||
if ok:
|
||||
return True, f"主后端 {self.primary.name}: {detail}"
|
||||
if self.fallback:
|
||||
ok2, d2 = await self.fallback.health()
|
||||
return ok2, f"主后端不可用（{detail}）；备用 {self.fallback.name}: {d2}"
|
||||
return False, detail
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LLMProvider", "LLMError", "Chunk", "ChunkKind", "LLMResult", "ToolCall",
|
||||
"LocalVLLMProvider", "DeepSeekProvider", "FallbackProvider", "create_provider",
|
||||
]
|
||||
+308
@@ -0,0 +1,308 @@
|
||||
"""LLM 抽象层。
|
||||
|
||||
为什么需要这层：本机 vLLM 和 DeepSeek 虽然都是 OpenAI 兼容协议，
|
||||
但有三处实测差异，写死任何一个都会出问题：
|
||||
|
||||
1. 思考内容的字段名
|
||||
本机 vLLM : delta.reasoning
|
||||
DeepSeek : delta.reasoning_content
|
||||
只认一个会**静默丢弃**思考内容——不报错，只是看不见。
|
||||
|
||||
2. 关闭思考的方式
|
||||
本机 vLLM : chat_template_kwargs={"enable_thinking": false}
|
||||
DeepSeek : thinking={"type": "disabled"}
|
||||
|
||||
3. system 消息条数
|
||||
本机 vLLM : 只允许一条，且必须在开头，多条直接 400
|
||||
DeepSeek : 宽松
|
||||
所以统一由本层归一化成单条 system。
|
||||
|
||||
以上均来自实测，见 P0-本地LLM验证报告.md / P0-DeepSeek验证报告.md。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, AsyncIterator, Sequence
|
||||
|
||||
import aiohttp
|
||||
|
||||
from ..config import LLMConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChunkKind(str, Enum):
|
||||
REASONING = "reasoning"
|
||||
CONTENT = "content"
|
||||
TOOL_CALL = "tool_call"
|
||||
DONE = "done"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
kind: ChunkKind
|
||||
text: str = ""
|
||||
tool_index: int = 0
|
||||
tool_id: str = ""
|
||||
tool_name: str = ""
|
||||
tool_arguments: str = ""
|
||||
finish_reason: str | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCall:
|
||||
id: str
|
||||
name: str
|
||||
arguments: str
|
||||
|
||||
def parsed_arguments(self) -> dict[str, Any]:
|
||||
try:
|
||||
return json.loads(self.arguments or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLMResult:
|
||||
content: str = ""
|
||||
reasoning: str = ""
|
||||
tool_calls: list[ToolCall] = field(default_factory=list)
|
||||
finish_reason: str | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
# 计时：首 token 是"思考"还是"回答"要分开，否则会误判延迟。
|
||||
# 实测开思考时 138ms 就出首块，但那是思考内容，用户要听的回答还没开始。
|
||||
first_reasoning_ms: float | None = None
|
||||
first_content_ms: float | None = None
|
||||
total_ms: float = 0.0
|
||||
backend: str = ""
|
||||
|
||||
@property
|
||||
def has_tool_calls(self) -> bool:
|
||||
return bool(self.tool_calls)
|
||||
|
||||
|
||||
class LLMProvider(ABC):
|
||||
"""一个 LLM 后端。
|
||||
|
||||
子类只需实现 `_build_body`（怎么关思考、怎么放参数）；
|
||||
消息归一化、SSE 解析、计时、取消都在基类做，保证各后端行为一致。
|
||||
"""
|
||||
|
||||
#: 是否支持图片输入。本机 vLLM 实例实测不支持（"At most 0 image(s)"）。
|
||||
supports_vision: bool = False
|
||||
|
||||
def __init__(self, cfg: LLMConfig, session: aiohttp.ClientSession | None = None):
|
||||
self.cfg = cfg
|
||||
self.name = cfg.name
|
||||
self._session = session
|
||||
self._owns_session = session is None
|
||||
|
||||
# ------------------------------------------------------------ 供子类覆写
|
||||
|
||||
@abstractmethod
|
||||
def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None,
|
||||
max_tokens: int) -> dict:
|
||||
"""构造请求体。各后端的思考开关差异在这里处理。"""
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {self.cfg.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
@property
|
||||
def endpoint(self) -> str:
|
||||
return self.cfg.base_url.rstrip("/") + "/chat/completions"
|
||||
|
||||
# ------------------------------------------------------------ 消息归一化
|
||||
|
||||
@staticmethod
|
||||
def normalize_messages(messages: Sequence[dict]) -> list[dict]:
|
||||
"""把多条 system 合并成一条并置于开头。
|
||||
|
||||
本机 vLLM 对多条 system 直接 400（实测 "System message must be at the
|
||||
beginning."）。而记忆注入天然想用第二条 system，所以在这里统一抹平。
|
||||
|
||||
顺带这也满足方案 §8 的要求："注入的记忆每轮替换，不是追加"——
|
||||
调用方每轮重新构造 system 内容即可。
|
||||
"""
|
||||
systems, rest = [], []
|
||||
for m in messages:
|
||||
if m.get("role") == "system":
|
||||
c = m.get("content")
|
||||
if isinstance(c, str) and c.strip():
|
||||
systems.append(c.strip())
|
||||
else:
|
||||
rest.append(m)
|
||||
out = []
|
||||
if systems:
|
||||
out.append({"role": "system", "content": "\n\n".join(systems)})
|
||||
out.extend(rest)
|
||||
return out
|
||||
|
||||
# ------------------------------------------------------------ 流式请求
|
||||
|
||||
async def _ensure_session(self) -> aiohttp.ClientSession:
|
||||
if self._session is None or self._session.closed:
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._owns_session = True
|
||||
return self._session
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._owns_session and self._session and not self._session.closed:
|
||||
await self._session.close()
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
messages: Sequence[dict],
|
||||
*,
|
||||
tools: Sequence[dict] | None = None,
|
||||
max_tokens: int | None = None,
|
||||
) -> AsyncIterator[Chunk]:
|
||||
"""流式产出。调用方取消（CancelledError）时连接会随之关闭。
|
||||
|
||||
这就是方案 §5 要求的"打断时取消旧回复生成"的底层能力。
|
||||
"""
|
||||
body = self._build_body(
|
||||
self.normalize_messages(messages),
|
||||
tools=tools,
|
||||
max_tokens=max_tokens or self.cfg.max_tokens,
|
||||
)
|
||||
session = await self._ensure_session()
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
total=self.cfg.timeout_s,
|
||||
sock_connect=5.0,
|
||||
sock_read=self.cfg.first_token_timeout_s,
|
||||
)
|
||||
async with session.post(self.endpoint, json=body,
|
||||
headers=self._headers(), timeout=timeout) as resp:
|
||||
if resp.status != 200:
|
||||
detail = (await resp.text())[:400]
|
||||
raise LLMError(f"{self.name} HTTP {resp.status}: {detail}")
|
||||
async for chunk in self._iter_sse(resp):
|
||||
yield chunk
|
||||
|
||||
@staticmethod
|
||||
async def _iter_sse(resp: aiohttp.ClientResponse) -> AsyncIterator[Chunk]:
|
||||
"""解析 SSE。字段名差异在这里吸收。"""
|
||||
# 工具调用的参数是分片到达的，按 index 累积
|
||||
tool_acc: dict[int, dict] = {}
|
||||
async for raw in resp.content:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line or not line.startswith("data:"):
|
||||
continue
|
||||
payload = line[5:].strip()
|
||||
if payload == "[DONE]":
|
||||
break
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
usage = data.get("usage")
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
if usage:
|
||||
yield Chunk(kind=ChunkKind.DONE, usage=usage)
|
||||
continue
|
||||
|
||||
choice = choices[0]
|
||||
delta = choice.get("delta") or {}
|
||||
finish = choice.get("finish_reason")
|
||||
|
||||
# 两个字段名都认。少认一个就会静默丢思考内容。
|
||||
reasoning = delta.get("reasoning_content") or delta.get("reasoning")
|
||||
if reasoning:
|
||||
yield Chunk(kind=ChunkKind.REASONING, text=reasoning)
|
||||
|
||||
content = delta.get("content")
|
||||
if content:
|
||||
yield Chunk(kind=ChunkKind.CONTENT, text=content)
|
||||
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
idx = tc.get("index", 0)
|
||||
slot = tool_acc.setdefault(idx, {"id": "", "name": "", "arguments": ""})
|
||||
if tc.get("id"):
|
||||
slot["id"] = tc["id"]
|
||||
fn = tc.get("function") or {}
|
||||
if fn.get("name"):
|
||||
slot["name"] = fn["name"]
|
||||
if fn.get("arguments"):
|
||||
slot["arguments"] += fn["arguments"]
|
||||
yield Chunk(kind=ChunkKind.TOOL_CALL, tool_index=idx,
|
||||
tool_id=slot["id"], tool_name=slot["name"],
|
||||
tool_arguments=slot["arguments"])
|
||||
|
||||
if finish:
|
||||
yield Chunk(kind=ChunkKind.DONE, finish_reason=finish, usage=usage)
|
||||
# 注意：**不能在这里 return**。开了 stream_options.include_usage 后，
|
||||
# usage 是在 finish_reason 之后的**单独一帧**里到达的
|
||||
# （该帧 choices 为空）。提前返回会把 usage 丢掉。
|
||||
# 循环由 [DONE] 或流结束自然终止。
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
messages: Sequence[dict],
|
||||
*,
|
||||
tools: Sequence[dict] | None = None,
|
||||
max_tokens: int | None = None,
|
||||
) -> LLMResult:
|
||||
"""收完整个流，返回结构化结果。"""
|
||||
t0 = time.perf_counter()
|
||||
res = LLMResult(backend=self.name)
|
||||
parts: list[str] = []
|
||||
reason: list[str] = []
|
||||
acc: dict[int, dict] = {}
|
||||
async for ch in self.stream(messages, tools=tools, max_tokens=max_tokens):
|
||||
if ch.kind == ChunkKind.CONTENT:
|
||||
if res.first_content_ms is None:
|
||||
res.first_content_ms = (time.perf_counter() - t0) * 1000
|
||||
parts.append(ch.text)
|
||||
elif ch.kind == ChunkKind.REASONING:
|
||||
if res.first_reasoning_ms is None:
|
||||
res.first_reasoning_ms = (time.perf_counter() - t0) * 1000
|
||||
reason.append(ch.text)
|
||||
elif ch.kind == ChunkKind.TOOL_CALL:
|
||||
acc[ch.tool_index] = {"id": ch.tool_id, "name": ch.tool_name,
|
||||
"arguments": ch.tool_arguments}
|
||||
elif ch.kind == ChunkKind.DONE:
|
||||
res.finish_reason = ch.finish_reason
|
||||
if ch.usage:
|
||||
res.usage = ch.usage
|
||||
res.content = "".join(parts)
|
||||
res.reasoning = "".join(reason)
|
||||
res.tool_calls = [ToolCall(**v) for _, v in sorted(acc.items())]
|
||||
res.total_ms = (time.perf_counter() - t0) * 1000
|
||||
return res
|
||||
|
||||
async def health(self) -> tuple[bool, str]:
|
||||
"""启动自检：确认模型名有效。方案 §10 要求记录实际版本。"""
|
||||
try:
|
||||
session = await self._ensure_session()
|
||||
url = self.cfg.base_url.rstrip("/") + "/models"
|
||||
async with session.get(url, headers=self._headers(),
|
||||
timeout=aiohttp.ClientTimeout(total=10)) as r:
|
||||
if r.status != 200:
|
||||
return False, f"HTTP {r.status}"
|
||||
data = await r.json()
|
||||
ids = [m.get("id") for m in data.get("data", [])]
|
||||
if self.cfg.model in ids:
|
||||
return True, f"{self.cfg.model} 可用（清单 {len(ids)} 个）"
|
||||
return False, f"模型 {self.cfg.model!r} 不在清单 {ids} 中"
|
||||
except Exception as e: # noqa: BLE001 — 自检不该因异常类型漏判
|
||||
return False, f"{type(e).__name__}: {e}"
|
||||
|
||||
|
||||
class LLMError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["LLMProvider", "LLMError", "Chunk", "ChunkKind", "ToolCall", "LLMResult"]
|
||||
@@ -0,0 +1,47 @@
|
||||
"""DeepSeek 云 API（备用后端）。
|
||||
|
||||
实测数据（P0-DeepSeek验证报告.md）：
|
||||
首 token P50 786ms / P95 1100ms（关思考，N=8）
|
||||
视觉可用——正确读出图中数字
|
||||
模型清单只有 deepseek-flash 与 deepseek-v4-pro
|
||||
|
||||
约束（重要，踩过）：
|
||||
默认开启思考模式，此时 API 要求把上一轮 assistant 消息里的
|
||||
`reasoning_content` 原样回传，否则任何多轮/工具回灌一律 400：
|
||||
"The `reasoning_content` in the thinking mode must be passed back to the API."
|
||||
手工构造 assistant 消息（content=None / content="" / 不带 tools）三种写法
|
||||
实测全部 400，只有回传服务端原始对象才行。
|
||||
所以会话主路径必须显式关闭思考。
|
||||
|
||||
另：本服务思考字段名是 reasoning_content（本机 vLLM 是 reasoning），
|
||||
基类两个都认。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from .base import LLMProvider
|
||||
|
||||
|
||||
class DeepSeekProvider(LLMProvider):
|
||||
supports_vision = True
|
||||
|
||||
def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None,
|
||||
max_tokens: int) -> dict:
|
||||
body: dict = {
|
||||
"model": self.cfg.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"max_tokens": max_tokens,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
# 显式声明思考开关。不传会走默认（开），多轮与工具回灌会 400。
|
||||
body["thinking"] = {"type": "enabled" if self.cfg.thinking else "disabled"}
|
||||
if tools:
|
||||
body["tools"] = list(tools)
|
||||
body["tool_choice"] = "auto"
|
||||
return body
|
||||
|
||||
|
||||
__all__ = ["DeepSeekProvider"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""本机 Qwen3.8-27B（vLLM，OpenAI 兼容）。
|
||||
|
||||
实测数据（P0-本地LLM验证报告.md）：
|
||||
关思考 首 token 367ms / 完成 10 tokens
|
||||
开思考 首 token 1390ms / 完成 187 tokens，且**回答被 max_tokens 截断**
|
||||
（实测 200 预算下只吐出"我不"两个字）
|
||||
|
||||
开思考不只是慢——思考 token 会占用 max_tokens 预算。
|
||||
语音场景 max_tokens 本来就要设小，所以这不是"慢一点"的取舍，是"说不出话"。
|
||||
默认关闭。
|
||||
|
||||
约束：只接受一条 system 消息，多条返回 400。已在基类归一化。
|
||||
视觉：本实例不支持（"At most 0 image(s) may be provided"），
|
||||
模型目录里没有视觉塔。视觉要走 llama.cpp + mmproj 那条线。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from .base import LLMProvider
|
||||
|
||||
|
||||
class LocalVLLMProvider(LLMProvider):
|
||||
supports_vision = False
|
||||
|
||||
def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None,
|
||||
max_tokens: int) -> dict:
|
||||
body: dict = {
|
||||
"model": self.cfg.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"max_tokens": max_tokens,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
# 关闭思考。这是本机部署实测唯一有效的方式
|
||||
# （服务器上的 ctx_test.sh 也用同一写法）。
|
||||
# 注意：即使关掉，响应里仍可能出现空的 reasoning 字段，属正常。
|
||||
body["chat_template_kwargs"] = {"enable_thinking": self.cfg.thinking}
|
||||
if tools:
|
||||
body["tools"] = list(tools)
|
||||
body["tool_choice"] = "auto"
|
||||
return body
|
||||
|
||||
|
||||
__all__ = ["LocalVLLMProvider"]
|
||||
@@ -0,0 +1,16 @@
|
||||
"""记忆层。
|
||||
|
||||
对外只暴露 `MemoryStore` 接口和 `SQLiteMemoryStore` 实现。
|
||||
调用方依赖接口，不依赖 SQLite——将来多设备共享记忆时要搬到服务器，
|
||||
只换实现，调用方一行不改（前提是调用方按 async + 可能失败 来写）。
|
||||
"""
|
||||
|
||||
from .base import (
|
||||
ConfirmStatus, Memory, MemoryKind, MemoryStore, Scope, Turn,
|
||||
)
|
||||
from .sqlite_store import SCHEMA, SQLiteMemoryStore
|
||||
|
||||
__all__ = [
|
||||
"MemoryStore", "SQLiteMemoryStore", "Memory", "Turn",
|
||||
"Scope", "ConfirmStatus", "MemoryKind", "SCHEMA",
|
||||
]
|
||||
@@ -0,0 +1,185 @@
|
||||
"""记忆抽象层。
|
||||
|
||||
设计原则（方案 §8）：
|
||||
|
||||
1. **接口按"可能远程"设计**。SQLite 现在跑在本地，但将来多设备共享记忆
|
||||
时要搬到服务器。所以所有操作都是 async、可能超时、可能失败——
|
||||
写成同步直连本地文件的话，搬家要重写所有调用方。
|
||||
现在付这个成本几乎为零，将来付就很贵。
|
||||
|
||||
2. **查询和写入都强制带 owner_user_id**，没有"查全部"的口子。
|
||||
未知身份只给访客临时上下文，不落库。
|
||||
|
||||
3. **模型推测不当作确认事实**。自动抽取的记忆是 candidate，默认不参与检索；
|
||||
只有用户显式"记住"或工具调用写入的才是 confirmed。
|
||||
|
||||
4. **异步任务绑定创建时的身份**。写入任务在创建时捕获 user_id/session_id，
|
||||
绝不读"当前用户"——否则用户切了身份，上一轮迟到的写入会串到新人头上。
|
||||
|
||||
5. 视觉 track_id、会话 session_id、永久 user_id **三者分开**，不互相冒充。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Sequence
|
||||
|
||||
|
||||
class Scope(str, Enum):
|
||||
"""记忆的作用域。群体会话默认只用 shared。"""
|
||||
PRIVATE = "private"
|
||||
SHARED = "shared"
|
||||
|
||||
|
||||
class ConfirmStatus(str, Enum):
|
||||
"""确认状态。区分"用户明确说的"和"模型猜的"。"""
|
||||
CONFIRMED = "confirmed" # 用户显式记住 / 工具调用写入
|
||||
CANDIDATE = "candidate" # 回复后自动抽取，默认不参与检索
|
||||
|
||||
|
||||
class MemoryKind(str, Enum):
|
||||
PREFERENCE = "preference" # 明确偏好：结构化键值
|
||||
FACT = "fact" # 事实
|
||||
TASK = "task" # 临时事项，常带过期时间
|
||||
|
||||
|
||||
@dataclass
|
||||
class Memory:
|
||||
memory_id: int | None = None
|
||||
owner_user_id: str = ""
|
||||
scope: Scope = Scope.PRIVATE
|
||||
kind: MemoryKind = MemoryKind.FACT
|
||||
key: str = ""
|
||||
value: str = ""
|
||||
confirm_status: ConfirmStatus = ConfirmStatus.CONFIRMED
|
||||
source_session_id: str | None = None
|
||||
source_turn_id: str | None = None
|
||||
created_at: float = field(default_factory=time.time)
|
||||
updated_at: float = field(default_factory=time.time)
|
||||
expires_at: float | None = None
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
"""用于检索和注入的文本形式。"""
|
||||
return f"{self.key}：{self.value}" if self.key else self.value
|
||||
|
||||
def is_expired(self, now: float | None = None) -> bool:
|
||||
if self.expires_at is None:
|
||||
return False
|
||||
return (now or time.time()) > self.expires_at
|
||||
|
||||
|
||||
@dataclass
|
||||
class Turn:
|
||||
"""一轮对话。用于溯源，不默认长期保存逐字稿。"""
|
||||
turn_id: str
|
||||
session_id: str
|
||||
user_id: str
|
||||
role: str # "user" | "assistant"
|
||||
text: str
|
||||
created_at: float = field(default_factory=time.time)
|
||||
|
||||
|
||||
class MemoryStore(ABC):
|
||||
"""记忆存储。
|
||||
|
||||
所有方法都可能抛异常——调用方必须按"记忆失败不阻塞对话"处理
|
||||
（方案 §8）。超时由调用方用 asyncio.wait_for 控制。
|
||||
"""
|
||||
|
||||
# ---------------------------------------------------------- 生命周期
|
||||
|
||||
@abstractmethod
|
||||
async def open(self) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None: ...
|
||||
|
||||
# ---------------------------------------------------------- 用户与会话
|
||||
|
||||
@abstractmethod
|
||||
async def ensure_user(self, user_id: str, display_name: str = "",
|
||||
trust_level: str = "normal") -> None:
|
||||
"""建档。已存在则不动。身份认证与信任等级分开（方案 §6）。"""
|
||||
|
||||
@abstractmethod
|
||||
async def start_session(self, session_id: str, user_id: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def end_session(self, session_id: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def add_turn(self, turn: Turn) -> None: ...
|
||||
|
||||
# ---------------------------------------------------------- 记忆读写
|
||||
|
||||
@abstractmethod
|
||||
async def remember(self, mem: Memory) -> int:
|
||||
"""写入一条记忆，返回 memory_id。
|
||||
|
||||
同 owner + 同 kind + 同 key 的 confirmed 记忆，新值替换旧值
|
||||
（方案 §8："明确偏好…新值替换旧值"）。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def forget(self, owner_user_id: str, memory_id: int) -> bool:
|
||||
"""删除。必须带 owner_user_id——不能靠 memory_id 单独删，
|
||||
否则猜到一个 id 就能删别人的记忆。"""
|
||||
|
||||
@abstractmethod
|
||||
async def retrieve(self, owner_user_id: str, query: str, *,
|
||||
limit: int = 5,
|
||||
include_candidates: bool = False,
|
||||
scopes: Sequence[Scope] | None = None) -> list[Memory]:
|
||||
"""按相关性检索。
|
||||
|
||||
强制按 owner_user_id 过滤。unknown 身份应传一个访客 id，
|
||||
而不是绕过这个方法。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def list_memories(self, owner_user_id: str, *,
|
||||
include_candidates: bool = True) -> list[Memory]:
|
||||
"""列出全部，供"查看/删除"入口使用（方案 §8）。"""
|
||||
|
||||
@abstractmethod
|
||||
async def purge_user(self, owner_user_id: str) -> int:
|
||||
"""彻底删除该用户的一切：记忆、会话、轮次。
|
||||
对应方案 §8："删除同时覆盖会话上下文、摘要和未来索引"。"""
|
||||
|
||||
# ---------------------------------------------------------- 检索打分
|
||||
|
||||
@staticmethod
|
||||
def relevance(query: str, text: str) -> float:
|
||||
"""中文字符二元组重合度打分。
|
||||
|
||||
为什么不用 FTS5：SQLite 默认分词器对中文是按整串处理的，
|
||||
一句话会变成一个 token，检索不出来；trigram 分词器可用但要额外
|
||||
编译选项。在"记忆条数不多"这个实际规模下，在 Python 里按二元组
|
||||
打分更可控，也不依赖 SQLite 的编译配置。
|
||||
|
||||
将来真的需要向量检索时，换掉这个打分函数和 retrieve 的实现即可，
|
||||
接口不动。
|
||||
"""
|
||||
q, t = query.strip(), text.strip()
|
||||
if not q or not t:
|
||||
return 0.0
|
||||
if q in t:
|
||||
return 1.0
|
||||
qb = {q[i:i + 2] for i in range(len(q) - 1)} or {q}
|
||||
tb = {t[i:i + 2] for i in range(len(t) - 1)} or {t}
|
||||
if not qb:
|
||||
return 0.0
|
||||
hit = len(qb & tb)
|
||||
if hit == 0:
|
||||
return 0.0
|
||||
# 以查询覆盖率为准：记忆里多出来的字不该拉低分数
|
||||
return hit / len(qb)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryStore", "Memory", "Turn", "Scope", "ConfirmStatus", "MemoryKind",
|
||||
]
|
||||
@@ -0,0 +1,334 @@
|
||||
"""SQLite 记忆存储。
|
||||
|
||||
几个刻意的选择：
|
||||
|
||||
* **列名用 mem_key / mem_value**，不用 key / value——后者在 SQL 里
|
||||
容易和关键字撞，每个语句都得加引号，早晚漏一处。
|
||||
|
||||
* **WAL 模式**：读写不互相阻塞，将来多进程访问同一份库也不容易锁死。
|
||||
|
||||
* **外键级联删除**：删用户时记忆/会话/轮次一起走。方案 §8 要求
|
||||
"删除同时覆盖会话上下文、摘要和未来索引"，靠数据库约束比靠代码记得住。
|
||||
|
||||
* **所有 SQL 操作走 asyncio.to_thread**。本地 SQLite 其实微秒级，
|
||||
但接口承诺了 async，就得真的不阻塞事件循环——否则将来换成远程
|
||||
实现时，调用方那些"同步假设"会全部暴露成 bug。
|
||||
|
||||
* **每条查询都带 owner_user_id**。没有"按 memory_id 单独取"的口子：
|
||||
猜到 id 就能读别人的记忆，是这类系统最容易犯的错。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from .base import ConfirmStatus, Memory, MemoryKind, MemoryStore, Scope, Turn
|
||||
|
||||
#: 关键词全不命中时，允许"退回最近记忆"的记忆总量上限。
|
||||
#: 超过这个数就退回严格匹配——否则每轮都会把记忆全塞进 prompt，
|
||||
#: 既不再是"按需检索"，也会持续推高首 token 延迟。
|
||||
FALLBACK_MAX_CORPUS = 10
|
||||
|
||||
SCHEMA = """
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
user_id TEXT PRIMARY KEY,
|
||||
display_name TEXT NOT NULL DEFAULT '',
|
||||
trust_level TEXT NOT NULL DEFAULT 'normal',
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
started_at REAL NOT NULL,
|
||||
ended_at REAL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS turns (
|
||||
turn_id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
memory_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
owner_user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||
scope TEXT NOT NULL DEFAULT 'private',
|
||||
kind TEXT NOT NULL DEFAULT 'fact',
|
||||
mem_key TEXT NOT NULL DEFAULT '',
|
||||
mem_value TEXT NOT NULL,
|
||||
confirm_status TEXT NOT NULL DEFAULT 'confirmed',
|
||||
source_session_id TEXT,
|
||||
source_turn_id TEXT,
|
||||
created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL,
|
||||
expires_at REAL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mem_owner ON memories(owner_user_id, confirm_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_mem_key ON memories(owner_user_id, kind, mem_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_turns_sess ON turns(session_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_sess_user ON sessions(user_id);
|
||||
"""
|
||||
|
||||
|
||||
def _ensure_user_row(c: sqlite3.Connection, user_id: str,
|
||||
display_name: str = "") -> None:
|
||||
"""保证 users 里有这一行。
|
||||
|
||||
为什么放在存储层而不是让调用方自己 ensure：
|
||||
memories / turns / sessions 都有指向 users 的外键，
|
||||
「被引用的 user 必须存在」是这个 schema 的不变量。
|
||||
让每个调用点各自记得先建档，早晚漏一处——而且漏了之后
|
||||
报的是 "FOREIGN KEY constraint failed"，跟业务逻辑对不上号，
|
||||
排查起来很绕。由存储层自己兜住，调用方就不需要知道这个约束。
|
||||
|
||||
实测踩过：会话中途切到一个未建档的 user_id，
|
||||
第一轮写 turns 就炸在这里。
|
||||
"""
|
||||
c.execute("INSERT INTO users(user_id, display_name, created_at) VALUES(?,?,?) "
|
||||
"ON CONFLICT(user_id) DO NOTHING", (user_id, display_name, time.time()))
|
||||
|
||||
|
||||
def _row_to_memory(row: sqlite3.Row) -> Memory:
|
||||
return Memory(
|
||||
memory_id=row["memory_id"],
|
||||
owner_user_id=row["owner_user_id"],
|
||||
scope=Scope(row["scope"]),
|
||||
kind=MemoryKind(row["kind"]),
|
||||
key=row["mem_key"],
|
||||
value=row["mem_value"],
|
||||
confirm_status=ConfirmStatus(row["confirm_status"]),
|
||||
source_session_id=row["source_session_id"],
|
||||
source_turn_id=row["source_turn_id"],
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
expires_at=row["expires_at"],
|
||||
)
|
||||
|
||||
|
||||
class SQLiteMemoryStore(MemoryStore):
|
||||
def __init__(self, db_path: Path | str):
|
||||
self.db_path = Path(db_path)
|
||||
self._conn: sqlite3.Connection | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------- 内部
|
||||
|
||||
def _db(self) -> sqlite3.Connection:
|
||||
if self._conn is None:
|
||||
raise RuntimeError("MemoryStore 未 open()")
|
||||
return self._conn
|
||||
|
||||
def _exec(self, fn):
|
||||
"""在受控线程里跑同步 SQL，串行化以免共用连接出问题。"""
|
||||
with self._lock:
|
||||
return fn(self._db())
|
||||
|
||||
async def _run(self, fn):
|
||||
return await asyncio.to_thread(self._exec, fn)
|
||||
|
||||
# ---------------------------------------------------------- 生命周期
|
||||
|
||||
async def open(self) -> None:
|
||||
def _open() -> sqlite3.Connection:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA)
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
# 此处还没有并发使用者，不需要持锁跨 await
|
||||
self._conn = await asyncio.to_thread(_open)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._conn is None:
|
||||
return
|
||||
|
||||
def _close(c: sqlite3.Connection):
|
||||
c.commit()
|
||||
c.close()
|
||||
self._conn = None
|
||||
|
||||
await self._run(_close)
|
||||
|
||||
# ---------------------------------------------------------- 用户与会话
|
||||
|
||||
async def ensure_user(self, user_id: str, display_name: str = "",
|
||||
trust_level: str = "normal") -> None:
|
||||
def op(c: sqlite3.Connection):
|
||||
c.execute(
|
||||
"INSERT INTO users(user_id, display_name, trust_level, created_at) "
|
||||
"VALUES(?,?,?,?) ON CONFLICT(user_id) DO NOTHING",
|
||||
(user_id, display_name, trust_level, time.time()))
|
||||
c.commit()
|
||||
await self._run(op)
|
||||
|
||||
async def start_session(self, session_id: str, user_id: str) -> None:
|
||||
def op(c: sqlite3.Connection):
|
||||
c.execute("INSERT INTO users(user_id, created_at) VALUES(?,?) "
|
||||
"ON CONFLICT(user_id) DO NOTHING", (user_id, time.time()))
|
||||
c.execute("INSERT INTO sessions(session_id, user_id, started_at) "
|
||||
"VALUES(?,?,?) ON CONFLICT(session_id) DO NOTHING",
|
||||
(session_id, user_id, time.time()))
|
||||
c.commit()
|
||||
await self._run(op)
|
||||
|
||||
async def end_session(self, session_id: str) -> None:
|
||||
def op(c: sqlite3.Connection):
|
||||
c.execute("UPDATE sessions SET ended_at=? WHERE session_id=?",
|
||||
(time.time(), session_id))
|
||||
c.commit()
|
||||
await self._run(op)
|
||||
|
||||
async def add_turn(self, turn: Turn) -> None:
|
||||
def op(c: sqlite3.Connection):
|
||||
_ensure_user_row(c, turn.user_id)
|
||||
c.execute("INSERT INTO turns(turn_id, session_id, user_id, role, text, created_at) "
|
||||
"VALUES(?,?,?,?,?,?) ON CONFLICT(turn_id) DO NOTHING",
|
||||
(turn.turn_id, turn.session_id, turn.user_id, turn.role,
|
||||
turn.text, turn.created_at))
|
||||
c.commit()
|
||||
await self._run(op)
|
||||
|
||||
# ---------------------------------------------------------- 记忆读写
|
||||
|
||||
async def remember(self, mem: Memory) -> int:
|
||||
now = time.time()
|
||||
|
||||
def op(c: sqlite3.Connection):
|
||||
_ensure_user_row(c, mem.owner_user_id)
|
||||
# 已确认的同类同键记忆：新值替换旧值（方案 §8）
|
||||
if mem.confirm_status == ConfirmStatus.CONFIRMED and mem.key:
|
||||
row = c.execute(
|
||||
"SELECT memory_id FROM memories WHERE owner_user_id=? AND kind=? "
|
||||
"AND mem_key=? AND confirm_status=?",
|
||||
(mem.owner_user_id, mem.kind.value, mem.key,
|
||||
ConfirmStatus.CONFIRMED.value)).fetchone()
|
||||
if row:
|
||||
c.execute(
|
||||
"UPDATE memories SET mem_value=?, scope=?, updated_at=?, "
|
||||
"expires_at=?, source_session_id=?, source_turn_id=? "
|
||||
"WHERE memory_id=? AND owner_user_id=?",
|
||||
(mem.value, mem.scope.value, now, mem.expires_at,
|
||||
mem.source_session_id, mem.source_turn_id,
|
||||
row["memory_id"], mem.owner_user_id))
|
||||
c.commit()
|
||||
return row["memory_id"]
|
||||
|
||||
cur = c.execute(
|
||||
"INSERT INTO memories(owner_user_id, scope, kind, mem_key, mem_value, "
|
||||
"confirm_status, source_session_id, source_turn_id, created_at, "
|
||||
"updated_at, expires_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(mem.owner_user_id, mem.scope.value, mem.kind.value, mem.key,
|
||||
mem.value, mem.confirm_status.value, mem.source_session_id,
|
||||
mem.source_turn_id, mem.created_at or now, now, mem.expires_at))
|
||||
c.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
return await self._run(op)
|
||||
|
||||
async def forget(self, owner_user_id: str, memory_id: int) -> bool:
|
||||
def op(c: sqlite3.Connection):
|
||||
# 必须同时匹配 owner——不能只凭 memory_id 删
|
||||
cur = c.execute("DELETE FROM memories WHERE memory_id=? AND owner_user_id=?",
|
||||
(memory_id, owner_user_id))
|
||||
c.commit()
|
||||
return cur.rowcount > 0
|
||||
return await self._run(op)
|
||||
|
||||
async def retrieve(self, owner_user_id: str, query: str, *,
|
||||
limit: int = 5,
|
||||
include_candidates: bool = False,
|
||||
scopes: Sequence[Scope] | None = None) -> list[Memory]:
|
||||
allowed = [s.value for s in (scopes or [Scope.PRIVATE, Scope.SHARED])]
|
||||
statuses = [ConfirmStatus.CONFIRMED.value]
|
||||
if include_candidates:
|
||||
statuses.append(ConfirmStatus.CANDIDATE.value)
|
||||
placeholders = ",".join("?" * len(allowed))
|
||||
status_ph = ",".join("?" * len(statuses))
|
||||
|
||||
def op(c: sqlite3.Connection):
|
||||
rows = c.execute(
|
||||
f"SELECT * FROM memories WHERE owner_user_id=? "
|
||||
f"AND scope IN ({placeholders}) AND confirm_status IN ({status_ph})",
|
||||
(owner_user_id, *allowed, *statuses)).fetchall()
|
||||
now = time.time()
|
||||
scored = []
|
||||
for r in rows:
|
||||
m = _row_to_memory(r)
|
||||
if m.is_expired(now):
|
||||
continue
|
||||
score = self.relevance(query, m.text) if query.strip() else 0.0
|
||||
scored.append((score, m.updated_at, m))
|
||||
|
||||
hits = [x for x in scored if x[0] > 0]
|
||||
if not hits and 0 < len(scored) <= FALLBACK_MAX_CORPUS:
|
||||
# 关键词一条都没命中时，退回"最近更新的若干条"。
|
||||
#
|
||||
# 理由是实测出来的：查"我想喝点什么"匹配不上"喜欢喝美式咖啡"——
|
||||
# 两句共享"喝"字，但没有共同的二元组，按字面打分是 0。
|
||||
# 中文口语短句和记忆文本之间"意思相关但用词不同"极其常见，
|
||||
# 纯字面匹配漏得厉害。
|
||||
#
|
||||
# 但兜底必须**限量**：不加限制就变成"每轮无条件注入记忆"，
|
||||
# 那就不是方案 §8 说的"按需检索"了，而且记忆一多会持续
|
||||
# 拉长 prompt、推高首 token 延迟。
|
||||
#
|
||||
# 所以只在记忆总量很小时兜底——这个阶段"全给"和"按需给"
|
||||
# 结果差不多，而漏给的代价（助手像不认识你）更大。
|
||||
# 超过阈值就退回严格匹配，接受漏召。
|
||||
#
|
||||
# 这是权宜之计，不是终局：真正的解法是向量检索
|
||||
# （方案 §8 把它列在"真实使用中检索不足再评估"）。
|
||||
# 到那时换掉打分和这段逻辑即可，接口不动。
|
||||
hits = scored
|
||||
|
||||
hits.sort(key=lambda x: (x[0], x[1]), reverse=True)
|
||||
return [m for _, _, m in hits[:limit]]
|
||||
|
||||
return await self._run(op)
|
||||
|
||||
async def list_memories(self, owner_user_id: str, *,
|
||||
include_candidates: bool = True) -> list[Memory]:
|
||||
statuses = [ConfirmStatus.CONFIRMED.value]
|
||||
if include_candidates:
|
||||
statuses.append(ConfirmStatus.CANDIDATE.value)
|
||||
status_ph = ",".join("?" * len(statuses))
|
||||
|
||||
def op(c: sqlite3.Connection):
|
||||
rows = c.execute(
|
||||
f"SELECT * FROM memories WHERE owner_user_id=? "
|
||||
f"AND confirm_status IN ({status_ph}) ORDER BY updated_at DESC",
|
||||
(owner_user_id, *statuses)).fetchall()
|
||||
now = time.time()
|
||||
return [m for m in (_row_to_memory(r) for r in rows) if not m.is_expired(now)]
|
||||
|
||||
return await self._run(op)
|
||||
|
||||
async def purge_user(self, owner_user_id: str) -> int:
|
||||
def op(c: sqlite3.Connection):
|
||||
cur = c.execute("DELETE FROM memories WHERE owner_user_id=?", (owner_user_id,))
|
||||
n = cur.rowcount
|
||||
c.execute("DELETE FROM turns WHERE user_id=?", (owner_user_id,))
|
||||
c.execute("DELETE FROM sessions WHERE user_id=?", (owner_user_id,))
|
||||
# users 放最后：外键级联会把残留一并带走
|
||||
c.execute("DELETE FROM users WHERE user_id=?", (owner_user_id,))
|
||||
c.commit()
|
||||
return n
|
||||
return await self._run(op)
|
||||
|
||||
|
||||
__all__ = ["SQLiteMemoryStore", "SCHEMA"]
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""提示词构造。
|
||||
|
||||
关键约束（实测，P0-本地LLM验证报告.md §四）：
|
||||
|
||||
本机 vLLM **只接受一条 system 消息**，多条返回
|
||||
400 "System message must be at the beginning."
|
||||
|
||||
所以记忆不能作为"第二条 system"注入，必须**并进同一条里、每轮重建**。
|
||||
|
||||
这其实比追加式更干净，也正好满足方案 §8 的要求：
|
||||
"切换身份清理上一人的私有检索上下文"、"注入的记忆每轮替换，不是追加"——
|
||||
每轮从零拼出来的 system 天然不会有上一人的残留。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from .identity.base import IdentityDecision
|
||||
from .memory.base import Memory
|
||||
|
||||
#: 语音助手的回复风格。方案 §5："默认简短自然回复"。
|
||||
#: 这几条不是"风格偏好"，是**功能性的**——放出去的文本会直接进 TTS，
|
||||
#: 带 Markdown 的话会被逐字念出来（"井号井号 第一步"）。
|
||||
PERSONA_RULES = """\
|
||||
你是一个桌面陪伴助手，通过语音和用户交流。用户听到的是合成语音，不是文字。
|
||||
|
||||
必须遵守：
|
||||
1. 回复简短自然，适合朗读，默认一到两句话。不要长篇解释。
|
||||
2. 绝对不要使用 Markdown、星号、井号、列表、表格、表情符号或括号注释。
|
||||
3. 数字、日期、金额按中文口语写法，例如"一千二百元""下午三点半"。
|
||||
4. 不确定的事直说不确定，不要编造。没做过的事不要说已经做了。
|
||||
5. 需要操作设备时调用工具。不要假装已经执行，也不要说"正在为您打开"。
|
||||
6. 用户打断时立刻停止，不要坚持把上一句说完。"""
|
||||
|
||||
|
||||
def build_system_prompt(
|
||||
identity: IdentityDecision,
|
||||
memories: Sequence[Memory] = (),
|
||||
*,
|
||||
extra: str = "",
|
||||
) -> str:
|
||||
"""拼出本轮唯一的 system 消息。
|
||||
|
||||
每轮调用一次，不要缓存后复用——身份或记忆一变，缓存就会串。
|
||||
"""
|
||||
parts = [PERSONA_RULES]
|
||||
|
||||
if identity.confirmed:
|
||||
parts.append(f"\n当前用户：{identity.user_id}（已确认身份）。")
|
||||
else:
|
||||
# 方案 §6：未知身份只用访客上下文，不做个性化
|
||||
parts.append("\n当前用户身份未确认，按访客对待。"
|
||||
"不要提及任何私人信息，也不要假定你知道对方是谁。")
|
||||
|
||||
if memories:
|
||||
lines = ["\n已知的用户信息（供参考，不要逐条复述）："]
|
||||
for m in memories:
|
||||
lines.append(f"- {m.text}")
|
||||
parts.append("\n".join(lines))
|
||||
|
||||
if identity.suggestions and not identity.confirmed:
|
||||
# 有人脸候选但不自动切换。让助手可以自然地问一句，
|
||||
# 而不是自作主张切换档案（方案 §6："看到熟人不等于确认他正在说话"）。
|
||||
names = "、".join(
|
||||
s.candidate_user_id for s in identity.suggestions[:2])
|
||||
parts.append(
|
||||
f"\n注意：摄像头可能认出了 {names}，但这不代表对方正在说话。"
|
||||
f"不要据此切换身份或提及该用户的私人信息。")
|
||||
|
||||
if extra:
|
||||
parts.append("\n" + extra.strip())
|
||||
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def trim_history(
|
||||
history: Sequence[dict],
|
||||
*,
|
||||
max_turns: int,
|
||||
max_chars: int = 4000,
|
||||
) -> list[dict]:
|
||||
"""裁掉过旧的对话轮次。
|
||||
|
||||
方案 §8 要求"对话摘要限长、按需检索"，不默认永久保存逐字稿。
|
||||
这里做最简单的一档：保留最近 max_turns 轮，再按字符数兜底。
|
||||
上下文越短，本地 LLM 的首 token 越短——这不是省资源，是直接买延迟。
|
||||
"""
|
||||
msgs = list(history)
|
||||
# 一轮 = 一问一答，所以保留的消息数是轮数的两倍
|
||||
keep = max_turns * 2
|
||||
if len(msgs) > keep:
|
||||
msgs = msgs[-keep:]
|
||||
|
||||
# 按字符数从旧到新裁，保证裁完仍然从 user 消息开始
|
||||
while msgs and sum(len(m.get("content") or "") for m in msgs) > max_chars:
|
||||
msgs.pop(0)
|
||||
while msgs and msgs[0].get("role") != "user":
|
||||
msgs.pop(0)
|
||||
return msgs
|
||||
|
||||
|
||||
__all__ = ["build_system_prompt", "trim_history", "PERSONA_RULES"]
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""端到端测试：音频/文本 → STT → 身份 → 记忆 → LLM → TTS → 音频。
|
||||
|
||||
**全程不需要麦克风、音箱、摄像头、ESP32。**
|
||||
这是"服务器即助手"架构的直接好处：链路可以在任何有 CPU 的地方验证，
|
||||
硬件到货后再接最后一段（真实音频进出）即可。
|
||||
|
||||
跑完会把合成音频写到 out/ 目录，可以人工试听整条链路的实际效果。
|
||||
|
||||
运行：
|
||||
cd <项目根>
|
||||
~/venvs/ai_assistant/bin/python app/tests/test_engine_e2e.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.config import AppConfig # noqa: E402
|
||||
from app.engine import ConversationEngine, EventKind # noqa: E402
|
||||
from app.identity import GUEST_USER_ID, ManualIdentityProvider # noqa: E402
|
||||
from app.llm import FallbackProvider, create_provider # noqa: E402
|
||||
from app.memory import SQLiteMemoryStore # noqa: E402
|
||||
from app.voice import StreamingSTT, create_tts # noqa: E402
|
||||
|
||||
OK, FAIL = [], []
|
||||
OUT = ROOT / "out"
|
||||
|
||||
|
||||
def check(name: str, cond: bool, detail: str = "") -> None:
|
||||
(OK if cond else FAIL).append(name)
|
||||
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
||||
|
||||
|
||||
def save_wav(path: Path, audio: np.ndarray, sr: int) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
pcm = (np.clip(audio, -1, 1) * 32767).astype(np.int16)
|
||||
with wave.open(str(path), "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(sr)
|
||||
w.writeframes(pcm.tobytes())
|
||||
|
||||
|
||||
async def collect(engine: ConversationEngine, text: str) -> tuple[str, np.ndarray, list]:
|
||||
"""跑一轮，返回 (回复文本, 拼接音频, 事件列表)。"""
|
||||
reply, audios, events = [], [], []
|
||||
async for ev in engine.respond_to_text(text):
|
||||
events.append(ev)
|
||||
if ev.kind == EventKind.RESPONSE_DELTA:
|
||||
reply.append(ev.text)
|
||||
elif ev.kind == EventKind.SPEAK and ev.audio is not None:
|
||||
audios.append(ev.audio)
|
||||
audio = np.concatenate(audios) if audios else np.zeros(0, dtype=np.float32)
|
||||
return "".join(reply).strip(), audio, events
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
cfg = AppConfig.load()
|
||||
tmp = Path(tempfile.mkdtemp(prefix="e2e_"))
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print("=== 0. 装配 ===")
|
||||
llm = FallbackProvider(create_provider(cfg.llm),
|
||||
create_provider(cfg.llm_fallback))
|
||||
mem = SQLiteMemoryStore(tmp / "memory.db")
|
||||
await mem.open()
|
||||
tts = create_tts(cfg.voice)
|
||||
stt = StreamingSTT(cfg.voice)
|
||||
ident = ManualIdentityProvider()
|
||||
engine = ConversationEngine(cfg, llm, mem, ident, tts, stt)
|
||||
print(f" LLM={cfg.llm.model} 备用={cfg.llm_fallback.model}")
|
||||
tts_desc = (f"{cfg.voice.tts_backend}/{cfg.voice.tts_speaker}"
|
||||
if cfg.voice.tts_backend == "remote" else f"local/{cfg.voice.tts_threads}线程")
|
||||
print(f" STT={cfg.voice.stt_threads}线程 TTS={tts_desc}")
|
||||
check("四个组件装配成功", True)
|
||||
|
||||
ok, detail = await llm.health()
|
||||
check("LLM 可用", ok, detail)
|
||||
if not ok:
|
||||
await mem.close()
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
return 1
|
||||
|
||||
await engine.start_session("e2e_sess", user_id="u_ylhan")
|
||||
|
||||
# ---------------------------------------------------------- 1
|
||||
print("\n=== 1. 纯文本一轮（LLM → 分句 → TTS）===")
|
||||
reply, audio, events = await collect(engine, "现在几点了？随便说一句就行。")
|
||||
print(f" 回复: {reply[:60]!r}")
|
||||
m = [e for e in events if e.kind == EventKind.METRICS][0].metrics
|
||||
print(f" {m.summary()}")
|
||||
check("有回复", bool(reply))
|
||||
check("有音频输出", len(audio) > 0, f"{len(audio)} 采样点")
|
||||
check("回复不含 Markdown",
|
||||
not any(c in reply for c in "*#`|[]"), reply[:40])
|
||||
# 首音延迟这里只做"没离谱"的兜底检查，不设严格阈值：
|
||||
# 本机 N5095 的 TTS RTF 是 1.86（实测），首音必然被 TTS 拖到秒级；
|
||||
# 服务器上同一模型是 0.09，首音在 150ms 量级。
|
||||
# 真正该卡指标的是部署机（服务器），所以那个断言放在服务器上的验收里。
|
||||
check("首音延迟未失控", m.first_audio_ms < 8000, f"{m.first_audio_ms:.0f}ms")
|
||||
save_wav(OUT / "e2e_1_text.wav", audio, tts.sample_rate)
|
||||
|
||||
# ---------------------------------------------------------- 2
|
||||
print("\n=== 2. 多句回复按句播（每句独立音频）===")
|
||||
_, _, events2 = await collect(
|
||||
engine, "用三句话介绍一下你自己，每句短一点。")
|
||||
speaks = [e for e in events2 if e.kind == EventKind.SPEAK]
|
||||
print(f" 分 {len(speaks)} 段: {[e.text[:14] for e in speaks]}")
|
||||
check("按句切成了多段", len(speaks) >= 2, f"{len(speaks)} 段")
|
||||
audio2 = np.concatenate([e.audio for e in speaks]) if speaks else np.zeros(0, np.float32)
|
||||
save_wav(OUT / "e2e_2_multisentence.wav", audio2, tts.sample_rate)
|
||||
|
||||
# ---------------------------------------------------------- 3
|
||||
print("\n=== 3. 记忆：显式记住 → 下一轮被检索到 ===")
|
||||
await collect(engine, "记住我喜欢喝美式咖啡，不加糖。")
|
||||
await asyncio.sleep(0.5) # 等异步写入落库
|
||||
stored = await mem.list_memories("u_ylhan", include_candidates=True)
|
||||
print(f" 库中记忆 {len(stored)} 条: {[m.text[:20] for m in stored]}")
|
||||
check("记忆已落库", len(stored) >= 1, f"{len(stored)} 条")
|
||||
check("自动抽取的是候选状态",
|
||||
all(m.confirm_status.value == "candidate" for m in stored))
|
||||
|
||||
# 手动确认一条，看它是否参与检索
|
||||
await mem.remember(type(stored[0])(
|
||||
owner_user_id="u_ylhan", kind=stored[0].kind, key="",
|
||||
value="用户喜欢喝美式咖啡，不加糖",
|
||||
confirm_status=__import__("app.memory", fromlist=["ConfirmStatus"]).ConfirmStatus.CONFIRMED))
|
||||
_, _, events3 = await collect(engine, "我想喝点东西，推荐一下？")
|
||||
used = [e for e in events3 if e.kind == EventKind.MEMORY_USED]
|
||||
n_used = len(used[0].memories) if used else 0
|
||||
print(f" 本轮用到记忆 {n_used} 条")
|
||||
check("确认的记忆参与检索", n_used >= 1, f"{n_used} 条")
|
||||
|
||||
# ---------------------------------------------------------- 4
|
||||
print("\n=== 4. 身份隔离：换人后拿不到上一个人的记忆 ===")
|
||||
ident.switch("u_other", reason="测试切换")
|
||||
_, _, events4 = await collect(engine, "我想喝点什么？")
|
||||
used4 = [e for e in events4 if e.kind == EventKind.MEMORY_USED]
|
||||
n4 = len(used4[0].memories) if used4 else 0
|
||||
check("新身份没有旧记忆", n4 == 0, f"{n4} 条")
|
||||
reply4 = "".join(e.text for e in events4 if e.kind == EventKind.RESPONSE_DELTA)
|
||||
check("回复未泄露咖啡偏好", "美式" not in reply4 and "咖啡" not in reply4,
|
||||
reply4[:50])
|
||||
|
||||
# ---------------------------------------------------------- 5
|
||||
print("\n=== 5. 访客身份不做个性化 ===")
|
||||
ident.reset()
|
||||
print(f" 当前身份: {ident.current().user_id} (confirmed={ident.current().confirmed})")
|
||||
_, _, events5 = await collect(engine, "我喜欢什么？")
|
||||
used5 = [e for e in events5 if e.kind == EventKind.MEMORY_USED]
|
||||
check("访客不检索私人记忆", len(used5) == 0 or not used5[0].memories)
|
||||
reply5 = "".join(e.text for e in events5 if e.kind == EventKind.RESPONSE_DELTA)
|
||||
check("访客回复不编造偏好", "美式" not in reply5, reply5[:50])
|
||||
|
||||
# ---------------------------------------------------------- 6
|
||||
print("\n=== 6. 音频输入（真人录音 → STT → LLM → TTS）===")
|
||||
# 测试音频不在代码目录里，和模型放在一起（都是可重新下载的产物）。
|
||||
# 用模型目录反推，注意要 resolve()——服务器上模型目录是个软链，
|
||||
# 不解析的话 .parent 会指到 ~/ai-companion 而不是真实位置。
|
||||
wav_path = None
|
||||
for cand in (
|
||||
cfg.voice.model_dir.resolve().parent / "wavs" / "stt_1.wav",
|
||||
ROOT / "p0" / "stt_tts_bench" / "wavs" / "stt_1.wav",
|
||||
):
|
||||
if cand.is_file():
|
||||
wav_path = cand
|
||||
break
|
||||
if wav_path is not None:
|
||||
with wave.open(str(wav_path)) as w:
|
||||
samples = np.frombuffer(w.readframes(w.getnframes()),
|
||||
dtype=np.int16).astype(np.float32) / 32768.0
|
||||
ident.switch("u_ylhan", reason="回到主人")
|
||||
t0 = time.perf_counter()
|
||||
transcript, audios, events6 = [], [], []
|
||||
async for ev in engine.respond_to_audio(samples):
|
||||
events6.append(ev)
|
||||
if ev.kind == EventKind.TRANSCRIPT:
|
||||
transcript.append(ev.text)
|
||||
elif ev.kind == EventKind.SPEAK and ev.audio is not None:
|
||||
audios.append(ev.audio)
|
||||
got = "".join(transcript)
|
||||
m6 = [e for e in events6 if e.kind == EventKind.METRICS]
|
||||
print(f" STT: {got[:50]!r}")
|
||||
if m6 and m6[0].metrics:
|
||||
print(f" {m6[0].metrics.summary()}")
|
||||
check("音频链路跑通（能出转写）", bool(got.strip()))
|
||||
if audios:
|
||||
save_wav(OUT / "e2e_6_fromaudio.wav", np.concatenate(audios), tts.sample_rate)
|
||||
else:
|
||||
check("音频链路跑通", False, f"测试音频不存在: {wav_path}")
|
||||
|
||||
# ---------------------------------------------------------- 7
|
||||
print("\n=== 7. 打断：取消正在进行的回复 ===")
|
||||
cancelled = "未取消"
|
||||
chunks_seen = 0
|
||||
|
||||
async def long_turn():
|
||||
# 必须把整个流消费完才谈得上"取消"。
|
||||
# 第一版写成 `async for _ in ...: return "已开始"`——
|
||||
# 第一个事件就 return，生成器随之关闭，根本没有正在进行的回复可取消，
|
||||
# 测试于是永远"失败"，但失败原因跟取消能力毫无关系。
|
||||
nonlocal chunks_seen
|
||||
async for _ev in engine.respond_to_text("请详细讲一讲中国历史，越长越好。"):
|
||||
chunks_seen += 1
|
||||
return "已完成"
|
||||
|
||||
task = asyncio.create_task(long_turn())
|
||||
await asyncio.sleep(0.7)
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
cancelled = "已取消"
|
||||
check("长回复可被取消", cancelled == "已取消" and chunks_seen > 0,
|
||||
f"{cancelled}，取消前已收到 {chunks_seen} 个事件")
|
||||
|
||||
await engine.end_session()
|
||||
await mem.close()
|
||||
await llm.aclose()
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
print(f"\n合成音频已写入 {OUT}/ ，可人工试听：")
|
||||
for f in sorted(OUT.glob("e2e_*.wav")):
|
||||
print(f" {f.name}")
|
||||
|
||||
print("\n" + "=" * 62)
|
||||
print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}")
|
||||
for n in FAIL:
|
||||
print(f" 未通过: {n}")
|
||||
return 0 if not FAIL else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""身份层测试。
|
||||
|
||||
最关键的一条：**人脸认出来不等于切身份**。
|
||||
方案 §6 写得很直白："看到熟人不等于确认他正在说话"。
|
||||
做错的后果是——路人从摄像头前走过，助手就把你的私人记忆念了出来。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
||||
|
||||
from app.identity import ( # noqa: E402
|
||||
GUEST_USER_ID, IdentityClaim, ManualIdentityProvider,
|
||||
)
|
||||
|
||||
OK, FAIL = [], []
|
||||
|
||||
|
||||
def check(name: str, cond: bool, detail: str = "") -> None:
|
||||
(OK if cond else FAIL).append(name)
|
||||
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print("=== 1. 初始状态 ===")
|
||||
p = ManualIdentityProvider()
|
||||
d = p.current()
|
||||
check("默认是访客", d.user_id == GUEST_USER_ID and d.is_guest, d.user_id)
|
||||
check("访客信任等级独立", d.trust_level == "guest")
|
||||
|
||||
print("\n=== 2. 显式切换（首版唯一可靠入口）===")
|
||||
d = p.switch("u_ylhan", reason="对话确认")
|
||||
check("切换生效", d.user_id == "u_ylhan" and d.confirmed)
|
||||
check("信任等级为 normal", d.trust_level == "normal")
|
||||
check("版本号 +1", p.version == 1, str(p.version))
|
||||
v = p.version
|
||||
p.switch("u_ylhan") # 切到同一个人
|
||||
check("切到同一人不增版本号", p.version == v, f"{v} → {p.version}")
|
||||
|
||||
print("\n=== 3. 人脸候选不得自动切换（关键）===")
|
||||
p2 = ManualIdentityProvider("u_ylhan")
|
||||
before = p2.current().user_id
|
||||
d = p2.resolve([IdentityClaim(source="face", candidate_user_id="u_other",
|
||||
confidence=0.95, track_id="track_7")])
|
||||
check("身份未被自动切换", d.user_id == before, f"{before} → {d.user_id}")
|
||||
check("但仍保留建议", any(s.candidate_user_id == "u_other" for s in d.suggestions),
|
||||
d.reason)
|
||||
check("track_id 未污染 user_id", d.user_id != "track_7")
|
||||
|
||||
# 访客状态下也不应变
|
||||
p3 = ManualIdentityProvider()
|
||||
d3 = p3.resolve([IdentityClaim(source="face", candidate_user_id="u_ylhan",
|
||||
confidence=0.99)])
|
||||
check("访客状态也不自动切换", d3.user_id == GUEST_USER_ID and d3.is_guest, d3.reason)
|
||||
|
||||
print("\n=== 4. 低置信度连建议都不给 ===")
|
||||
p4 = ManualIdentityProvider()
|
||||
d4 = p4.resolve([IdentityClaim(source="face", candidate_user_id="u_x", confidence=0.3)])
|
||||
check("低置信度无建议", len(d4.suggestions) == 0, f"{len(d4.suggestions)} 条")
|
||||
|
||||
print("\n=== 5. 权威来源可以切换 ===")
|
||||
p5 = ManualIdentityProvider()
|
||||
d5 = p5.resolve([IdentityClaim(source="client_login", candidate_user_id="u_phone",
|
||||
confidence=1.0)])
|
||||
check("客户端登录态可切身份", d5.user_id == "u_phone" and d5.confirmed, d5.reason)
|
||||
|
||||
d5b = p5.resolve([IdentityClaim(source="manual", candidate_user_id="u_ylhan")])
|
||||
check("显式来源可切身份", d5b.user_id == "u_ylhan")
|
||||
|
||||
print("\n=== 6. 权威线索互相矛盾时不切 ===")
|
||||
p6 = ManualIdentityProvider("u_ylhan")
|
||||
d6 = p6.resolve([
|
||||
IdentityClaim(source="manual", candidate_user_id="u_a", confidence=1.0),
|
||||
IdentityClaim(source="client_login", candidate_user_id="u_b", confidence=1.0),
|
||||
])
|
||||
check("矛盾时保持原身份", d6.user_id == "u_ylhan", d6.reason)
|
||||
|
||||
print("\n=== 7. reset 回访客 ===")
|
||||
p7 = ManualIdentityProvider("u_ylhan")
|
||||
v7 = p7.version
|
||||
d7 = p7.reset()
|
||||
check("回到访客", d7.user_id == GUEST_USER_ID and d7.is_guest)
|
||||
check("版本号 +1（上层据此丢弃旧上下文）", p7.version == v7 + 1,
|
||||
f"{v7} → {p7.version}")
|
||||
|
||||
print("\n=== 8. 空 user_id 应报错而非静默接受 ===")
|
||||
try:
|
||||
p7.switch("")
|
||||
check("拒绝空 user_id", False, "没报错")
|
||||
except ValueError:
|
||||
check("拒绝空 user_id", True)
|
||||
|
||||
print("\n" + "=" * 62)
|
||||
print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}")
|
||||
for n in FAIL:
|
||||
print(f" 未通过: {n}")
|
||||
return 0 if not FAIL else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LLM 抽象层测试。
|
||||
|
||||
重点验证实测踩过的三个坑是否真的被吸收：
|
||||
1. 思考字段名差异（reasoning / reasoning_content）
|
||||
2. 关闭思考的方式差异
|
||||
3. 多条 system 消息归一化
|
||||
|
||||
运行：
|
||||
~/venvs/ai_assistant/bin/python -m app.tests.test_llm
|
||||
或
|
||||
cd <项目根> && ~/venvs/ai_assistant/bin/python app/tests/test_llm.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
||||
|
||||
from app.config import AppConfig # noqa: E402
|
||||
from app.llm import ( # noqa: E402
|
||||
ChunkKind, DeepSeekProvider, FallbackProvider, LocalVLLMProvider, create_provider,
|
||||
)
|
||||
from app.llm.base import LLMProvider # noqa: E402
|
||||
|
||||
OK, FAIL = [], []
|
||||
|
||||
|
||||
def check(name: str, cond: bool, detail: str = "") -> None:
|
||||
(OK if cond else FAIL).append(name)
|
||||
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 纯本地逻辑（无需网络）
|
||||
|
||||
def test_normalize_messages() -> None:
|
||||
print("\n=== 1. 消息归一化（多条 system → 一条，置于开头）===")
|
||||
msgs = [
|
||||
{"role": "system", "content": "你是助手。"},
|
||||
{"role": "user", "content": "你好"},
|
||||
{"role": "system", "content": "记忆：用户叫 ylhan。"},
|
||||
]
|
||||
out = LLMProvider.normalize_messages(msgs)
|
||||
check("只剩一条 system", sum(1 for m in out if m["role"] == "system") == 1,
|
||||
f"实际 {sum(1 for m in out if m['role']=='system')} 条")
|
||||
check("system 在开头", out[0]["role"] == "system")
|
||||
check("两条 system 内容都保留",
|
||||
"你是助手。" in out[0]["content"] and "ylhan" in out[0]["content"])
|
||||
check("非 system 消息顺序不变",
|
||||
[m["role"] for m in out[1:]] == ["user"])
|
||||
|
||||
# 无 system 的情况不应凭空造一条
|
||||
out2 = LLMProvider.normalize_messages([{"role": "user", "content": "hi"}])
|
||||
check("无 system 时不新增", len(out2) == 1 and out2[0]["role"] == "user")
|
||||
|
||||
# 空 system 应被丢弃
|
||||
out3 = LLMProvider.normalize_messages(
|
||||
[{"role": "system", "content": " "}, {"role": "user", "content": "hi"}])
|
||||
check("空白 system 被丢弃", len(out3) == 1)
|
||||
|
||||
|
||||
def test_request_bodies() -> None:
|
||||
print("\n=== 2. 各后端的请求体（关思考方式不同）===")
|
||||
cfg_local = AppConfig.load().llm
|
||||
cfg_ds = AppConfig.load().llm_fallback
|
||||
cfg_local.thinking = False
|
||||
cfg_ds.thinking = False
|
||||
|
||||
b_local = LocalVLLMProvider(cfg_local)._build_body(
|
||||
[{"role": "user", "content": "hi"}], tools=None, max_tokens=100)
|
||||
check("本机用 chat_template_kwargs",
|
||||
b_local.get("chat_template_kwargs") == {"enable_thinking": False},
|
||||
str(b_local.get("chat_template_kwargs")))
|
||||
|
||||
b_ds = DeepSeekProvider(cfg_ds)._build_body(
|
||||
[{"role": "user", "content": "hi"}], tools=None, max_tokens=100)
|
||||
check("DeepSeek 用 thinking.type",
|
||||
b_ds.get("thinking") == {"type": "disabled"},
|
||||
str(b_ds.get("thinking")))
|
||||
|
||||
check("两者模型名不同", b_local["model"] != b_ds["model"],
|
||||
f"{b_local['model']} vs {b_ds['model']}")
|
||||
|
||||
|
||||
def test_sse_parsing() -> None:
|
||||
print("\n=== 3. SSE 解析：两种思考字段名都认 ===")
|
||||
|
||||
class FakeResp:
|
||||
def __init__(self, lines): self.content = self._gen(lines)
|
||||
async def _gen(self, lines):
|
||||
for l in lines:
|
||||
yield (l + "\n").encode()
|
||||
|
||||
async def run(lines):
|
||||
return [c async for c in LLMProvider._iter_sse(FakeResp(lines))]
|
||||
|
||||
async def main():
|
||||
# reasoning_content（DeepSeek 写法）
|
||||
c1 = await run([
|
||||
'data: {"choices":[{"delta":{"reasoning_content":"想一想"}}]}',
|
||||
'data: {"choices":[{"delta":{"content":"你好"}}]}',
|
||||
'data: [DONE]',
|
||||
])
|
||||
check("认 reasoning_content",
|
||||
any(c.kind == ChunkKind.REASONING and c.text == "想一想" for c in c1))
|
||||
check("content 正常", any(c.kind == ChunkKind.CONTENT and c.text == "你好" for c in c1))
|
||||
|
||||
# reasoning（本机 vLLM 写法）
|
||||
c2 = await run([
|
||||
'data: {"choices":[{"delta":{"reasoning":"想一下"}}]}',
|
||||
'data: {"choices":[{"delta":{"content":"在的"}}]}',
|
||||
'data: [DONE]',
|
||||
])
|
||||
check("认 reasoning（本机 vLLM 字段）",
|
||||
any(c.kind == ChunkKind.REASONING and c.text == "想一下" for c in c2))
|
||||
|
||||
# 工具调用分片累积
|
||||
c3 = await run([
|
||||
'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","function":{"name":"set_light","arguments":"{\\"ro"}}]}}]}',
|
||||
'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"om\\":\\"客厅\\"}"}}]}}]}',
|
||||
'data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}',
|
||||
])
|
||||
tc = [c for c in c3 if c.kind == ChunkKind.TOOL_CALL]
|
||||
check("工具调用参数分片拼接", tc and tc[-1].tool_arguments == '{"room":"客厅"}',
|
||||
tc[-1].tool_arguments if tc else "无")
|
||||
check("工具名保留", tc and tc[-1].tool_name == "set_light")
|
||||
|
||||
# 坏帧不应崩溃
|
||||
c4 = await run(['data: {坏json', 'data: {"choices":[{"delta":{"content":"ok"}}]}', 'data: [DONE]'])
|
||||
check("坏帧被跳过而非崩溃",
|
||||
any(c.kind == ChunkKind.CONTENT and c.text == "ok" for c in c4))
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 需要网络
|
||||
|
||||
async def test_live() -> None:
|
||||
cfg = AppConfig.load()
|
||||
provider = create_provider(cfg.llm)
|
||||
print(f"\n=== 4. 实连本机 vLLM（{cfg.llm.model} @ {cfg.llm.base_url}）===")
|
||||
ok, detail = await provider.health()
|
||||
check("健康检查", ok, detail)
|
||||
if not ok:
|
||||
await provider.aclose()
|
||||
return
|
||||
|
||||
msgs = [
|
||||
{"role": "system", "content":
|
||||
"你是桌面陪伴助手，回复必须简短自然、适合朗读，不超过两句话，"
|
||||
"不要用 Markdown 或表情符号。"},
|
||||
{"role": "user", "content": "今天天气怎么样"},
|
||||
{"role": "assistant", "content": "今天北京晴，十八到二十六度。"},
|
||||
{"role": "user", "content": "那明天呢"},
|
||||
]
|
||||
res = await provider.complete(msgs, max_tokens=120)
|
||||
print(f" 回答: {res.content.strip()[:70]!r}")
|
||||
print(f" 首token(回答)={res.first_content_ms:.0f}ms 总={res.total_ms:.0f}ms")
|
||||
print(f" 思考={len(res.reasoning)}字 完成tok={res.usage.get('completion_tokens') if res.usage else '?'}")
|
||||
check("有回答内容", bool(res.content.strip()))
|
||||
check("关思考后无思考内容", len(res.reasoning) == 0, f"{len(res.reasoning)} 字")
|
||||
check("首token在 1 秒内", (res.first_content_ms or 9e9) < 1000,
|
||||
f"{res.first_content_ms:.0f}ms")
|
||||
check("回复足够短", len(res.content) < 100, f"{len(res.content)} 字")
|
||||
|
||||
# 工具调用
|
||||
tools = [{"type": "function", "function": {
|
||||
"name": "set_light", "description": "开关指定房间的灯",
|
||||
"parameters": {"type": "object", "properties": {
|
||||
"room": {"type": "string"}, "on": {"type": "boolean"}},
|
||||
"required": ["room", "on"]}}}]
|
||||
res2 = await provider.complete([{"role": "user", "content": "把客厅的灯打开"}],
|
||||
tools=tools, max_tokens=128)
|
||||
check("工具调用正确", res2.has_tool_calls and
|
||||
res2.tool_calls[0].parsed_arguments().get("room") == "客厅",
|
||||
f"{res2.tool_calls[0].name if res2.tool_calls else '无'}")
|
||||
|
||||
# 取消：模拟打断
|
||||
async def cancel_midway():
|
||||
async for _ch in provider.stream(
|
||||
[{"role": "user", "content": "请详细讲一下中国历史"}], max_tokens=800):
|
||||
return "cancelled"
|
||||
return "finished"
|
||||
|
||||
got = await asyncio.wait_for(cancel_midway(), timeout=30)
|
||||
check("可中途取消（打断）", got == "cancelled", got)
|
||||
await provider.aclose()
|
||||
|
||||
|
||||
async def test_fallback_logic() -> None:
|
||||
print("\n=== 5. 降级逻辑（不发真实请求）===")
|
||||
cfg = AppConfig.load()
|
||||
|
||||
class BoomProvider(LocalVLLMProvider):
|
||||
def __init__(self, c, err): super().__init__(c); self._err = err
|
||||
async def stream(self, *a, **kw):
|
||||
raise self._err
|
||||
yield # noqa: 使其成为生成器
|
||||
|
||||
fallback = DeepSeekProvider(cfg.llm_fallback)
|
||||
# 连接类错误应降级
|
||||
from app.llm.base import LLMError
|
||||
p = FallbackProvider(BoomProvider(cfg.llm, ConnectionError("连不上")), fallback)
|
||||
check("连接错误判为可降级", p._should_fallback(ConnectionError("x")))
|
||||
check("超时判为可降级", p._should_fallback(TimeoutError("x")))
|
||||
check("503 判为可降级", p._should_fallback(LLMError("local HTTP 503: busy")))
|
||||
check("400 判为不可降级", not p._should_fallback(LLMError("local HTTP 400: bad")))
|
||||
check("401 判为不可降级", not p._should_fallback(LLMError("local HTTP 401: auth")))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
test_normalize_messages()
|
||||
test_request_bodies()
|
||||
test_sse_parsing()
|
||||
asyncio.run(test_live())
|
||||
asyncio.run(test_fallback_logic())
|
||||
|
||||
print("\n" + "=" * 62)
|
||||
print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}")
|
||||
for n in FAIL:
|
||||
print(f" 未通过: {n}")
|
||||
return 0 if not FAIL else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""记忆层测试。
|
||||
|
||||
重点不是"能存能取"，而是方案 §8 反复强调的隔离性质：
|
||||
- 未知身份不碰私人记忆
|
||||
- 删别人的记忆删不掉
|
||||
- 自动抽取的候选不参与检索
|
||||
- 删用户要连会话和摘要一起删干净
|
||||
|
||||
这些是"错了很难发现、发现了很严重"的类型，所以单独测。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
||||
|
||||
from app.memory import ( # noqa: E402
|
||||
ConfirmStatus, Memory, MemoryKind, Scope, SQLiteMemoryStore, Turn,
|
||||
)
|
||||
from app.memory.sqlite_store import FALLBACK_MAX_CORPUS # noqa: E402
|
||||
|
||||
OK, FAIL = [], []
|
||||
|
||||
|
||||
def check(name: str, cond: bool, detail: str = "") -> None:
|
||||
(OK if cond else FAIL).append(name)
|
||||
print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
||||
|
||||
|
||||
async def run(store: SQLiteMemoryStore) -> None:
|
||||
await store.open()
|
||||
|
||||
print("\n=== 1. 基本读写 ===")
|
||||
await store.ensure_user("u_ylhan", "ylhan")
|
||||
await store.ensure_user("u_guest", "访客")
|
||||
mid = await store.remember(Memory(
|
||||
owner_user_id="u_ylhan", kind=MemoryKind.PREFERENCE,
|
||||
key="回复风格", value="喜欢简短直接，不要长篇解释"))
|
||||
check("写入返回 memory_id", isinstance(mid, int) and mid > 0, str(mid))
|
||||
got = await store.retrieve("u_ylhan", "回复风格")
|
||||
check("能检索到", len(got) == 1 and got[0].key == "回复风格")
|
||||
check("字段完整", got[0].kind == MemoryKind.PREFERENCE and got[0].value.startswith("喜欢"))
|
||||
|
||||
print("\n=== 2. 身份隔离（最关键）===")
|
||||
other = await store.retrieve("u_guest", "回复风格")
|
||||
check("别人检索不到", len(other) == 0, f"拿到 {len(other)} 条")
|
||||
listed = await store.list_memories("u_guest")
|
||||
check("别人列不出", len(listed) == 0)
|
||||
|
||||
# 用 memory_id 越权删
|
||||
deleted = await store.forget("u_guest", mid)
|
||||
check("越权删除失败", deleted is False)
|
||||
still = await store.retrieve("u_ylhan", "回复风格")
|
||||
check("原主记忆还在", len(still) == 1)
|
||||
|
||||
print("\n=== 3. 同键新值替换旧值 ===")
|
||||
await store.remember(Memory(
|
||||
owner_user_id="u_ylhan", kind=MemoryKind.PREFERENCE,
|
||||
key="回复风格", value="喜欢更活泼一点的说法"))
|
||||
rows = await store.retrieve("u_ylhan", "回复风格")
|
||||
check("同键只剩一条", len(rows) == 1, f"{len(rows)} 条")
|
||||
check("值已更新", "活泼" in rows[0].value, rows[0].value)
|
||||
|
||||
# 不同 key 不应互相覆盖
|
||||
await store.remember(Memory(owner_user_id="u_ylhan", kind=MemoryKind.PREFERENCE,
|
||||
key="称呼", value="叫他老韩"))
|
||||
rows2 = await store.list_memories("u_ylhan")
|
||||
check("不同 key 各自保留", len(rows2) == 2, f"{len(rows2)} 条")
|
||||
|
||||
print("\n=== 4. 候选记忆默认不参与检索 ===")
|
||||
# 判据是"这条候选不在结果里"，**不是"结果为空"**。
|
||||
# 后者是实现细节：检索在关键词不命中时会退回最近记忆
|
||||
# （见 sqlite_store.FALLBACK_MAX_CORPUS），结果本来就可能非空。
|
||||
CAND = "用户好像养了一只叫豆豆的猫"
|
||||
await store.remember(Memory(
|
||||
owner_user_id="u_ylhan", kind=MemoryKind.FACT, key="", value=CAND,
|
||||
confirm_status=ConfirmStatus.CANDIDATE))
|
||||
hit = await store.retrieve("u_ylhan", "豆豆")
|
||||
check("候选不出现在检索结果里", all(m.value != CAND for m in hit),
|
||||
f"拿到 {len(hit)} 条：{[m.value[:12] for m in hit]}")
|
||||
hit2 = await store.retrieve("u_ylhan", "豆豆", include_candidates=True)
|
||||
check("显式要求时可检索到", any(m.value == CAND for m in hit2),
|
||||
f"拿到 {len(hit2)} 条")
|
||||
allm = await store.list_memories("u_ylhan")
|
||||
check("候选确实已落库", any(m.confirm_status == ConfirmStatus.CANDIDATE for m in allm))
|
||||
|
||||
print("\n=== 5. 过期 ===")
|
||||
EXPIRED = "十分钟后提醒我关火"
|
||||
await store.remember(Memory(
|
||||
owner_user_id="u_ylhan", kind=MemoryKind.TASK, key="", value=EXPIRED,
|
||||
expires_at=time.time() - 1)) # 已过期
|
||||
expired = await store.retrieve("u_ylhan", "关火")
|
||||
check("过期记忆不出现在结果里", all(m.value != EXPIRED for m in expired),
|
||||
f"拿到 {len(expired)} 条")
|
||||
check("过期记忆也不在列表里",
|
||||
all(m.value != EXPIRED for m in await store.list_memories("u_ylhan")))
|
||||
|
||||
print("\n=== 6. 会话与轮次 ===")
|
||||
await store.start_session("sess_1", "u_ylhan")
|
||||
for i, (role, text) in enumerate([("user", "今天天气怎么样"),
|
||||
("assistant", "今天晴，十八到二十六度。")]):
|
||||
await store.add_turn(Turn(turn_id=f"t{i}", session_id="sess_1",
|
||||
user_id="u_ylhan", role=role, text=text))
|
||||
check("会话与轮次写入无异常", True)
|
||||
|
||||
print("\n=== 7. 中文检索打分 ===")
|
||||
r = SQLiteMemoryStore.relevance("客厅的灯", "客厅有一盏米家吸顶灯")
|
||||
check("相关文本有分", r > 0.3, f"{r:.2f}")
|
||||
r2 = SQLiteMemoryStore.relevance("客厅的灯", "喜欢听爵士乐")
|
||||
check("无关文本为 0", r2 == 0.0, f"{r2:.2f}")
|
||||
r3 = SQLiteMemoryStore.relevance("回复风格", "回复风格：喜欢简短")
|
||||
check("子串直接命中", r3 == 1.0, f"{r3:.2f}")
|
||||
|
||||
print("\n=== 8. 删除用户要清干净 ===")
|
||||
n = await store.purge_user("u_ylhan")
|
||||
check("返回删除的记忆数", n >= 3, f"{n} 条")
|
||||
check("记忆已清空", len(await store.list_memories("u_ylhan")) == 0)
|
||||
check("检索也拿不到", len(await store.retrieve("u_ylhan", "回复风格")) == 0)
|
||||
# 会话和轮次靠外键级联删除
|
||||
rows = await store._run(lambda c: c.execute(
|
||||
"SELECT COUNT(*) AS n FROM turns WHERE user_id=?", ("u_ylhan",)).fetchone()["n"])
|
||||
check("轮次一并删除", rows == 0, f"残留 {rows} 条")
|
||||
rows2 = await store._run(lambda c: c.execute(
|
||||
"SELECT COUNT(*) AS n FROM sessions WHERE user_id=?", ("u_ylhan",)).fetchone()["n"])
|
||||
check("会话一并删除", rows2 == 0, f"残留 {rows2} 条")
|
||||
|
||||
print("\n=== 9. 重复建用户不报错、不覆盖 ===")
|
||||
await store.ensure_user("u_x", "第一次")
|
||||
await store.ensure_user("u_x", "第二次")
|
||||
name = await store._run(lambda c: c.execute(
|
||||
"SELECT display_name FROM users WHERE user_id=?", ("u_x",)).fetchone()["display_name"])
|
||||
check("已存在则不改", name == "第一次", name)
|
||||
|
||||
print("\n=== 10. 兜底检索要限量（否则就成了每轮全量注入）===")
|
||||
await store.ensure_user("u_big")
|
||||
await store.remember(Memory(owner_user_id="u_big", kind=MemoryKind.FACT,
|
||||
value="常喝拿铁"))
|
||||
# "推荐个饮品"和"常喝拿铁"没有任何共同字符，字面打分为 0
|
||||
got = await store.retrieve("u_big", "推荐个饮品")
|
||||
check("记忆少时兜底返回最近一条", len(got) == 1, f"{len(got)} 条")
|
||||
|
||||
for i in range(FALLBACK_MAX_CORPUS + 2):
|
||||
await store.remember(Memory(owner_user_id="u_big", kind=MemoryKind.FACT,
|
||||
value=f"无关记录编号{i}"))
|
||||
got2 = await store.retrieve("u_big", "推荐个饮品")
|
||||
check("记忆多时不再无条件兜底", len(got2) == 0, f"{len(got2)} 条")
|
||||
got3 = await store.retrieve("u_big", "拿铁")
|
||||
check("关键词命中仍生效", any("拿铁" in m.value for m in got3), f"{len(got3)} 条")
|
||||
|
||||
await store.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
tmp = Path(tempfile.mkdtemp(prefix="memtest_"))
|
||||
try:
|
||||
store = SQLiteMemoryStore(tmp / "memory.db")
|
||||
asyncio.run(run(store))
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
print("\n" + "=" * 62)
|
||||
print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}")
|
||||
for n in FAIL:
|
||||
print(f" 未通过: {n}")
|
||||
return 0 if not FAIL else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
"""语音层：STT / TTS。
|
||||
|
||||
TTS 有两个后端，接口一致（synthesize / synthesize_sentences），
|
||||
切换只需改配置 AI_TTS_BACKEND：
|
||||
|
||||
local — sherpa-onnx vits-zh-ll，RTF 0.09，很快但音色用户不满意
|
||||
remote — Qwen3-TTS 独立服务，RTF 0.55–0.94，音质是另一档
|
||||
|
||||
为什么 remote 要走 HTTP 而不是进程内：qwen-tts 与助手/vllm-omni 的依赖互斥
|
||||
（transformers 4.57 vs 5.14、hf_hub 0.36 vs 1.31），同 venv 会互相破坏。
|
||||
|
||||
模型文件不随代码走（体积大、可重新下载），由 VoiceConfig.model_dir 指向。
|
||||
缺失时构造对象直接报错并提示下载脚本，不静默降级——静默降级会让
|
||||
"没声音"变成一个很难查的问题。
|
||||
"""
|
||||
|
||||
from .stt import STTError, STTStream, StreamingSTT
|
||||
from .tts import MAX_CHARS_PER_SEGMENT, LocalTTS, TTSError, split_sentences
|
||||
from .remote_tts import RemoteTTS, RemoteTTSError
|
||||
|
||||
|
||||
def create_tts(cfg):
|
||||
"""按配置造 TTS。返回对象统一提供 synthesize / synthesize_sentences。"""
|
||||
if cfg.tts_backend == "remote":
|
||||
return RemoteTTS(cfg.tts_service_url, voice=cfg.tts_speaker,
|
||||
instructions=cfg.tts_instructions,
|
||||
language=cfg.tts_language)
|
||||
return LocalTTS(cfg)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"StreamingSTT", "STTStream", "STTError",
|
||||
"LocalTTS", "TTSError", "split_sentences", "MAX_CHARS_PER_SEGMENT",
|
||||
"RemoteTTS", "RemoteTTSError", "create_tts",
|
||||
]
|
||||
@@ -0,0 +1,138 @@
|
||||
"""远端 TTS 适配器（HTTP 调 Qwen3-TTS 服务）。
|
||||
|
||||
**为什么是 HTTP 而不是进程内调用**：qwen-tts 和 vllm-omni/助手的依赖互斥
|
||||
（transformers 4.57 vs 5.14、hf_hub 0.36 vs 1.31），装一个 venv 里会互相破坏，
|
||||
实测把 vllm-omni 搞崩过。所以 TTS 跑在独立进程，用 HTTP 通信。
|
||||
|
||||
接口和 `LocalTTS` 保持一致（同步的 synthesize / synthesize_sentences），
|
||||
这样引擎那边一行不用改——切换本地 VITS 和远端 Qwen3-TTS 只是换一个类。
|
||||
|
||||
实测延迟（RTX 3090，Serena 音色，非流式）：
|
||||
|
||||
文本长度 音频 合成耗时 RTF
|
||||
8 字 1.52s 1429ms 0.94 ← 短句最差
|
||||
19 字 3.36s 1881ms 0.56
|
||||
26 字 4.64s 2527ms 0.55
|
||||
|
||||
**注意短句 RTF 反而最差**：每次调用有约 1 秒的固定开销，
|
||||
与文本长度无关（talker 模型自回归解码的启动成本 + 声码器解码）。
|
||||
所以"让模型先说个短句"并不能降低首音延迟——反而比中长句更不划算。
|
||||
|
||||
流式可以绕开这个固定开销，但 qwen-tts 的封装把 codes→音频 的解码放在
|
||||
全部 codes 生成完之后，拿不到增量。要走流式需要接管底层解码循环。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import wave
|
||||
from typing import Iterator
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .tts import split_sentences # 复用分句逻辑
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RemoteTTSError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class RemoteTTS:
|
||||
"""通过 HTTP 调用远端 TTS 服务。接口与 LocalTTS 一致。"""
|
||||
|
||||
def __init__(self, base_url: str, *, voice: str = "Serena",
|
||||
instructions: str = "用轻快活泼的语气说",
|
||||
language: str = "Chinese", timeout_s: float = 60.0):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.voice = voice
|
||||
self.instructions = instructions
|
||||
self.language = language
|
||||
self.timeout_s = timeout_s
|
||||
self.sample_rate = 24000 # Qwen3-TTS 输出 24kHz，稍后按需重采样
|
||||
self._checked = False
|
||||
|
||||
# ------------------------------------------------------------ 健康检查
|
||||
|
||||
def health(self) -> tuple[bool, str]:
|
||||
try:
|
||||
with urllib.request.urlopen(f"{self.base_url}/health",
|
||||
timeout=10) as r:
|
||||
d = json.loads(r.read().decode())
|
||||
if d.get("model_loaded"):
|
||||
return True, f"模型已加载，默认音色 {d.get('default_voice')}"
|
||||
return False, f"服务在但模型未加载：{d.get('status')}"
|
||||
except Exception as e: # noqa: BLE001
|
||||
return False, f"{type(e).__name__}: {e}"
|
||||
|
||||
def list_voices(self) -> dict:
|
||||
with urllib.request.urlopen(f"{self.base_url}/v1/voices", timeout=10) as r:
|
||||
return json.loads(r.read().decode())
|
||||
|
||||
# ------------------------------------------------------------ 合成
|
||||
|
||||
def synthesize(self, text: str, *, instructions: str | None = None) -> np.ndarray:
|
||||
"""合成一段文本，返回 float32 单声道。"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
|
||||
payload = json.dumps({
|
||||
"input": text,
|
||||
"voice": self.voice,
|
||||
"instructions": instructions if instructions is not None else self.instructions,
|
||||
"language": self.language,
|
||||
"response_format": "wav",
|
||||
}).encode("utf-8")
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{self.base_url}/v1/audio/speech",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=self.timeout_s) as r:
|
||||
wav_bytes = r.read()
|
||||
sr = int(r.headers.get("X-Sample-Rate", self.sample_rate))
|
||||
gen_ms = r.headers.get("X-Gen-Ms", "?")
|
||||
rtf = r.headers.get("X-RTF", "?")
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = e.read().decode("utf-8", "replace")[:200]
|
||||
raise RemoteTTSError(f"TTS 服务 HTTP {e.code}: {detail}") from e
|
||||
except Exception as e: # noqa: BLE001
|
||||
raise RemoteTTSError(f"TTS 服务不可达：{type(e).__name__}: {e}") from e
|
||||
|
||||
self.sample_rate = sr
|
||||
log.debug("合成 %d 字：服务端 %sms（RTF %s），往返 %.0fms",
|
||||
len(text), gen_ms, rtf, (time.perf_counter() - t0) * 1000)
|
||||
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as w:
|
||||
n = w.getnframes()
|
||||
ch = w.getnchannels()
|
||||
pcm = w.readframes(n)
|
||||
audio = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
if ch > 1: # 服务端是单声道，防御一下
|
||||
audio = audio.reshape(-1, ch).mean(axis=1)
|
||||
return audio
|
||||
|
||||
def synthesize_sentences(self, text: str) -> Iterator[tuple[str, np.ndarray]]:
|
||||
"""按句合成，逐句返回。调用方拿到第一句就能开播。"""
|
||||
for seg in split_sentences(text):
|
||||
yield seg, self.synthesize(seg)
|
||||
|
||||
def synthesize_all(self, text: str) -> np.ndarray:
|
||||
chunks = [a for _, a in self.synthesize_sentences(text) if len(a)]
|
||||
if not chunks:
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
return np.concatenate(chunks)
|
||||
|
||||
|
||||
__all__ = ["RemoteTTS", "RemoteTTSError"]
|
||||
@@ -0,0 +1,125 @@
|
||||
"""本地流式 STT（sherpa-onnx Zipformer 中英双语）。
|
||||
|
||||
实测（P0-语音基准报告.md）：
|
||||
|
||||
N5095（4核，无 AVX2） RTF 0.22 @2线程 ／ 0.86 @4线程
|
||||
服务器（i9-14900K） RTF 0.03
|
||||
|
||||
**线程数是实测出来的，不能按"核多给多"猜。** N5095 上 4 线程比 2 线程
|
||||
慢 3.9 倍（小算子多，每算子一次线程同步，线程越多同步开销越大）。
|
||||
所以线程数由配置决定，不写死。
|
||||
|
||||
另外两条实测结论写进了代码：
|
||||
|
||||
* **冲刷耗时 ≈ 0**：流式解码过程中已持续输出，`input_finished()` 后
|
||||
立即拿到最终结果。"说完到出结果"的开销在 VAD 判定，不在识别器。
|
||||
* **并非每块都解码**：约每 3 块解一次（模型 chunk 约 300ms）。
|
||||
所以单块耗时偶尔超过块长是正常的，判断能否实时要看 RTF 而不是单块。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
|
||||
from ..config import VoiceConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class STTError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class StreamingSTT:
|
||||
"""流式识别器。线程安全地创建多条流（多客户端场景）。"""
|
||||
|
||||
def __init__(self, cfg: VoiceConfig):
|
||||
self.cfg = cfg
|
||||
d = Path(cfg.model_dir) / "stt"
|
||||
files = {
|
||||
"tokens": d / "tokens.txt",
|
||||
"encoder": d / "encoder-epoch-99-avg-1.int8.onnx",
|
||||
"decoder": d / "decoder-epoch-99-avg-1.int8.onnx",
|
||||
"joiner": d / "joiner-epoch-99-avg-1.int8.onnx",
|
||||
}
|
||||
missing = [k for k, p in files.items() if not p.is_file()]
|
||||
if missing:
|
||||
raise STTError(
|
||||
f"STT 模型文件缺失：{missing}（目录 {d}）。"
|
||||
f"先跑 p0/stt_tts_bench/download_models.sh")
|
||||
|
||||
self._rec = sherpa_onnx.OnlineRecognizer.from_transducer(
|
||||
tokens=str(files["tokens"]),
|
||||
encoder=str(files["encoder"]),
|
||||
decoder=str(files["decoder"]),
|
||||
joiner=str(files["joiner"]),
|
||||
num_threads=cfg.stt_threads,
|
||||
sample_rate=cfg.sample_rate,
|
||||
feature_dim=80,
|
||||
decoding_method="greedy_search",
|
||||
# 端点检测留给上层的 VAD 处理：方案 §5 要求
|
||||
# "语音活动检测、说完判断、唤醒是不同环节"，不合并。
|
||||
enable_endpoint_detection=False,
|
||||
)
|
||||
log.info("STT 就绪：%d 线程，采样率 %d", cfg.stt_threads, cfg.sample_rate)
|
||||
|
||||
def new_stream(self) -> "STTStream":
|
||||
return STTStream(self, self._rec.create_stream())
|
||||
|
||||
|
||||
class STTStream:
|
||||
"""一条识别流。"""
|
||||
|
||||
def __init__(self, owner: StreamingSTT, stream):
|
||||
self._owner = owner
|
||||
self._s = stream
|
||||
self._sr = owner.cfg.sample_rate
|
||||
self._consumed = 0
|
||||
|
||||
def accept(self, samples: np.ndarray) -> None:
|
||||
"""喂入 float32 单声道音频（取值 [-1,1]）。"""
|
||||
if samples.dtype != np.float32:
|
||||
samples = samples.astype(np.float32)
|
||||
self._s.accept_waveform(self._sr, samples)
|
||||
self._consumed += len(samples)
|
||||
rec = self._owner._rec
|
||||
while rec.is_ready(self._s):
|
||||
rec.decode_stream(self._s)
|
||||
|
||||
def accept_pcm16(self, pcm: bytes) -> None:
|
||||
"""喂入 16bit PCM 字节流（麦克风常见格式）。"""
|
||||
self.accept(np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0)
|
||||
|
||||
@property
|
||||
def result(self) -> str:
|
||||
"""当前结果。流式过程中会不断变长，说完后即为最终结果。"""
|
||||
return self._owner._rec.get_result(self._s)
|
||||
|
||||
@property
|
||||
def consumed_seconds(self) -> float:
|
||||
"""已喂入的音频时长。用于判断"喂多少音频才出第一个字"（算法延迟）。"""
|
||||
return self._consumed / self._sr
|
||||
|
||||
def finish(self) -> str:
|
||||
"""结束输入并冲刷。
|
||||
|
||||
实测冲刷耗时 ≈ 0ms——流式解码已持续输出，这里不额外花钱。
|
||||
"""
|
||||
rec = self._owner._rec
|
||||
self._s.input_finished()
|
||||
while rec.is_ready(self._s):
|
||||
rec.decode_stream(self._s)
|
||||
return rec.get_result(self._s)
|
||||
|
||||
def reset(self) -> None:
|
||||
"""丢弃当前流，重新开始（打断后重新听）。"""
|
||||
self._owner._rec.reset(self._s)
|
||||
self._consumed = 0
|
||||
|
||||
|
||||
__all__ = ["StreamingSTT", "STTStream", "STTError"]
|
||||
@@ -0,0 +1,157 @@
|
||||
"""本地 TTS（sherpa-onnx VITS 中文）。
|
||||
|
||||
选型依据（P0-语音基准报告.md §六之二）：
|
||||
|
||||
vits-zh-ll 服务器 RTF 0.09 / N5095 RTF 1.86 ← 用户试听选定，音质最好
|
||||
piper-huayan-med 服务器 0.36 / N5095 0.32 （音质次之）
|
||||
kokoro 1.96 以上，排除
|
||||
|
||||
**两个必须写进代码的实测结论：**
|
||||
|
||||
1. **VITS 是非自回归的，没有"边合成边播"。**
|
||||
实测回调只在合成结束后触发一次，所以首音延迟 == 整句合成耗时。
|
||||
提前出声的唯一手段是**按语义短句逐句合成**——把长回复切成短句，
|
||||
第一句合成完就能播。
|
||||
|
||||
2. **`max_num_sentences=1` 会静默截断多句输入。**
|
||||
传含多个句号的文本只会合成第一句，**不报错**。
|
||||
第一版测"长段"时只出 0.94 秒音频，差点当成正常结果。
|
||||
所以这里必须先分句，逐句送。
|
||||
|
||||
另外按方案 §5 要求，输出是 16kHz 单声道 float32；调用方负责重采样。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
|
||||
from ..config import VoiceConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: 句末标点。中英文都要，方案 §5 要求覆盖中英混读。
|
||||
_SENT_END = "。！？；!?;"
|
||||
#: 从句标点，长句再切一刀，避免单句合成太久
|
||||
_CLAUSE = "，,、：:"
|
||||
|
||||
#: 单句最大字符数。超过就按从句标点再切——
|
||||
#: 句子越长，首音延迟越大（首音 == 整句合成耗时）。
|
||||
MAX_CHARS_PER_SEGMENT = 30
|
||||
|
||||
|
||||
class TTSError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def split_sentences(text: str, *, max_chars: int = MAX_CHARS_PER_SEGMENT) -> list[str]:
|
||||
"""按语义切分，返回可逐句合成的片段。
|
||||
|
||||
先按句末标点切，过长的再按从句标点切，仍过长就硬切。
|
||||
标点保留在片段末尾（TTS 需要它来判断语调）。
|
||||
|
||||
这个函数存在的原因是 `max_num_sentences=1` 会静默丢内容——
|
||||
不自己切，就会"念到一半没了"。
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
|
||||
parts = re.split(rf"(?<=[{_SENT_END}])", text)
|
||||
out: list[str] = []
|
||||
for p in parts:
|
||||
p = p.strip()
|
||||
if not p:
|
||||
continue
|
||||
if len(p) <= max_chars:
|
||||
out.append(p)
|
||||
continue
|
||||
# 过长：按从句标点再切
|
||||
subs = re.split(rf"(?<=[{_CLAUSE}])", p)
|
||||
buf = ""
|
||||
for s in subs:
|
||||
if not s:
|
||||
continue
|
||||
if len(buf) + len(s) <= max_chars:
|
||||
buf += s
|
||||
else:
|
||||
if buf:
|
||||
out.append(buf)
|
||||
# 单从句仍然过长，硬切
|
||||
while len(s) > max_chars:
|
||||
out.append(s[:max_chars])
|
||||
s = s[max_chars:]
|
||||
buf = s
|
||||
if buf:
|
||||
out.append(buf)
|
||||
return out
|
||||
|
||||
|
||||
class LocalTTS:
|
||||
def __init__(self, cfg: VoiceConfig):
|
||||
self.cfg = cfg
|
||||
d = Path(cfg.model_dir) / "tts"
|
||||
root = Path(cfg.model_dir)
|
||||
files = {
|
||||
"model": d / "model.onnx",
|
||||
"lexicon": d / "lexicon.txt",
|
||||
"tokens": d / "tokens.txt",
|
||||
}
|
||||
missing = [k for k, p in files.items() if not p.is_file()]
|
||||
if missing:
|
||||
raise TTSError(
|
||||
f"TTS 模型文件缺失：{missing}（目录 {d}）。"
|
||||
f"先跑 p0/stt_tts_bench/download_models.sh")
|
||||
|
||||
rule_fsts = ",".join(
|
||||
str(d / f) for f in ("date.fst", "number.fst", "phone.fst", "new_heteronym.fst")
|
||||
if (d / f).is_file())
|
||||
|
||||
self._tts = sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig(
|
||||
model=sherpa_onnx.OfflineTtsModelConfig(
|
||||
vits=sherpa_onnx.OfflineTtsVitsModelConfig(
|
||||
model=str(files["model"]),
|
||||
lexicon=str(files["lexicon"]),
|
||||
tokens=str(files["tokens"]),
|
||||
dict_dir=str(d / "dict"),
|
||||
),
|
||||
num_threads=cfg.tts_threads,
|
||||
provider="cpu",
|
||||
),
|
||||
rule_fsts=rule_fsts,
|
||||
# 逐句生成：这是唯一能提前出声的手段（见模块文档）
|
||||
max_num_sentences=1,
|
||||
))
|
||||
self.sample_rate = self._tts.sample_rate
|
||||
log.info("TTS 就绪：%d 线程，输出 %d Hz", cfg.tts_threads, self.sample_rate)
|
||||
|
||||
def synthesize(self, text: str) -> np.ndarray:
|
||||
"""合成一段文本（内部已确保是单句）。"""
|
||||
if not text.strip():
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
audio = self._tts.generate(text, sid=0, speed=1.0)
|
||||
return np.asarray(audio.samples, dtype=np.float32)
|
||||
|
||||
def synthesize_sentences(self, text: str) -> Iterator[tuple[str, np.ndarray]]:
|
||||
"""按句合成，逐句 yield (句子, 音频)。
|
||||
|
||||
调用方拿到第一句就能开播，不必等整段——
|
||||
这正是 VITS 非自回归下唯一的低延迟路径。
|
||||
"""
|
||||
for seg in split_sentences(text):
|
||||
yield seg, self.synthesize(seg)
|
||||
|
||||
def synthesize_all(self, text: str) -> np.ndarray:
|
||||
"""合成整段并拼接。用于离线/测试，不适合实时播放。"""
|
||||
chunks = [a for _, a in self.synthesize_sentences(text) if len(a)]
|
||||
if not chunks:
|
||||
return np.zeros(0, dtype=np.float32)
|
||||
return np.concatenate(chunks)
|
||||
|
||||
|
||||
__all__ = ["LocalTTS", "TTSError", "split_sentences", "MAX_CHARS_PER_SEGMENT"]
|
||||
Reference in New Issue
Block a user