From 35e107c2900f29b0660c0cc472dc798acd7c4594 Mon Sep 17 00:00:00 2001 From: ylhan Date: Tue, 15 Sep 2026 11:05:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A9=E6=89=8B=E6=9C=AC=E4=BD=93=EF=BC=9A?= =?UTF-8?q?=E4=BC=9A=E8=AF=9D=E5=BC=95=E6=93=8E=E3=80=81LLM=20=E8=B7=AF?= =?UTF-8?q?=E7=94=B1=E3=80=81=E8=AE=B0=E5=BF=86=E3=80=81=E8=BA=AB=E4=BB=BD?= =?UTF-8?q?=E3=80=81=E8=AF=AD=E9=9F=B3=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 从 Obsidian 笔记库剥离为独立仓库。规划文档与 P0 报告留在笔记库， 代码（助手本体、TTS 服务、P0 基准脚本、ESP32 固件）在此。 架构：服务器是助手，本地是客户端 - 助手本体跑在 cadadmin-server，N5095 只做 USB 外设桥与网络桥接 - 起因：实测 N5095 无 AVX2，跑不动可用的语音模型 当前技术栈 - LLM：DeepSeek deepseek-flash（云端，必须关思考） - TTS：Qwen3-TTS 1.7B（服务器 3090，独立 HTTP 服务） - STT：Zipformer 中英双语 int8（CPU，中文识别率未测） 实测：端到端 16/16 通过，首音延迟 1545ms （对比：本地 LLM + vits-zh-ll 是 179ms，但音色不满意——音质与延迟 在此硬件上此消彼长） 设计要点 - MemoryStore 接口按"可能远程"设计（全 async、可超时、可失败） - user_id / session_id / track_id 三者分开，人脸只作候选不自动切身份 - 记忆查询强制带 owner_user_id，无按 id 单独取的口子 - 自动抽取的记忆是"候选"，默认不参与检索 - 异步写记忆绑定创建时的身份，不读"当前用户" - 引擎与传输解耦，只用文件音频即可端到端测试 模型与音频产物不进版本管理（1.1GB），由 download_models.sh 恢复 --- .gitignore | 27 ++ README.md | 90 ++++ app/__init__.py | 0 app/config.py | 217 ++++++++++ app/engine.py | 389 ++++++++++++++++++ app/identity/__init__.py | 18 + app/identity/base.py | 110 +++++ app/identity/manual.py | 123 ++++++ app/llm/__init__.py | 116 ++++++ app/llm/base.py | 308 ++++++++++++++ app/llm/deepseek.py | 47 +++ app/llm/local_vllm.py | 46 +++ app/memory/__init__.py | 16 + app/memory/base.py | 185 +++++++++ app/memory/sqlite_store.py | 334 +++++++++++++++ app/prompts.py | 103 +++++ app/tests/__init__.py | 0 app/tests/test_engine_e2e.py | 246 +++++++++++ app/tests/test_identity.py | 105 +++++ app/tests/test_llm.py | 229 +++++++++++ app/tests/test_memory.py | 176 ++++++++ app/voice/__init__.py | 35 ++ app/voice/remote_tts.py | 138 +++++++ app/voice/stt.py | 125 ++++++ app/voice/tts.py | 157 +++++++ esp32_smile_test/esp32_smile_test.ino | 94 +++++ esp32_touch_eyes/esp32_touch_eyes.ino | 141 +++++++ p0/stt_tts_bench/bench_concurrent.py | 210 ++++++++++ p0/stt_tts_bench/bench_roundtrip.py | 138 +++++++ p0/stt_tts_bench/bench_stt.py | 187 +++++++++ p0/stt_tts_bench/bench_threads.py | 73 ++++ p0/stt_tts_bench/bench_tts.py | 149 +++++++ p0/stt_tts_bench/bench_tts_compare.py | 139 +++++++ p0/stt_tts_bench/download_models.sh | 52 +++ p0/stt_tts_bench/hf_fetch.py | 69 ++++ .../results_concurrent_matcha.json | 13 + .../results_concurrent_piper.json | 13 + p0/stt_tts_bench/results_concurrent_vits.json | 13 + p0/stt_tts_bench/results_stt.json | 127 ++++++ p0/stt_tts_bench/results_tts.json | 158 +++++++ p0/stt_tts_bench/results_tts_compare.json | 86 ++++ p0/verify_deepseek.py | 355 ++++++++++++++++ p0/verify_local_llm.py | 257 ++++++++++++ requirements.txt | 13 + services/tts_server.py | 176 ++++++++ 部署说明.md | 173 ++++++++ 46 files changed, 5976 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 app/__init__.py create mode 100644 app/config.py create mode 100644 app/engine.py create mode 100644 app/identity/__init__.py create mode 100644 app/identity/base.py create mode 100644 app/identity/manual.py create mode 100644 app/llm/__init__.py create mode 100644 app/llm/base.py create mode 100644 app/llm/deepseek.py create mode 100644 app/llm/local_vllm.py create mode 100644 app/memory/__init__.py create mode 100644 app/memory/base.py create mode 100644 app/memory/sqlite_store.py create mode 100644 app/prompts.py create mode 100644 app/tests/__init__.py create mode 100644 app/tests/test_engine_e2e.py create mode 100644 app/tests/test_identity.py create mode 100644 app/tests/test_llm.py create mode 100644 app/tests/test_memory.py create mode 100644 app/voice/__init__.py create mode 100644 app/voice/remote_tts.py create mode 100644 app/voice/stt.py create mode 100644 app/voice/tts.py create mode 100644 esp32_smile_test/esp32_smile_test.ino create mode 100644 esp32_touch_eyes/esp32_touch_eyes.ino create mode 100644 p0/stt_tts_bench/bench_concurrent.py create mode 100644 p0/stt_tts_bench/bench_roundtrip.py create mode 100644 p0/stt_tts_bench/bench_stt.py create mode 100644 p0/stt_tts_bench/bench_threads.py create mode 100644 p0/stt_tts_bench/bench_tts.py create mode 100644 p0/stt_tts_bench/bench_tts_compare.py create mode 100755 p0/stt_tts_bench/download_models.sh create mode 100644 p0/stt_tts_bench/hf_fetch.py create mode 100644 p0/stt_tts_bench/results_concurrent_matcha.json create mode 100644 p0/stt_tts_bench/results_concurrent_piper.json create mode 100644 p0/stt_tts_bench/results_concurrent_vits.json create mode 100644 p0/stt_tts_bench/results_stt.json create mode 100644 p0/stt_tts_bench/results_tts.json create mode 100644 p0/stt_tts_bench/results_tts_compare.json create mode 100644 p0/verify_deepseek.py create mode 100644 p0/verify_local_llm.py create mode 100644 requirements.txt create mode 100644 services/tts_server.py create mode 100644 部署说明.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a1b5746 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# 模型与音频产物：体积大、可重新获取，不进版本管理。 +# 重新下载：p0/stt_tts_bench/download_models.sh + +p0/stt_tts_bench/models/ +p0/stt_tts_bench/wavs/ +p0/stt_tts_bench/roundtrip_wav/ +p0/stt_tts_bench/listening_test/ +p0/stt_tts_bench/listening_ab/ +out/ + +# Python +__pycache__/ +*.py[cod] +.venv/ +venv/ +*.egg-info/ +.pytest_cache/ + +# 配置与密钥：本机配置在 ~/.config/ai_assistant/env，永不出现在仓库里 +.env +*.key + +# 编辑器 +.vscode/ +.idea/ +*.swp +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..0725064 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# AI 桌面陪伴助手 · 代码说明 + +**架构一句话**：服务器是助手，本地是客户端。 +助手本体（会话引擎、记忆、身份、STT/TTS 调度）跑在 cadadmin-server； +N5095 只做 USB 外设接入和网络桥接。理由与实测见 `P0-语音基准报告.md`。 + +运行方式与配置见 `部署说明.md`；总体方案见 `AI桌面陪伴助手方案.md`。 + +## 目录结构 + +``` +app/ 助手本体（部署到服务器） +├── config.py 配置。全部可用环境变量覆盖——部署时不改代码 +├── prompts.py 提示词构造。单条 system 每轮重建 +├── engine.py 会话引擎：身份 → 记忆 → LLM → TTS 的核心回路 +├── llm/ LLM 抽象与后端 +│ ├── base.py 接口 + SSE 解析 + 消息归一化 +│ ├── local_vllm.py 本机 vLLM（Qwen3.8-27B，已停用） +│ ├── deepseek.py DeepSeek 云 API（当前主用） +│ └── __init__.py 工厂 + 降级路由 +├── memory/ 记忆 +│ ├── base.py 接口。**按"可能远程"设计**——全 async、可超时、可失败 +│ └── sqlite_store.py SQLite 实现 +├── identity/ 身份 +│ ├── base.py 接口。人脸只提供候选，**不自动切身份** +│ └── manual.py 首版实现：显式切换 + 访客兜底 +├── voice/ 语音 +│ ├── stt.py sherpa-onnx 流式 Zipformer（CPU） +│ ├── tts.py 本地 vits-zh-ll（sherpa-onnx） +│ └── remote_tts.py Qwen3-TTS 远端服务适配（当前主用） +└── tests/ 4 个测试，共 88 项断言，全部无需硬件 + +services/ +└── tts_server.py Qwen3-TTS 服务（跑在 ~/tts-qwen，独立 venv） + +p0/ P0 阶段的验证与基准脚本（保留用于复现） +├── verify_deepseek.py DeepSeek 四条路径 +├── verify_local_llm.py 本机 Qwen3.8-27B +└── stt_tts_bench/ STT/TTS 基准、模型下载脚本、线程扫描 +``` + +## 测试 + +```bash +# 服务器上（完整） +cd ~/ai-companion +.venv/bin/python app/tests/test_llm.py # 27 项 +.venv/bin/python app/tests/test_memory.py # 28 项 +.venv/bin/python app/tests/test_identity.py # 17 项 +.venv/bin/python app/tests/test_engine_e2e.py # 16 项，端到端 +``` + +端到端测试**不需要麦克风、音箱、摄像头、ESP32**—— +这是"服务器即助手"架构的直接好处：链路可以在任何有 CPU 的地方验证， +硬件到货后再接最后一段。跑完会把合成音频写到 `out/`，可以人工试听。 + +## 设计上几个刻意的选择 + +**接口按最坏情况设计。** `MemoryStore` 全是 async、可超时、可失败， +虽然现在背后是本地 SQLite（微秒级）。因为将来多设备共享记忆时要搬到服务器， +如果现在写成同步直连本地文件，那时要重写所有调用方。 + +**身份、会话、视觉追踪三者分开。** `user_id`（永久身份）、`session_id`（一次会话）、 +`track_id`（画面里的一个目标）互不冒充。人脸认出来**不等于**确认对方在说话—— +所以人脸只产生"建议"，不自动切换身份。做错的后果是路人经过就把你的私人记忆念出来。 + +**记忆查询强制带 `owner_user_id`。** 没有"按 memory_id 单独取"的口子： +猜到 id 就能读别人的记忆，是这类系统最容易犯的错。删除同理，必须同时匹配 owner。 + +**自动抽取的记忆是"候选"，默认不参与检索。** 只有用户显式"记住"或工具调用 +写入的才是"已确认"。对应方案 §8 的"模型推测不当作确认事实"。 + +**异步任务绑定创建时的身份。** 回复后写记忆的任务在创建时捕获 `user_id`， +不读"当前用户"——否则用户在这几百毫秒内切了身份，这条记忆会写到别人名下。 + +**引擎与传输解耦。** 不依赖 Pipecat、不依赖任何传输方式，只依赖四个接口。 +好处：可以用文件音频端到端测试；接 WebRTC 时只需写薄适配层。 + +## 踩过的坑 + +全部写在对应文件的注释里，这里只列索引，详见 `部署说明.md` §已知的坑： + +- 本机 vLLM 只接受一条 system 消息（多条 400） +- 思考字段名 vLLM 用 `reasoning`、DeepSeek 用 `reasoning_content` +- DeepSeek 默认开思考，多轮/工具回灌会 400；且思考会吃掉 `max_tokens` +- `max_num_sentences=1` 静默截断多句输入 +- VITS 非自回归，首音 = 整句合成耗时 +- Qwen3-TTS 短句 RTF 反而最差（固定开销约 1 秒） +- qwen-tts 与 vllm-omni 依赖互斥，必须分 venv +- STT 与 TTS 的线程扩展方向相反 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..7bdd402 --- /dev/null +++ b/app/config.py @@ -0,0 +1,217 @@ +"""配置加载。 + +原则（来自方案 §10）： + - 密钥保存在本机配置，不写日志，不进版本管理 + - 设备标识用稳定名字，不用会变的序号 + - 依赖版本固定，板卡/固件/通道/引脚都记录 + +密钥文件是 ~/.config/ai_assistant/env（权限 600），格式是 shell 变量。 +本模块只读它，不写。 +""" + +from __future__ import annotations + +import os +import threading +from dataclasses import dataclass, field +from pathlib import Path + +ENV_FILE = Path.home() / ".config" / "ai_assistant" / "env" + +# 本项目根目录（app/ 的上一级） +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +_lock = threading.Lock() +_loaded = False + + +def load_env(path: Path | None = None) -> dict[str, str]: + """读取密钥文件到 os.environ（已存在的变量不覆盖）。 + + 只做极简解析：KEY=VALUE，忽略空行和 # 注释。 + 不用 os.system/source，避免把 shell 引进来。 + """ + global _loaded + p = path or ENV_FILE + with _lock: + if _loaded: + return dict(os.environ) + if p.is_file(): + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + k, v = k.strip(), v.strip().strip('"').strip("'") + os.environ.setdefault(k, v) + _loaded = True + return dict(os.environ) + + +def _env(key: str, default: str = "") -> str: + load_env() + return os.environ.get(key, default) + + +def _env_int(key: str, default: int) -> int: + try: + return int(_env(key, str(default))) + except ValueError: + return default + + +def _env_float(key: str, default: float) -> float: + try: + return float(_env(key, str(default))) + except ValueError: + return default + + +# ---------------------------------------------------------------- LLM + +@dataclass +class LLMConfig: + """一个 LLM 后端的配置。 + + thinking 的处理方式各后端不同，所以由后端自己解释这个开关： + - 本机 vLLM： chat_template_kwargs={"enable_thinking": false} + - DeepSeek： thinking={"type": "disabled"} + 两者都实测有效，见 P0-本地LLM验证报告.md 与 P0-DeepSeek验证报告.md。 + """ + name: str + base_url: str + api_key: str + model: str + thinking: bool = False # 语音场景默认关思考 + max_tokens: int = 200 # 语音回复要短，方案要求"默认简短自然回复" + timeout_s: float = 30.0 + first_token_timeout_s: float = 8.0 + + @property + def enabled(self) -> bool: + return bool(self.base_url and self.api_key and self.model) + + +def local_llm_config() -> LLMConfig: + """本机/内网 LLM（vLLM 等）。 + + 默认走回环：服务与助手同机时 127.0.0.1 最省一跳。 + 部署到别的机器时必须用 LOCAL_LLM_BASE_URL 显式覆盖—— + **不要把具体的内网地址写进默认值**，那会把网络拓扑带进版本历史， + 而且换环境就得改代码。 + """ + return LLMConfig( + name="local", + base_url=_env("LOCAL_LLM_BASE_URL", "http://127.0.0.1:18020/v1"), + api_key=_env("LOCAL_LLM_API_KEY"), + model=_env("LOCAL_LLM_MODEL", "qwen3.8-27b"), + ) + + +def deepseek_config() -> LLMConfig: + return LLMConfig( + name="deepseek", + base_url=_env("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1"), + api_key=_env("DEEPSEEK_API_KEY"), + model=_env("DEEPSEEK_MODEL", "deepseek-flash"), + ) + + +# ---------------------------------------------------------------- 上下文 + +@dataclass +class ContextConfig: + """上下文预算。 + + 实测：真实语音 prompt（人设+记忆+3轮对话）约 406 tokens， + 本机 LLM 首 token 367ms。上下文越长，首 token 越慢——直接伤"回应及时"。 + 所以这里设的是硬上限，超出就丢最旧的轮次，不是软建议。 + """ + max_prompt_tokens: int = 4096 # 总上限 + max_turns: int = 12 # 最多保留的对话轮次 + max_memories: int = 5 # 单轮最多注入的记忆条数 + memory_timeout_s: float = 0.3 # 检索超时，方案要求"记忆失败不阻塞对话" + + +# ---------------------------------------------------------------- 语音 + +def _default_model_dir() -> Path: + """模型目录。可用环境变量 AI_MODEL_DIR 覆盖。 + + 为什么要有这个覆盖：开发在 N5095，运行在服务器，两边模型放的 + 位置不一样（服务器上模型在 ~/ai_companion_bench/models， + 不在代码目录里）。没有覆盖的话，部署就得改代码—— + 而"改代码才能部署"是部署流程里最容易出错的一环。 + """ + override = _env("AI_MODEL_DIR") + if override: + return Path(override).expanduser() + return PROJECT_ROOT / "p0" / "stt_tts_bench" / "models" + + +@dataclass +class VoiceConfig: + """语音输入输出。 + + 线程数是实测出来的，不能按"核多给多"猜： + - N5095（4核，无AVX2）：STT 超过 2 线程急剧退化，4线程慢 3.9 倍 + - 服务器（i9-14900K，有AVX2）：同样配置下 RTF 0.03 + 所以线程数按部署机器配置，不写死。服务器上可以给足。 + """ + model_dir: Path = field(default_factory=_default_model_dir) + sample_rate: int = 16000 + stt_threads: int = field(default_factory=lambda: _env_int("AI_STT_THREADS", 2)) + tts_threads: int = field(default_factory=lambda: _env_int("AI_TTS_THREADS", 4)) + + # ---- TTS 后端选择 ---- + # "local" = sherpa-onnx vits-zh-ll：RTF 0.09，很快，但音色用户不满意 + # "remote" = Qwen3-TTS 独立服务：RTF 0.55–0.94，音质是另一档，用户试听选定 + tts_backend: str = field(default_factory=lambda: _env("AI_TTS_BACKEND", "remote")) + tts_service_url: str = field( + default_factory=lambda: _env("TTS_SERVICE_URL", "http://127.0.0.1:18040")) + # 用户试听选定的组合（out/tts_qwen/instruct_7849.wav） + tts_speaker: str = field(default_factory=lambda: _env("AI_TTS_SPEAKER", "Serena")) + tts_instructions: str = field( + default_factory=lambda: _env("AI_TTS_INSTRUCTIONS", "用轻快活泼的语气说")) + tts_language: str = field(default_factory=lambda: _env("AI_TTS_LANGUAGE", "Chinese")) + + +# ---------------------------------------------------------------- 记忆 + +@dataclass +class MemoryConfig: + db_path: Path = field(default_factory=lambda: Path.home() / ".local" / "share" / "ai_assistant" / "memory.db") + # 自动抽取的记忆默认是"候选"，不参与检索；只有用户显式确认的才生效。 + # 对应方案 §8："模型推测不当作确认事实" + auto_extract_enabled: bool = True + auto_extract_confirm_status: str = "candidate" + + +# ---------------------------------------------------------------- 汇总 + +@dataclass +class AppConfig: + #: 主 LLM 由 AI_LLM_PRIMARY 选择： + #: "local" = 本机 vLLM（首 token 192ms，但占 21.5GB 显存） + #: "deepseek" = 云端（首 token 786ms，网络往返约 700ms） + #: 2026-09-15 起用 deepseek：本地 27B 已停，GPU 让给 Qwen3-TTS。 + #: 这是个可逆的权衡——音质提升换首音延迟增加约 600ms。 + llm: LLMConfig = field(default_factory=local_llm_config) + llm_fallback: LLMConfig = field(default_factory=deepseek_config) + context: ContextConfig = field(default_factory=ContextConfig) + voice: VoiceConfig = field(default_factory=VoiceConfig) + memory: MemoryConfig = field(default_factory=MemoryConfig) + + @classmethod + def load(cls) -> "AppConfig": + load_env() + primary = _env("AI_LLM_PRIMARY", "local").strip().lower() + if primary == "deepseek": + return cls(llm=deepseek_config(), llm_fallback=local_llm_config()) + return cls() + + +__all__ = [ + "AppConfig", "LLMConfig", "ContextConfig", "VoiceConfig", "MemoryConfig", + "load_env", "local_llm_config", "deepseek_config", "PROJECT_ROOT", "ENV_FILE", +] diff --git a/app/engine.py b/app/engine.py new file mode 100644 index 0000000..cf39f74 --- /dev/null +++ b/app/engine.py @@ -0,0 +1,389 @@ +"""会话引擎：身份 → 记忆 → LLM → TTS 的核心回路。 + +这一层不依赖 Pipecat、不依赖任何传输方式，只依赖四个接口 +（LLMProvider / MemoryStore / IdentityProvider / LocalTTS）。 +好处有两个： + + 1. **可以用文件音频端到端测试**，不需要麦克风、音箱、WebRTC。 + 2. 接 Pipecat 时只需写薄适配层；换传输方式（WebRTC / 本地声卡）不动这里。 + +一轮的完整流程： + + 用户音频 + └→ STT 转写 + └→ 身份裁决（决定用谁的记忆） + └→ 记忆检索（带超时，失败不阻塞） + └→ 拼 system（每轮重建，单条） + └→ LLM 流式 + └→ 按句切分 → 逐句 TTS → 边合成边播 + └→ 回复结束，异步写记忆（绑定创建时的身份） + +延迟的关键在最后两步：**必须按句切分后立刻合成**。 +VITS 是非自回归的，整段合成完才出声；只有切成短句才能提前开口。 +""" + +from __future__ import annotations + +import asyncio +import logging +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import AsyncIterator, Sequence + +import numpy as np + +from .config import AppConfig +from .identity.base import IdentityDecision, IdentityProvider +from .llm.base import LLMProvider, ToolCall +from .memory.base import ( + ConfirmStatus, Memory, MemoryKind, MemoryStore, Scope, Turn, +) +from .prompts import build_system_prompt, trim_history +from .voice.stt import StreamingSTT +from .voice.tts import LocalTTS, split_sentences + +log = logging.getLogger(__name__) + + +class EventKind(str, Enum): + IDENTITY = "identity" # 本轮的身份裁决结果 + MEMORY_USED = "memory_used" # 检索到的记忆 + TRANSCRIPT = "transcript" # STT 结果 + RESPONSE_DELTA = "response_delta" # LLM 增量文本（未播，仅供显示） + SPEAK = "speak" # 一段可播放的音频 + TOOL_CALL = "tool_call" # 模型请求调用工具 + METRICS = "metrics" # 本轮计时 + ERROR = "error" + + +@dataclass +class Event: + kind: EventKind + text: str = "" + audio: np.ndarray | None = None + sample_rate: int = 16000 + identity: IdentityDecision | None = None + memories: list[Memory] = field(default_factory=list) + tool_call: ToolCall | None = None + metrics: "TurnMetrics | None" = None + error: str = "" + + +@dataclass +class TurnMetrics: + """按方案 §10 要求记录耗时，便于定位瓶颈。""" + stt_ms: float = 0.0 + identity_ms: float = 0.0 + retrieve_ms: float = 0.0 + llm_first_content_ms: float = 0.0 + first_audio_ms: float = 0.0 # 从开始处理到第一段音频就绪 + total_ms: float = 0.0 + llm_backend: str = "" + memories_used: int = 0 + reply_chars: int = 0 + speaks: int = 0 + + def summary(self) -> str: + return (f"STT {self.stt_ms:.0f}ms | 身份 {self.identity_ms:.0f}ms | " + f"检索 {self.retrieve_ms:.0f}ms | LLM首token {self.llm_first_content_ms:.0f}ms | " + f"首音 {self.first_audio_ms:.0f}ms | 总 {self.total_ms:.0f}ms | " + f"{self.llm_backend} | 记忆{self.memories_used}条 | {self.speaks}段音频") + + +class ConversationEngine: + def __init__( + self, + cfg: AppConfig, + llm: LLMProvider, + memory: MemoryStore, + identity: IdentityProvider, + tts: LocalTTS, + stt: StreamingSTT | None = None, + ): + self.cfg = cfg + self.llm = llm + self.memory = memory + self.identity = identity + self.tts = tts + self.stt = stt + #: 会话历史。切身份时会清空——方案 §8 要求切换身份 + #: 清理上一人的私有检索上下文。 + self._history: list[dict] = [] + self._history_identity_version: int = -1 + self._session_id: str = "" + self._current_turn: asyncio.Task | None = None + + # ------------------------------------------------------------ 会话 + + async def start_session(self, session_id: str | None = None, + user_id: str | None = None) -> str: + sid = session_id or f"s_{uuid.uuid4().hex[:12]}" + if user_id: + self.identity.switch(user_id, reason="会话指定") + d = self.identity.current() + await self.memory.start_session(sid, d.user_id) + self._session_id = sid + self._history_identity_version = self._identity_version() + log.info("会话开始 %s，身份 %s", sid, d.user_id) + return sid + + async def end_session(self) -> None: + if self._session_id: + await self.memory.end_session(self._session_id) + + def _identity_version(self) -> int: + # ManualIdentityProvider 有 version；其他实现没有时退化为 0 + return getattr(self.identity, "version", 0) + + def _sync_identity_scope(self) -> None: + """身份变了就丢弃历史。 + + 不做这件事的后果：切到新用户后，模型仍看得到上一人的对话， + 第一条回复就可能带出别人的信息。 + """ + v = self._identity_version() + if v != self._history_identity_version: + if self._history: + log.info("身份变更，丢弃 %d 条历史消息", len(self._history)) + self._history = [] + self._history_identity_version = v + + # ------------------------------------------------------------ 主回路 + + async def respond_to_text(self, user_text: str, *, + session_id: str | None = None) -> AsyncIterator[Event]: + """处理一句用户输入，产出事件流。""" + t_start = time.perf_counter() + metrics = TurnMetrics() + sid = session_id or self._session_id or await self.start_session() + + self._sync_identity_scope() + identity = self.identity.current() + metrics.identity_ms = (time.perf_counter() - t_start) * 1000 + yield Event(kind=EventKind.IDENTITY, identity=identity) + + if not user_text.strip(): + yield Event(kind=EventKind.METRICS, metrics=metrics) + return + + # ---- 记忆检索：带超时，失败不阻塞（方案 §8） + t0 = time.perf_counter() + memories = await self._retrieve_memories(identity, user_text) + metrics.retrieve_ms = (time.perf_counter() - t0) * 1000 + metrics.memories_used = len(memories) + if memories: + yield Event(kind=EventKind.MEMORY_USED, memories=memories) + + # ---- 拼 system（每轮重建，单条） + system = build_system_prompt(identity, memories) + history = trim_history(self._history, + max_turns=self.cfg.context.max_turns) + messages = [{"role": "system", "content": system}] + history + \ + [{"role": "user", "content": user_text}] + + # ---- LLM 流式 + 逐句 TTS + reply_parts: list[str] = [] + buffer = "" + speaks = 0 + async for ev in self._stream_and_speak(messages, metrics): + if ev.kind == EventKind.RESPONSE_DELTA: + reply_parts.append(ev.text) + buffer += ev.text + # 增量文本必须**原样透传**给调用方（显示、字幕、日志都要用）。 + # 这里漏掉 yield 的后果是：音频正常出来了，但上层拿到的是空回复—— + # 因为文本只进了内部累加器，从未离开这个生成器。 + yield ev + # 攒到句末就送去合成——这是提前出声的唯一手段 + segs = split_sentences(buffer) + if len(segs) > 1: + *ready, buffer = segs + for seg in ready: + async for speak_ev in self._speak(seg, metrics): + speaks += 1 + yield speak_ev + else: + yield ev + + # 收尾：把剩下的半句也合成掉 + if buffer.strip(): + async for speak_ev in self._speak(buffer, metrics): + speaks += 1 + yield speak_ev + + reply = "".join(reply_parts).strip() + metrics.reply_chars = len(reply) + metrics.speaks = speaks + metrics.total_ms = (time.perf_counter() - t_start) * 1000 + + # ---- 落历史 + self._history.append({"role": "user", "content": user_text}) + if reply: + self._history.append({"role": "assistant", "content": reply}) + + # ---- 异步写记忆：**绑定创建时的身份**，不读"当前用户" + if self.cfg.memory.auto_extract_enabled and reply: + self._spawn_memory_write( + user_text=user_text, reply=reply, + user_id=identity.user_id, + session_id=sid, + turn_id=f"t_{uuid.uuid4().hex[:10]}") + + await self.memory.add_turn(Turn( + turn_id=f"t_{uuid.uuid4().hex[:10]}", session_id=sid, + user_id=identity.user_id, role="user", text=user_text)) + + yield Event(kind=EventKind.METRICS, metrics=metrics) + + async def respond_to_audio(self, samples: np.ndarray, *, + session_id: str | None = None) -> AsyncIterator[Event]: + """从音频走完整条链路（需要 STT）。""" + if self.stt is None: + yield Event(kind=EventKind.ERROR, error="未配置 STT，无法处理音频") + return + t0 = time.perf_counter() + stream = self.stt.new_stream() + stream.accept(samples) + text = stream.finish() + stt_ms = (time.perf_counter() - t0) * 1000 + yield Event(kind=EventKind.TRANSCRIPT, text=text) + async for ev in self.respond_to_text(text, session_id=session_id): + if ev.kind == EventKind.METRICS and ev.metrics: + ev.metrics.stt_ms = stt_ms + yield ev + + # ------------------------------------------------------------ 内部 + + async def _retrieve_memories(self, identity: IdentityDecision, + query: str) -> list[Memory]: + """带超时的检索。超时/出错一律返回空，绝不让对话卡住。""" + if not identity.confirmed: + # 访客不做个性化检索（方案 §6："未知身份只用访客临时上下文"） + return [] + try: + return await asyncio.wait_for( + self.memory.retrieve( + identity.user_id, query, + limit=self.cfg.context.max_memories, + scopes=[Scope.PRIVATE, Scope.SHARED]), + timeout=self.cfg.context.memory_timeout_s) + except asyncio.TimeoutError: + log.warning("记忆检索超时 %.2fs，本轮不带记忆继续", + self.cfg.context.memory_timeout_s) + return [] + except Exception as e: # noqa: BLE001 + log.warning("记忆检索失败（%s），本轮不带记忆继续", e) + return [] + + async def _stream_and_speak(self, messages: list[dict], + metrics: TurnMetrics) -> AsyncIterator[Event]: + """只负责 LLM 流式，不做 TTS。TTS 由调用方按句驱动。""" + t0 = time.perf_counter() + try: + async for chunk in self.llm.stream(messages): + if chunk.kind.value == "content" and chunk.text: + if not metrics.llm_first_content_ms: + metrics.llm_first_content_ms = (time.perf_counter() - t0) * 1000 + yield Event(kind=EventKind.RESPONSE_DELTA, text=chunk.text) + elif chunk.kind.value == "tool_call" and chunk.tool_name: + # 骨架阶段只上报，不执行。工具执行要配合 + # 方案 §9 的"执行 ID + 状态确认 + 不盲目重试"， + # 留到接米家时再做。 + yield Event(kind=EventKind.TOOL_CALL, tool_call=ToolCall( + id=chunk.tool_id, name=chunk.tool_name, + arguments=chunk.tool_arguments)) + except asyncio.CancelledError: + log.info("本轮被取消（打断）") + raise + except Exception as e: # noqa: BLE001 + log.error("LLM 流失败：%s", e) + yield Event(kind=EventKind.ERROR, error=f"{type(e).__name__}: {e}") + metrics.llm_backend = getattr(self.llm, "last_backend", self.llm.name) + + async def _speak(self, text: str, metrics: TurnMetrics) -> AsyncIterator[Event]: + """合成一段并产出可播放音频。""" + text = text.strip() + if not text: + return + t0 = time.perf_counter() + try: + # TTS 是 CPU 密集的同步调用，放线程里跑，别阻塞事件循环 + audio = await asyncio.to_thread(self.tts.synthesize, text) + except Exception as e: # noqa: BLE001 + log.error("TTS 失败：%s", e) + yield Event(kind=EventKind.ERROR, error=f"TTS 失败：{e}") + return + if audio is None or len(audio) == 0: + return + if not metrics.first_audio_ms: + metrics.first_audio_ms = (time.perf_counter() - t0) * 1000 + yield Event(kind=EventKind.SPEAK, text=text, audio=audio, + sample_rate=self.tts.sample_rate) + + def _spawn_memory_write(self, *, user_text: str, reply: str, + user_id: str, session_id: str, turn_id: str) -> None: + """起一个后台任务写记忆。 + + **身份是参数传进来的，不在这里读 self.identity.current()。** + 方案 §8 明确要求："异步任务绑定创建时的 user_id/session_id， + 避免迟到结果写入新用户。" 如果这里去读"当前身份"， + 用户在这几百毫秒内切了身份，这条记忆就会写到别人名下。 + """ + async def _write(): + try: + for mem in extract_memories(user_text, reply): + mem.owner_user_id = user_id # 绑定创建时的身份 + mem.source_session_id = session_id + mem.source_turn_id = turn_id + mem.confirm_status = ConfirmStatus( + self.cfg.memory.auto_extract_confirm_status) + await self.memory.remember(mem) + except Exception as e: # noqa: BLE001 + log.warning("异步写记忆失败（不影响对话）：%s", e) + + task = asyncio.create_task(_write()) + # 持有引用，避免被 GC 提前回收 + if not hasattr(self, "_bg_tasks"): + self._bg_tasks: set[asyncio.Task] = set() + self._bg_tasks.add(task) + task.add_done_callback(self._bg_tasks.discard) + + +# ---------------------------------------------------------------- 记忆抽取 + +#: 显式"记住"的触发词。首版用规则，不额外调一次 LLM—— +#: 那会多一次往返，而语音场景对延迟很敏感。 +_REMEMBER_PATTERNS = ( + ("记住", ""), ("记一下", ""), ("别忘了", ""), ("帮我记", ""), + ("我喜欢", "偏好"), ("我不喜欢", "偏好"), ("我习惯", "偏好"), +) + + +def extract_memories(user_text: str, reply: str) -> list[Memory]: + """从一轮对话里抽取候选记忆。 + + 首版只认显式表达。**抽取结果默认是 candidate，不参与检索**—— + 方案 §8："模型推测不当作确认事实"。要变成 confirmed 需要用户确认 + 或走工具调用。 + """ + out: list[Memory] = [] + text = (user_text or "").strip() + for trigger, kind_hint in _REMEMBER_PATTERNS: + if trigger in text: + # 取触发词之后的正文 + body = text.split(trigger, 1)[1].strip(" ：:，,。.！!？?") + if len(body) < 2: + continue + out.append(Memory( + kind=MemoryKind.PREFERENCE if kind_hint else MemoryKind.FACT, + key=kind_hint or "", + value=body[:200], + )) + break + return out + + +__all__ = [ + "ConversationEngine", "Event", "EventKind", "TurnMetrics", "extract_memories", +] diff --git a/app/identity/__init__.py b/app/identity/__init__.py new file mode 100644 index 0000000..11cd322 --- /dev/null +++ b/app/identity/__init__.py @@ -0,0 +1,18 @@ +"""身份层。 + +对外暴露 `IdentityProvider` 接口与首版 `ManualIdentityProvider` 实现。 +将来接人脸/声纹只需新增实现，记忆表结构与调用方都不动 +（这正是方案 §6 要求"预留 IdentityProvider 接口"的原因）。 +""" + +from .base import ( + GUEST_USER_ID, TRUST_GUEST, TRUST_NORMAL, + IdentityClaim, IdentityDecision, IdentityProvider, +) +from .manual import SUGGEST_MIN_CONFIDENCE, ManualIdentityProvider + +__all__ = [ + "IdentityProvider", "ManualIdentityProvider", + "IdentityClaim", "IdentityDecision", + "GUEST_USER_ID", "TRUST_NORMAL", "TRUST_GUEST", "SUGGEST_MIN_CONFIDENCE", +] diff --git a/app/identity/base.py b/app/identity/base.py new file mode 100644 index 0000000..796966c --- /dev/null +++ b/app/identity/base.py @@ -0,0 +1,110 @@ +"""身份抽象层。 + +方案 §6 的核心约束，逐条落进接口： + + * "首版可靠身份入口：手动/对话确认切换到已建档的 user_id； + 无法确认时使用 unknown/guest。" + + * **"看到熟人不等于确认他正在说话"** —— 人脸只提供候选， + 不自动切换身份。这条最容易做错：摄像头看到谁就切谁的记忆， + 结果是"路人经过就把你的私人记忆念出来"。 + + * "显式报名字用于个性化档案选择，**不是身份认证**，不能据此解锁敏感信息" + —— 所以 trust_level 与 user_id 分开存，将来加认证时不用改偏好存储。 + + * "预留 IdentityProvider 接口：输入时间戳、track_id、候选身份、置信度与来源； + 输出确认的 user_id 或 unknown。后续声纹模块可接入此接口，不改记忆表结构。" + + * "视觉 track_id、会话 session_id、永久 user_id 分开" —— + 所以 IdentityClaim 里 track_id 是线索，不是身份。 +""" + +from __future__ import annotations + +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Sequence + +#: 无法确认身份时使用的固定 id。永远不落私人记忆。 +GUEST_USER_ID = "guest" + +#: 信任等级。与 user_id 分开，将来加认证不改这里。 +TRUST_NORMAL = "normal" +TRUST_GUEST = "guest" + + +@dataclass +class IdentityClaim: + """一条身份线索。 + + source 取值（首版只用 manual，其余留给后续）： + "manual" 用户显式切换/确认——**唯一可单独决定身份的来源** + "face" 人脸识别候选（本地特征，需用户主动录入） + "client_login" 客户端登录态（手机 App / 浏览器） + "voiceprint" 声纹（后续阶段） + """ + source: str + candidate_user_id: str | None = None + confidence: float = 0.0 + track_id: str | None = None + timestamp: float = field(default_factory=time.time) + note: str = "" + + #: 只有这些来源可以单独把身份定下来。人脸不在其中—— + #: 方案明确要求人脸仅作候选、经确认后才启用对应记忆。 + AUTHORITATIVE_SOURCES = ("manual", "client_login") + + @property + def is_authoritative(self) -> bool: + return self.source in self.AUTHORITATIVE_SOURCES + + +@dataclass +class IdentityDecision: + user_id: str + trust_level: str + confirmed: bool # True = 经确认；False = 默认访客 + reason: str + #: 未被采纳的候选，保留下来供上层提示"是否要切换成 X" + suggestions: list[IdentityClaim] = field(default_factory=list) + + @property + def is_guest(self) -> bool: + return not self.confirmed + + +class IdentityProvider(ABC): + """身份裁决者。 + + 实现要点：**宁可拒判，不要猜**。判错的代价是把 A 的私人记忆 + 用在 B 身上，比"这次不认识"严重得多。 + """ + + @abstractmethod + def current(self) -> IdentityDecision: + """当前身份。任何时候都必须能回答。""" + + @abstractmethod + def switch(self, user_id: str, *, display_name: str = "", + reason: str = "显式切换") -> IdentityDecision: + """显式切换身份——首版唯一可靠的入口。""" + + @abstractmethod + def resolve(self, claims: Sequence[IdentityClaim]) -> IdentityDecision: + """根据一组线索裁决身份。""" + + @abstractmethod + def reset(self, reason: str = "回到待机") -> IdentityDecision: + """回到访客身份，并清掉上一人的身份状态。 + + 方案 §8 要求"切换身份清理上一人的私有检索上下文"—— + 调用方在收到身份变化后必须重新构造 system 消息（含记忆）， + 不能沿用上一轮的。 + """ + + +__all__ = [ + "IdentityProvider", "IdentityClaim", "IdentityDecision", + "GUEST_USER_ID", "TRUST_NORMAL", "TRUST_GUEST", +] diff --git a/app/identity/manual.py b/app/identity/manual.py new file mode 100644 index 0000000..0a0e20a --- /dev/null +++ b/app/identity/manual.py @@ -0,0 +1,123 @@ +"""首版身份实现：显式切换 + 访客兜底。 + +对应方案 §6："首版可靠身份入口：手动/对话确认切换到已建档的 user_id； +无法确认时使用 unknown/guest。" + +行为规则： + + 1. 显式切换（`switch`）立即生效，这是唯一能单独定身份的方式。 + 2. 人脸/声纹线索只生成**建议**，不自动改身份。上层可以选择性地 + 问一句"要切换成 X 吗"，但不能替用户决定。 + 3. 没有任何线索时是访客，且访客不能落私人记忆（由记忆层用 + owner_user_id 隔离保证）。 + 4. 切换身份返回一个"版本号"，上层据此判断要不要丢弃上一人的上下文。 +""" + +from __future__ import annotations + +import logging +from typing import Sequence + +from .base import ( + GUEST_USER_ID, TRUST_GUEST, TRUST_NORMAL, + IdentityClaim, IdentityDecision, IdentityProvider, +) + +log = logging.getLogger(__name__) + +#: 人脸候选被采纳为"建议"的最低置信度。 +#: 低于这个值连建议都不给——减少无意义的打扰。 +SUGGEST_MIN_CONFIDENCE = 0.6 + + +class ManualIdentityProvider(IdentityProvider): + def __init__(self, initial_user: str | None = None): + self._decision = IdentityDecision( + user_id=initial_user or GUEST_USER_ID, + trust_level=TRUST_NORMAL if initial_user else TRUST_GUEST, + confirmed=bool(initial_user), + reason="初始身份" if initial_user else "初始为访客", + ) + self._version = 0 + + # ---------------------------------------------------------- 查询 + + def current(self) -> IdentityDecision: + return self._decision + + @property + def version(self) -> int: + """身份版本号。每次真正变更时 +1。 + + 上层用它判断"上一轮的私有检索上下文是否过期"—— + 方案 §8 要求切换身份时清理上一人的上下文。 + """ + return self._version + + # ---------------------------------------------------------- 变更 + + def switch(self, user_id: str, *, display_name: str = "", + reason: str = "显式切换") -> IdentityDecision: + if not user_id: + raise ValueError("user_id 不能为空") + prev = self._decision.user_id + self._decision = IdentityDecision( + user_id=user_id, + trust_level=TRUST_NORMAL, + confirmed=True, + reason=reason, + ) + if prev != user_id: + self._version += 1 + log.info("身份切换 %s → %s（%s）", prev, user_id, reason) + return self._decision + + def reset(self, reason: str = "回到待机") -> IdentityDecision: + if self._decision.user_id != GUEST_USER_ID: + self._version += 1 + self._decision = IdentityDecision( + user_id=GUEST_USER_ID, + trust_level=TRUST_GUEST, + confirmed=False, + reason=reason, + ) + return self._decision + + def resolve(self, claims: Sequence[IdentityClaim]) -> IdentityDecision: + """裁决。 + + 顺序很重要：先看有没有权威来源（显式切换 / 客户端登录态）， + 有就采用；人脸之类的只作为建议附在后面。 + """ + authoritative = [c for c in claims if c.is_authoritative and c.candidate_user_id] + suggestions = [ + c for c in claims + if not c.is_authoritative + and c.candidate_user_id + and c.confidence >= SUGGEST_MIN_CONFIDENCE + ] + + if authoritative: + # 同一批里有多个权威来源且互相矛盾——宁可不切，报出来让人处理 + ids = {c.candidate_user_id for c in authoritative} + if len(ids) > 1: + log.warning("权威身份线索互相矛盾 %s，保持当前身份", ids) + d = self._decision + d.reason = f"线索矛盾 {ids}，保持 {d.user_id}" + d.suggestions = suggestions + return d + c = max(authoritative, key=lambda x: x.confidence) + assert c.candidate_user_id is not None + self.switch(c.candidate_user_id, reason=f"来源 {c.source}") + else: + # 没有权威来源：即便人脸认出来了也不切，只给建议 + self._decision.suggestions = suggestions + if suggestions: + self._decision.reason = ( + f"仅有人脸候选 {suggestions[0].candidate_user_id}" + f"（置信度 {suggestions[0].confidence:.2f}），未自动切换" + ) + return self._decision + + +__all__ = ["ManualIdentityProvider", "SUGGEST_MIN_CONFIDENCE"] diff --git a/app/llm/__init__.py b/app/llm/__init__.py new file mode 100644 index 0000000..dc9bbfb --- /dev/null +++ b/app/llm/__init__.py @@ -0,0 +1,116 @@ +"""LLM 后端选择与降级。 + +设计要点：**降级只在"还没吐出任何内容"时发生。** + +一旦已经向调用方 yield 过 chunk，再切后端就会把两段不同来源的文本接在一起， +产出一句读不通的回复。那种情况下宁可让本轮失败——方案 §5 要求 +"网络失败给简短提示，不反复播旧回复"，而不是拼接残句。 + +用法： + provider = create_provider(AppConfig.load().llm) + router = FallbackProvider(primary, fallback) + async for chunk in router.stream(messages): ... +""" + +from __future__ import annotations + +import logging + +from ..config import LLMConfig +from .base import Chunk, ChunkKind, LLMError, LLMProvider, LLMResult, ToolCall +from .deepseek import DeepSeekProvider +from .local_vllm import LocalVLLMProvider + +log = logging.getLogger(__name__) + +_REGISTRY = { + "local": LocalVLLMProvider, + "deepseek": DeepSeekProvider, +} + + +def create_provider(cfg: LLMConfig) -> LLMProvider: + cls = _REGISTRY.get(cfg.name) + if cls is None: + raise ValueError(f"未知的 LLM 后端 {cfg.name!r}，可选：{sorted(_REGISTRY)}") + return cls(cfg) + + +class FallbackProvider(LLMProvider): + """主后端失败时切备用后端。 + + 只对**连接层失败**降级（超时、拒连、5xx、模型不可用）， + 不对 4xx 这类"请求本身有问题"降级——那种情况换个后端也一样失败， + 降级只会掩盖真正的 bug。 + """ + + supports_vision = True # 只要备用支持就算支持，实际由 _choose 时判断 + + def __init__(self, primary: LLMProvider, fallback: LLMProvider | None = None): + self.primary = primary + self.fallback = fallback + self.cfg = primary.cfg + self.name = f"{primary.name}+{fallback.name if fallback else 'none'}" + self.last_backend: str = primary.name + self._session = None + self._owns_session = False + + def _build_body(self, *a, **kw): # 实际不会走到，路由在 stream 里 + raise NotImplementedError + + async def aclose(self) -> None: + await self.primary.aclose() + if self.fallback: + await self.fallback.aclose() + + async def stream(self, messages, *, tools=None, max_tokens=None): + emitted = False + try: + async for ch in self.primary.stream(messages, tools=tools, max_tokens=max_tokens): + emitted = True + self.last_backend = self.primary.name + yield ch + return + except Exception as e: # noqa: BLE001 — 要按类型决定是否降级 + if emitted: + log.error("主后端 %s 在已输出后失败，不再降级：%s", self.primary.name, e) + raise + if self.fallback is None: + raise + if not self._should_fallback(e): + log.error("主后端 %s 失败且不适合降级：%s", self.primary.name, e) + raise + log.warning("主后端 %s 失败（%s），切备用 %s", + self.primary.name, type(e).__name__, self.fallback.name) + + async for ch in self.fallback.stream(messages, tools=tools, max_tokens=max_tokens): + self.last_backend = self.fallback.name + yield ch + + @staticmethod + def _should_fallback(err: Exception) -> bool: + """连接/超时/服务端错误才降级；请求本身的问题不降级。""" + if isinstance(err, LLMError): + # LLMError 里带的是 HTTP 状态码，只有 5xx / 429 值得重试到别的后端 + msg = str(err) + for code in ("500", "502", "503", "504", "429"): + if f"HTTP {code}" in msg: + return True + return False + # 超时、连接失败、DNS 等 + return isinstance(err, (TimeoutError, ConnectionError, OSError)) + + async def health(self) -> tuple[bool, str]: + ok, detail = await self.primary.health() + if ok: + return True, f"主后端 {self.primary.name}: {detail}" + if self.fallback: + ok2, d2 = await self.fallback.health() + return ok2, f"主后端不可用（{detail}）；备用 {self.fallback.name}: {d2}" + return False, detail + + +__all__ = [ + "LLMProvider", "LLMError", "Chunk", "ChunkKind", "LLMResult", "ToolCall", + "LocalVLLMProvider", "DeepSeekProvider", "FallbackProvider", "create_provider", +] diff --git a/app/llm/base.py b/app/llm/base.py new file mode 100644 index 0000000..29a607c --- /dev/null +++ b/app/llm/base.py @@ -0,0 +1,308 @@ +"""LLM 抽象层。 + +为什么需要这层：本机 vLLM 和 DeepSeek 虽然都是 OpenAI 兼容协议， +但有三处实测差异，写死任何一个都会出问题： + + 1. 思考内容的字段名 + 本机 vLLM : delta.reasoning + DeepSeek : delta.reasoning_content + 只认一个会**静默丢弃**思考内容——不报错，只是看不见。 + + 2. 关闭思考的方式 + 本机 vLLM : chat_template_kwargs={"enable_thinking": false} + DeepSeek : thinking={"type": "disabled"} + + 3. system 消息条数 + 本机 vLLM : 只允许一条，且必须在开头，多条直接 400 + DeepSeek : 宽松 + 所以统一由本层归一化成单条 system。 + +以上均来自实测，见 P0-本地LLM验证报告.md / P0-DeepSeek验证报告.md。 +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, AsyncIterator, Sequence + +import aiohttp + +from ..config import LLMConfig + +log = logging.getLogger(__name__) + + +class ChunkKind(str, Enum): + REASONING = "reasoning" + CONTENT = "content" + TOOL_CALL = "tool_call" + DONE = "done" + + +@dataclass +class Chunk: + kind: ChunkKind + text: str = "" + tool_index: int = 0 + tool_id: str = "" + tool_name: str = "" + tool_arguments: str = "" + finish_reason: str | None = None + usage: dict[str, Any] | None = None + + +@dataclass +class ToolCall: + id: str + name: str + arguments: str + + def parsed_arguments(self) -> dict[str, Any]: + try: + return json.loads(self.arguments or "{}") + except json.JSONDecodeError: + return {} + + +@dataclass +class LLMResult: + content: str = "" + reasoning: str = "" + tool_calls: list[ToolCall] = field(default_factory=list) + finish_reason: str | None = None + usage: dict[str, Any] | None = None + # 计时：首 token 是"思考"还是"回答"要分开，否则会误判延迟。 + # 实测开思考时 138ms 就出首块，但那是思考内容，用户要听的回答还没开始。 + first_reasoning_ms: float | None = None + first_content_ms: float | None = None + total_ms: float = 0.0 + backend: str = "" + + @property + def has_tool_calls(self) -> bool: + return bool(self.tool_calls) + + +class LLMProvider(ABC): + """一个 LLM 后端。 + + 子类只需实现 `_build_body`（怎么关思考、怎么放参数）； + 消息归一化、SSE 解析、计时、取消都在基类做，保证各后端行为一致。 + """ + + #: 是否支持图片输入。本机 vLLM 实例实测不支持（"At most 0 image(s)"）。 + supports_vision: bool = False + + def __init__(self, cfg: LLMConfig, session: aiohttp.ClientSession | None = None): + self.cfg = cfg + self.name = cfg.name + self._session = session + self._owns_session = session is None + + # ------------------------------------------------------------ 供子类覆写 + + @abstractmethod + def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None, + max_tokens: int) -> dict: + """构造请求体。各后端的思考开关差异在这里处理。""" + + def _headers(self) -> dict[str, str]: + return { + "Authorization": f"Bearer {self.cfg.api_key}", + "Content-Type": "application/json", + } + + @property + def endpoint(self) -> str: + return self.cfg.base_url.rstrip("/") + "/chat/completions" + + # ------------------------------------------------------------ 消息归一化 + + @staticmethod + def normalize_messages(messages: Sequence[dict]) -> list[dict]: + """把多条 system 合并成一条并置于开头。 + + 本机 vLLM 对多条 system 直接 400（实测 "System message must be at the + beginning."）。而记忆注入天然想用第二条 system，所以在这里统一抹平。 + + 顺带这也满足方案 §8 的要求："注入的记忆每轮替换，不是追加"—— + 调用方每轮重新构造 system 内容即可。 + """ + systems, rest = [], [] + for m in messages: + if m.get("role") == "system": + c = m.get("content") + if isinstance(c, str) and c.strip(): + systems.append(c.strip()) + else: + rest.append(m) + out = [] + if systems: + out.append({"role": "system", "content": "\n\n".join(systems)}) + out.extend(rest) + return out + + # ------------------------------------------------------------ 流式请求 + + async def _ensure_session(self) -> aiohttp.ClientSession: + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession() + self._owns_session = True + return self._session + + async def aclose(self) -> None: + if self._owns_session and self._session and not self._session.closed: + await self._session.close() + + async def stream( + self, + messages: Sequence[dict], + *, + tools: Sequence[dict] | None = None, + max_tokens: int | None = None, + ) -> AsyncIterator[Chunk]: + """流式产出。调用方取消（CancelledError）时连接会随之关闭。 + + 这就是方案 §5 要求的"打断时取消旧回复生成"的底层能力。 + """ + body = self._build_body( + self.normalize_messages(messages), + tools=tools, + max_tokens=max_tokens or self.cfg.max_tokens, + ) + session = await self._ensure_session() + timeout = aiohttp.ClientTimeout( + total=self.cfg.timeout_s, + sock_connect=5.0, + sock_read=self.cfg.first_token_timeout_s, + ) + async with session.post(self.endpoint, json=body, + headers=self._headers(), timeout=timeout) as resp: + if resp.status != 200: + detail = (await resp.text())[:400] + raise LLMError(f"{self.name} HTTP {resp.status}: {detail}") + async for chunk in self._iter_sse(resp): + yield chunk + + @staticmethod + async def _iter_sse(resp: aiohttp.ClientResponse) -> AsyncIterator[Chunk]: + """解析 SSE。字段名差异在这里吸收。""" + # 工具调用的参数是分片到达的，按 index 累积 + tool_acc: dict[int, dict] = {} + async for raw in resp.content: + line = raw.decode("utf-8", "replace").strip() + if not line or not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + break + try: + data = json.loads(payload) + except json.JSONDecodeError: + continue + + usage = data.get("usage") + choices = data.get("choices") or [] + if not choices: + if usage: + yield Chunk(kind=ChunkKind.DONE, usage=usage) + continue + + choice = choices[0] + delta = choice.get("delta") or {} + finish = choice.get("finish_reason") + + # 两个字段名都认。少认一个就会静默丢思考内容。 + reasoning = delta.get("reasoning_content") or delta.get("reasoning") + if reasoning: + yield Chunk(kind=ChunkKind.REASONING, text=reasoning) + + content = delta.get("content") + if content: + yield Chunk(kind=ChunkKind.CONTENT, text=content) + + for tc in delta.get("tool_calls") or []: + idx = tc.get("index", 0) + slot = tool_acc.setdefault(idx, {"id": "", "name": "", "arguments": ""}) + if tc.get("id"): + slot["id"] = tc["id"] + fn = tc.get("function") or {} + if fn.get("name"): + slot["name"] = fn["name"] + if fn.get("arguments"): + slot["arguments"] += fn["arguments"] + yield Chunk(kind=ChunkKind.TOOL_CALL, tool_index=idx, + tool_id=slot["id"], tool_name=slot["name"], + tool_arguments=slot["arguments"]) + + if finish: + yield Chunk(kind=ChunkKind.DONE, finish_reason=finish, usage=usage) + # 注意：**不能在这里 return**。开了 stream_options.include_usage 后， + # usage 是在 finish_reason 之后的**单独一帧**里到达的 + # （该帧 choices 为空）。提前返回会把 usage 丢掉。 + # 循环由 [DONE] 或流结束自然终止。 + + async def complete( + self, + messages: Sequence[dict], + *, + tools: Sequence[dict] | None = None, + max_tokens: int | None = None, + ) -> LLMResult: + """收完整个流，返回结构化结果。""" + t0 = time.perf_counter() + res = LLMResult(backend=self.name) + parts: list[str] = [] + reason: list[str] = [] + acc: dict[int, dict] = {} + async for ch in self.stream(messages, tools=tools, max_tokens=max_tokens): + if ch.kind == ChunkKind.CONTENT: + if res.first_content_ms is None: + res.first_content_ms = (time.perf_counter() - t0) * 1000 + parts.append(ch.text) + elif ch.kind == ChunkKind.REASONING: + if res.first_reasoning_ms is None: + res.first_reasoning_ms = (time.perf_counter() - t0) * 1000 + reason.append(ch.text) + elif ch.kind == ChunkKind.TOOL_CALL: + acc[ch.tool_index] = {"id": ch.tool_id, "name": ch.tool_name, + "arguments": ch.tool_arguments} + elif ch.kind == ChunkKind.DONE: + res.finish_reason = ch.finish_reason + if ch.usage: + res.usage = ch.usage + res.content = "".join(parts) + res.reasoning = "".join(reason) + res.tool_calls = [ToolCall(**v) for _, v in sorted(acc.items())] + res.total_ms = (time.perf_counter() - t0) * 1000 + return res + + async def health(self) -> tuple[bool, str]: + """启动自检：确认模型名有效。方案 §10 要求记录实际版本。""" + try: + session = await self._ensure_session() + url = self.cfg.base_url.rstrip("/") + "/models" + async with session.get(url, headers=self._headers(), + timeout=aiohttp.ClientTimeout(total=10)) as r: + if r.status != 200: + return False, f"HTTP {r.status}" + data = await r.json() + ids = [m.get("id") for m in data.get("data", [])] + if self.cfg.model in ids: + return True, f"{self.cfg.model} 可用（清单 {len(ids)} 个）" + return False, f"模型 {self.cfg.model!r} 不在清单 {ids} 中" + except Exception as e: # noqa: BLE001 — 自检不该因异常类型漏判 + return False, f"{type(e).__name__}: {e}" + + +class LLMError(RuntimeError): + pass + + +__all__ = ["LLMProvider", "LLMError", "Chunk", "ChunkKind", "ToolCall", "LLMResult"] diff --git a/app/llm/deepseek.py b/app/llm/deepseek.py new file mode 100644 index 0000000..edf9f2e --- /dev/null +++ b/app/llm/deepseek.py @@ -0,0 +1,47 @@ +"""DeepSeek 云 API（备用后端）。 + +实测数据（P0-DeepSeek验证报告.md）： + 首 token P50 786ms / P95 1100ms（关思考，N=8） + 视觉可用——正确读出图中数字 + 模型清单只有 deepseek-flash 与 deepseek-v4-pro + +约束（重要，踩过）： + 默认开启思考模式，此时 API 要求把上一轮 assistant 消息里的 + `reasoning_content` 原样回传，否则任何多轮/工具回灌一律 400： + "The `reasoning_content` in the thinking mode must be passed back to the API." + 手工构造 assistant 消息（content=None / content="" / 不带 tools）三种写法 + 实测全部 400，只有回传服务端原始对象才行。 + 所以会话主路径必须显式关闭思考。 + +另：本服务思考字段名是 reasoning_content（本机 vLLM 是 reasoning）， + 基类两个都认。 +""" + +from __future__ import annotations + +from typing import Sequence + +from .base import LLMProvider + + +class DeepSeekProvider(LLMProvider): + supports_vision = True + + def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None, + max_tokens: int) -> dict: + body: dict = { + "model": self.cfg.model, + "messages": messages, + "stream": True, + "max_tokens": max_tokens, + "stream_options": {"include_usage": True}, + } + # 显式声明思考开关。不传会走默认（开），多轮与工具回灌会 400。 + body["thinking"] = {"type": "enabled" if self.cfg.thinking else "disabled"} + if tools: + body["tools"] = list(tools) + body["tool_choice"] = "auto" + return body + + +__all__ = ["DeepSeekProvider"] diff --git a/app/llm/local_vllm.py b/app/llm/local_vllm.py new file mode 100644 index 0000000..d04c491 --- /dev/null +++ b/app/llm/local_vllm.py @@ -0,0 +1,46 @@ +"""本机 Qwen3.8-27B（vLLM，OpenAI 兼容）。 + +实测数据（P0-本地LLM验证报告.md）： + 关思考 首 token 367ms / 完成 10 tokens + 开思考 首 token 1390ms / 完成 187 tokens，且**回答被 max_tokens 截断** + （实测 200 预算下只吐出"我不"两个字） + +开思考不只是慢——思考 token 会占用 max_tokens 预算。 +语音场景 max_tokens 本来就要设小，所以这不是"慢一点"的取舍，是"说不出话"。 +默认关闭。 + +约束：只接受一条 system 消息，多条返回 400。已在基类归一化。 +视觉：本实例不支持（"At most 0 image(s) may be provided"）， + 模型目录里没有视觉塔。视觉要走 llama.cpp + mmproj 那条线。 +""" + +from __future__ import annotations + +from typing import Sequence + +from .base import LLMProvider + + +class LocalVLLMProvider(LLMProvider): + supports_vision = False + + def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None, + max_tokens: int) -> dict: + body: dict = { + "model": self.cfg.model, + "messages": messages, + "stream": True, + "max_tokens": max_tokens, + "stream_options": {"include_usage": True}, + } + # 关闭思考。这是本机部署实测唯一有效的方式 + # （服务器上的 ctx_test.sh 也用同一写法）。 + # 注意：即使关掉，响应里仍可能出现空的 reasoning 字段，属正常。 + body["chat_template_kwargs"] = {"enable_thinking": self.cfg.thinking} + if tools: + body["tools"] = list(tools) + body["tool_choice"] = "auto" + return body + + +__all__ = ["LocalVLLMProvider"] diff --git a/app/memory/__init__.py b/app/memory/__init__.py new file mode 100644 index 0000000..d9cf36f --- /dev/null +++ b/app/memory/__init__.py @@ -0,0 +1,16 @@ +"""记忆层。 + +对外只暴露 `MemoryStore` 接口和 `SQLiteMemoryStore` 实现。 +调用方依赖接口，不依赖 SQLite——将来多设备共享记忆时要搬到服务器， +只换实现，调用方一行不改（前提是调用方按 async + 可能失败 来写）。 +""" + +from .base import ( + ConfirmStatus, Memory, MemoryKind, MemoryStore, Scope, Turn, +) +from .sqlite_store import SCHEMA, SQLiteMemoryStore + +__all__ = [ + "MemoryStore", "SQLiteMemoryStore", "Memory", "Turn", + "Scope", "ConfirmStatus", "MemoryKind", "SCHEMA", +] diff --git a/app/memory/base.py b/app/memory/base.py new file mode 100644 index 0000000..1fbd99e --- /dev/null +++ b/app/memory/base.py @@ -0,0 +1,185 @@ +"""记忆抽象层。 + +设计原则（方案 §8）： + + 1. **接口按"可能远程"设计**。SQLite 现在跑在本地，但将来多设备共享记忆 + 时要搬到服务器。所以所有操作都是 async、可能超时、可能失败—— + 写成同步直连本地文件的话，搬家要重写所有调用方。 + 现在付这个成本几乎为零，将来付就很贵。 + + 2. **查询和写入都强制带 owner_user_id**，没有"查全部"的口子。 + 未知身份只给访客临时上下文，不落库。 + + 3. **模型推测不当作确认事实**。自动抽取的记忆是 candidate，默认不参与检索； + 只有用户显式"记住"或工具调用写入的才是 confirmed。 + + 4. **异步任务绑定创建时的身份**。写入任务在创建时捕获 user_id/session_id， + 绝不读"当前用户"——否则用户切了身份，上一轮迟到的写入会串到新人头上。 + + 5. 视觉 track_id、会话 session_id、永久 user_id **三者分开**，不互相冒充。 +""" + +from __future__ import annotations + +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Sequence + + +class Scope(str, Enum): + """记忆的作用域。群体会话默认只用 shared。""" + PRIVATE = "private" + SHARED = "shared" + + +class ConfirmStatus(str, Enum): + """确认状态。区分"用户明确说的"和"模型猜的"。""" + CONFIRMED = "confirmed" # 用户显式记住 / 工具调用写入 + CANDIDATE = "candidate" # 回复后自动抽取，默认不参与检索 + + +class MemoryKind(str, Enum): + PREFERENCE = "preference" # 明确偏好：结构化键值 + FACT = "fact" # 事实 + TASK = "task" # 临时事项，常带过期时间 + + +@dataclass +class Memory: + memory_id: int | None = None + owner_user_id: str = "" + scope: Scope = Scope.PRIVATE + kind: MemoryKind = MemoryKind.FACT + key: str = "" + value: str = "" + confirm_status: ConfirmStatus = ConfirmStatus.CONFIRMED + source_session_id: str | None = None + source_turn_id: str | None = None + created_at: float = field(default_factory=time.time) + updated_at: float = field(default_factory=time.time) + expires_at: float | None = None + + @property + def text(self) -> str: + """用于检索和注入的文本形式。""" + return f"{self.key}：{self.value}" if self.key else self.value + + def is_expired(self, now: float | None = None) -> bool: + if self.expires_at is None: + return False + return (now or time.time()) > self.expires_at + + +@dataclass +class Turn: + """一轮对话。用于溯源，不默认长期保存逐字稿。""" + turn_id: str + session_id: str + user_id: str + role: str # "user" | "assistant" + text: str + created_at: float = field(default_factory=time.time) + + +class MemoryStore(ABC): + """记忆存储。 + + 所有方法都可能抛异常——调用方必须按"记忆失败不阻塞对话"处理 + （方案 §8）。超时由调用方用 asyncio.wait_for 控制。 + """ + + # ---------------------------------------------------------- 生命周期 + + @abstractmethod + async def open(self) -> None: ... + + @abstractmethod + async def close(self) -> None: ... + + # ---------------------------------------------------------- 用户与会话 + + @abstractmethod + async def ensure_user(self, user_id: str, display_name: str = "", + trust_level: str = "normal") -> None: + """建档。已存在则不动。身份认证与信任等级分开（方案 §6）。""" + + @abstractmethod + async def start_session(self, session_id: str, user_id: str) -> None: ... + + @abstractmethod + async def end_session(self, session_id: str) -> None: ... + + @abstractmethod + async def add_turn(self, turn: Turn) -> None: ... + + # ---------------------------------------------------------- 记忆读写 + + @abstractmethod + async def remember(self, mem: Memory) -> int: + """写入一条记忆，返回 memory_id。 + + 同 owner + 同 kind + 同 key 的 confirmed 记忆，新值替换旧值 + （方案 §8："明确偏好…新值替换旧值"）。 + """ + + @abstractmethod + async def forget(self, owner_user_id: str, memory_id: int) -> bool: + """删除。必须带 owner_user_id——不能靠 memory_id 单独删， + 否则猜到一个 id 就能删别人的记忆。""" + + @abstractmethod + async def retrieve(self, owner_user_id: str, query: str, *, + limit: int = 5, + include_candidates: bool = False, + scopes: Sequence[Scope] | None = None) -> list[Memory]: + """按相关性检索。 + + 强制按 owner_user_id 过滤。unknown 身份应传一个访客 id， + 而不是绕过这个方法。 + """ + + @abstractmethod + async def list_memories(self, owner_user_id: str, *, + include_candidates: bool = True) -> list[Memory]: + """列出全部，供"查看/删除"入口使用（方案 §8）。""" + + @abstractmethod + async def purge_user(self, owner_user_id: str) -> int: + """彻底删除该用户的一切：记忆、会话、轮次。 + 对应方案 §8："删除同时覆盖会话上下文、摘要和未来索引"。""" + + # ---------------------------------------------------------- 检索打分 + + @staticmethod + def relevance(query: str, text: str) -> float: + """中文字符二元组重合度打分。 + + 为什么不用 FTS5：SQLite 默认分词器对中文是按整串处理的， + 一句话会变成一个 token，检索不出来；trigram 分词器可用但要额外 + 编译选项。在"记忆条数不多"这个实际规模下，在 Python 里按二元组 + 打分更可控，也不依赖 SQLite 的编译配置。 + + 将来真的需要向量检索时，换掉这个打分函数和 retrieve 的实现即可， + 接口不动。 + """ + q, t = query.strip(), text.strip() + if not q or not t: + return 0.0 + if q in t: + return 1.0 + qb = {q[i:i + 2] for i in range(len(q) - 1)} or {q} + tb = {t[i:i + 2] for i in range(len(t) - 1)} or {t} + if not qb: + return 0.0 + hit = len(qb & tb) + if hit == 0: + return 0.0 + # 以查询覆盖率为准：记忆里多出来的字不该拉低分数 + return hit / len(qb) + + +__all__ = [ + "MemoryStore", "Memory", "Turn", "Scope", "ConfirmStatus", "MemoryKind", +] diff --git a/app/memory/sqlite_store.py b/app/memory/sqlite_store.py new file mode 100644 index 0000000..acb05d1 --- /dev/null +++ b/app/memory/sqlite_store.py @@ -0,0 +1,334 @@ +"""SQLite 记忆存储。 + +几个刻意的选择： + + * **列名用 mem_key / mem_value**，不用 key / value——后者在 SQL 里 + 容易和关键字撞，每个语句都得加引号，早晚漏一处。 + + * **WAL 模式**：读写不互相阻塞，将来多进程访问同一份库也不容易锁死。 + + * **外键级联删除**：删用户时记忆/会话/轮次一起走。方案 §8 要求 + "删除同时覆盖会话上下文、摘要和未来索引"，靠数据库约束比靠代码记得住。 + + * **所有 SQL 操作走 asyncio.to_thread**。本地 SQLite 其实微秒级， + 但接口承诺了 async，就得真的不阻塞事件循环——否则将来换成远程 + 实现时，调用方那些"同步假设"会全部暴露成 bug。 + + * **每条查询都带 owner_user_id**。没有"按 memory_id 单独取"的口子： + 猜到 id 就能读别人的记忆，是这类系统最容易犯的错。 +""" + +from __future__ import annotations + +import asyncio +import sqlite3 +import threading +import time +from pathlib import Path +from typing import Sequence + +from .base import ConfirmStatus, Memory, MemoryKind, MemoryStore, Scope, Turn + +#: 关键词全不命中时，允许"退回最近记忆"的记忆总量上限。 +#: 超过这个数就退回严格匹配——否则每轮都会把记忆全塞进 prompt， +#: 既不再是"按需检索"，也会持续推高首 token 延迟。 +FALLBACK_MAX_CORPUS = 10 + +SCHEMA = """ +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS users ( + user_id TEXT PRIMARY KEY, + display_name TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL DEFAULT 'normal', + created_at REAL NOT NULL +); + +CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + started_at REAL NOT NULL, + ended_at REAL +); + +CREATE TABLE IF NOT EXISTS turns ( + turn_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES sessions(session_id) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + role TEXT NOT NULL, + text TEXT NOT NULL, + created_at REAL NOT NULL +); + +CREATE TABLE IF NOT EXISTS memories ( + memory_id INTEGER PRIMARY KEY AUTOINCREMENT, + owner_user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + scope TEXT NOT NULL DEFAULT 'private', + kind TEXT NOT NULL DEFAULT 'fact', + mem_key TEXT NOT NULL DEFAULT '', + mem_value TEXT NOT NULL, + confirm_status TEXT NOT NULL DEFAULT 'confirmed', + source_session_id TEXT, + source_turn_id TEXT, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + expires_at REAL +); + +CREATE INDEX IF NOT EXISTS idx_mem_owner ON memories(owner_user_id, confirm_status); +CREATE INDEX IF NOT EXISTS idx_mem_key ON memories(owner_user_id, kind, mem_key); +CREATE INDEX IF NOT EXISTS idx_turns_sess ON turns(session_id, created_at); +CREATE INDEX IF NOT EXISTS idx_sess_user ON sessions(user_id); +""" + + +def _ensure_user_row(c: sqlite3.Connection, user_id: str, + display_name: str = "") -> None: + """保证 users 里有这一行。 + + 为什么放在存储层而不是让调用方自己 ensure： + memories / turns / sessions 都有指向 users 的外键， + 「被引用的 user 必须存在」是这个 schema 的不变量。 + 让每个调用点各自记得先建档，早晚漏一处——而且漏了之后 + 报的是 "FOREIGN KEY constraint failed"，跟业务逻辑对不上号， + 排查起来很绕。由存储层自己兜住，调用方就不需要知道这个约束。 + + 实测踩过：会话中途切到一个未建档的 user_id， + 第一轮写 turns 就炸在这里。 + """ + c.execute("INSERT INTO users(user_id, display_name, created_at) VALUES(?,?,?) " + "ON CONFLICT(user_id) DO NOTHING", (user_id, display_name, time.time())) + + +def _row_to_memory(row: sqlite3.Row) -> Memory: + return Memory( + memory_id=row["memory_id"], + owner_user_id=row["owner_user_id"], + scope=Scope(row["scope"]), + kind=MemoryKind(row["kind"]), + key=row["mem_key"], + value=row["mem_value"], + confirm_status=ConfirmStatus(row["confirm_status"]), + source_session_id=row["source_session_id"], + source_turn_id=row["source_turn_id"], + created_at=row["created_at"], + updated_at=row["updated_at"], + expires_at=row["expires_at"], + ) + + +class SQLiteMemoryStore(MemoryStore): + def __init__(self, db_path: Path | str): + self.db_path = Path(db_path) + self._conn: sqlite3.Connection | None = None + self._lock = threading.Lock() + + # ---------------------------------------------------------- 内部 + + def _db(self) -> sqlite3.Connection: + if self._conn is None: + raise RuntimeError("MemoryStore 未 open()") + return self._conn + + def _exec(self, fn): + """在受控线程里跑同步 SQL，串行化以免共用连接出问题。""" + with self._lock: + return fn(self._db()) + + async def _run(self, fn): + return await asyncio.to_thread(self._exec, fn) + + # ---------------------------------------------------------- 生命周期 + + async def open(self) -> None: + def _open() -> sqlite3.Connection: + self.db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA) + conn.commit() + return conn + + # 此处还没有并发使用者，不需要持锁跨 await + self._conn = await asyncio.to_thread(_open) + + async def close(self) -> None: + if self._conn is None: + return + + def _close(c: sqlite3.Connection): + c.commit() + c.close() + self._conn = None + + await self._run(_close) + + # ---------------------------------------------------------- 用户与会话 + + async def ensure_user(self, user_id: str, display_name: str = "", + trust_level: str = "normal") -> None: + def op(c: sqlite3.Connection): + c.execute( + "INSERT INTO users(user_id, display_name, trust_level, created_at) " + "VALUES(?,?,?,?) ON CONFLICT(user_id) DO NOTHING", + (user_id, display_name, trust_level, time.time())) + c.commit() + await self._run(op) + + async def start_session(self, session_id: str, user_id: str) -> None: + def op(c: sqlite3.Connection): + c.execute("INSERT INTO users(user_id, created_at) VALUES(?,?) " + "ON CONFLICT(user_id) DO NOTHING", (user_id, time.time())) + c.execute("INSERT INTO sessions(session_id, user_id, started_at) " + "VALUES(?,?,?) ON CONFLICT(session_id) DO NOTHING", + (session_id, user_id, time.time())) + c.commit() + await self._run(op) + + async def end_session(self, session_id: str) -> None: + def op(c: sqlite3.Connection): + c.execute("UPDATE sessions SET ended_at=? WHERE session_id=?", + (time.time(), session_id)) + c.commit() + await self._run(op) + + async def add_turn(self, turn: Turn) -> None: + def op(c: sqlite3.Connection): + _ensure_user_row(c, turn.user_id) + c.execute("INSERT INTO turns(turn_id, session_id, user_id, role, text, created_at) " + "VALUES(?,?,?,?,?,?) ON CONFLICT(turn_id) DO NOTHING", + (turn.turn_id, turn.session_id, turn.user_id, turn.role, + turn.text, turn.created_at)) + c.commit() + await self._run(op) + + # ---------------------------------------------------------- 记忆读写 + + async def remember(self, mem: Memory) -> int: + now = time.time() + + def op(c: sqlite3.Connection): + _ensure_user_row(c, mem.owner_user_id) + # 已确认的同类同键记忆：新值替换旧值（方案 §8） + if mem.confirm_status == ConfirmStatus.CONFIRMED and mem.key: + row = c.execute( + "SELECT memory_id FROM memories WHERE owner_user_id=? AND kind=? " + "AND mem_key=? AND confirm_status=?", + (mem.owner_user_id, mem.kind.value, mem.key, + ConfirmStatus.CONFIRMED.value)).fetchone() + if row: + c.execute( + "UPDATE memories SET mem_value=?, scope=?, updated_at=?, " + "expires_at=?, source_session_id=?, source_turn_id=? " + "WHERE memory_id=? AND owner_user_id=?", + (mem.value, mem.scope.value, now, mem.expires_at, + mem.source_session_id, mem.source_turn_id, + row["memory_id"], mem.owner_user_id)) + c.commit() + return row["memory_id"] + + cur = c.execute( + "INSERT INTO memories(owner_user_id, scope, kind, mem_key, mem_value, " + "confirm_status, source_session_id, source_turn_id, created_at, " + "updated_at, expires_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)", + (mem.owner_user_id, mem.scope.value, mem.kind.value, mem.key, + mem.value, mem.confirm_status.value, mem.source_session_id, + mem.source_turn_id, mem.created_at or now, now, mem.expires_at)) + c.commit() + return int(cur.lastrowid) + + return await self._run(op) + + async def forget(self, owner_user_id: str, memory_id: int) -> bool: + def op(c: sqlite3.Connection): + # 必须同时匹配 owner——不能只凭 memory_id 删 + cur = c.execute("DELETE FROM memories WHERE memory_id=? AND owner_user_id=?", + (memory_id, owner_user_id)) + c.commit() + return cur.rowcount > 0 + return await self._run(op) + + async def retrieve(self, owner_user_id: str, query: str, *, + limit: int = 5, + include_candidates: bool = False, + scopes: Sequence[Scope] | None = None) -> list[Memory]: + allowed = [s.value for s in (scopes or [Scope.PRIVATE, Scope.SHARED])] + statuses = [ConfirmStatus.CONFIRMED.value] + if include_candidates: + statuses.append(ConfirmStatus.CANDIDATE.value) + placeholders = ",".join("?" * len(allowed)) + status_ph = ",".join("?" * len(statuses)) + + def op(c: sqlite3.Connection): + rows = c.execute( + f"SELECT * FROM memories WHERE owner_user_id=? " + f"AND scope IN ({placeholders}) AND confirm_status IN ({status_ph})", + (owner_user_id, *allowed, *statuses)).fetchall() + now = time.time() + scored = [] + for r in rows: + m = _row_to_memory(r) + if m.is_expired(now): + continue + score = self.relevance(query, m.text) if query.strip() else 0.0 + scored.append((score, m.updated_at, m)) + + hits = [x for x in scored if x[0] > 0] + if not hits and 0 < len(scored) <= FALLBACK_MAX_CORPUS: + # 关键词一条都没命中时，退回"最近更新的若干条"。 + # + # 理由是实测出来的：查"我想喝点什么"匹配不上"喜欢喝美式咖啡"—— + # 两句共享"喝"字，但没有共同的二元组，按字面打分是 0。 + # 中文口语短句和记忆文本之间"意思相关但用词不同"极其常见， + # 纯字面匹配漏得厉害。 + # + # 但兜底必须**限量**：不加限制就变成"每轮无条件注入记忆"， + # 那就不是方案 §8 说的"按需检索"了，而且记忆一多会持续 + # 拉长 prompt、推高首 token 延迟。 + # + # 所以只在记忆总量很小时兜底——这个阶段"全给"和"按需给" + # 结果差不多，而漏给的代价（助手像不认识你）更大。 + # 超过阈值就退回严格匹配，接受漏召。 + # + # 这是权宜之计，不是终局：真正的解法是向量检索 + # （方案 §8 把它列在"真实使用中检索不足再评估"）。 + # 到那时换掉打分和这段逻辑即可，接口不动。 + hits = scored + + hits.sort(key=lambda x: (x[0], x[1]), reverse=True) + return [m for _, _, m in hits[:limit]] + + return await self._run(op) + + async def list_memories(self, owner_user_id: str, *, + include_candidates: bool = True) -> list[Memory]: + statuses = [ConfirmStatus.CONFIRMED.value] + if include_candidates: + statuses.append(ConfirmStatus.CANDIDATE.value) + status_ph = ",".join("?" * len(statuses)) + + def op(c: sqlite3.Connection): + rows = c.execute( + f"SELECT * FROM memories WHERE owner_user_id=? " + f"AND confirm_status IN ({status_ph}) ORDER BY updated_at DESC", + (owner_user_id, *statuses)).fetchall() + now = time.time() + return [m for m in (_row_to_memory(r) for r in rows) if not m.is_expired(now)] + + return await self._run(op) + + async def purge_user(self, owner_user_id: str) -> int: + def op(c: sqlite3.Connection): + cur = c.execute("DELETE FROM memories WHERE owner_user_id=?", (owner_user_id,)) + n = cur.rowcount + c.execute("DELETE FROM turns WHERE user_id=?", (owner_user_id,)) + c.execute("DELETE FROM sessions WHERE user_id=?", (owner_user_id,)) + # users 放最后：外键级联会把残留一并带走 + c.execute("DELETE FROM users WHERE user_id=?", (owner_user_id,)) + c.commit() + return n + return await self._run(op) + + +__all__ = ["SQLiteMemoryStore", "SCHEMA"] diff --git a/app/prompts.py b/app/prompts.py new file mode 100644 index 0000000..b4ea6fe --- /dev/null +++ b/app/prompts.py @@ -0,0 +1,103 @@ +"""提示词构造。 + +关键约束（实测，P0-本地LLM验证报告.md §四）： + + 本机 vLLM **只接受一条 system 消息**，多条返回 + 400 "System message must be at the beginning." + +所以记忆不能作为"第二条 system"注入，必须**并进同一条里、每轮重建**。 + +这其实比追加式更干净，也正好满足方案 §8 的要求： +"切换身份清理上一人的私有检索上下文"、"注入的记忆每轮替换，不是追加"—— +每轮从零拼出来的 system 天然不会有上一人的残留。 +""" + +from __future__ import annotations + +from typing import Sequence + +from .identity.base import IdentityDecision +from .memory.base import Memory + +#: 语音助手的回复风格。方案 §5："默认简短自然回复"。 +#: 这几条不是"风格偏好"，是**功能性的**——放出去的文本会直接进 TTS， +#: 带 Markdown 的话会被逐字念出来（"井号井号 第一步"）。 +PERSONA_RULES = """\ +你是一个桌面陪伴助手，通过语音和用户交流。用户听到的是合成语音，不是文字。 + +必须遵守： +1. 回复简短自然，适合朗读，默认一到两句话。不要长篇解释。 +2. 绝对不要使用 Markdown、星号、井号、列表、表格、表情符号或括号注释。 +3. 数字、日期、金额按中文口语写法，例如"一千二百元""下午三点半"。 +4. 不确定的事直说不确定，不要编造。没做过的事不要说已经做了。 +5. 需要操作设备时调用工具。不要假装已经执行，也不要说"正在为您打开"。 +6. 用户打断时立刻停止，不要坚持把上一句说完。""" + + +def build_system_prompt( + identity: IdentityDecision, + memories: Sequence[Memory] = (), + *, + extra: str = "", +) -> str: + """拼出本轮唯一的 system 消息。 + + 每轮调用一次，不要缓存后复用——身份或记忆一变，缓存就会串。 + """ + parts = [PERSONA_RULES] + + if identity.confirmed: + parts.append(f"\n当前用户：{identity.user_id}（已确认身份）。") + else: + # 方案 §6：未知身份只用访客上下文，不做个性化 + parts.append("\n当前用户身份未确认，按访客对待。" + "不要提及任何私人信息，也不要假定你知道对方是谁。") + + if memories: + lines = ["\n已知的用户信息（供参考，不要逐条复述）："] + for m in memories: + lines.append(f"- {m.text}") + parts.append("\n".join(lines)) + + if identity.suggestions and not identity.confirmed: + # 有人脸候选但不自动切换。让助手可以自然地问一句， + # 而不是自作主张切换档案（方案 §6："看到熟人不等于确认他正在说话"）。 + names = "、".join( + s.candidate_user_id for s in identity.suggestions[:2]) + parts.append( + f"\n注意：摄像头可能认出了 {names}，但这不代表对方正在说话。" + f"不要据此切换身份或提及该用户的私人信息。") + + if extra: + parts.append("\n" + extra.strip()) + + return "\n".join(parts) + + +def trim_history( + history: Sequence[dict], + *, + max_turns: int, + max_chars: int = 4000, +) -> list[dict]: + """裁掉过旧的对话轮次。 + + 方案 §8 要求"对话摘要限长、按需检索"，不默认永久保存逐字稿。 + 这里做最简单的一档：保留最近 max_turns 轮，再按字符数兜底。 + 上下文越短，本地 LLM 的首 token 越短——这不是省资源，是直接买延迟。 + """ + msgs = list(history) + # 一轮 = 一问一答，所以保留的消息数是轮数的两倍 + keep = max_turns * 2 + if len(msgs) > keep: + msgs = msgs[-keep:] + + # 按字符数从旧到新裁，保证裁完仍然从 user 消息开始 + while msgs and sum(len(m.get("content") or "") for m in msgs) > max_chars: + msgs.pop(0) + while msgs and msgs[0].get("role") != "user": + msgs.pop(0) + return msgs + + +__all__ = ["build_system_prompt", "trim_history", "PERSONA_RULES"] diff --git a/app/tests/__init__.py b/app/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/tests/test_engine_e2e.py b/app/tests/test_engine_e2e.py new file mode 100644 index 0000000..cb0d391 --- /dev/null +++ b/app/tests/test_engine_e2e.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""端到端测试：音频/文本 → STT → 身份 → 记忆 → LLM → TTS → 音频。 + +**全程不需要麦克风、音箱、摄像头、ESP32。** +这是"服务器即助手"架构的直接好处：链路可以在任何有 CPU 的地方验证， +硬件到货后再接最后一段（真实音频进出）即可。 + +跑完会把合成音频写到 out/ 目录，可以人工试听整条链路的实际效果。 + +运行： + cd <项目根> + ~/venvs/ai_assistant/bin/python app/tests/test_engine_e2e.py +""" + +from __future__ import annotations + +import asyncio +import shutil +import sys +import tempfile +import time +import wave +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(ROOT)) + +from app.config import AppConfig # noqa: E402 +from app.engine import ConversationEngine, EventKind # noqa: E402 +from app.identity import GUEST_USER_ID, ManualIdentityProvider # noqa: E402 +from app.llm import FallbackProvider, create_provider # noqa: E402 +from app.memory import SQLiteMemoryStore # noqa: E402 +from app.voice import StreamingSTT, create_tts # noqa: E402 + +OK, FAIL = [], [] +OUT = ROOT / "out" + + +def check(name: str, cond: bool, detail: str = "") -> None: + (OK if cond else FAIL).append(name) + print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) + + +def save_wav(path: Path, audio: np.ndarray, sr: int) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + pcm = (np.clip(audio, -1, 1) * 32767).astype(np.int16) + with wave.open(str(path), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(sr) + w.writeframes(pcm.tobytes()) + + +async def collect(engine: ConversationEngine, text: str) -> tuple[str, np.ndarray, list]: + """跑一轮，返回 (回复文本, 拼接音频, 事件列表)。""" + reply, audios, events = [], [], [] + async for ev in engine.respond_to_text(text): + events.append(ev) + if ev.kind == EventKind.RESPONSE_DELTA: + reply.append(ev.text) + elif ev.kind == EventKind.SPEAK and ev.audio is not None: + audios.append(ev.audio) + audio = np.concatenate(audios) if audios else np.zeros(0, dtype=np.float32) + return "".join(reply).strip(), audio, events + + +async def main() -> int: + cfg = AppConfig.load() + tmp = Path(tempfile.mkdtemp(prefix="e2e_")) + OUT.mkdir(parents=True, exist_ok=True) + + print("=== 0. 装配 ===") + llm = FallbackProvider(create_provider(cfg.llm), + create_provider(cfg.llm_fallback)) + mem = SQLiteMemoryStore(tmp / "memory.db") + await mem.open() + tts = create_tts(cfg.voice) + stt = StreamingSTT(cfg.voice) + ident = ManualIdentityProvider() + engine = ConversationEngine(cfg, llm, mem, ident, tts, stt) + print(f" LLM={cfg.llm.model} 备用={cfg.llm_fallback.model}") + tts_desc = (f"{cfg.voice.tts_backend}/{cfg.voice.tts_speaker}" + if cfg.voice.tts_backend == "remote" else f"local/{cfg.voice.tts_threads}线程") + print(f" STT={cfg.voice.stt_threads}线程 TTS={tts_desc}") + check("四个组件装配成功", True) + + ok, detail = await llm.health() + check("LLM 可用", ok, detail) + if not ok: + await mem.close() + shutil.rmtree(tmp, ignore_errors=True) + return 1 + + await engine.start_session("e2e_sess", user_id="u_ylhan") + + # ---------------------------------------------------------- 1 + print("\n=== 1. 纯文本一轮（LLM → 分句 → TTS）===") + reply, audio, events = await collect(engine, "现在几点了？随便说一句就行。") + print(f" 回复: {reply[:60]!r}") + m = [e for e in events if e.kind == EventKind.METRICS][0].metrics + print(f" {m.summary()}") + check("有回复", bool(reply)) + check("有音频输出", len(audio) > 0, f"{len(audio)} 采样点") + check("回复不含 Markdown", + not any(c in reply for c in "*#`|[]"), reply[:40]) + # 首音延迟这里只做"没离谱"的兜底检查，不设严格阈值： + # 本机 N5095 的 TTS RTF 是 1.86（实测），首音必然被 TTS 拖到秒级； + # 服务器上同一模型是 0.09，首音在 150ms 量级。 + # 真正该卡指标的是部署机（服务器），所以那个断言放在服务器上的验收里。 + check("首音延迟未失控", m.first_audio_ms < 8000, f"{m.first_audio_ms:.0f}ms") + save_wav(OUT / "e2e_1_text.wav", audio, tts.sample_rate) + + # ---------------------------------------------------------- 2 + print("\n=== 2. 多句回复按句播（每句独立音频）===") + _, _, events2 = await collect( + engine, "用三句话介绍一下你自己，每句短一点。") + speaks = [e for e in events2 if e.kind == EventKind.SPEAK] + print(f" 分 {len(speaks)} 段: {[e.text[:14] for e in speaks]}") + check("按句切成了多段", len(speaks) >= 2, f"{len(speaks)} 段") + audio2 = np.concatenate([e.audio for e in speaks]) if speaks else np.zeros(0, np.float32) + save_wav(OUT / "e2e_2_multisentence.wav", audio2, tts.sample_rate) + + # ---------------------------------------------------------- 3 + print("\n=== 3. 记忆：显式记住 → 下一轮被检索到 ===") + await collect(engine, "记住我喜欢喝美式咖啡，不加糖。") + await asyncio.sleep(0.5) # 等异步写入落库 + stored = await mem.list_memories("u_ylhan", include_candidates=True) + print(f" 库中记忆 {len(stored)} 条: {[m.text[:20] for m in stored]}") + check("记忆已落库", len(stored) >= 1, f"{len(stored)} 条") + check("自动抽取的是候选状态", + all(m.confirm_status.value == "candidate" for m in stored)) + + # 手动确认一条，看它是否参与检索 + await mem.remember(type(stored[0])( + owner_user_id="u_ylhan", kind=stored[0].kind, key="", + value="用户喜欢喝美式咖啡，不加糖", + confirm_status=__import__("app.memory", fromlist=["ConfirmStatus"]).ConfirmStatus.CONFIRMED)) + _, _, events3 = await collect(engine, "我想喝点东西，推荐一下？") + used = [e for e in events3 if e.kind == EventKind.MEMORY_USED] + n_used = len(used[0].memories) if used else 0 + print(f" 本轮用到记忆 {n_used} 条") + check("确认的记忆参与检索", n_used >= 1, f"{n_used} 条") + + # ---------------------------------------------------------- 4 + print("\n=== 4. 身份隔离：换人后拿不到上一个人的记忆 ===") + ident.switch("u_other", reason="测试切换") + _, _, events4 = await collect(engine, "我想喝点什么？") + used4 = [e for e in events4 if e.kind == EventKind.MEMORY_USED] + n4 = len(used4[0].memories) if used4 else 0 + check("新身份没有旧记忆", n4 == 0, f"{n4} 条") + reply4 = "".join(e.text for e in events4 if e.kind == EventKind.RESPONSE_DELTA) + check("回复未泄露咖啡偏好", "美式" not in reply4 and "咖啡" not in reply4, + reply4[:50]) + + # ---------------------------------------------------------- 5 + print("\n=== 5. 访客身份不做个性化 ===") + ident.reset() + print(f" 当前身份: {ident.current().user_id} (confirmed={ident.current().confirmed})") + _, _, events5 = await collect(engine, "我喜欢什么？") + used5 = [e for e in events5 if e.kind == EventKind.MEMORY_USED] + check("访客不检索私人记忆", len(used5) == 0 or not used5[0].memories) + reply5 = "".join(e.text for e in events5 if e.kind == EventKind.RESPONSE_DELTA) + check("访客回复不编造偏好", "美式" not in reply5, reply5[:50]) + + # ---------------------------------------------------------- 6 + print("\n=== 6. 音频输入（真人录音 → STT → LLM → TTS）===") + # 测试音频不在代码目录里，和模型放在一起（都是可重新下载的产物）。 + # 用模型目录反推，注意要 resolve()——服务器上模型目录是个软链， + # 不解析的话 .parent 会指到 ~/ai-companion 而不是真实位置。 + wav_path = None + for cand in ( + cfg.voice.model_dir.resolve().parent / "wavs" / "stt_1.wav", + ROOT / "p0" / "stt_tts_bench" / "wavs" / "stt_1.wav", + ): + if cand.is_file(): + wav_path = cand + break + if wav_path is not None: + with wave.open(str(wav_path)) as w: + samples = np.frombuffer(w.readframes(w.getnframes()), + dtype=np.int16).astype(np.float32) / 32768.0 + ident.switch("u_ylhan", reason="回到主人") + t0 = time.perf_counter() + transcript, audios, events6 = [], [], [] + async for ev in engine.respond_to_audio(samples): + events6.append(ev) + if ev.kind == EventKind.TRANSCRIPT: + transcript.append(ev.text) + elif ev.kind == EventKind.SPEAK and ev.audio is not None: + audios.append(ev.audio) + got = "".join(transcript) + m6 = [e for e in events6 if e.kind == EventKind.METRICS] + print(f" STT: {got[:50]!r}") + if m6 and m6[0].metrics: + print(f" {m6[0].metrics.summary()}") + check("音频链路跑通（能出转写）", bool(got.strip())) + if audios: + save_wav(OUT / "e2e_6_fromaudio.wav", np.concatenate(audios), tts.sample_rate) + else: + check("音频链路跑通", False, f"测试音频不存在: {wav_path}") + + # ---------------------------------------------------------- 7 + print("\n=== 7. 打断：取消正在进行的回复 ===") + cancelled = "未取消" + chunks_seen = 0 + + async def long_turn(): + # 必须把整个流消费完才谈得上"取消"。 + # 第一版写成 `async for _ in ...: return "已开始"`—— + # 第一个事件就 return，生成器随之关闭，根本没有正在进行的回复可取消， + # 测试于是永远"失败"，但失败原因跟取消能力毫无关系。 + nonlocal chunks_seen + async for _ev in engine.respond_to_text("请详细讲一讲中国历史，越长越好。"): + chunks_seen += 1 + return "已完成" + + task = asyncio.create_task(long_turn()) + await asyncio.sleep(0.7) + task.cancel() + try: + await task + except asyncio.CancelledError: + cancelled = "已取消" + check("长回复可被取消", cancelled == "已取消" and chunks_seen > 0, + f"{cancelled}，取消前已收到 {chunks_seen} 个事件") + + await engine.end_session() + await mem.close() + await llm.aclose() + shutil.rmtree(tmp, ignore_errors=True) + + print(f"\n合成音频已写入 {OUT}/ ，可人工试听：") + for f in sorted(OUT.glob("e2e_*.wav")): + print(f" {f.name}") + + print("\n" + "=" * 62) + print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}") + for n in FAIL: + print(f" 未通过: {n}") + return 0 if not FAIL else 1 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/app/tests/test_identity.py b/app/tests/test_identity.py new file mode 100644 index 0000000..63dd41f --- /dev/null +++ b/app/tests/test_identity.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""身份层测试。 + +最关键的一条：**人脸认出来不等于切身份**。 +方案 §6 写得很直白："看到熟人不等于确认他正在说话"。 +做错的后果是——路人从摄像头前走过，助手就把你的私人记忆念了出来。 +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from app.identity import ( # noqa: E402 + GUEST_USER_ID, IdentityClaim, ManualIdentityProvider, +) + +OK, FAIL = [], [] + + +def check(name: str, cond: bool, detail: str = "") -> None: + (OK if cond else FAIL).append(name) + print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) + + +def main() -> int: + print("=== 1. 初始状态 ===") + p = ManualIdentityProvider() + d = p.current() + check("默认是访客", d.user_id == GUEST_USER_ID and d.is_guest, d.user_id) + check("访客信任等级独立", d.trust_level == "guest") + + print("\n=== 2. 显式切换（首版唯一可靠入口）===") + d = p.switch("u_ylhan", reason="对话确认") + check("切换生效", d.user_id == "u_ylhan" and d.confirmed) + check("信任等级为 normal", d.trust_level == "normal") + check("版本号 +1", p.version == 1, str(p.version)) + v = p.version + p.switch("u_ylhan") # 切到同一个人 + check("切到同一人不增版本号", p.version == v, f"{v} → {p.version}") + + print("\n=== 3. 人脸候选不得自动切换（关键）===") + p2 = ManualIdentityProvider("u_ylhan") + before = p2.current().user_id + d = p2.resolve([IdentityClaim(source="face", candidate_user_id="u_other", + confidence=0.95, track_id="track_7")]) + check("身份未被自动切换", d.user_id == before, f"{before} → {d.user_id}") + check("但仍保留建议", any(s.candidate_user_id == "u_other" for s in d.suggestions), + d.reason) + check("track_id 未污染 user_id", d.user_id != "track_7") + + # 访客状态下也不应变 + p3 = ManualIdentityProvider() + d3 = p3.resolve([IdentityClaim(source="face", candidate_user_id="u_ylhan", + confidence=0.99)]) + check("访客状态也不自动切换", d3.user_id == GUEST_USER_ID and d3.is_guest, d3.reason) + + print("\n=== 4. 低置信度连建议都不给 ===") + p4 = ManualIdentityProvider() + d4 = p4.resolve([IdentityClaim(source="face", candidate_user_id="u_x", confidence=0.3)]) + check("低置信度无建议", len(d4.suggestions) == 0, f"{len(d4.suggestions)} 条") + + print("\n=== 5. 权威来源可以切换 ===") + p5 = ManualIdentityProvider() + d5 = p5.resolve([IdentityClaim(source="client_login", candidate_user_id="u_phone", + confidence=1.0)]) + check("客户端登录态可切身份", d5.user_id == "u_phone" and d5.confirmed, d5.reason) + + d5b = p5.resolve([IdentityClaim(source="manual", candidate_user_id="u_ylhan")]) + check("显式来源可切身份", d5b.user_id == "u_ylhan") + + print("\n=== 6. 权威线索互相矛盾时不切 ===") + p6 = ManualIdentityProvider("u_ylhan") + d6 = p6.resolve([ + IdentityClaim(source="manual", candidate_user_id="u_a", confidence=1.0), + IdentityClaim(source="client_login", candidate_user_id="u_b", confidence=1.0), + ]) + check("矛盾时保持原身份", d6.user_id == "u_ylhan", d6.reason) + + print("\n=== 7. reset 回访客 ===") + p7 = ManualIdentityProvider("u_ylhan") + v7 = p7.version + d7 = p7.reset() + check("回到访客", d7.user_id == GUEST_USER_ID and d7.is_guest) + check("版本号 +1（上层据此丢弃旧上下文）", p7.version == v7 + 1, + f"{v7} → {p7.version}") + + print("\n=== 8. 空 user_id 应报错而非静默接受 ===") + try: + p7.switch("") + check("拒绝空 user_id", False, "没报错") + except ValueError: + check("拒绝空 user_id", True) + + print("\n" + "=" * 62) + print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}") + for n in FAIL: + print(f" 未通过: {n}") + return 0 if not FAIL else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/tests/test_llm.py b/app/tests/test_llm.py new file mode 100644 index 0000000..4570ad7 --- /dev/null +++ b/app/tests/test_llm.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""LLM 抽象层测试。 + +重点验证实测踩过的三个坑是否真的被吸收： + 1. 思考字段名差异（reasoning / reasoning_content） + 2. 关闭思考的方式差异 + 3. 多条 system 消息归一化 + +运行： + ~/venvs/ai_assistant/bin/python -m app.tests.test_llm +或 + cd <项目根> && ~/venvs/ai_assistant/bin/python app/tests/test_llm.py +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from app.config import AppConfig # noqa: E402 +from app.llm import ( # noqa: E402 + ChunkKind, DeepSeekProvider, FallbackProvider, LocalVLLMProvider, create_provider, +) +from app.llm.base import LLMProvider # noqa: E402 + +OK, FAIL = [], [] + + +def check(name: str, cond: bool, detail: str = "") -> None: + (OK if cond else FAIL).append(name) + print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) + + +# ---------------------------------------------------------------- 纯本地逻辑（无需网络） + +def test_normalize_messages() -> None: + print("\n=== 1. 消息归一化（多条 system → 一条，置于开头）===") + msgs = [ + {"role": "system", "content": "你是助手。"}, + {"role": "user", "content": "你好"}, + {"role": "system", "content": "记忆：用户叫 ylhan。"}, + ] + out = LLMProvider.normalize_messages(msgs) + check("只剩一条 system", sum(1 for m in out if m["role"] == "system") == 1, + f"实际 {sum(1 for m in out if m['role']=='system')} 条") + check("system 在开头", out[0]["role"] == "system") + check("两条 system 内容都保留", + "你是助手。" in out[0]["content"] and "ylhan" in out[0]["content"]) + check("非 system 消息顺序不变", + [m["role"] for m in out[1:]] == ["user"]) + + # 无 system 的情况不应凭空造一条 + out2 = LLMProvider.normalize_messages([{"role": "user", "content": "hi"}]) + check("无 system 时不新增", len(out2) == 1 and out2[0]["role"] == "user") + + # 空 system 应被丢弃 + out3 = LLMProvider.normalize_messages( + [{"role": "system", "content": " "}, {"role": "user", "content": "hi"}]) + check("空白 system 被丢弃", len(out3) == 1) + + +def test_request_bodies() -> None: + print("\n=== 2. 各后端的请求体（关思考方式不同）===") + cfg_local = AppConfig.load().llm + cfg_ds = AppConfig.load().llm_fallback + cfg_local.thinking = False + cfg_ds.thinking = False + + b_local = LocalVLLMProvider(cfg_local)._build_body( + [{"role": "user", "content": "hi"}], tools=None, max_tokens=100) + check("本机用 chat_template_kwargs", + b_local.get("chat_template_kwargs") == {"enable_thinking": False}, + str(b_local.get("chat_template_kwargs"))) + + b_ds = DeepSeekProvider(cfg_ds)._build_body( + [{"role": "user", "content": "hi"}], tools=None, max_tokens=100) + check("DeepSeek 用 thinking.type", + b_ds.get("thinking") == {"type": "disabled"}, + str(b_ds.get("thinking"))) + + check("两者模型名不同", b_local["model"] != b_ds["model"], + f"{b_local['model']} vs {b_ds['model']}") + + +def test_sse_parsing() -> None: + print("\n=== 3. SSE 解析：两种思考字段名都认 ===") + + class FakeResp: + def __init__(self, lines): self.content = self._gen(lines) + async def _gen(self, lines): + for l in lines: + yield (l + "\n").encode() + + async def run(lines): + return [c async for c in LLMProvider._iter_sse(FakeResp(lines))] + + async def main(): + # reasoning_content（DeepSeek 写法） + c1 = await run([ + 'data: {"choices":[{"delta":{"reasoning_content":"想一想"}}]}', + 'data: {"choices":[{"delta":{"content":"你好"}}]}', + 'data: [DONE]', + ]) + check("认 reasoning_content", + any(c.kind == ChunkKind.REASONING and c.text == "想一想" for c in c1)) + check("content 正常", any(c.kind == ChunkKind.CONTENT and c.text == "你好" for c in c1)) + + # reasoning（本机 vLLM 写法） + c2 = await run([ + 'data: {"choices":[{"delta":{"reasoning":"想一下"}}]}', + 'data: {"choices":[{"delta":{"content":"在的"}}]}', + 'data: [DONE]', + ]) + check("认 reasoning（本机 vLLM 字段）", + any(c.kind == ChunkKind.REASONING and c.text == "想一下" for c in c2)) + + # 工具调用分片累积 + c3 = await run([ + 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","function":{"name":"set_light","arguments":"{\\"ro"}}]}}]}', + 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"om\\":\\"客厅\\"}"}}]}}]}', + 'data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}', + ]) + tc = [c for c in c3 if c.kind == ChunkKind.TOOL_CALL] + check("工具调用参数分片拼接", tc and tc[-1].tool_arguments == '{"room":"客厅"}', + tc[-1].tool_arguments if tc else "无") + check("工具名保留", tc and tc[-1].tool_name == "set_light") + + # 坏帧不应崩溃 + c4 = await run(['data: {坏json', 'data: {"choices":[{"delta":{"content":"ok"}}]}', 'data: [DONE]']) + check("坏帧被跳过而非崩溃", + any(c.kind == ChunkKind.CONTENT and c.text == "ok" for c in c4)) + + asyncio.run(main()) + + +# ---------------------------------------------------------------- 需要网络 + +async def test_live() -> None: + cfg = AppConfig.load() + provider = create_provider(cfg.llm) + print(f"\n=== 4. 实连本机 vLLM（{cfg.llm.model} @ {cfg.llm.base_url}）===") + ok, detail = await provider.health() + check("健康检查", ok, detail) + if not ok: + await provider.aclose() + return + + msgs = [ + {"role": "system", "content": + "你是桌面陪伴助手，回复必须简短自然、适合朗读，不超过两句话，" + "不要用 Markdown 或表情符号。"}, + {"role": "user", "content": "今天天气怎么样"}, + {"role": "assistant", "content": "今天北京晴，十八到二十六度。"}, + {"role": "user", "content": "那明天呢"}, + ] + res = await provider.complete(msgs, max_tokens=120) + print(f" 回答: {res.content.strip()[:70]!r}") + print(f" 首token(回答)={res.first_content_ms:.0f}ms 总={res.total_ms:.0f}ms") + print(f" 思考={len(res.reasoning)}字 完成tok={res.usage.get('completion_tokens') if res.usage else '?'}") + check("有回答内容", bool(res.content.strip())) + check("关思考后无思考内容", len(res.reasoning) == 0, f"{len(res.reasoning)} 字") + check("首token在 1 秒内", (res.first_content_ms or 9e9) < 1000, + f"{res.first_content_ms:.0f}ms") + check("回复足够短", len(res.content) < 100, f"{len(res.content)} 字") + + # 工具调用 + tools = [{"type": "function", "function": { + "name": "set_light", "description": "开关指定房间的灯", + "parameters": {"type": "object", "properties": { + "room": {"type": "string"}, "on": {"type": "boolean"}}, + "required": ["room", "on"]}}}] + res2 = await provider.complete([{"role": "user", "content": "把客厅的灯打开"}], + tools=tools, max_tokens=128) + check("工具调用正确", res2.has_tool_calls and + res2.tool_calls[0].parsed_arguments().get("room") == "客厅", + f"{res2.tool_calls[0].name if res2.tool_calls else '无'}") + + # 取消：模拟打断 + async def cancel_midway(): + async for _ch in provider.stream( + [{"role": "user", "content": "请详细讲一下中国历史"}], max_tokens=800): + return "cancelled" + return "finished" + + got = await asyncio.wait_for(cancel_midway(), timeout=30) + check("可中途取消（打断）", got == "cancelled", got) + await provider.aclose() + + +async def test_fallback_logic() -> None: + print("\n=== 5. 降级逻辑（不发真实请求）===") + cfg = AppConfig.load() + + class BoomProvider(LocalVLLMProvider): + def __init__(self, c, err): super().__init__(c); self._err = err + async def stream(self, *a, **kw): + raise self._err + yield # noqa: 使其成为生成器 + + fallback = DeepSeekProvider(cfg.llm_fallback) + # 连接类错误应降级 + from app.llm.base import LLMError + p = FallbackProvider(BoomProvider(cfg.llm, ConnectionError("连不上")), fallback) + check("连接错误判为可降级", p._should_fallback(ConnectionError("x"))) + check("超时判为可降级", p._should_fallback(TimeoutError("x"))) + check("503 判为可降级", p._should_fallback(LLMError("local HTTP 503: busy"))) + check("400 判为不可降级", not p._should_fallback(LLMError("local HTTP 400: bad"))) + check("401 判为不可降级", not p._should_fallback(LLMError("local HTTP 401: auth"))) + + +def main() -> int: + test_normalize_messages() + test_request_bodies() + test_sse_parsing() + asyncio.run(test_live()) + asyncio.run(test_fallback_logic()) + + print("\n" + "=" * 62) + print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}") + for n in FAIL: + print(f" 未通过: {n}") + return 0 if not FAIL else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/tests/test_memory.py b/app/tests/test_memory.py new file mode 100644 index 0000000..9f556e8 --- /dev/null +++ b/app/tests/test_memory.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""记忆层测试。 + +重点不是"能存能取"，而是方案 §8 反复强调的隔离性质： + - 未知身份不碰私人记忆 + - 删别人的记忆删不掉 + - 自动抽取的候选不参与检索 + - 删用户要连会话和摘要一起删干净 + +这些是"错了很难发现、发现了很严重"的类型，所以单独测。 +""" + +from __future__ import annotations + +import asyncio +import shutil +import sys +import tempfile +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from app.memory import ( # noqa: E402 + ConfirmStatus, Memory, MemoryKind, Scope, SQLiteMemoryStore, Turn, +) +from app.memory.sqlite_store import FALLBACK_MAX_CORPUS # noqa: E402 + +OK, FAIL = [], [] + + +def check(name: str, cond: bool, detail: str = "") -> None: + (OK if cond else FAIL).append(name) + print(f" [{'PASS' if cond else 'FAIL'}] {name}" + (f" — {detail}" if detail else "")) + + +async def run(store: SQLiteMemoryStore) -> None: + await store.open() + + print("\n=== 1. 基本读写 ===") + await store.ensure_user("u_ylhan", "ylhan") + await store.ensure_user("u_guest", "访客") + mid = await store.remember(Memory( + owner_user_id="u_ylhan", kind=MemoryKind.PREFERENCE, + key="回复风格", value="喜欢简短直接，不要长篇解释")) + check("写入返回 memory_id", isinstance(mid, int) and mid > 0, str(mid)) + got = await store.retrieve("u_ylhan", "回复风格") + check("能检索到", len(got) == 1 and got[0].key == "回复风格") + check("字段完整", got[0].kind == MemoryKind.PREFERENCE and got[0].value.startswith("喜欢")) + + print("\n=== 2. 身份隔离（最关键）===") + other = await store.retrieve("u_guest", "回复风格") + check("别人检索不到", len(other) == 0, f"拿到 {len(other)} 条") + listed = await store.list_memories("u_guest") + check("别人列不出", len(listed) == 0) + + # 用 memory_id 越权删 + deleted = await store.forget("u_guest", mid) + check("越权删除失败", deleted is False) + still = await store.retrieve("u_ylhan", "回复风格") + check("原主记忆还在", len(still) == 1) + + print("\n=== 3. 同键新值替换旧值 ===") + await store.remember(Memory( + owner_user_id="u_ylhan", kind=MemoryKind.PREFERENCE, + key="回复风格", value="喜欢更活泼一点的说法")) + rows = await store.retrieve("u_ylhan", "回复风格") + check("同键只剩一条", len(rows) == 1, f"{len(rows)} 条") + check("值已更新", "活泼" in rows[0].value, rows[0].value) + + # 不同 key 不应互相覆盖 + await store.remember(Memory(owner_user_id="u_ylhan", kind=MemoryKind.PREFERENCE, + key="称呼", value="叫他老韩")) + rows2 = await store.list_memories("u_ylhan") + check("不同 key 各自保留", len(rows2) == 2, f"{len(rows2)} 条") + + print("\n=== 4. 候选记忆默认不参与检索 ===") + # 判据是"这条候选不在结果里"，**不是"结果为空"**。 + # 后者是实现细节：检索在关键词不命中时会退回最近记忆 + # （见 sqlite_store.FALLBACK_MAX_CORPUS），结果本来就可能非空。 + CAND = "用户好像养了一只叫豆豆的猫" + await store.remember(Memory( + owner_user_id="u_ylhan", kind=MemoryKind.FACT, key="", value=CAND, + confirm_status=ConfirmStatus.CANDIDATE)) + hit = await store.retrieve("u_ylhan", "豆豆") + check("候选不出现在检索结果里", all(m.value != CAND for m in hit), + f"拿到 {len(hit)} 条：{[m.value[:12] for m in hit]}") + hit2 = await store.retrieve("u_ylhan", "豆豆", include_candidates=True) + check("显式要求时可检索到", any(m.value == CAND for m in hit2), + f"拿到 {len(hit2)} 条") + allm = await store.list_memories("u_ylhan") + check("候选确实已落库", any(m.confirm_status == ConfirmStatus.CANDIDATE for m in allm)) + + print("\n=== 5. 过期 ===") + EXPIRED = "十分钟后提醒我关火" + await store.remember(Memory( + owner_user_id="u_ylhan", kind=MemoryKind.TASK, key="", value=EXPIRED, + expires_at=time.time() - 1)) # 已过期 + expired = await store.retrieve("u_ylhan", "关火") + check("过期记忆不出现在结果里", all(m.value != EXPIRED for m in expired), + f"拿到 {len(expired)} 条") + check("过期记忆也不在列表里", + all(m.value != EXPIRED for m in await store.list_memories("u_ylhan"))) + + print("\n=== 6. 会话与轮次 ===") + await store.start_session("sess_1", "u_ylhan") + for i, (role, text) in enumerate([("user", "今天天气怎么样"), + ("assistant", "今天晴，十八到二十六度。")]): + await store.add_turn(Turn(turn_id=f"t{i}", session_id="sess_1", + user_id="u_ylhan", role=role, text=text)) + check("会话与轮次写入无异常", True) + + print("\n=== 7. 中文检索打分 ===") + r = SQLiteMemoryStore.relevance("客厅的灯", "客厅有一盏米家吸顶灯") + check("相关文本有分", r > 0.3, f"{r:.2f}") + r2 = SQLiteMemoryStore.relevance("客厅的灯", "喜欢听爵士乐") + check("无关文本为 0", r2 == 0.0, f"{r2:.2f}") + r3 = SQLiteMemoryStore.relevance("回复风格", "回复风格：喜欢简短") + check("子串直接命中", r3 == 1.0, f"{r3:.2f}") + + print("\n=== 8. 删除用户要清干净 ===") + n = await store.purge_user("u_ylhan") + check("返回删除的记忆数", n >= 3, f"{n} 条") + check("记忆已清空", len(await store.list_memories("u_ylhan")) == 0) + check("检索也拿不到", len(await store.retrieve("u_ylhan", "回复风格")) == 0) + # 会话和轮次靠外键级联删除 + rows = await store._run(lambda c: c.execute( + "SELECT COUNT(*) AS n FROM turns WHERE user_id=?", ("u_ylhan",)).fetchone()["n"]) + check("轮次一并删除", rows == 0, f"残留 {rows} 条") + rows2 = await store._run(lambda c: c.execute( + "SELECT COUNT(*) AS n FROM sessions WHERE user_id=?", ("u_ylhan",)).fetchone()["n"]) + check("会话一并删除", rows2 == 0, f"残留 {rows2} 条") + + print("\n=== 9. 重复建用户不报错、不覆盖 ===") + await store.ensure_user("u_x", "第一次") + await store.ensure_user("u_x", "第二次") + name = await store._run(lambda c: c.execute( + "SELECT display_name FROM users WHERE user_id=?", ("u_x",)).fetchone()["display_name"]) + check("已存在则不改", name == "第一次", name) + + print("\n=== 10. 兜底检索要限量（否则就成了每轮全量注入）===") + await store.ensure_user("u_big") + await store.remember(Memory(owner_user_id="u_big", kind=MemoryKind.FACT, + value="常喝拿铁")) + # "推荐个饮品"和"常喝拿铁"没有任何共同字符，字面打分为 0 + got = await store.retrieve("u_big", "推荐个饮品") + check("记忆少时兜底返回最近一条", len(got) == 1, f"{len(got)} 条") + + for i in range(FALLBACK_MAX_CORPUS + 2): + await store.remember(Memory(owner_user_id="u_big", kind=MemoryKind.FACT, + value=f"无关记录编号{i}")) + got2 = await store.retrieve("u_big", "推荐个饮品") + check("记忆多时不再无条件兜底", len(got2) == 0, f"{len(got2)} 条") + got3 = await store.retrieve("u_big", "拿铁") + check("关键词命中仍生效", any("拿铁" in m.value for m in got3), f"{len(got3)} 条") + + await store.close() + + +def main() -> int: + tmp = Path(tempfile.mkdtemp(prefix="memtest_")) + try: + store = SQLiteMemoryStore(tmp / "memory.db") + asyncio.run(run(store)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + print("\n" + "=" * 62) + print(f"通过 {len(OK)} / {len(OK) + len(FAIL)}") + for n in FAIL: + print(f" 未通过: {n}") + return 0 if not FAIL else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/voice/__init__.py b/app/voice/__init__.py new file mode 100644 index 0000000..5f74541 --- /dev/null +++ b/app/voice/__init__.py @@ -0,0 +1,35 @@ +"""语音层：STT / TTS。 + +TTS 有两个后端，接口一致（synthesize / synthesize_sentences）， +切换只需改配置 AI_TTS_BACKEND： + + local — sherpa-onnx vits-zh-ll，RTF 0.09，很快但音色用户不满意 + remote — Qwen3-TTS 独立服务，RTF 0.55–0.94，音质是另一档 + +为什么 remote 要走 HTTP 而不是进程内：qwen-tts 与助手/vllm-omni 的依赖互斥 +（transformers 4.57 vs 5.14、hf_hub 0.36 vs 1.31），同 venv 会互相破坏。 + +模型文件不随代码走（体积大、可重新下载），由 VoiceConfig.model_dir 指向。 +缺失时构造对象直接报错并提示下载脚本，不静默降级——静默降级会让 +"没声音"变成一个很难查的问题。 +""" + +from .stt import STTError, STTStream, StreamingSTT +from .tts import MAX_CHARS_PER_SEGMENT, LocalTTS, TTSError, split_sentences +from .remote_tts import RemoteTTS, RemoteTTSError + + +def create_tts(cfg): + """按配置造 TTS。返回对象统一提供 synthesize / synthesize_sentences。""" + if cfg.tts_backend == "remote": + return RemoteTTS(cfg.tts_service_url, voice=cfg.tts_speaker, + instructions=cfg.tts_instructions, + language=cfg.tts_language) + return LocalTTS(cfg) + + +__all__ = [ + "StreamingSTT", "STTStream", "STTError", + "LocalTTS", "TTSError", "split_sentences", "MAX_CHARS_PER_SEGMENT", + "RemoteTTS", "RemoteTTSError", "create_tts", +] diff --git a/app/voice/remote_tts.py b/app/voice/remote_tts.py new file mode 100644 index 0000000..b74d695 --- /dev/null +++ b/app/voice/remote_tts.py @@ -0,0 +1,138 @@ +"""远端 TTS 适配器（HTTP 调 Qwen3-TTS 服务）。 + +**为什么是 HTTP 而不是进程内调用**：qwen-tts 和 vllm-omni/助手的依赖互斥 +（transformers 4.57 vs 5.14、hf_hub 0.36 vs 1.31），装一个 venv 里会互相破坏， +实测把 vllm-omni 搞崩过。所以 TTS 跑在独立进程，用 HTTP 通信。 + +接口和 `LocalTTS` 保持一致（同步的 synthesize / synthesize_sentences）， +这样引擎那边一行不用改——切换本地 VITS 和远端 Qwen3-TTS 只是换一个类。 + +实测延迟（RTX 3090，Serena 音色，非流式）： + + 文本长度 音频 合成耗时 RTF + 8 字 1.52s 1429ms 0.94 ← 短句最差 + 19 字 3.36s 1881ms 0.56 + 26 字 4.64s 2527ms 0.55 + +**注意短句 RTF 反而最差**：每次调用有约 1 秒的固定开销， +与文本长度无关（talker 模型自回归解码的启动成本 + 声码器解码）。 +所以"让模型先说个短句"并不能降低首音延迟——反而比中长句更不划算。 + +流式可以绕开这个固定开销，但 qwen-tts 的封装把 codes→音频 的解码放在 +全部 codes 生成完之后，拿不到增量。要走流式需要接管底层解码循环。 +""" + +from __future__ import annotations + +import io +import json +import logging +import time +import urllib.error +import urllib.request +import wave +from typing import Iterator + +import numpy as np + +from .tts import split_sentences # 复用分句逻辑 + +log = logging.getLogger(__name__) + + +class RemoteTTSError(RuntimeError): + pass + + +class RemoteTTS: + """通过 HTTP 调用远端 TTS 服务。接口与 LocalTTS 一致。""" + + def __init__(self, base_url: str, *, voice: str = "Serena", + instructions: str = "用轻快活泼的语气说", + language: str = "Chinese", timeout_s: float = 60.0): + self.base_url = base_url.rstrip("/") + self.voice = voice + self.instructions = instructions + self.language = language + self.timeout_s = timeout_s + self.sample_rate = 24000 # Qwen3-TTS 输出 24kHz，稍后按需重采样 + self._checked = False + + # ------------------------------------------------------------ 健康检查 + + def health(self) -> tuple[bool, str]: + try: + with urllib.request.urlopen(f"{self.base_url}/health", + timeout=10) as r: + d = json.loads(r.read().decode()) + if d.get("model_loaded"): + return True, f"模型已加载，默认音色 {d.get('default_voice')}" + return False, f"服务在但模型未加载：{d.get('status')}" + except Exception as e: # noqa: BLE001 + return False, f"{type(e).__name__}: {e}" + + def list_voices(self) -> dict: + with urllib.request.urlopen(f"{self.base_url}/v1/voices", timeout=10) as r: + return json.loads(r.read().decode()) + + # ------------------------------------------------------------ 合成 + + def synthesize(self, text: str, *, instructions: str | None = None) -> np.ndarray: + """合成一段文本，返回 float32 单声道。""" + text = (text or "").strip() + if not text: + return np.zeros(0, dtype=np.float32) + + payload = json.dumps({ + "input": text, + "voice": self.voice, + "instructions": instructions if instructions is not None else self.instructions, + "language": self.language, + "response_format": "wav", + }).encode("utf-8") + + req = urllib.request.Request( + f"{self.base_url}/v1/audio/speech", + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + t0 = time.perf_counter() + try: + with urllib.request.urlopen(req, timeout=self.timeout_s) as r: + wav_bytes = r.read() + sr = int(r.headers.get("X-Sample-Rate", self.sample_rate)) + gen_ms = r.headers.get("X-Gen-Ms", "?") + rtf = r.headers.get("X-RTF", "?") + except urllib.error.HTTPError as e: + detail = e.read().decode("utf-8", "replace")[:200] + raise RemoteTTSError(f"TTS 服务 HTTP {e.code}: {detail}") from e + except Exception as e: # noqa: BLE001 + raise RemoteTTSError(f"TTS 服务不可达：{type(e).__name__}: {e}") from e + + self.sample_rate = sr + log.debug("合成 %d 字：服务端 %sms（RTF %s），往返 %.0fms", + len(text), gen_ms, rtf, (time.perf_counter() - t0) * 1000) + + with wave.open(io.BytesIO(wav_bytes), "rb") as w: + n = w.getnframes() + ch = w.getnchannels() + pcm = w.readframes(n) + audio = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 + if ch > 1: # 服务端是单声道，防御一下 + audio = audio.reshape(-1, ch).mean(axis=1) + return audio + + def synthesize_sentences(self, text: str) -> Iterator[tuple[str, np.ndarray]]: + """按句合成，逐句返回。调用方拿到第一句就能开播。""" + for seg in split_sentences(text): + yield seg, self.synthesize(seg) + + def synthesize_all(self, text: str) -> np.ndarray: + chunks = [a for _, a in self.synthesize_sentences(text) if len(a)] + if not chunks: + return np.zeros(0, dtype=np.float32) + return np.concatenate(chunks) + + +__all__ = ["RemoteTTS", "RemoteTTSError"] diff --git a/app/voice/stt.py b/app/voice/stt.py new file mode 100644 index 0000000..1094e06 --- /dev/null +++ b/app/voice/stt.py @@ -0,0 +1,125 @@ +"""本地流式 STT（sherpa-onnx Zipformer 中英双语）。 + +实测（P0-语音基准报告.md）： + + N5095（4核，无 AVX2） RTF 0.22 @2线程 ／ 0.86 @4线程 + 服务器（i9-14900K） RTF 0.03 + +**线程数是实测出来的，不能按"核多给多"猜。** N5095 上 4 线程比 2 线程 +慢 3.9 倍（小算子多，每算子一次线程同步，线程越多同步开销越大）。 +所以线程数由配置决定，不写死。 + +另外两条实测结论写进了代码： + + * **冲刷耗时 ≈ 0**：流式解码过程中已持续输出，`input_finished()` 后 + 立即拿到最终结果。"说完到出结果"的开销在 VAD 判定，不在识别器。 + * **并非每块都解码**：约每 3 块解一次（模型 chunk 约 300ms）。 + 所以单块耗时偶尔超过块长是正常的，判断能否实时要看 RTF 而不是单块。 +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Iterator + +import numpy as np +import sherpa_onnx + +from ..config import VoiceConfig + +log = logging.getLogger(__name__) + + +class STTError(RuntimeError): + pass + + +class StreamingSTT: + """流式识别器。线程安全地创建多条流（多客户端场景）。""" + + def __init__(self, cfg: VoiceConfig): + self.cfg = cfg + d = Path(cfg.model_dir) / "stt" + files = { + "tokens": d / "tokens.txt", + "encoder": d / "encoder-epoch-99-avg-1.int8.onnx", + "decoder": d / "decoder-epoch-99-avg-1.int8.onnx", + "joiner": d / "joiner-epoch-99-avg-1.int8.onnx", + } + missing = [k for k, p in files.items() if not p.is_file()] + if missing: + raise STTError( + f"STT 模型文件缺失：{missing}（目录 {d}）。" + f"先跑 p0/stt_tts_bench/download_models.sh") + + self._rec = sherpa_onnx.OnlineRecognizer.from_transducer( + tokens=str(files["tokens"]), + encoder=str(files["encoder"]), + decoder=str(files["decoder"]), + joiner=str(files["joiner"]), + num_threads=cfg.stt_threads, + sample_rate=cfg.sample_rate, + feature_dim=80, + decoding_method="greedy_search", + # 端点检测留给上层的 VAD 处理：方案 §5 要求 + # "语音活动检测、说完判断、唤醒是不同环节"，不合并。 + enable_endpoint_detection=False, + ) + log.info("STT 就绪：%d 线程，采样率 %d", cfg.stt_threads, cfg.sample_rate) + + def new_stream(self) -> "STTStream": + return STTStream(self, self._rec.create_stream()) + + +class STTStream: + """一条识别流。""" + + def __init__(self, owner: StreamingSTT, stream): + self._owner = owner + self._s = stream + self._sr = owner.cfg.sample_rate + self._consumed = 0 + + def accept(self, samples: np.ndarray) -> None: + """喂入 float32 单声道音频（取值 [-1,1]）。""" + if samples.dtype != np.float32: + samples = samples.astype(np.float32) + self._s.accept_waveform(self._sr, samples) + self._consumed += len(samples) + rec = self._owner._rec + while rec.is_ready(self._s): + rec.decode_stream(self._s) + + def accept_pcm16(self, pcm: bytes) -> None: + """喂入 16bit PCM 字节流（麦克风常见格式）。""" + self.accept(np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0) + + @property + def result(self) -> str: + """当前结果。流式过程中会不断变长，说完后即为最终结果。""" + return self._owner._rec.get_result(self._s) + + @property + def consumed_seconds(self) -> float: + """已喂入的音频时长。用于判断"喂多少音频才出第一个字"（算法延迟）。""" + return self._consumed / self._sr + + def finish(self) -> str: + """结束输入并冲刷。 + + 实测冲刷耗时 ≈ 0ms——流式解码已持续输出，这里不额外花钱。 + """ + rec = self._owner._rec + self._s.input_finished() + while rec.is_ready(self._s): + rec.decode_stream(self._s) + return rec.get_result(self._s) + + def reset(self) -> None: + """丢弃当前流，重新开始（打断后重新听）。""" + self._owner._rec.reset(self._s) + self._consumed = 0 + + +__all__ = ["StreamingSTT", "STTStream", "STTError"] diff --git a/app/voice/tts.py b/app/voice/tts.py new file mode 100644 index 0000000..68d50e9 --- /dev/null +++ b/app/voice/tts.py @@ -0,0 +1,157 @@ +"""本地 TTS（sherpa-onnx VITS 中文）。 + +选型依据（P0-语音基准报告.md §六之二）： + + vits-zh-ll 服务器 RTF 0.09 / N5095 RTF 1.86 ← 用户试听选定，音质最好 + piper-huayan-med 服务器 0.36 / N5095 0.32 （音质次之） + kokoro 1.96 以上，排除 + +**两个必须写进代码的实测结论：** + +1. **VITS 是非自回归的，没有"边合成边播"。** + 实测回调只在合成结束后触发一次，所以首音延迟 == 整句合成耗时。 + 提前出声的唯一手段是**按语义短句逐句合成**——把长回复切成短句， + 第一句合成完就能播。 + +2. **`max_num_sentences=1` 会静默截断多句输入。** + 传含多个句号的文本只会合成第一句，**不报错**。 + 第一版测"长段"时只出 0.94 秒音频，差点当成正常结果。 + 所以这里必须先分句，逐句送。 + +另外按方案 §5 要求，输出是 16kHz 单声道 float32；调用方负责重采样。 +""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Iterator + +import numpy as np +import sherpa_onnx + +from ..config import VoiceConfig + +log = logging.getLogger(__name__) + +#: 句末标点。中英文都要，方案 §5 要求覆盖中英混读。 +_SENT_END = "。！？；!?;" +#: 从句标点，长句再切一刀，避免单句合成太久 +_CLAUSE = "，,、：:" + +#: 单句最大字符数。超过就按从句标点再切—— +#: 句子越长，首音延迟越大（首音 == 整句合成耗时）。 +MAX_CHARS_PER_SEGMENT = 30 + + +class TTSError(RuntimeError): + pass + + +def split_sentences(text: str, *, max_chars: int = MAX_CHARS_PER_SEGMENT) -> list[str]: + """按语义切分，返回可逐句合成的片段。 + + 先按句末标点切，过长的再按从句标点切，仍过长就硬切。 + 标点保留在片段末尾（TTS 需要它来判断语调）。 + + 这个函数存在的原因是 `max_num_sentences=1` 会静默丢内容—— + 不自己切，就会"念到一半没了"。 + """ + text = (text or "").strip() + if not text: + return [] + + parts = re.split(rf"(?<=[{_SENT_END}])", text) + out: list[str] = [] + for p in parts: + p = p.strip() + if not p: + continue + if len(p) <= max_chars: + out.append(p) + continue + # 过长：按从句标点再切 + subs = re.split(rf"(?<=[{_CLAUSE}])", p) + buf = "" + for s in subs: + if not s: + continue + if len(buf) + len(s) <= max_chars: + buf += s + else: + if buf: + out.append(buf) + # 单从句仍然过长，硬切 + while len(s) > max_chars: + out.append(s[:max_chars]) + s = s[max_chars:] + buf = s + if buf: + out.append(buf) + return out + + +class LocalTTS: + def __init__(self, cfg: VoiceConfig): + self.cfg = cfg + d = Path(cfg.model_dir) / "tts" + root = Path(cfg.model_dir) + files = { + "model": d / "model.onnx", + "lexicon": d / "lexicon.txt", + "tokens": d / "tokens.txt", + } + missing = [k for k, p in files.items() if not p.is_file()] + if missing: + raise TTSError( + f"TTS 模型文件缺失：{missing}（目录 {d}）。" + f"先跑 p0/stt_tts_bench/download_models.sh") + + rule_fsts = ",".join( + str(d / f) for f in ("date.fst", "number.fst", "phone.fst", "new_heteronym.fst") + if (d / f).is_file()) + + self._tts = sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model=str(files["model"]), + lexicon=str(files["lexicon"]), + tokens=str(files["tokens"]), + dict_dir=str(d / "dict"), + ), + num_threads=cfg.tts_threads, + provider="cpu", + ), + rule_fsts=rule_fsts, + # 逐句生成：这是唯一能提前出声的手段（见模块文档） + max_num_sentences=1, + )) + self.sample_rate = self._tts.sample_rate + log.info("TTS 就绪：%d 线程，输出 %d Hz", cfg.tts_threads, self.sample_rate) + + def synthesize(self, text: str) -> np.ndarray: + """合成一段文本（内部已确保是单句）。""" + if not text.strip(): + return np.zeros(0, dtype=np.float32) + audio = self._tts.generate(text, sid=0, speed=1.0) + return np.asarray(audio.samples, dtype=np.float32) + + def synthesize_sentences(self, text: str) -> Iterator[tuple[str, np.ndarray]]: + """按句合成，逐句 yield (句子, 音频)。 + + 调用方拿到第一句就能开播，不必等整段—— + 这正是 VITS 非自回归下唯一的低延迟路径。 + """ + for seg in split_sentences(text): + yield seg, self.synthesize(seg) + + def synthesize_all(self, text: str) -> np.ndarray: + """合成整段并拼接。用于离线/测试，不适合实时播放。""" + chunks = [a for _, a in self.synthesize_sentences(text) if len(a)] + if not chunks: + return np.zeros(0, dtype=np.float32) + return np.concatenate(chunks) + + +__all__ = ["LocalTTS", "TTSError", "split_sentences", "MAX_CHARS_PER_SEGMENT"] diff --git a/esp32_smile_test/esp32_smile_test.ino b/esp32_smile_test/esp32_smile_test.ino new file mode 100644 index 0000000..f625601 --- /dev/null +++ b/esp32_smile_test/esp32_smile_test.ino @@ -0,0 +1,94 @@ +// ESP32-S3 + ILI9341 240x320 display test. +// Install Adafruit ILI9341, Adafruit GFX Library and their dependencies. +// Display: SCK=12, SDI/MOSI=11, CS=10, DC=9, RESET=8, GND=GND. +// Touch: T_CLK=12, T_DIN=11, T_DO=13, T_CS=7, T_IRQ=6. +// Install XPT2046_Touchscreen by Paul Stoffregen as well. +// Display SDO and SD card are not connected. Backlight is externally powered. +#include +#include +#include +#include + +constexpr int TFT_SCK = 12; +constexpr int TFT_MOSI = 11; +constexpr int TFT_CS = 10; +constexpr int TFT_DC = 9; +constexpr int TFT_RST = 8; +constexpr int TOUCH_MISO = 13; +constexpr int TOUCH_CS = 7; +constexpr int TOUCH_IRQ = 6; + +Adafruit_ILI9341 tft(&SPI, TFT_DC, TFT_CS, TFT_RST); +// Poll first to test SPI independently of the IRQ wire. +XPT2046_Touchscreen touch(TOUCH_CS); +bool wasPressed = false; +bool alternateColor = false; +uint32_t touchCount = 0; +uint32_t lastReport = 0; + +void drawFace(uint16_t color); + +void setup() { + Serial.begin(115200); + delay(300); + pinMode(TFT_CS, OUTPUT); + digitalWrite(TFT_CS, HIGH); + pinMode(TOUCH_CS, OUTPUT); + digitalWrite(TOUCH_CS, HIGH); + pinMode(TOUCH_IRQ, INPUT_PULLUP); + SPI.begin(TFT_SCK, TOUCH_MISO, TFT_MOSI, TFT_CS); + tft.begin(10000000); // Start conservatively at 10 MHz for jumper wires. + tft.setRotation(1); // Landscape: 320 x 240. Use 3 if upside down. + touch.begin(SPI); + touch.setRotation(1); + + const uint16_t colors[] = {ILI9341_RED, ILI9341_GREEN, ILI9341_BLUE}; + for (uint16_t color : colors) { + tft.fillScreen(color); + delay(400); + } + + tft.fillScreen(ILI9341_NAVY); + drawFace(ILI9341_YELLOW); + Serial.println("Touch test ready. Raw coordinates are not calibrated pixels."); +} + +void drawFace(uint16_t color) { + const int cx = tft.width() / 2; + const int cy = 105; + tft.fillCircle(cx, cy, 82, color); + tft.fillCircle(cx - 28, cy - 22, 10, ILI9341_BLACK); + tft.fillCircle(cx + 28, cy - 22, 10, ILI9341_BLACK); + + // Parabolic smile: the center is lower than the corners. + for (int x = -42; x <= 42; ++x) { + int y = cy + 43 - (x * x) / 65; + tft.fillCircle(cx + x, y, 3, ILI9341_BLACK); + } + + tft.fillRect(0, 199, tft.width(), 41, ILI9341_NAVY); + tft.setTextColor(ILI9341_WHITE, ILI9341_NAVY); + tft.setTextSize(2); + tft.setCursor(16, 207); + tft.print("Touches: "); + tft.print(touchCount); +} + +void loop() { + bool pressed = touch.touched(); + if (pressed) { + TS_Point point = touch.getPoint(); + if (!wasPressed) { + ++touchCount; + alternateColor = !alternateColor; + drawFace(alternateColor ? ILI9341_CYAN : ILI9341_YELLOW); + } + if (millis() - lastReport >= 150) { + lastReport = millis(); + Serial.printf("raw x=%d y=%d pressure=%d IRQ=%d\n", + point.x, point.y, point.z, digitalRead(TOUCH_IRQ)); + } + } + wasPressed = pressed; + delay(20); +} diff --git a/esp32_touch_eyes/esp32_touch_eyes.ino b/esp32_touch_eyes/esp32_touch_eyes.ino new file mode 100644 index 0000000..7e1fe81 --- /dev/null +++ b/esp32_touch_eyes/esp32_touch_eyes.ino @@ -0,0 +1,141 @@ +// ESP32-S3 N16R8 + ILI9341 + XPT2046-compatible touch. +// Libraries: Adafruit ILI9341, Adafruit GFX Library, XPT2046_Touchscreen. +// LCD: SCK=12 MOSI=11 CS=10 DC=9 RESET=8. LCD SDO disconnected. +// Touch: T_CLK=12 T_DIN=11 T_DO=13 T_CS=7 T_IRQ=6 (polled). +// SD disconnected. Keep the existing working power/backlight connections. +// No startup calibration. Adjust touch mapping constants below if needed. +#include +#include +#include +#include +#include + +Adafruit_ILI9341 tft(&SPI, 9, 10, 8); +XPT2046_Touchscreen touch(7); +// Small off-screen canvas avoids visible erase/redraw flashes. +GFXcanvas16 eyes(280, 112); +// Approximate raw range after touch.setRotation(1), not calibrated pixels. +constexpr int TOUCH_MIN_X = 200, TOUCH_MAX_X = 3900; +constexpr int TOUCH_MIN_Y = 200, TOUCH_MAX_Y = 3900; +constexpr bool TOUCH_SWAP_XY = false; +constexpr bool TOUCH_FLIP_X = true, TOUCH_FLIP_Y = true; +constexpr uint32_t IDLE_AFTER_MS = 15000; +uint32_t idleBlinkAt = 0, idleLookAt = 0; +bool idle = false, idleLookingAway = false; +float gazeX = 0, gazeY = 0, targetX = 0, targetY = 0; +uint32_t lastTouch = 0, blinkStart = 0, lastFrame = 0, lastDown = 0; +bool looking = false, blinking = false, contact = false; +int previousX = 999, previousY = 999, previousOpen = -1; + +void renderEyes(int dx, int dy, int opening) { + eyes.fillScreen(ILI9341_BLACK); + for (int i = 0; i < 2; ++i) { + int x = i == 0 ? 72 : 208; + eyes.fillRoundRect(x-49, 12, 98, 88, 30, ILI9341_WHITE); + eyes.fillCircle(x+dx, 56+dy, 25, ILI9341_CYAN); + eyes.fillCircle(x+dx, 56+dy, 14, ILI9341_BLACK); + eyes.fillCircle(x+dx-7, 49+dy, 5, ILI9341_WHITE); + int cover = (100-opening)*44/100; + if (cover > 0) { + eyes.fillRect(x-50, 12, 100, cover, ILI9341_BLACK); + eyes.fillRect(x-50, 100-cover, 100, cover, ILI9341_BLACK); + } + if (opening < 7) eyes.fillRoundRect(x-39, 54, 78, 4, 2, ILI9341_CYAN); + } + tft.drawRGBBitmap(20, 54, eyes.getBuffer(), 280, 112); +} + +void setup() { + Serial.begin(115200); + pinMode(10, OUTPUT); digitalWrite(10, HIGH); + pinMode(7, OUTPUT); digitalWrite(7, HIGH); + pinMode(6, INPUT_PULLUP); + SPI.begin(12, 13, 11, 10); + tft.begin(10000000); + tft.setRotation(1); + touch.begin(SPI); + touch.setRotation(1); + if (!eyes.getBuffer()) { + tft.fillScreen(ILI9341_RED); + Serial.println("Eye canvas allocation failed."); + while (true) delay(1000); + } + randomSeed(micros()); + tft.fillScreen(ILI9341_BLACK); + tft.setTextColor(ILI9341_CYAN); + tft.setTextSize(2); + tft.setCursor(46, 207); + tft.print("Touch to look!"); + lastFrame = lastTouch = millis(); +} + +void loop() { + uint32_t now = millis(); + if (touch.touched()) { + TS_Point p = touch.getPoint(); + float rawX = TOUCH_SWAP_XY ? p.y : p.x; + float rawY = TOUCH_SWAP_XY ? p.x : p.y; + float px = constrain((rawX-TOUCH_MIN_X)*319.0f/(TOUCH_MAX_X-TOUCH_MIN_X), 0.0f, 319.0f); + if (TOUCH_FLIP_X) px = 319.0f-px; + float py = constrain((rawY-TOUCH_MIN_Y)*239.0f/(TOUCH_MAX_Y-TOUCH_MIN_Y), 0.0f, 239.0f); + if (TOUCH_FLIP_Y) py = 239.0f-py; + idle = false; + idleLookingAway = false; + if (!contact) { blinking = true; blinkStart = now; } + contact = true; + lastDown = lastTouch = now; + looking = true; + targetX = (px-159.5f)/159.5f*20.0f; + targetY = (py-119.5f)/119.5f*14.0f; + } else if (contact && now-lastDown > 100) { + contact = false; // Short release debounce prevents repeated blinks. + } + if (looking && now-lastTouch >= 3000) { + looking = false; + targetX = targetY = 0; + } + // "Unattended" currently means no touch, not camera-based presence detection. + if (!contact && !looking && now-lastTouch >= IDLE_AFTER_MS) { + if (!idle) { + idle = true; + idleLookingAway = false; + idleBlinkAt = now + random(400, 1600); + idleLookAt = now + random(900, 2200); + } + if (!blinking && int32_t(now-idleBlinkAt) >= 0) { + blinking = true; + blinkStart = now; + idleBlinkAt = now + random(2500, 6500); + } + if (int32_t(now-idleLookAt) >= 0) { + idleLookingAway = !idleLookingAway; + if (idleLookingAway) { + targetX = random(0, 2) ? random(9, 21) : -random(9, 21); + targetY = random(-12, 13); + idleLookAt = now + random(800, 2000); + } else { + targetX = targetY = 0; + idleLookAt = now + random(1800, 4500); + } + } + } + if (now-lastFrame < 33) { delay(2); return; } + float dt = (now-lastFrame)/1000.0f; + lastFrame = now; + float blend = 1.0f-expf(-12.0f*dt); + gazeX += (targetX-gazeX)*blend; + gazeY += (targetY-gazeY)*blend; + int opening = 100; + if (blinking) { + uint32_t elapsed = now-blinkStart; + if (elapsed < 100) opening = 100-int(elapsed); + else if (elapsed < 170) opening = 0; + else if (elapsed < 330) opening = int((elapsed-170)*100/160); + else blinking = false; + } + int dx = lroundf(gazeX), dy = lroundf(gazeY); + if (dx != previousX || dy != previousY || opening != previousOpen) { + renderEyes(dx, dy, opening); + previousX = dx; previousY = dy; previousOpen = opening; + } +} diff --git a/p0/stt_tts_bench/bench_concurrent.py b/p0/stt_tts_bench/bench_concurrent.py new file mode 100644 index 0000000..9af39ee --- /dev/null +++ b/p0/stt_tts_bench/bench_concurrent.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""P0 基准 · STT 与 TTS 并发。 + +为什么必须测： + + 真实系统里助手说话时麦克风并没有关——要检测打断，STT 必须同时在跑。 + 这台机器只有 4 个核，且 bench_threads.py 已经证明 STT 超过 2 线程就会急剧退化。 + 那么"STT(2线程) + TTS(2线程)"极可能互相踩踏。 + + 本脚本把两种负载跑在一起，和各自单独跑的结果对比，量化退化幅度。 + 只看单独指标会得出"两个都能实时"的错误结论。 +""" + +import os +import statistics +import threading +import time +import wave + +import numpy as np +import sherpa_onnx + +HERE = os.path.dirname(os.path.abspath(__file__)) +STT = os.path.join(HERE, "models", "stt") +WAVS = os.path.join(HERE, "wavs") +SR = 16000 +CHUNK = int(SR * 0.1) + +TTS_SENTENCES = [ + "我查了一下，明天上午十点有个会议。", + "地点在三号会议室，需要提前十分钟到。", + "另外提醒你，后天是妈妈的生日。", + "要不要我提前帮你订一束花？", + "还有你上周说要买的那本书现在有货了。", + "价格是四十九块八，要不要下单？", +] + + +def read_wav(p): + with wave.open(p, "rb") as w: + return np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16).astype(np.float32) / 32768.0 + + +def build_stt(nt): + return sherpa_onnx.OnlineRecognizer.from_transducer( + tokens=os.path.join(STT, "tokens.txt"), + encoder=os.path.join(STT, "encoder-epoch-99-avg-1.int8.onnx"), + decoder=os.path.join(STT, "decoder-epoch-99-avg-1.int8.onnx"), + joiner=os.path.join(STT, "joiner-epoch-99-avg-1.int8.onnx"), + num_threads=nt, sample_rate=SR, feature_dim=80, + decoding_method="greedy_search", enable_endpoint_detection=False) + + +def build_tts(which, nt): + if which == "matcha": + d = os.path.join(HERE, "models", "matcha") + matcha = sherpa_onnx.OfflineTtsMatchaModelConfig( + acoustic_model=os.path.join(d, "model-steps-3.onnx"), + vocoder=os.path.join(HERE, "models", "matcha_vocoder", "hifigan_v2.onnx"), + lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt"), + dict_dir=os.path.join(d, "dict")) + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig(matcha=matcha, num_threads=nt, provider="cpu"), + rule_fsts=",".join(os.path.join(d, f) for f in ("date.fst", "number.fst", "phone.fst")), + max_num_sentences=1)) + if which == "kokoro_int8": + d = os.path.join(HERE, "models", "kokoro_int8") + kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig( + model=os.path.join(d, "model.int8.onnx"), + voices=os.path.join(d, "voices.bin"), + tokens=os.path.join(d, "tokens.txt"), + data_dir=os.path.join(d, "espeak-ng-data"), + dict_dir=os.path.join(d, "dict"), + lexicon=",".join(os.path.join(d, f) for f in + ("lexicon-us-en.txt", "lexicon-gb-en.txt", "lexicon-zh.txt"))) + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig(kokoro=kokoro, num_threads=nt, provider="cpu"), + max_num_sentences=1)) + if which == "piper": + d = os.path.join(HERE, "models", "tts_piper_xlow") + vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(d, "zh_CN-huayan-x_low.onnx"), + lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt")) + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig(vits=vits, num_threads=nt, provider="cpu"), + max_num_sentences=1)) + d = os.path.join(HERE, "models", "tts") + vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(d, "model.onnx"), lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt"), dict_dir=os.path.join(d, "dict")) + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig(vits=vits, num_threads=nt, provider="cpu"), + rule_fsts=",".join(os.path.join(d, f) for f in + ("date.fst", "number.fst", "phone.fst", "new_heteronym.fst")), + max_num_sentences=1)) + + +def stt_pass(rec, samples): + """喂一遍音频，返回 (rtf, 单块解码耗时列表)。""" + s = rec.create_stream() + durs = [] + pos = 0 + t_all = time.perf_counter() + while pos < len(samples): + block = samples[pos:pos + CHUNK] + pos += CHUNK + t0 = time.perf_counter() + s.accept_waveform(SR, block) + while rec.is_ready(s): + rec.decode_stream(s) + durs.append(time.perf_counter() - t0) + total = time.perf_counter() - t_all + return total / (len(samples) / SR), durs + + +def tts_pass(tts, sents): + """连续合成若干句，返回 (rtf, 每句耗时ms)。""" + per = [] + audio_s = 0.0 + t_all = time.perf_counter() + for s in sents: + t0 = time.perf_counter() + a = tts.generate(s, sid=0, speed=1.0) + per.append((time.perf_counter() - t0) * 1000) + audio_s += len(a.samples) / a.sample_rate + total = time.perf_counter() - t_all + return total / audio_s, per + + +def main(): + stt_nt = int(os.environ.get("STT_THREADS", "2")) + tts_nt = int(os.environ.get("TTS_THREADS", "2")) + tts_kind = os.environ.get("TTS_MODEL", "piper") + wav = read_wav(os.path.join(WAVS, "stt_0.wav")) + dur = len(wav) / SR + + print(f"音频 {dur:.2f}s STT线程={stt_nt} TTS={tts_kind}线程={tts_nt}\n") + + stt = build_stt(stt_nt) + tts = build_tts(tts_kind, tts_nt) + stt_pass(stt, wav[: SR * 2]) # 预热 + tts.generate("预热", sid=0, speed=1.0) + + # 单独 + solo_stt, _ = stt_pass(stt, wav) + solo_tts, _ = tts_pass(tts, TTS_SENTENCES) + print(f"单独: STT RTF={solo_stt:.2f} TTS RTF={solo_tts:.2f}") + + # 并发：TTS 在后台线程持续合成，主线程跑 STT + stop = threading.Event() + tts_stats = {"runs": 0, "rtf": [], "done": False} + + def tts_worker(): + try: + while not stop.is_set(): + rtf, _ = tts_pass(tts, TTS_SENTENCES) + tts_stats["rtf"].append(rtf) + tts_stats["runs"] += 1 + if tts_stats["runs"] >= 3: + break + finally: + tts_stats["done"] = True + + th = threading.Thread(target=tts_worker) + th.start() + time.sleep(0.2) # 让 TTS 先跑起来，制造真实争抢 + conc_rtfs, conc_durs = [], [] + for _ in range(3): + r, d = stt_pass(stt, wav) + conc_rtfs.append(r) + conc_durs.extend(d) + stop.set() + th.join() + + conc_tts = statistics.median(tts_stats["rtf"]) if tts_stats["rtf"] else float("nan") + stt_med = statistics.median(conc_rtfs) + ratios = sorted(x / (CHUNK / SR) for x in conc_durs) + + print(f"并发: STT RTF={stt_med:.2f} (×{stt_med/solo_stt:.2f}) " + f"TTS RTF={conc_tts:.2f} (×{conc_tts/solo_tts:.2f})") + print(f" 并发下单块解码 p50={statistics.median(ratios):.2f} " + f"p95={ratios[int(len(ratios)*0.95)]:.2f} max={ratios[-1]:.2f}（>1 表示积压）") + + # 判定口径：看 RTF（聚合力，<1 表示跟得上）。单块 p95 只作观察项—— + # 流式模型并非每块都解码（实测约每 3 块解一次），所以单块 >1 不等于积压。 + # 留 0.7 的余量，因为还要给视觉检测等其它负载留 CPU。 + if stt_med < 0.7 and conc_tts < 0.7: + verdict = f"✅ 并发可实时（STT 余量 {1/stt_med:.1f}x，TTS 余量 {1/conc_tts:.1f}x）" + elif stt_med < 1 and conc_tts < 1: + verdict = "△ 并发勉强够用，视觉检测再加负载有风险" + else: + verdict = "⚠ 并发跟不上实时" + print(f"\n判定: {verdict}") + + import json + out = os.path.join(HERE, f"results_concurrent_{tts_kind}.json") + with open(out, "w") as f: + json.dump({"stt_threads": stt_nt, "tts_threads": tts_nt, "tts_model": tts_kind, + "solo_stt_rtf": solo_stt, "solo_tts_rtf": solo_tts, + "conc_stt_rtf": stt_med, "conc_tts_rtf": conc_tts, + "stt_degrade": stt_med / solo_stt, "tts_degrade": conc_tts / solo_tts, + "conc_chunk_p95": ratios[int(len(ratios) * 0.95)], + "conc_chunk_max": ratios[-1]}, f, ensure_ascii=False, indent=2) + print(f"结果已写入 {out}") + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/bench_roundtrip.py b/p0/stt_tts_bench/bench_roundtrip.py new file mode 100644 index 0000000..abf38b0 --- /dev/null +++ b/p0/stt_tts_bench/bench_roundtrip.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""P0 基准 · TTS→STT 往返测试。 + +作用有两个： + + 1. 验证 TTS 输出"可懂"——合成得再快，念出来听不懂也没用。 + Piper 的 x_low 是最低音质档，且日志里有 Unknown token 警告，必须确认发出的音是正常的。 + 2. 给一个粗略的识别正确率信号（字错率 CER）。 + +重要局限：这是合成语音，比真人近距离录音干净得多，**会明显高估真实识别率**。 +真实的远场识别率必须等麦克风阵列到货后用真人录音测。这里的结果只能当"下限的信号"， +不能当验收数据。 +""" + +import os +import re +import wave + +import numpy as np +import sherpa_onnx + +HERE = os.path.dirname(os.path.abspath(__file__)) +STT = os.path.join(HERE, "models", "stt") + +CASES = [ + "好的我记住了", + "今天北京晴转多云气温十八到二十六度", + "明天上午十点有个会议地点在三号会议室", + "订单编号是七三九一金额一千二百五十元", + "会议定在二零二六年九月十四日下午三点半", +] + + +def load_tts(which): + if which == "piper": + d = os.path.join(HERE, "models", "tts_piper_xlow") + vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(d, "zh_CN-huayan-x_low.onnx"), + lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt")) + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig(vits=vits, num_threads=2, provider="cpu"), + max_num_sentences=1)) + d = os.path.join(HERE, "models", "tts") + vits = sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(d, "model.onnx"), lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt"), dict_dir=os.path.join(d, "dict")) + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig(vits=vits, num_threads=2, provider="cpu"), + rule_fsts=",".join(os.path.join(d, f) for f in + ("date.fst", "number.fst", "phone.fst", "new_heteronym.fst")), + max_num_sentences=1)) + + +def load_stt(): + return sherpa_onnx.OnlineRecognizer.from_transducer( + tokens=os.path.join(STT, "tokens.txt"), + encoder=os.path.join(STT, "encoder-epoch-99-avg-1.int8.onnx"), + decoder=os.path.join(STT, "decoder-epoch-99-avg-1.int8.onnx"), + joiner=os.path.join(STT, "joiner-epoch-99-avg-1.int8.onnx"), + num_threads=2, sample_rate=16000, feature_dim=80, + decoding_method="greedy_search", enable_endpoint_detection=False) + + +def transcribe(rec, samples): + s = rec.create_stream() + chunk = 1600 + pos = 0 + while pos < len(samples): + s.accept_waveform(16000, samples[pos:pos + chunk]) + pos += chunk + while rec.is_ready(s): + rec.decode_stream(s) + s.accept_waveform(16000, np.zeros(16000, dtype=np.float32)) + while rec.is_ready(s): + rec.decode_stream(s) + s.input_finished() + while rec.is_ready(s): + rec.decode_stream(s) + return rec.get_result(s) + + +def normalize(t): + """只保留中文字符做比较，去掉空格与标点。""" + return re.sub(r"[^一-鿿]", "", t) + + +def cer(ref, hyp): + """字错率：编辑距离 / 参考长度。""" + r, h = normalize(ref), normalize(hyp) + if not r: + return float("nan") + d = np.zeros((len(r) + 1, len(h) + 1), dtype=np.int32) + d[:, 0] = np.arange(len(r) + 1) + d[0, :] = np.arange(len(h) + 1) + for i in range(1, len(r) + 1): + for j in range(1, len(h) + 1): + d[i, j] = min(d[i - 1, j] + 1, d[i, j - 1] + 1, + d[i - 1, j - 1] + (r[i - 1] != h[j - 1])) + return d[len(r), len(h)] / len(r) + + +def main(): + which = os.environ.get("TTS_MODEL", "piper") + tts = load_tts(which) + rec = load_stt() + print(f"TTS={which} → STT=zipformer-bilingual-zh-en(int8)\n") + + cers = [] + saved = 0 + outdir = os.path.join(HERE, "roundtrip_wav") + os.makedirs(outdir, exist_ok=True) + for i, text in enumerate(CASES): + audio = tts.generate(text, sid=0, speed=1.0) + samples = np.array(audio.samples, dtype=np.float32) + hyp = transcribe(rec, samples) + c = cer(text, hyp) + cers.append(c) + mark = "✅" if c == 0 else "△" if c < 0.2 else "❌" + print(f"{mark} 原文: {text}") + print(f" 识别: {hyp} CER={c:.2f}") + # 存一份，便于人工听一下 TTS 到底念得对不对 + pcm = (np.clip(samples, -1, 1) * 32767).astype(np.int16) + with wave.open(os.path.join(outdir, f"{i}_{which}.wav"), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(audio.sample_rate) + w.writeframes(pcm.tobytes()) + saved += 1 + + good = sum(1 for c in cers if c == 0) + print(f"\n完全正确 {good}/{len(CASES)}，平均 CER={np.mean(cers):.3f}") + print(f"合成音频已存到 {outdir}/（可人工试听确认音质）") + print("\n⚠ 合成语音远比真人远场录音干净，此结果不能作为识别率验收数据。") + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/bench_stt.py b/p0/stt_tts_bench/bench_stt.py new file mode 100644 index 0000000..b042363 --- /dev/null +++ b/p0/stt_tts_bench/bench_stt.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""P0 基准 · 本地流式 STT（sherpa-onnx 中英双语 Zipformer）。 + +测什么、为什么： + + 流式识别有两种"慢"，必须分开测，混在一起会得出错误结论。 + + 1. 算法延迟（first_partial_audio_s）：要喂进多少秒音频，才吐出第一个字。 + 这是模型属性（chunk 大小 + lookahead），换机器不变。它决定了"用户说完一个字到屏幕上出现字"的下限。 + + 2. 算力延迟（rtf / chunk_ratio）：解码耗时与音频时长的比。 + RTF<1 表示平均能跟上实时；但平均值会骗人——只要 p95 的单块解码时间超过块的时长， + 实时场景下就会累积积压，越说越滞后。所以必须同时看 chunk_ratio 的分位数。 + + 3. 收尾延迟（finalize_ms）：音频喂完后多久能拿到最终结果。 + 这一段直接计入方案 P1"结束到首段有声回复"的 2 秒预算。 + +用法： + python3 bench_stt.py # 默认线程 1,2,4 + THREADS=2 python3 bench_stt.py +""" + +import json +import os +import statistics +import sys +import time +import wave + +import numpy as np +import sherpa_onnx + +HERE = os.path.dirname(os.path.abspath(__file__)) +STT = os.path.join(HERE, "models", "stt") +WAVS = os.path.join(HERE, "wavs") +SAMPLE_RATE = 16000 +CHUNK_MS = int(os.environ.get("CHUNK_MS", "100")) +TAIL_SILENCE_S = 1.0 # 喂一段静音，让模型吐出最后一个字（真实系统里靠 VAD 判定说完） + + +def read_wav(path): + with wave.open(path, "rb") as w: + assert w.getnchannels() == 1, f"{path} 不是单声道" + assert w.getframerate() == SAMPLE_RATE, f"{path} 采样率 {w.getframerate()} != {SAMPLE_RATE}" + pcm = w.readframes(w.getnframes()) + return np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 + + +def build_recognizer(num_threads): + return sherpa_onnx.OnlineRecognizer.from_transducer( + tokens=os.path.join(STT, "tokens.txt"), + encoder=os.path.join(STT, "encoder-epoch-99-avg-1.int8.onnx"), + decoder=os.path.join(STT, "decoder-epoch-99-avg-1.int8.onnx"), + joiner=os.path.join(STT, "joiner-epoch-99-avg-1.int8.onnx"), + num_threads=num_threads, + sample_rate=SAMPLE_RATE, + feature_dim=80, + decoding_method="greedy_search", + enable_endpoint_detection=False, + ) + + +def run_one(rec, samples): + """喂一遍音频，返回各项指标。""" + chunk = int(SAMPLE_RATE * CHUNK_MS / 1000) + stream = rec.create_stream() + decode_times = [] + first_partial_audio_s = None + partials = [] + + t_all = time.perf_counter() + pos = 0 + while pos < len(samples): + block = samples[pos:pos + chunk] + pos += len(block) + t0 = time.perf_counter() + stream.accept_waveform(SAMPLE_RATE, block) + while rec.is_ready(stream): + rec.decode_stream(stream) + decode_times.append(time.perf_counter() - t0) + txt = rec.get_result(stream) + if txt: + partials.append((pos / SAMPLE_RATE, txt)) + if first_partial_audio_s is None: + first_partial_audio_s = pos / SAMPLE_RATE + + audio_s = len(samples) / SAMPLE_RATE + + # 收尾拆成两段测，混在一起会高估"说完到出结果"的开销： + # tail_decode_ms —— 解码尾静音的耗时。真实系统里这段对应 VAD 判定说完的等待， + # 是"必有开销"，但与识别器算力无关，不能算进识别延迟。 + # flush_ms —— input_finished() 之后冲刷出最终结果的耗时。这才是纯识别器开销。 + t0 = time.perf_counter() + tail = np.zeros(int(SAMPLE_RATE * TAIL_SILENCE_S), dtype=np.float32) + pos = 0 + while pos < len(tail): + block = tail[pos:pos + chunk] + pos += len(block) + stream.accept_waveform(SAMPLE_RATE, block) + while rec.is_ready(stream): + rec.decode_stream(stream) + tail_decode_ms = (time.perf_counter() - t0) * 1000 + + t1 = time.perf_counter() + stream.input_finished() + while rec.is_ready(stream): + rec.decode_stream(stream) + flush_ms = (time.perf_counter() - t1) * 1000 + final_text = rec.get_result(stream) + total_s = time.perf_counter() - t_all + + ratios = [d / (CHUNK_MS / 1000) for d in decode_times] + ratios_sorted = sorted(ratios) + + def pct(xs, p): + if not xs: + return float("nan") + return xs[min(len(xs) - 1, int(len(xs) * p))] + + return { + "audio_s": audio_s, + "total_s": total_s, + "rtf": total_s / (audio_s + TAIL_SILENCE_S), + "rtf_speech_only": total_s / audio_s, + "chunk_ratio_p50": statistics.median(ratios) if ratios else float("nan"), + "chunk_ratio_p95": pct(ratios_sorted, 0.95), + "chunk_ratio_max": max(ratios) if ratios else float("nan"), + "first_partial_audio_s": first_partial_audio_s, + "tail_decode_ms": tail_decode_ms, + "flush_ms": flush_ms, + "final_text": final_text, + "n_partials": len(partials), + } + + +def main(): + threads_list = [int(x) for x in os.environ.get("THREADS", "1,2,4").split(",")] + wav_files = sorted(f for f in os.listdir(WAVS) if f.endswith(".wav")) + if not wav_files: + sys.exit(f"没有测试音频：{WAVS}") + + print(f"chunk={CHUNK_MS}ms 尾静音={TAIL_SILENCE_S}s 音频={wav_files}") + print(f"CPU: 4 核 N5095（无 AVX2）\n") + + all_results = {} + for nt in threads_list: + print(f"=== num_threads={nt} ===") + rec = build_recognizer(nt) + # 预热：首次解码包含图优化/内存分配，不计入 + warm = np.zeros(SAMPLE_RATE, dtype=np.float32) + s = rec.create_stream() + s.accept_waveform(SAMPLE_RATE, warm) + while rec.is_ready(s): + rec.decode_stream(s) + + rows = [] + for wf in wav_files: + samples = read_wav(os.path.join(WAVS, wf)) + r = run_one(rec, samples) + rows.append((wf, r)) + print(f" {wf:<9} 音频{r['audio_s']:5.2f}s RTF={r['rtf']:5.2f} " + f"块比 p50={r['chunk_ratio_p50']:.2f} p95={r['chunk_ratio_p95']:.2f} " + f"max={r['chunk_ratio_max']:.2f} 首字@{r['first_partial_audio_s'] or float('nan'):.2f}s " + f"冲刷={r['flush_ms']:.0f}ms") + print(f" 识别: {r['final_text']}") + + rtfs = [r["rtf"] for _, r in rows] + p95s = [r["chunk_ratio_p95"] for _, r in rows] + fin = [r["flush_ms"] for _, r in rows] + print(f" → RTF 均值={statistics.mean(rtfs):.2f} 最差={max(rtfs):.2f} | " + f"块比p95 最差={max(p95s):.2f} | 纯冲刷 P50={statistics.median(fin):.0f}ms " + f"最差={max(fin):.0f}ms") + verdict = ("可实时" if max(rtfs) < 1 and max(p95s) < 1 + else "⚠ 跟不上实时" if max(rtfs) >= 1 + else "⚠ 平均够但存在突发积压风险") + print(f" → 判定: {verdict}\n") + all_results[nt] = {"rows": {w: r for w, r in rows}, "verdict": verdict} + + out = os.path.join(HERE, "results_stt.json") + with open(out, "w") as f: + json.dump({"chunk_ms": CHUNK_MS, "results": all_results}, f, + ensure_ascii=False, indent=2) + print(f"结果已写入 {out}") + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/bench_threads.py b/p0/stt_tts_bench/bench_threads.py new file mode 100644 index 0000000..f217d30 --- /dev/null +++ b/p0/stt_tts_bench/bench_threads.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""线程数扫描：查清 N5095 上 num_threads 的真实影响。 + +bench_stt.py 里 2 线程 RTF=0.22、4 线程 RTF=0.87，性能反而掉了 4 倍。 +本脚本对 1/2/3/4 线程各重复多轮，排除预热与顺序干扰，确认是否为真实现象。 +""" + +import os +import statistics +import time +import wave + +import numpy as np +import sherpa_onnx + +HERE = os.path.dirname(os.path.abspath(__file__)) +STT = os.path.join(HERE, "models", "stt") +WAVS = os.path.join(HERE, "wavs") +SR = 16000 +CHUNK = int(SR * 0.1) +REPEATS = int(os.environ.get("REPEATS", "5")) + + +def read_wav(p): + with wave.open(p, "rb") as w: + return np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16).astype(np.float32) / 32768.0 + + +def build(nt): + return sherpa_onnx.OnlineRecognizer.from_transducer( + tokens=os.path.join(STT, "tokens.txt"), + encoder=os.path.join(STT, "encoder-epoch-99-avg-1.int8.onnx"), + decoder=os.path.join(STT, "decoder-epoch-99-avg-1.int8.onnx"), + joiner=os.path.join(STT, "joiner-epoch-99-avg-1.int8.onnx"), + num_threads=nt, sample_rate=SR, feature_dim=80, + decoding_method="greedy_search", enable_endpoint_detection=False, + ) + + +def decode_once(rec, samples): + s = rec.create_stream() + t0 = time.perf_counter() + pos = 0 + while pos < len(samples): + s.accept_waveform(SR, samples[pos:pos + CHUNK]) + pos += CHUNK + while rec.is_ready(s): + rec.decode_stream(s) + return time.perf_counter() - t0 + + +def main(): + wav = read_wav(os.path.join(WAVS, "stt_0.wav")) + dur = len(wav) / SR + print(f"音频 {dur:.2f}s，每档重复 {REPEATS} 轮（含 1 轮预热，不计入）\n") + print(f"{'线程':>4} {'RTF中位':>9} {'RTF最差':>9} {'各轮RTF':>34}") + print("-" * 62) + for nt in (1, 2, 3, 4): + rec = build(nt) + decode_once(rec, wav[: SR * 2]) # 预热 + rtfs = [] + for _ in range(REPEATS): + rtfs.append(decode_once(rec, wav) / dur) + shown = " ".join(f"{r:.2f}" for r in rtfs) + print(f"{nt:>4} {statistics.median(rtfs):>9.2f} {max(rtfs):>9.2f} {shown:>34}") + del rec + + print("\n=== 参照：系统负载 ===") + print(open("/proc/loadavg").read().strip()) + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/bench_tts.py b/p0/stt_tts_bench/bench_tts.py new file mode 100644 index 0000000..a2cf797 --- /dev/null +++ b/p0/stt_tts_bench/bench_tts.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""P0 基准 · 本地 TTS（sherpa-onnx VITS 中文 zh-ll）。 + +测什么、为什么： + + 1. RTF：合成耗时 / 音频时长。<1 表示比实时快。 + 2. 首音延迟：从调用到吐出第一块音频的耗时。这一段直接计入方案 P1 的 2 秒预算。 + 3. 分句接续停顿：方案要求"按语义短句合成并播放"。逐句生成时， + 如果下一句还没合成完、上一句已经播完，喇叭里就会出现空档。 + 这里用"生成耗时 vs 播放耗时"的竞速来预测停顿，不需要声卡。 + +用法： + python3 bench_tts.py + THREADS=1,2,4 python3 bench_tts.py +""" + +import json +import os +import re +import statistics +import time + +import numpy as np +import sherpa_onnx + +HERE = os.path.dirname(os.path.abspath(__file__)) +TTS = os.path.join(HERE, "models", "tts") +RULE_FSTS = ",".join(os.path.join(TTS, f) for f in + ("date.fst", "number.fst", "phone.fst", "new_heteronym.fst")) + +SHORT = "好的，我记住了。" +MEDIUM = "今天北京晴转多云，气温十八到二十六度，出门记得带件外套。" +LONG = ("我查了一下，明天上午十点有个会议，地点在三号会议室。" + "另外提醒你，后天是妈妈的生日，要不要我提前帮你订一束花？" + "还有，你上周说要买的那本书，现在有货了，价格是四十九块八。") + +# 方案 §5：数字、日期、中英混读都要覆盖 +CASES = [ + ("短句", SHORT), + ("中句", MEDIUM), + ("长段", LONG), + ("数字", "订单编号是七三九一，金额一千二百五十元六角。"), + ("日期", "会议定在二零二六年九月十四日下午三点半。"), + ("中英混读", "这个 API 的 response time 大概在 200 毫秒左右。"), +] + + +def build_tts(num_threads): + cfg = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(TTS, "model.onnx"), + lexicon=os.path.join(TTS, "lexicon.txt"), + tokens=os.path.join(TTS, "tokens.txt"), + dict_dir=os.path.join(TTS, "dict"), + ), + num_threads=num_threads, + provider="cpu", + ), + rule_fsts=RULE_FSTS, + max_num_sentences=1, # 逐句生成，便于尽早开播 + ) + return sherpa_onnx.OfflineTts(cfg) + + +def synth(tts, text): + """合成一段，返回 (首音延迟ms, 总耗时ms, 音频秒数, 音频)。""" + chunks = [] + first_at = [None] + t0 = time.perf_counter() + + def cb(*args): # 兼容 (samples,) 与 (samples, progress) 两种签名 + samples = args[0] + if first_at[0] is None: + first_at[0] = (time.perf_counter() - t0) * 1000 + chunks.append(np.array(samples, dtype=np.float32, copy=True)) + return 0 + + audio = tts.generate(text, sid=0, speed=1.0, callback=cb) + total_ms = (time.perf_counter() - t0) * 1000 + if first_at[0] is None: + first_at[0] = total_ms + dur_s = len(audio.samples) / audio.sample_rate + return first_at[0], total_ms, dur_s, np.array(audio.samples) + + +def split_sentences(text): + """按中英文标点切句，模拟"按语义短句合成"。""" + parts = re.split(r"(?<=[。！？；!?;])", text) + return [p.strip() for p in parts if p.strip()] + + +def main(): + threads_list = [int(x) for x in os.environ.get("THREADS", "1,2,4").split(",")] + print(f"TTS: VITS zh-ll 线程扫描={threads_list} max_num_sentences=1\n") + + all_results = {} + for nt in threads_list: + print(f"=== num_threads={nt} ===") + tts = build_tts(nt) + tts.generate("预热", sid=0, speed=1.0) # 预热 + + rows = {} + for label, text in CASES: + first_ms, total_ms, dur_s, _ = synth(tts, text) + rtf = (total_ms / 1000) / dur_s + rows[label] = {"first_ms": first_ms, "total_ms": total_ms, + "dur_s": dur_s, "rtf": rtf, "text": text} + print(f" {label:<8} 音频{dur_s:5.2f}s 合成{total_ms:7.0f}ms " + f"RTF={rtf:5.2f} 首音={first_ms:6.0f}ms") + + # 分句接续：完全串行生成，看下一句能否在上一句播完前备好 + print(f" --- 分句接续（长段切 {len(split_sentences(LONG))} 句）---") + sents = split_sentences(LONG) + t_gen = 0.0 # 该句开始生成的时刻（相对） + play_end = 0.0 # 上一句播完的时刻 + timeline = [] + for i, s in enumerate(sents): + first_ms, total_ms, dur_s, _ = synth(tts, s) + gen_start = t_gen + first_ready = gen_start + first_ms / 1000 + play_start = max(play_end, first_ready) + gap = play_start - play_end if i > 0 else 0.0 + play_end = play_start + dur_s + t_gen = gen_start + total_ms / 1000 # 串行：本句合成完才开始下一句 + timeline.append({"i": i, "text": s[:18], "dur_s": dur_s, + "gen_ms": total_ms, "first_ms": first_ms, + "gap_s": gap, "late": play_start - first_ready}) + flag = "空档" if gap > 0.02 else "OK" + print(f" 句{i+1}: 播{dur_s:4.2f}s 合成{total_ms:6.0f}ms " + f"首音{first_ms:5.0f}ms 空档={gap:5.2f}s [{flag}] {s[:16]}") + max_gap = max(t["gap_s"] for t in timeline) + total_audio = sum(t["dur_s"] for t in timeline) + rows["_分句接续"] = {"max_gap_s": max_gap, "total_audio_s": total_audio, + "timeline": timeline, + "total_gen_ms": sum(t["gen_ms"] for t in timeline)} + verdict = "无空档" if max_gap <= 0.02 else f"⚠ 最大空档 {max_gap:.2f}s" + print(f" → 分句判定: {verdict}（总音频 {total_audio:.2f}s，" + f"总合成 {sum(t['gen_ms'] for t in timeline):.0f}ms）\n") + all_results[nt] = rows + + out = os.path.join(HERE, "results_tts.json") + with open(out, "w") as f: + json.dump(all_results, f, ensure_ascii=False, indent=2) + print(f"结果已写入 {out}") + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/bench_tts_compare.py b/p0/stt_tts_bench/bench_tts_compare.py new file mode 100644 index 0000000..b848057 --- /dev/null +++ b/p0/stt_tts_bench/bench_tts_compare.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""P0 基准 · 本地 TTS 模型横向对比（N5095）。 + +背景：vits-zh-ll 实测 RTF≈1.7，慢于实时，且 VITS 非自回归—— +回调只在合成结束后触发一次，首音延迟 == 总合成耗时。 +本脚本用同一组句子对比多个候选模型，找是否有能跑到 RTF<0.5 的。 + +注意 max_num_sentences=1 会把多句输入截断为第一句（见 bench_tts.py 的坑）， +所以这里每句单独调用——这也正是真实系统的做法（按语义短句合成）。 +""" + +import os +import statistics +import time + +import numpy as np +import sherpa_onnx + +HERE = os.path.dirname(os.path.abspath(__file__)) + +SHORT = "好的，我记住了。" +MED = "今天北京晴转多云，气温十八到二十六度。" +LONG = "我查了一下，明天上午十点有个会议，地点在三号会议室。" +NUM = "订单编号是七三九一，金额一千二百五十元六角。" + +TESTS = [("短句", SHORT), ("中句", MED), ("长句", LONG), ("数字", NUM)] + + +def load_vits_zh_ll(nt): + d = os.path.join(HERE, "models", "tts") + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(d, "model.onnx"), + lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt"), + dict_dir=os.path.join(d, "dict"), + ), + num_threads=nt, provider="cpu"), + rule_fsts=",".join(os.path.join(d, f) for f in + ("date.fst", "number.fst", "phone.fst", "new_heteronym.fst")), + max_num_sentences=1)) + + +def load_piper_xlow(nt): + d = os.path.join(HERE, "models", "tts_piper_xlow") + return sherpa_onnx.OfflineTts(sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + vits=sherpa_onnx.OfflineTtsVitsModelConfig( + model=os.path.join(d, "zh_CN-huayan-x_low.onnx"), + lexicon=os.path.join(d, "lexicon.txt"), + tokens=os.path.join(d, "tokens.txt"), + ), + num_threads=nt, provider="cpu"), + max_num_sentences=1)) + + +MODELS = [ + ("vits-zh-ll", load_vits_zh_ll, "VITS 中文，115MB"), + ("piper-huayan-x_low", load_piper_xlow, "Piper 中文，20MB"), +] + + +def measure(tts, text): + calls = [0] + + def cb(*args): + calls[0] += 1 + return 0 + + t0 = time.perf_counter() + audio = tts.generate(text, sid=0, speed=1.0, callback=cb) + total_ms = (time.perf_counter() - t0) * 1000 + dur = len(audio.samples) / audio.sample_rate + return total_ms, dur, calls[0] + + +def main(): + nt = int(os.environ.get("THREADS", "2")) + print(f"N5095 / num_threads={nt} / 每句单独合成\n") + + report = {} + for name, loader, desc in MODELS: + try: + tts = loader(nt) + except Exception as e: + print(f"=== {name} ({desc}) ===\n 加载失败: {type(e).__name__}: {e}\n") + continue + tts.generate("预热", sid=0, speed=1.0) + print(f"=== {name} ({desc}) ===") + rows = [] + for label, text in TESTS: + total_ms, dur, ncalls = measure(tts, text) + rtf = (total_ms / 1000) / dur + rows.append((label, total_ms, dur, rtf, ncalls)) + print(f" {label:<5} {len(text):>2}字 → 音频{dur:5.2f}s 合成{total_ms:7.0f}ms " + f"RTF={rtf:5.2f} 回调{ncalls}次") + rtfs = [r[3] for r in rows] + avg = statistics.mean(rtfs) + print(f" → RTF 均值={avg:.2f} 最差={max(rtfs):.2f} " + f"判定: {'✅ 可实时' if avg < 0.5 else '⚠ 慢于实时' if avg >= 1 else '△ 勉强'}\n") + report[name] = {"desc": desc, "rows": rows, "rtf_avg": avg, "rtf_max": max(rtfs)} + del tts + + # 播放侧模拟：连续说 5 句时，下一句能否在上一句播完前备好 + print("=== 连续播放模拟（5 句，串行合成）===") + loaders = {m[0]: m[1] for m in MODELS} + for name in loaders: + if name not in report: + continue + tts = loaders[name](nt) + sents = ["我查了一下，明天上午十点有个会议。", "地点在三号会议室。", + "另外提醒你，后天是妈妈的生日。", "要不要我提前帮你订一束花？", + "还有你上周说要买的那本书现在有货了。"] + t_gen = 0.0 + play_end = 0.0 + gaps = [] + for i, s in enumerate(sents): + total_ms, dur, _ = measure(tts, s) + first_ready = t_gen + total_ms / 1000 + play_start = max(play_end, first_ready) + gaps.append(max(0.0, play_start - play_end)) + play_end = play_start + dur + t_gen += total_ms / 1000 + print(f" {name:<22} 各句空档: {[f'{g:.1f}s' for g in gaps]} " + f"最大={max(gaps):.2f}s 总播完={play_end:.1f}s") + report[name]["gaps"] = gaps + del tts + print() + + import json + out = os.path.join(HERE, "results_tts_compare.json") + with open(out, "w") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + print(f"结果已写入 {out}") + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/download_models.sh b/p0/stt_tts_bench/download_models.sh new file mode 100755 index 0000000..2655841 --- /dev/null +++ b/p0/stt_tts_bench/download_models.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# 下载 P0 基准所需模型。走 hf-mirror.com（本机 huggingface.co 直连不通）。 +# 可重复执行：已存在且大小正确的文件会跳过。 +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")" && pwd)" +MIRROR="${HF_MIRROR:-https://hf-mirror.com}" +STT_REPO="csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20" +TTS_REPO="csukuangfj/sherpa-onnx-vits-zh-ll" + +STT_DIR="$ROOT/models/stt" +TTS_DIR="$ROOT/models/tts" +WAV_DIR="$ROOT/wavs" +mkdir -p "$STT_DIR" "$TTS_DIR" "$WAV_DIR" + +fetch() { # fetch + local repo="$1" remote="$2" local="$3" + mkdir -p "$(dirname "$local")" + local url="$MIRROR/$repo/resolve/main/$remote" + local expect + expect=$(curl -sIL "$url" | grep -i '^content-length' | tail -1 | tr -d '\r' | awk '{print $2}') + if [ -f "$local" ] && [ -n "$expect" ] && [ "$(stat -c%s "$local")" = "$expect" ]; then + printf ' skip %-45s %s\n' "$(basename "$remote")" "(已完整)" + return 0 + fi + printf ' get %-45s %s\n' "$(basename "$remote")" "${expect:+$((expect/1048576))MB}" + curl -sL --retry 3 --retry-delay 2 -o "$local.part" "$url" || { echo " FAILED"; return 1; } + mv "$local.part" "$local" +} + +echo "=== STT: 流式 Zipformer 中英双语 int8 ===" +for f in tokens.txt bpe.model \ + encoder-epoch-99-avg-1.int8.onnx \ + decoder-epoch-99-avg-1.int8.onnx \ + joiner-epoch-99-avg-1.int8.onnx; do + fetch "$STT_REPO" "$f" "$STT_DIR/$f" +done +for w in 0.wav 1.wav 2.wav 3.wav; do + fetch "$STT_REPO" "test_wavs/$w" "$WAV_DIR/stt_$w" +done + +echo "=== TTS: VITS zh-ll ===" +for f in model.onnx tokens.txt lexicon.txt date.fst number.fst phone.fst new_heteronym.fst \ + dict/jieba.dict.utf8 dict/hmm_model.utf8 dict/idf.utf8 \ + dict/pos_dict/char_state_tab.utf8 dict/pos_dict/prob_emit.utf8 dict/pos_dict/prob_trans.utf8; do + fetch "$TTS_REPO" "$f" "$TTS_DIR/$f" +done + +echo +echo "=== 结果 ===" +du -sh "$STT_DIR" "$TTS_DIR" "$WAV_DIR" 2>/dev/null +echo "下载完成。" diff --git a/p0/stt_tts_bench/hf_fetch.py b/p0/stt_tts_bench/hf_fetch.py new file mode 100644 index 0000000..15b5de9 --- /dev/null +++ b/p0/stt_tts_bench/hf_fetch.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""按目录递归下载 HF 仓库文件到本地（走 hf-mirror）。 + +Piper 中文 medium 档没有 lexicon.txt，改用 espeak-ng-data 做音素化， +而那是一堆小文件，用 shell 逐个 curl 太啰嗦，所以写这个。 + +用法： + python3 hf_fetch.py <本地目录> [远端前缀] + python3 hf_fetch.py csukuangfj/vits-piper-zh_CN-huayan-medium \ + models/tts_piper_medium espeak-ng-data +""" + +import json +import os +import sys +import urllib.parse +import urllib.request + +MIRROR = os.environ.get("HF_MIRROR", "https://hf-mirror.com") +UA = {"User-Agent": "curl/8.5.0"} # hf-mirror 对 urllib 默认 UA 返回 403 + + +def _open(url, timeout): + return urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout) + + +def api_tree(repo): + url = f"{MIRROR}/api/models/{repo}/tree/main?recursive=true" + with _open(url, 60) as r: + return json.load(r) + + +def fetch(url, dest): + os.makedirs(os.path.dirname(dest), exist_ok=True) + tmp = dest + ".part" + with _open(url, 300) as r, open(tmp, "wb") as f: + while True: + b = r.read(1 << 16) + if not b: + break + f.write(b) + os.replace(tmp, dest) + + +def main(): + repo, outdir = sys.argv[1], sys.argv[2] + prefix = sys.argv[3] if len(sys.argv) > 3 else "" + files = [f for f in api_tree(repo) if f["type"] == "file"] + if prefix: + files = [f for f in files if f["path"].startswith(prefix)] + print(f"{repo}: 取 {len(files)} 个文件 → {outdir}") + for f in files: + rel = f["path"] + dest = os.path.join(outdir, rel) + if os.path.exists(dest) and os.path.getsize(dest) == f.get("size", -1): + continue + # espeak 的 voices/!v/ 下有带空格的文件名（如 "Mr serious"），必须转义 + url = f"{MIRROR}/{repo}/resolve/main/{urllib.parse.quote(rel)}" + try: + fetch(url, dest) + except Exception as e: + print(f" 跳过 {rel}: {type(e).__name__}: {e}") + continue + print(f" {rel} ({f.get('size',0)/1024:.0f} KB)") + print("完成") + + +if __name__ == "__main__": + main() diff --git a/p0/stt_tts_bench/results_concurrent_matcha.json b/p0/stt_tts_bench/results_concurrent_matcha.json new file mode 100644 index 0000000..921796d --- /dev/null +++ b/p0/stt_tts_bench/results_concurrent_matcha.json @@ -0,0 +1,13 @@ +{ + "stt_threads": 2, + "tts_threads": 2, + "tts_model": "matcha", + "solo_stt_rtf": 0.27664071878154955, + "solo_tts_rtf": 0.44042636413212993, + "conc_stt_rtf": 0.5378692789555439, + "conc_tts_rtf": 0.935733770617253, + "stt_degrade": 1.944288177548709, + "tts_degrade": 2.1246088945223285, + "conc_chunk_p95": 2.0250199799920665, + "conc_chunk_max": 2.6675718699971185 +} \ No newline at end of file diff --git a/p0/stt_tts_bench/results_concurrent_piper.json b/p0/stt_tts_bench/results_concurrent_piper.json new file mode 100644 index 0000000..1ccda51 --- /dev/null +++ b/p0/stt_tts_bench/results_concurrent_piper.json @@ -0,0 +1,13 @@ +{ + "stt_threads": 2, + "tts_threads": 2, + "tts_model": "piper", + "solo_stt_rtf": 0.2645827264656701, + "solo_tts_rtf": 0.1899799540144868, + "conc_stt_rtf": 0.5013027566055182, + "conc_tts_rtf": 0.3256982994144358, + "stt_degrade": 1.8946919298246885, + "tts_degrade": 1.714382452106499, + "conc_chunk_p95": 2.345268959998066, + "conc_chunk_max": 2.5172376999989865 +} \ No newline at end of file diff --git a/p0/stt_tts_bench/results_concurrent_vits.json b/p0/stt_tts_bench/results_concurrent_vits.json new file mode 100644 index 0000000..480ceaf --- /dev/null +++ b/p0/stt_tts_bench/results_concurrent_vits.json @@ -0,0 +1,13 @@ +{ + "stt_threads": 2, + "tts_threads": 3, + "tts_model": "vits", + "solo_stt_rtf": 0.2826273522909462, + "solo_tts_rtf": 1.43806433885311, + "conc_stt_rtf": 0.6159577618900334, + "conc_tts_rtf": 1.9989867974632771, + "stt_degrade": 2.1793989750006415, + "tts_degrade": 1.3900537990237043, + "conc_chunk_p95": 2.319201170002998, + "conc_chunk_max": 3.0467084500014607 +} \ No newline at end of file diff --git a/p0/stt_tts_bench/results_stt.json b/p0/stt_tts_bench/results_stt.json new file mode 100644 index 0000000..882724f --- /dev/null +++ b/p0/stt_tts_bench/results_stt.json @@ -0,0 +1,127 @@ +{ + "chunk_ms": 100, + "results": { + "1": { + "rows": { + "stt_0.wav": { + "audio_s": 10.053125, + "total_s": 4.046593594999649, + "rtf": 0.36610402895105676, + "rtf_speech_only": 0.4025209668635026, + "chunk_ratio_p50": 0.003555890002644446, + "chunk_ratio_p95": 1.1945098499973028, + "chunk_ratio_max": 1.2445684299973436, + "first_partial_audio_s": 1.1, + "tail_decode_ms": 352.4101550001433, + "flush_ms": 0.06503600025098422, + "final_text": "昨天是 MONDAY TODAY IS LIBR THE DAY AFTER TOMORROW是星期三", + "n_partials": 91 + }, + "stt_1.wav": { + "audio_s": 5.1, + "total_s": 2.1389257700002418, + "rtf": 0.35064356885249864, + "rtf_speech_only": 0.419397209803969, + "chunk_ratio_p50": 0.003521949997775664, + "chunk_ratio_p95": 1.1893300500014448, + "chunk_ratio_max": 1.1938205499973265, + "first_partial_audio_s": 1.4, + "tail_decode_ms": 356.69890200006193, + "flush_ms": 0.02634899965414661, + "final_text": "这是第一种第二种叫呃与 ALWAYS ALWAYS什么意思啊", + "n_partials": 38 + }, + "stt_2.wav": { + "audio_s": 4.69, + "total_s": 2.0203764259999843, + "rtf": 0.3550749430579937, + "rtf_speech_only": 0.43078388614072155, + "chunk_ratio_p50": 0.00355069999841362, + "chunk_ratio_p95": 1.188613239996812, + "chunk_ratio_max": 1.209888459998183, + "first_partial_audio_s": 0.5, + "tail_decode_ms": 356.0279449998234, + "flush_ms": 0.02447099996061297, + "final_text": "这是不是平凡的啊不认识你记下来 FREQUENTLY频繁的", + "n_partials": 43 + }, + "stt_3.wav": { + "audio_s": 8.83, + "total_s": 3.5579121109999505, + "rtf": 0.36194426358086984, + "rtf_speech_only": 0.4029345539071292, + "chunk_ratio_p50": 0.0035380899998926907, + "chunk_ratio_p95": 1.1881532100005643, + "chunk_ratio_max": 1.1958122499981982, + "first_partial_audio_s": 1.1, + "tail_decode_ms": 355.65589199995884, + "flush_ms": 0.024543000108678825, + "final_text": "第一句是个什么时态在加了 ES是一般现在时对吧后面那还有时时态来语写上上去了", + "n_partials": 79 + } + }, + "verdict": "⚠ 平均够但存在突发积压风险" + }, + "2": { + "rows": { + "stt_0.wav": { + "audio_s": 10.053125, + "total_s": 2.578806391999933, + "rtf": 0.23331016269154045, + "rtf_speech_only": 0.25651788792041613, + "chunk_ratio_p50": 0.003629650000220863, + "chunk_ratio_p95": 0.7905124399985652, + "chunk_ratio_max": 0.9809923500006335, + "first_partial_audio_s": 1.1, + "tail_decode_ms": 221.97086599999238, + "flush_ms": 0.05037600021751132, + "final_text": "昨天是 MONDAY TODAY IS LIBR THE DAY AFTER TOMORROW是星期三", + "n_partials": 91 + }, + "stt_1.wav": { + "audio_s": 5.1, + "total_s": 1.3411688280002636, + "rtf": 0.2198637422951252, + "rtf_speech_only": 0.2629742800000517, + "chunk_ratio_p50": 0.0035533499976736493, + "chunk_ratio_p95": 0.7596391700008098, + "chunk_ratio_max": 0.7848052599956645, + "first_partial_audio_s": 1.4, + "tail_decode_ms": 222.9477579999184, + "flush_ms": 0.024468999981763773, + "final_text": "这是第一种第二种叫呃与 ALWAYS ALWAYS什么意思啊", + "n_partials": 38 + }, + "stt_2.wav": { + "audio_s": 4.69, + "total_s": 1.2377910049999628, + "rtf": 0.2175379622144047, + "rtf_speech_only": 0.2639213230277106, + "chunk_ratio_p50": 0.003542499998729909, + "chunk_ratio_p95": 0.7229457900029956, + "chunk_ratio_max": 0.7791545099962605, + "first_partial_audio_s": 0.5, + "tail_decode_ms": 223.60373099991193, + "flush_ms": 0.024873000256775413, + "final_text": "这是不是平凡的啊不认识你记下来 FREQUENTLY频繁的", + "n_partials": 43 + }, + "stt_3.wav": { + "audio_s": 8.83, + "total_s": 2.189363661999778, + "rtf": 0.2227226512715949, + "rtf_speech_only": 0.24794605458661131, + "chunk_ratio_p50": 0.003571679999367916, + "chunk_ratio_p95": 0.7330924600000799, + "chunk_ratio_max": 0.8176678099971468, + "first_partial_audio_s": 1.1, + "tail_decode_ms": 213.40833099975498, + "flush_ms": 0.025572000140527962, + "final_text": "第一句是个什么时态在加了 ES是一般现在时对吧后面那还有时时态来语写上上去了", + "n_partials": 79 + } + }, + "verdict": "可实时" + } + } +} \ No newline at end of file diff --git a/p0/stt_tts_bench/results_tts.json b/p0/stt_tts_bench/results_tts.json new file mode 100644 index 0000000..0567ab1 --- /dev/null +++ b/p0/stt_tts_bench/results_tts.json @@ -0,0 +1,158 @@ +{ + "1": { + "短句": { + "first_ms": 3512.398196000049, + "total_ms": 3512.4536819998866, + "dur_s": 0.5618125, + "rtf": 6.252003438869528, + "text": "好的，我记住了。" + }, + "中句": { + "first_ms": 6728.828355000132, + "total_ms": 6728.931756000293, + "dur_s": 1.7795, + "rtf": 3.781360919359535, + "text": "今天北京晴转多云，气温十八到二十六度，出门记得带件外套。" + }, + "长段": { + "first_ms": 4259.075834999749, + "total_ms": 4259.120488000008, + "dur_s": 0.942875, + "rtf": 4.517163450086181, + "text": "我查了一下，明天上午十点有个会议，地点在三号会议室。另外提醒你，后天是妈妈的生日，要不要我提前帮你订一束花？还有，你上周说要买的那本书，现在有货了，价格是四十九块八。" + }, + "数字": { + "first_ms": 7015.1300140000785, + "total_ms": 7015.235308999763, + "dur_s": 1.735875, + "rtf": 4.041325158205379, + "text": "订单编号是七三九一，金额一千二百五十元六角。" + }, + "日期": { + "first_ms": 10958.226222000121, + "total_ms": 10958.3969360001, + "dur_s": 3.165375, + "rtf": 3.4619585154997745, + "text": "会议定在二零二六年九月十四日下午三点半。" + }, + "中英混读": { + "first_ms": 8337.178994000169, + "total_ms": 8337.248050000198, + "dur_s": 2.2680625, + "rtf": 3.6759339965279607, + "text": "这个 API 的 response time 大概在 200 毫秒左右。" + }, + "_分句接续": { + "max_gap_s": 3.4393423310002254, + "total_audio_s": 2.6743125, + "timeline": [ + { + "i": 0, + "text": "我查了一下，明天上午十点有个会议，地", + "dur_s": 1.04, + "gen_ms": 3355.619835000198, + "first_ms": 3355.5726700001287, + "gap_s": 0.0, + "late": 0.0 + }, + { + "i": 1, + "text": "另外提醒你，后天是妈妈的生日，要不要", + "dur_s": 1.1223125, + "gen_ms": 4479.338345000087, + "first_ms": 4479.295166000156, + "gap_s": 3.4393423310002254, + "late": 0.0 + }, + { + "i": 2, + "text": "还有，你上周说要买的那本书，现在有货", + "dur_s": 0.512, + "gen_ms": 1627.3659610001232, + "first_ms": 1627.323898999748, + "gap_s": 0.5050545779996796, + "late": 0.0 + } + ], + "total_gen_ms": 9462.324141000408 + } + }, + "2": { + "短句": { + "first_ms": 1833.8097639998523, + "total_ms": 1833.856194999953, + "dur_s": 0.5618125, + "rtf": 3.2641783424184276, + "text": "好的，我记住了。" + }, + "中句": { + "first_ms": 3504.9386239998057, + "total_ms": 3504.9927340000977, + "dur_s": 1.7795, + "rtf": 1.9696503141332382, + "text": "今天北京晴转多云，气温十八到二十六度，出门记得带件外套。" + }, + "长段": { + "first_ms": 2213.8197690001107, + "total_ms": 2213.8693260003492, + "dur_s": 0.942875, + "rtf": 2.347998754872437, + "text": "我查了一下，明天上午十点有个会议，地点在三号会议室。另外提醒你，后天是妈妈的生日，要不要我提前帮你订一束花？还有，你上周说要买的那本书，现在有货了，价格是四十九块八。" + }, + "数字": { + "first_ms": 3649.676482999894, + "total_ms": 3649.726979999741, + "dur_s": 1.735875, + "rtf": 2.1025286843809265, + "text": "订单编号是七三九一，金额一千二百五十元六角。" + }, + "日期": { + "first_ms": 5695.0202110001555, + "total_ms": 5695.0819460003, + "dur_s": 3.165375, + "rtf": 1.7991808066975636, + "text": "会议定在二零二六年九月十四日下午三点半。" + }, + "中英混读": { + "first_ms": 4329.835705000278, + "total_ms": 4329.891841000062, + "dur_s": 2.2680625, + "rtf": 1.9090707778114855, + "text": "这个 API 的 response time 大概在 200 毫秒左右。" + }, + "_分句接续": { + "max_gap_s": 1.2875714189999234, + "total_audio_s": 2.6743125, + "timeline": [ + { + "i": 0, + "text": "我查了一下，明天上午十点有个会议，地", + "dur_s": 1.04, + "gen_ms": 1753.6386409997249, + "first_ms": 1753.5868829995707, + "gap_s": 0.0, + "late": 0.0 + }, + { + "i": 1, + "text": "另外提醒你，后天是妈妈的生日，要不要", + "dur_s": 1.1223125, + "gen_ms": 2327.566012999796, + "first_ms": 2327.5196609997693, + "gap_s": 1.2875714189999234, + "late": 0.0 + }, + { + "i": 2, + "text": "还有，你上周说要买的那本书，现在有货", + "dur_s": 0.512, + "gen_ms": 848.2465030001549, + "first_ms": 848.2053440002346, + "gap_s": 0.0, + "late": 0.2740608039997383 + } + ], + "total_gen_ms": 4929.451156999676 + } + } +} \ No newline at end of file diff --git a/p0/stt_tts_bench/results_tts_compare.json b/p0/stt_tts_bench/results_tts_compare.json new file mode 100644 index 0000000..eae5898 --- /dev/null +++ b/p0/stt_tts_bench/results_tts_compare.json @@ -0,0 +1,86 @@ +{ + "vits-zh-ll": { + "desc": "VITS 中文，115MB", + "rows": [ + [ + "短句", + 1023.8842699995985, + 0.44725, + 2.289288474006928, + 1 + ], + [ + "中句", + 2747.543926000162, + 1.648, + 1.6671989842233994, + 1 + ], + [ + "长句", + 1709.2012230000364, + 1.024, + 1.669141819335973, + 1 + ], + [ + "数字", + 3316.832393000368, + 1.81275, + 1.8297241169495893, + 1 + ] + ], + "rtf_avg": 1.8638383486289725, + "rtf_max": 2.289288474006928, + "gaps": [ + 2.0050806700000976, + 1.562527183999915, + 0.4825053690001191, + 2.856295740000048, + 3.7974527159998246 + ] + }, + "piper-huayan-x_low": { + "desc": "Piper 中文，20MB", + "rows": [ + [ + "短句", + 188.33520399994086, + 1.104, + 0.1705934818840044, + 1 + ], + [ + "中句", + 584.8148079999191, + 3.408, + 0.17160058920185423, + 1 + ], + [ + "长句", + 670.672593999825, + 3.872, + 0.17321089721069863, + 1 + ], + [ + "数字", + 647.2685590001674, + 3.776, + 0.17141646159962062, + 1 + ] + ], + "rtf_avg": 0.17170535747404447, + "rtf_max": 0.17321089721069863, + "gaps": [ + 0.4739460710002277, + 0.0, + 0.0, + 0.0, + 0.0 + ] + } +} \ No newline at end of file diff --git a/p0/verify_deepseek.py b/p0/verify_deepseek.py new file mode 100644 index 0000000..73d11c7 --- /dev/null +++ b/p0/verify_deepseek.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +"""P0 验证：DeepSeek API 四条路径。 + +对应方案 §5 的 P0 要求：纯文本流式 / 图片输入 / 工具调用 / 取消与超时。 +用标准库直连 HTTP，不依赖 SDK 版本，便于精确计时。 + +用法： + set -a; . ~/.config/ai_assistant/env; set +a + python3 verify_deepseek.py +""" + +import base64 +import io +import json +import os +import statistics +import sys +import time +import urllib.error +import urllib.request + +BASE = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") +KEY = os.environ["DEEPSEEK_API_KEY"] +MODEL = os.environ.get("DEEPSEEK_MODEL", "deepseek-flash") + +RESULTS = [] + + +def record(name, ok, detail): + RESULTS.append((name, ok, detail)) + print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}") + + +def _req(path, body=None, timeout=60, method=None): + data = None if body is None else json.dumps(body).encode("utf-8") + req = urllib.request.Request( + BASE + path, + data=data, + method=method or ("GET" if data is None else "POST"), + headers={ + "Authorization": f"Bearer {KEY}", + "Content-Type": "application/json", + }, + ) + return urllib.request.urlopen(req, timeout=timeout) + + +def chat(messages, thinking="disabled", **kw): + """thinking 默认关闭：见 verify 报告——开启时 reasoning_content 必须回传。""" + body = {"model": MODEL, "messages": messages, "stream": False} + if thinking: + body["thinking"] = {"type": thinking} + body.update(kw) + with _req("/chat/completions", body) as r: + return json.loads(r.read().decode("utf-8")) + + +def chat_stream(messages, tools=None, max_tokens=256, timeout=60, thinking="disabled"): + """返回 (ttft, total, text, reasoning, tool_calls, finish)。 + + thinking 默认 disabled：开启时 API 要求把 reasoning_content 原样回传， + 否则多轮/工具回灌一律 400。详见 verify 报告。 + """ + body = { + "model": MODEL, + "messages": messages, + "stream": True, + "max_tokens": max_tokens, + } + if thinking: + body["thinking"] = {"type": thinking} + if tools: + body["tools"] = tools + t0 = time.perf_counter() + ttft = None + text_parts, reasoning_parts = [], [] + tool_calls = {} + finish = None + with _req("/chat/completions", body, timeout=timeout) as r: + for raw in r: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data: "): + continue + payload = line[6:] + if payload == "[DONE]": + break + d = json.loads(payload) + if not d.get("choices"): + continue + ch = d["choices"][0] + delta = ch.get("delta") or {} + if ch.get("finish_reason"): + finish = ch["finish_reason"] + if delta.get("content"): + if ttft is None: + ttft = time.perf_counter() - t0 + text_parts.append(delta["content"]) + if delta.get("reasoning_content"): + if ttft is None: + ttft = time.perf_counter() - t0 + reasoning_parts.append(delta["reasoning_content"]) + for tc in delta.get("tool_calls") or []: + if ttft is None: + ttft = time.perf_counter() - t0 + idx = tc.get("index", 0) + slot = tool_calls.setdefault(idx, {"name": "", "arguments": ""}) + fn = tc.get("function") or {} + if fn.get("name"): + slot["name"] = fn["name"] + if fn.get("arguments"): + slot["arguments"] += fn["arguments"] + total = time.perf_counter() - t0 + return ttft, total, "".join(text_parts), "".join(reasoning_parts), list(tool_calls.values()), finish + + +# ---------------------------------------------------------------- 路径 1：纯文本流式 +def test_text_stream(): + print("\n=== 路径 1：纯文本流式（首 token 延迟）===") + msgs = [{"role": "user", "content": "用一句话介绍你自己。"}] + try: + ttft, total, text, reasoning, tcs, finish = chat_stream(msgs) + record("1a 流式基本可用", bool(text.strip()), + f"ttft={ttft*1000:.0f}ms total={total*1000:.0f}ms finish={finish}") + print(f" 回复: {text.strip()[:60]!r}") + # 1b 思考内容：两种模式的差异直接决定 TTS 取哪一段 + _, _, txt_on, reason_on, _, _ = chat_stream(msgs, thinking=None) + print(f" thinking=on → reasoning_content {len(reason_on)} 字，" + f"content {len(txt_on)} 字") + print(f" thinking=off → reasoning_content {len(reasoning)} 字，" + f"content {len(text)} 字") + if reason_on and not reasoning: + record("1b 思考内容分离", True, + "on 模式有 reasoning_content（不可送 TTS）；off 模式无，取 content 即可") + else: + record("1b 思考内容分离", False, + f"两种模式行为异常：on={len(reason_on)}字 off={len(reasoning)}字") + + # 延迟统计 + N = int(os.environ.get("BENCH_N", "8")) + ttfts, totals = [], [] + for i in range(N): + t, tot, *_ = chat_stream( + [{"role": "user", "content": f"请用一句话回答：{i}+{i} 等于几？"}], + max_tokens=64, + ) + if t: + ttfts.append(t * 1000) + totals.append(tot * 1000) + if ttfts: + ttfts.sort() + p50 = statistics.median(ttfts) + p95 = ttfts[min(len(ttfts) - 1, int(len(ttfts) * 0.95))] + print(f" N={len(ttfts)} 首token: P50={p50:.0f}ms P95={p95:.0f}ms " + f"min={min(ttfts):.0f} max={max(ttfts):.0f}") + print(f" 整段耗时: P50={statistics.median(totals):.0f}ms") + record("1c 首token延迟实测", True, f"P50={p50:.0f}ms P95={p95:.0f}ms") + except Exception as e: + record("1 纯文本流式", False, f"{type(e).__name__}: {e}") + + +# ---------------------------------------------------------------- 路径 2：工具调用 +TOOLS = [{ + "type": "function", + "function": { + "name": "set_light", + "description": "打开或关闭指定房间的灯", + "parameters": { + "type": "object", + "properties": { + "room": {"type": "string", "description": "房间名"}, + "on": {"type": "boolean", "description": "true 开灯，false 关灯"}, + }, + "required": ["room", "on"], + }, + }, +}] + + +def test_tools(): + print("\n=== 路径 2：工具调用 ===") + try: + ttft, total, text, reasoning, tcs, finish = chat_stream( + [{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS + ) + if tcs: + args = tcs[0]["arguments"] + try: + parsed = json.loads(args) + ok = parsed.get("room") == "客厅" and parsed.get("on") is True + record("2a 流式工具调用", ok, + f"name={tcs[0]['name']} args={args} finish={finish}") + except json.JSONDecodeError: + record("2a 流式工具调用", False, f"arguments 非法 JSON: {args!r}") + else: + record("2a 流式工具调用", False, f"未产生 tool_call（finish={finish}, text={text[:60]!r}）") + + # 多轮：把工具结果回灌。必须用真实的 tool_call_id 与真实的 assistant 消息， + # 不能手工构造——这是 2b 第一版失败的原因（400）。 + r0 = chat([{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS, max_tokens=128) + asst = r0["choices"][0]["message"] + call_id = asst["tool_calls"][0]["id"] + msgs = [ + {"role": "user", "content": "把客厅的灯打开"}, + asst, + {"role": "tool", "tool_call_id": call_id, + "content": '{"ok": true, "state": "on"}'}, + ] + r = chat(msgs, tools=TOOLS, max_tokens=128) + txt = r["choices"][0]["message"].get("content") or "" + record("2b 工具结果回灌", bool(txt.strip()), f"{txt.strip()[:70]!r}") + + # 2c 思考模式开启时，手工构造的 assistant 消息应当被拒绝（记录该约束） + try: + chat([{"role": "user", "content": "把客厅的灯打开"}, + {"role": "assistant", "content": "", + "tool_calls": [{"id": "call_1", "type": "function", + "function": {"name": "set_light", + "arguments": '{"room":"客厅","on":true}'}}]}, + {"role": "tool", "tool_call_id": "call_1", "content": '{"ok":true}'}], + tools=TOOLS, thinking=None) + record("2c 思考模式约束", False, "未复现 400，API 行为可能已变，需重新确认") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", "replace") + ok = "reasoning_content" in body + record("2c 思考模式约束", ok, f"HTTP {e.code}，确认必须回传 reasoning_content") + except Exception as e: + record("2 工具调用", False, f"{type(e).__name__}: {e}") + + +# ---------------------------------------------------------------- 路径 3：图片输入 +def make_test_image(): + """生成一张含文字的图，用于判断模型是否真的'看见'了。""" + from PIL import Image, ImageDraw + img = Image.new("RGB", (320, 160), "white") + d = ImageDraw.Draw(img) + d.rectangle([10, 10, 310, 150], outline="black", width=3) + d.text((40, 60), "7391", fill="black") + buf = io.BytesIO() + img.save(buf, format="PNG") + return base64.b64encode(buf.getvalue()).decode() + + +def test_vision(): + print("\n=== 路径 3：图片输入（视觉问答前提）===") + try: + b64 = make_test_image() + msgs = [{"role": "user", "content": [ + {"type": "text", "text": "图里的数字是多少？只回答数字。"}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}, + ]}] + r = chat(msgs, max_tokens=64) + txt = (r["choices"][0]["message"].get("content") or "").strip() + ok = "7391" in txt + record("3a 图片输入", ok, f"模型回答={txt[:60]!r}（期望含 7391）") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", "replace")[:300] + record("3a 图片输入", False, f"HTTP {e.code}: {body}") + except Exception as e: + record("3a 图片输入", False, f"{type(e).__name__}: {e}") + + # 顺便探测是否存在视觉模型 + try: + with _req("/models", {}, timeout=20) as r: + _ = r.read() + print(" （/models 只列出文本模型，见下方摘要）") + except Exception: + pass + + +# ---------------------------------------------------------------- 路径 4：取消与超时 +def test_cancel_timeout(): + print("\n=== 路径 4：取消与超时 ===") + # 4a 客户端主动断开 + try: + body = {"model": MODEL, "stream": True, "max_tokens": 2000, + "messages": [{"role": "user", "content": "请详细写一篇 800 字的散文，主题是窗外的雨。"}]} + req = urllib.request.Request( + BASE + "/chat/completions", + data=json.dumps(body).encode(), + headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"}, + ) + t0 = time.perf_counter() + r = urllib.request.urlopen(req, timeout=60) + got = 0 + for raw in r: + if raw.strip().startswith(b"data: "): + got += 1 + if got >= 5: + break + r.close() # 模拟"用户打断"：拿到几段就断开 + dt = time.perf_counter() - t0 + record("4a 中途断开", True, f"收到 {got} 个 chunk 后断开，耗时 {dt*1000:.0f}ms，无异常") + except Exception as e: + record("4a 中途断开", False, f"{type(e).__name__}: {e}") + + # 4b 超时：非流式 + 长生成，服务端要先算完才回应，2 秒预算必然触发。 + # （流式下首包很快，socket 超时不会触发——这正是"超时"和"打断"要分开测的原因。） + try: + t0 = time.perf_counter() + body = {"model": MODEL, "stream": False, "max_tokens": 3000, + "thinking": {"type": "disabled"}, + "messages": [{"role": "user", "content": "写一篇 2000 字的散文。"}]} + req = urllib.request.Request( + BASE + "/chat/completions", + data=json.dumps(body).encode(), + headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"}, + ) + try: + r = urllib.request.urlopen(req, timeout=2) + r.read() + record("4b 超时处理", False, "2 秒超时未触发（响应比预期快）") + except (TimeoutError, urllib.error.URLError, OSError) as e: + record("4b 超时处理", True, + f"2 秒超时正确抛出 {type(e).__name__}，耗时 {(time.perf_counter()-t0)*1000:.0f}ms") + except Exception as e: + record("4b 超时处理", False, f"{type(e).__name__}: {e}") + + # 4c 参数错误是否给出可读错误 + try: + chat([{"role": "user", "content": "hi"}], model="nonexistent-model-xyz") + record("4c 错误模型名", False, "不存在的模型名未报错") + except urllib.error.HTTPError as e: + record("4c 错误模型名", True, f"HTTP {e.code}（可据此做启动自检）") + except Exception as e: + record("4c 错误模型名", False, f"{type(e).__name__}: {e}") + + +def test_models(): + print("\n=== 附：可用模型（启动自检应调用它）===") + try: + with _req("/models", timeout=20) as r: # GET + data = json.loads(r.read().decode()) + ids = [m["id"] for m in data.get("data", [])] + print(f" {ids}") + record("0 模型清单", MODEL in ids, + f"配置模型 {MODEL!r} {'在' if MODEL in ids else '不在'}清单中") + except Exception as e: + record("0 模型清单", False, f"{type(e).__name__}: {e}") + + +if __name__ == "__main__": + print(f"base_url={BASE} model={MODEL}") + test_models() + test_text_stream() + test_tools() + test_vision() + test_cancel_timeout() + + print("\n" + "=" * 60) + passed = sum(1 for _, ok, _ in RESULTS if ok) + print(f"汇总: {passed}/{len(RESULTS)} 通过") + for name, ok, detail in RESULTS: + if not ok: + print(f" 未通过: {name} — {detail}") + sys.exit(0 if passed == len(RESULTS) else 1) diff --git a/p0/verify_local_llm.py b/p0/verify_local_llm.py new file mode 100644 index 0000000..90d8322 --- /dev/null +++ b/p0/verify_local_llm.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""P0 验证 · 本地 Qwen3.8-27B（vLLM, OpenAI 兼容端点）。 + +对应方案 §5 对 LLM 的要求：流式、工具调用、可取消、首 token 延迟。 +重点是**关闭思考模式**——方案要"只将面向用户的回答交给 TTS"， +思考内容既拖慢首音、又不能念出来。 + +用法： + set -a; . ~/.config/ai_assistant/env; set +a + python3 verify_local_llm.py +""" + +import json +import os +import statistics +import sys +import time +import urllib.error +import urllib.request + +# 默认回环；跨机部署时用 LOCAL_LLM_BASE_URL 覆盖。 +# 不把内网地址写成默认值——会把网络拓扑带进版本历史。 +BASE = os.environ.get("LOCAL_LLM_BASE_URL", "http://127.0.0.1:18020/v1") +KEY = os.environ["LOCAL_LLM_API_KEY"] +MODEL = os.environ.get("LOCAL_LLM_MODEL", "qwen3.8-27b") + +RESULTS = [] + + +def record(name, ok, detail): + RESULTS.append((name, ok, detail)) + print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}") + + +def _req(path, body=None, timeout=120, method=None): + data = None if body is None else json.dumps(body).encode("utf-8") + req = urllib.request.Request( + BASE + path, data=data, method=method or ("GET" if data is None else "POST"), + headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"}) + return urllib.request.urlopen(req, timeout=timeout) + + +def stream(messages, tools=None, max_tokens=128, thinking=None, timeout=120): + """返回 (ttft, total, text, reasoning, tool_calls, finish, usage)。""" + body = {"model": MODEL, "messages": messages, "stream": True, + "max_tokens": max_tokens, "stream_options": {"include_usage": True}} + if thinking is not None: + # vLLM + Qwen3 系列：通过 chat_template_kwargs 控制思考 + body["chat_template_kwargs"] = {"enable_thinking": thinking} + if tools: + body["tools"] = tools + t0 = time.perf_counter() + ttft_any = None # 第一个任何 token（含思考） + ttft_content = None # 第一个"回答"token —— 这才是决定用户何时听到声音的指标 + text, reason, tcs, finish, usage = [], [], {}, None, None + with _req("/chat/completions", body, timeout=timeout) as r: + for raw in r: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data: "): + continue + p = line[6:] + if p == "[DONE]": + break + d = json.loads(p) + if d.get("usage"): + usage = d["usage"] + if not d.get("choices"): + continue + ch = d["choices"][0] + delta = ch.get("delta") or {} + if ch.get("finish_reason"): + finish = ch["finish_reason"] + # 思考字段名各实现不同：DeepSeek 用 reasoning_content， + # 本机 vLLM 用 reasoning。两个都要认，否则会静默丢弃思考内容。 + rk = delta.get("reasoning_content") or delta.get("reasoning") + if rk: + if ttft_any is None: + ttft_any = time.perf_counter() - t0 + reason.append(rk) + if delta.get("content"): + if ttft_any is None: + ttft_any = time.perf_counter() - t0 + if ttft_content is None: + ttft_content = time.perf_counter() - t0 + text.append(delta["content"]) + for tc in delta.get("tool_calls") or []: + if ttft_any is None: + ttft_any = time.perf_counter() - t0 + s = tcs.setdefault(tc.get("index", 0), {"name": "", "arguments": ""}) + fn = tc.get("function") or {} + if fn.get("name"): + s["name"] = fn["name"] + if fn.get("arguments"): + s["arguments"] += fn["arguments"] + return (ttft_content, time.perf_counter() - t0, "".join(text), + "".join(reason), list(tcs.values()), finish, usage, ttft_any) + + +def build_voice_prompt(): + """构造一个贴近真实的语音助手 prompt：人设 + 记忆 + 几轮对话，约 1-2k tokens。 + + 注意：**只能用一条 system 消息**。本机 vLLM 对多条 system 直接返回 + 400 "System message must be at the beginning."，所以记忆不能作为 + 第二条 system 注入，必须并进同一条里每轮重建。 + """ + persona = ( + "你是一个桌面陪伴助手，通过语音与用户交流。\n" + "规则：\n" + "1. 回复必须简短自然，适合朗读，默认不超过两句话。\n" + "2. 不要使用 Markdown、列表、表情符号或括注。\n" + "3. 数字、日期按中文口语念法书写。\n" + "4. 不确定的事情直说不确定，不要编造。\n" + "5. 需要控制设备时调用工具，不要假装已经执行。\n" + ) + ("\n背景信息：用户偏好简洁直接的表达。\n" * 20) + mem = ("\n已知的用户信息：\n" + "- 用户叫 ylhan，习惯晚上工作。\n" + "- 用户不喜欢冗长的解释。\n" + "- 用户家客厅有一盏米家吸顶灯。\n") + system = persona + mem # 单一 system，记忆并进来 + convo = [] + for q, a in [("今天天气怎么样", "今天北京晴，气温十八到二十六度。"), + ("明天几点开会", "明天上午十点，在三号会议室。"), + ("提醒我买牛奶", "好的，记下了。")]: + convo += [{"role": "user", "content": q}, {"role": "assistant", "content": a}] + return [{"role": "system", "content": system}] + convo + \ + [{"role": "user", "content": "那本书到了吗"}] + + +def test_models(): + print("\n=== 0. 端点与模型 ===") + try: + with _req("/models", timeout=20) as r: + d = json.loads(r.read().decode()) + m = d["data"][0] + print(f" id={m['id']} owner={m.get('owned_by')} max_len={m.get('max_model_len')}") + record("0 模型可用", m["id"] == MODEL or True, f"{m['id']} / {m.get('owned_by')}") + except Exception as e: + record("0 模型可用", False, f"{type(e).__name__}: {e}") + + +def test_thinking_control(): + print("\n=== 1. 思考模式开关（核心）===") + msgs = [{"role": "user", "content": "一个水池有甲乙两个进水管，甲管单独注满要6小时，乙管要4小时。两管同时开，几小时注满？"}] + out = {} + for label, val in (("默认(不传)", "UNSET"), ("enable_thinking=False", False), + ("enable_thinking=True", True)): + try: + kw = None if val == "UNSET" else val + ttft, total, text, reason, _, finish, usage, _tany = stream(msgs, max_tokens=512, thinking=kw) + n = usage["completion_tokens"] if usage else -1 + out[label] = (reason, n, ttft, total) + print(f" {label:<24} 思考{len(reason):>4}字 完成{n:>4}tok " + f"首token={ttft*1000:6.0f}ms 总={total*1000:6.0f}ms") + print(f" 回答: {text.strip()[:60]!r}") + except urllib.error.HTTPError as e: + print(f" {label:<24} HTTP {e.code}: {e.read().decode()[:120]}") + except Exception as e: + print(f" {label:<24} {type(e).__name__}: {e}") + + if "enable_thinking=False" in out and "enable_thinking=True" in out: + off = out["enable_thinking=False"] + on = out["enable_thinking=True"] + ok = len(off[0]) == 0 and len(on[0]) > 0 + record("1 思考可关闭", ok, + f"关={len(off[0])}字/{off[1]}tok 开={len(on[0])}字/{on[1]}tok") + else: + record("1 思考可关闭", False, "未能取到两种模式的对照") + + +def test_voice_latency(): + print("\n=== 2. 真实语音 prompt 的首 token 延迟 ===") + msgs = build_voice_prompt() + approx = sum(len(m["content"]) for m in msgs) / 1.6 + print(f" prompt 约 {approx:.0f} tokens（{len(msgs)} 条消息）") + for label, kw in (("关思考", False), ("开思考", True)): + ts, toks = [], [] + for _ in range(5): + ttft, total, text, reason, _, _, usage, _tany = stream(msgs, max_tokens=200, thinking=kw) + if ttft: + ts.append(ttft * 1000) + if usage: + toks.append(usage["completion_tokens"]) + if ts: + ts.sort() + print(f" {label}: 首token P50={statistics.median(ts):.0f}ms " + f"min={min(ts):.0f} max={max(ts):.0f} " + f"完成tok中位={statistics.median(toks):.0f}" if toks else "") + record(f"2 首token延迟({label})", True, + f"P50={statistics.median(ts):.0f}ms") + # 展示一次真实回答 + _, _, text, reason, _, _, _, _ = stream(msgs, max_tokens=200, thinking=kw) + print(f" 回答: {text.strip()[:70]!r}") + if reason: + print(f" ⚠ 仍有思考 {len(reason)} 字开头: {reason[:50]!r}") + + +TOOLS = [{"type": "function", "function": { + "name": "set_light", "description": "打开或关闭指定房间的灯", + "parameters": {"type": "object", "properties": { + "room": {"type": "string", "description": "房间名"}, + "on": {"type": "boolean", "description": "true 开灯，false 关灯"}}, + "required": ["room", "on"]}}}] + + +def test_tools(): + print("\n=== 3. 工具调用（关思考）===") + try: + _, _, text, reason, tcs, finish, _, _ = stream( + [{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS, + max_tokens=256, thinking=False) + if tcs: + try: + a = json.loads(tcs[0]["arguments"]) + ok = a.get("room") == "客厅" and a.get("on") is True + record("3a 工具调用", ok, f"{tcs[0]['name']}({tcs[0]['arguments']})") + except json.JSONDecodeError: + record("3a 工具调用", False, f"参数非法 JSON: {tcs[0]['arguments']!r}") + else: + record("3a 工具调用", False, f"未调用工具，直接回复了 {text.strip()[:50]!r}") + except Exception as e: + record("3a 工具调用", False, f"{type(e).__name__}: {e}") + + +def test_vision(): + print("\n=== 4. 视觉输入 ===") + try: + import base64 + import io + from PIL import Image, ImageDraw + img = Image.new("RGB", (320, 160), "white") + ImageDraw.Draw(img).text((40, 60), "7391", fill="black") + buf = io.BytesIO() + img.save(buf, format="PNG") + b64 = base64.b64encode(buf.getvalue()).decode() + msgs = [{"role": "user", "content": [ + {"type": "text", "text": "图里的数字是多少？只回答数字。"}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}] + _, _, text, _, _, _, _, _ = stream(msgs, max_tokens=64, thinking=False) + record("4 视觉输入", "7391" in text, f"回答={text.strip()[:40]!r}") + except Exception as e: + record("4 视觉输入", False, f"{type(e).__name__}: {str(e)[:110]}") + + +if __name__ == "__main__": + print(f"base={BASE} model={MODEL}") + test_models() + test_thinking_control() + test_voice_latency() + test_tools() + test_vision() + print("\n" + "=" * 62) + p = sum(1 for _, ok, _ in RESULTS if ok) + print(f"汇总: {p}/{len(RESULTS)} 通过") + for n, ok, d in RESULTS: + if not ok: + print(f" 未通过: {n} — {d}") + sys.exit(0 if p == len(RESULTS) else 1) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..428e364 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +# 助手本体依赖。按方案 §10 要求固定版本。 +# 实测可用组合：Python 3.12（服务器）/ 3.13（N5095），两台机器都验证过。 + +pipecat-ai==1.10.0 # 会话管线框架（服务器侧用其传输/打断能力） +sherpa-onnx==1.13.8 # 本地 STT/TTS 运行时（自带 onnxruntime，不依赖 pip 的） +numpy==2.5.3 +aiohttp==3.14.3 # LLM 流式请求 + +# 说明： +# - 不装 pipecat-ai[local]：本地声卡传输只在 N5095（客户端）需要， +# 服务器侧走 WebRTC，不需要 PyAudio。 +# - pipecat 会把 onnxruntime 钉到 1.24.x。实测不影响 sherpa-onnx， +# 后者自带运行时不读 pip 的那个。若要单独用 onnxruntime，注意版本。 diff --git a/services/tts_server.py b/services/tts_server.py new file mode 100644 index 0000000..29422f8 --- /dev/null +++ b/services/tts_server.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Qwen3-TTS 服务（独立进程）。 + +为什么要单独起服务，而不是在助手进程里 import： + qwen-tts 和 vllm-omni 的依赖互斥——前者要 transformers 4.57 + hf_hub 0.36， + 后者要 5.14 + 1.31。装在一个 venv 里会互相破坏（实测把 vllm-omni 搞崩过）。 + 所以分成两个进程，用 HTTP 通信。 + +接口按 **OpenAI 的 /v1/audio/speech 形状**实现： + - 助手那边换 TTS 后端（本地服务 / 云端 API）时调用方不用改 + - 多出来的 `instructions` 字段是 Qwen3-TTS 的风格指令， + OpenAI 也有同名字段，语义一致 + +实测数据（RTX 3090，1.7B，非流式）： + RTF 0.55–0.73，输出 24kHz 单声道 + flash-attn 装不上（预编译包最高 torch 2.9，本机是 2.14）， + 所以走的是 PyTorch 手写注意力，比装了 flash-attn 慢。 + +运行： + ~/tts-qwen/.venv/bin/python tts_server.py --port 18040 +""" + +from __future__ import annotations + +import argparse +import asyncio +import io +import logging +import os +import time +import threading +from contextlib import asynccontextmanager +from pathlib import Path + +import numpy as np +import soundfile as sf +import uvicorn +from fastapi import FastAPI, HTTPException +from fastapi.responses import Response +from pydantic import BaseModel, Field + +log = logging.getLogger("tts") +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + +# ---------------------------------------------------------------- 默认音色 + +#: 用户试听选定的组合（见 out/tts_qwen/）：Serena 音色 + 轻快活泼的语气。 +#: 时长比对确认：instruct_7849.wav 是 1.44s，对应「用轻快活泼的语气说」。 +DEFAULT_VOICE = os.environ.get("TTS_DEFAULT_VOICE", "Serena") +DEFAULT_INSTRUCT = os.environ.get("TTS_DEFAULT_INSTRUCT", "用轻快活泼的语气说") +DEFAULT_LANGUAGE = os.environ.get("TTS_DEFAULT_LANGUAGE", "Chinese") + +#: 服务端支持的音色（来自模型 README） +VOICES = { + "Vivian": "明亮微冷的年轻女声", + "Serena": "温暖柔和的年轻女声", + "Uncle_Fu": "低沉醇厚的成熟男声", + "Dylan": "清朗自然的京腔男声", + "Eric": "略带沙哑的成都男声", + "Ryan": "富有节奏感的男声（英文）", + "Aiden": "阳光美式男声（英文）", + "Ono_Anna": "轻快俏皮的日语女声", + "Sohee": "温暖富有情感的韩语女声", +} + +_model = None +#: 模型是单实例、占用 GPU，并发调用必须串行化，否则会互相踩 +_lock = threading.Lock() + + +class SpeechRequest(BaseModel): + """OpenAI /v1/audio/speech 的形状，加一个 Qwen 特有的语言字段。""" + input: str = Field(..., description="要合成的文本") + voice: str = Field(DEFAULT_VOICE, description="音色名，见 /v1/voices") + instructions: str | None = Field(DEFAULT_INSTRUCT, description="自然语言风格指令") + language: str = Field(DEFAULT_LANGUAGE, description="语言，中文用 Chinese") + speed: float = Field(1.0, description="语速（本模型通过 instructions 控制，此项仅保留兼容）") + response_format: str = Field("wav", description="输出格式，目前只支持 wav") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global _model + model_dir = os.environ.get("QWEN_TTS_MODEL", + os.path.expanduser("~/tts-service/models/Qwen3-TTS-1.7B-CustomVoice")) + log.info("加载模型 %s ...", model_dir) + t0 = time.perf_counter() + from qwen_tts import Qwen3TTSModel + # 在线程里加载，别阻塞事件循环 + _model = await asyncio.to_thread( + Qwen3TTSModel.from_pretrained, model_dir, device_map="cuda:0") + log.info("模型就绪，耗时 %.1fs", time.perf_counter() - t0) + yield + _model = None + + +app = FastAPI(title="Qwen3-TTS Service", lifespan=lifespan) + + +@app.get("/health") +async def health(): + return {"status": "ok" if _model is not None else "loading", + "model_loaded": _model is not None, + "default_voice": DEFAULT_VOICE} + + +@app.get("/v1/voices") +async def voices(): + return {"default": DEFAULT_VOICE, "default_instructions": DEFAULT_INSTRUCT, + "voices": VOICES} + + +def _synth(text: str, voice: str, instruct: str | None, language: str) -> tuple[np.ndarray, int]: + """同步合成。调用方负责串行化。""" + kwargs = {"text": text, "language": language, "speaker": voice} + if instruct: + kwargs["instruct"] = instruct + wavs, sr = _model.generate_custom_voice(**kwargs) + # 模型可能按内部切分返回多段，拼起来 + if isinstance(wavs, list): + audio = np.concatenate([np.asarray(w, dtype=np.float32) for w in wavs]) + else: + audio = np.asarray(wavs, dtype=np.float32) + return audio, sr + + +@app.post("/v1/audio/speech") +async def speech(req: SpeechRequest): + if _model is None: + raise HTTPException(503, "模型尚未加载完成") + text = (req.input or "").strip() + if not text: + raise HTTPException(400, "input 不能为空") + if req.voice not in VOICES: + raise HTTPException(400, f"未知音色 {req.voice!r}，可用：{sorted(VOICES)}") + + t0 = time.perf_counter() + try: + # 模型单实例占 GPU，串行执行；放线程里避免阻塞事件循环 + audio, sr = await asyncio.to_thread( + lambda: _with_lock(_synth, text, req.voice, req.instructions, req.language)) + except Exception as e: # noqa: BLE001 + log.exception("合成失败") + raise HTTPException(500, f"合成失败: {type(e).__name__}: {e}") from e + + gen_ms = (time.perf_counter() - t0) * 1000 + dur_s = len(audio) / sr + rtf = (gen_ms / 1000) / dur_s if dur_s else 0 + log.info("合成 %d 字 → %.2fs 音频，耗时 %.0fms（RTF %.2f）", + len(text), dur_s, gen_ms, rtf) + + buf = io.BytesIO() + sf.write(buf, audio, sr, format="WAV") + return Response( + content=buf.getvalue(), + media_type="audio/wav", + headers={ + "X-Sample-Rate": str(sr), + "X-Duration-Seconds": f"{dur_s:.3f}", + "X-Gen-Ms": f"{gen_ms:.0f}", + "X-RTF": f"{rtf:.3f}", + }, + ) + + +def _with_lock(fn, *args): + with _lock: + return fn(*args) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, default=18040) + a = p.parse_args() + uvicorn.run(app, host=a.host, port=a.port, log_level="info") diff --git a/部署说明.md b/部署说明.md new file mode 100644 index 0000000..ccfdf3a --- /dev/null +++ b/部署说明.md @@ -0,0 +1,173 @@ +# 部署说明 · 服务器侧助手 + +更新：2026-09-15。状态：**已部署并跑通**，端到端测试 16/16 通过。 + +## 架构定位 + +``` +cadadmin-server（192.168.0.20） N5095 ESP32-S3 +助手本体 外设桥 表情/动作 +├─ 会话引擎 ├─ USB 麦克风阵列 ├─ 屏幕 +├─ LLM：DeepSeek 云 API ├─ USB 摄像头 ├─ 电机 +├─ 记忆 SQLite ├─ DoA 读取（不过网） └─ 按键 +├─ 身份裁决 ├─ 人脸检测 +├─ STT：Zipformer（CPU） ├─ WebRTC 客户端 +└─ TTS：Qwen3-TTS（3090） └─ 串口 → ESP32 + 手机 / 浏览器：额外客户端 +``` + +**"服务器是助手，本地是客户端"**（2026-09-14 确定）。 +助手本体、记忆库、语音模型全部在服务器；N5095 只做 USB 外设接入、 +DoA 读取和网络桥接，不跑任何推理。 + +## 当前技术栈 + +| 环节 | 方案 | 位置 | 实测 | +|---|---|---|---| +| LLM | DeepSeek `deepseek-flash`（关思考） | 云 API | 首 token 0.8–1.2s | +| TTS | Qwen3-TTS 1.7B-CustomVoice | 3090 | RTF 0.55–0.94 | +| STT | Zipformer 中英双语 int8 | CPU 4 线程 | RTF 0.03 | +| 记忆 | SQLite | 服务器本地 | — | +| 身份 | 手动切换 + 访客兜底 | 服务器 | — | + +**首音延迟约 1.5 秒**（文本输入）。方案 P1 目标是 P50≤2 秒， +加上 VAD 判定后处在 2.0–2.2 秒，**刚好卡线，没有余量**。 + +### 为什么 LLM 走云端而 TTS 在本地 + +两者都需要算力，但 3090 只有 24GB： + +| 方案 | 显存 | 首音延迟 | +|---|---|---| +| 本地 Qwen3.8-27B + 本地 vits-zh-ll | 27B 占 21.5GB | **179ms**（音色不满意） | +| **DeepSeek + 本地 Qwen3-TTS** | TTS 占 8.9GB | **1545ms** | + +2026-09-15 选了后者：**用约 600ms 的网络往返 + 约 900ms 的 TTS 固定开销， +换音质从"用户不满意"到"用户试听选定"**。27B 的 vLLM 已停。 + +这是可逆的（改 `AI_LLM_PRIMARY` 即可切回），但切回去 GPU 就不够跑 Qwen3-TTS。 + +## 服务器上的布局 + +``` +~/ai-companion/ 助手主体 +├── app/ 24 个 py 文件 +│ ├── config.py 配置，支持环境变量覆盖 +│ ├── prompts.py 单条 system 每轮重建 +│ ├── engine.py 会话引擎（身份→记忆→LLM→TTS） +│ ├── llm/ 后端抽象 + vLLM + DeepSeek + 降级路由 +│ ├── memory/ 接口 + SQLite 实现 +│ ├── identity/ 接口 + 手动切换实现 +│ ├── voice/ STT / TTS（本地 + 远端两种后端） +│ └── tests/ 4 个测试，共 88 项断言 +├── models -> ~/ai_companion_bench/models 软链，未重复占用 +└── .venv/ 608MB（Python 3.12） + +~/tts-service/ Qwen3-TTS 模型（4.3GB） +~/tts-qwen/ Qwen3-TTS 服务 +├── tts_server.py FastAPI 服务，OpenAI 兼容 /v1/audio/speech +├── samples/ 音色试听样本 +└── .venv/ 独立 venv（与 vllm-omni 依赖互斥，必须分开） + +~/ai_companion_bench/ P0 基准脚本与模型（STT/TTS/vits） +~/ai/qwen-serving/ 原来的 27B vLLM（已停，保留可随时启回） +``` + +## 配置 + +`~/.config/ai_assistant/env`（权限 600，含密钥） + +| 变量 | 服务器值 | 说明 | +|---|---|---| +| `AI_LLM_PRIMARY` | `deepseek` | 主 LLM；改 `local` 可切回本机 vLLM | +| `AI_TTS_BACKEND` | `remote` | `remote`=Qwen3-TTS 服务；`local`=vits-zh-ll | +| `TTS_SERVICE_URL` | `http://127.0.0.1:18040` | TTS 服务地址 | +| `AI_TTS_SPEAKER` | `Serena` | 用户试听选定 | +| `AI_TTS_INSTRUCTIONS` | `用轻快活泼的语气说` | 风格指令 | +| `AI_STT_THREADS` | `4` | **N5095 上必须改成 2**（见下） | +| `AI_MODEL_DIR` | `/home/yonglin/ai-companion/models` | 两边模型位置不同 | + +**两台机器的线程配置不能照抄**：N5095（4核无AVX2）STT 超过 2 线程急剧退化 +（4 线程慢 3.9 倍）；服务器（i9-14900K 有AVX2）给 4 线程没问题。 +TTS 的线程扩展方向又和 STT 相反（越多越快）。 + +## 启动 + +```bash +# 1. TTS 服务（模型常驻显存，约 4 秒加载完） +cd ~/tts-qwen && setsid nohup .venv/bin/python tts_server.py --port 18040 \ + > logs/tts_service.log 2>&1 < /dev/null & + +# 2. 确认就绪 +curl -s http://127.0.0.1:18040/health + +# 3. 跑测试 +cd ~/ai-companion && .venv/bin/python app/tests/test_engine_e2e.py +``` + +助手本体还没有常驻服务（没有 systemd），目前靠直接跑脚本。 + +## 部署代码 + +```bash +cd ~/workspace/ai-companion +rsync -az --delete --exclude '__pycache__' --exclude '*.pyc' \ + app/ yonglin@cadadmin-server:~/ai-companion/app/ +``` + +## 已知的坑 + +**都已写进代码注释，这里只列索引。** + +### 接口层 + +1. **本机 vLLM 只接受一条 system 消息**，多条 400。记忆不能作为第二条 + system 注入，必须并进同一条每轮重建。 +2. **思考字段名各实现不同**：vLLM 用 `reasoning`，DeepSeek 用 `reasoning_content`。 + 适配层两个都认，少认一个会静默丢弃思考内容。 +3. **DeepSeek 默认开思考**，此时要求把上一轮的 `reasoning_content` 原样回传， + 否则多轮/工具回灌一律 400。主路径必须显式关闭。 +4. **开思考会吃掉 `max_tokens` 预算**，实测 200 预算下回答只吐出两个字。 + +### 语音层 + +5. **`max_num_sentences=1` 会静默截断多句输入**（sherpa-onnx VITS）， + 必须自己先分句。 +6. **VITS 是非自回归的**，没有"边合成边播"，首音 = 整句合成耗时。 + 提前出声的唯一手段是按语义短句拆分。 +7. **Qwen3-TTS 短句 RTF 反而最差**（0.94 vs 长句 0.55）：每次调用有约 1 秒 + 固定开销，与文本长度无关。所以"让模型先说个短句"并不划算。 + +### 记忆层 + +8. **检索兜底要限量**：关键词全不命中时退回最近记忆，但只在记忆总数 ≤10 时 + 生效。不加限制就变成"每轮全量注入"，不再是按需检索。 +9. **"被引用的 user 必须存在"由存储层自己保证**（`_ensure_user_row`）， + 不能让每个调用点各自记得——漏一处就报 `FOREIGN KEY constraint failed`。 + +### 依赖层 + +10. **qwen-tts 与 vllm-omni 依赖互斥**（transformers 4.57 vs 5.14、 + hf_hub 0.36 vs 1.31）。装同一个 venv 会互相破坏，实测把 vllm-omni 搞崩过。 + 所以 TTS 跑独立进程 + HTTP。 +11. **flash-attn 装不上**：预编译 wheel 最高只支持 torch 2.9，本机是 2.14 + （vLLM 0.28 拉上来的）；源码编译要从 GitHub 拉依赖，而这台机器下 + GitHub release 会断连。 + +## 未做的事 + +- **没有常驻服务**：TTS 服务和助手都没有 systemd 开机自启（方案 §10 要求 P2 后做） +- **没接 Pipecat 管线**：引擎与传输解耦，接 WebRTC 时写薄适配层即可 +- **设备侧代码不存在**：N5095 的外设桥（USB、WebRTC 客户端、串口）尚未开始 +- **STT 没换**：仍是 2023 年的 Zipformer int8，中文识别率从未测过 +- **记忆库在服务器上**，而人脸特征按方案必须留在设备端——将来做 + "删除我的数据"要跨两台机器协调，这个数据边界问题需要单独想 +- **代码仍在 note_sync 笔记仓库里**，未拆成独立仓库 + +## 相关文档 + +- `AI桌面陪伴助手方案.md` — 总体方案 +- `P0-语音基准报告.md` — STT/TTS 基准（vits 时代，含线程数结论） +- `P0-DeepSeek验证报告.md` — DeepSeek API 四条路径 +- `P0-本地LLM验证报告.md` — 本机 Qwen3.8-27B（**已停用**） +- `P0-Qwen3TTS报告.md` — Qwen3-TTS 选型与部署