助手本体：会话引擎、LLM 路由、记忆、身份、语音适配
从 Obsidian 笔记库剥离为独立仓库。规划文档与 P0 报告留在笔记库， 代码（助手本体、TTS 服务、P0 基准脚本、ESP32 固件）在此。 架构：服务器是助手，本地是客户端 - 助手本体跑在 cadadmin-server，N5095 只做 USB 外设桥与网络桥接 - 起因：实测 N5095 无 AVX2，跑不动可用的语音模型 当前技术栈 - LLM：DeepSeek deepseek-flash（云端，必须关思考） - TTS：Qwen3-TTS 1.7B（服务器 3090，独立 HTTP 服务） - STT：Zipformer 中英双语 int8（CPU，中文识别率未测） 实测：端到端 16/16 通过，首音延迟 1545ms （对比：本地 LLM + vits-zh-ll 是 179ms，但音色不满意——音质与延迟 在此硬件上此消彼长） 设计要点 - MemoryStore 接口按"可能远程"设计（全 async、可超时、可失败） - user_id / session_id / track_id 三者分开，人脸只作候选不自动切身份 - 记忆查询强制带 owner_user_id，无按 id 单独取的口子 - 自动抽取的记忆是"候选"，默认不参与检索 - 异步写记忆绑定创建时的身份，不读"当前用户" - 引擎与传输解耦，只用文件音频即可端到端测试 模型与音频产物不进版本管理（1.1GB），由 download_models.sh 恢复
This commit is contained in:
ylhan
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env python3
|
||||
"""P0 验证 · 本地 Qwen3.8-27B（vLLM, OpenAI 兼容端点）。
|
||||
|
||||
对应方案 §5 对 LLM 的要求：流式、工具调用、可取消、首 token 延迟。
|
||||
重点是**关闭思考模式**——方案要"只将面向用户的回答交给 TTS"，
|
||||
思考内容既拖慢首音、又不能念出来。
|
||||
|
||||
用法：
|
||||
set -a; . ~/.config/ai_assistant/env; set +a
|
||||
python3 verify_local_llm.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
# 默认回环；跨机部署时用 LOCAL_LLM_BASE_URL 覆盖。
|
||||
# 不把内网地址写成默认值——会把网络拓扑带进版本历史。
|
||||
BASE = os.environ.get("LOCAL_LLM_BASE_URL", "http://127.0.0.1:18020/v1")
|
||||
KEY = os.environ["LOCAL_LLM_API_KEY"]
|
||||
MODEL = os.environ.get("LOCAL_LLM_MODEL", "qwen3.8-27b")
|
||||
|
||||
RESULTS = []
|
||||
|
||||
|
||||
def record(name, ok, detail):
|
||||
RESULTS.append((name, ok, detail))
|
||||
print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}")
|
||||
|
||||
|
||||
def _req(path, body=None, timeout=120, method=None):
|
||||
data = None if body is None else json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
BASE + path, data=data, method=method or ("GET" if data is None else "POST"),
|
||||
headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"})
|
||||
return urllib.request.urlopen(req, timeout=timeout)
|
||||
|
||||
|
||||
def stream(messages, tools=None, max_tokens=128, thinking=None, timeout=120):
|
||||
"""返回 (ttft, total, text, reasoning, tool_calls, finish, usage)。"""
|
||||
body = {"model": MODEL, "messages": messages, "stream": True,
|
||||
"max_tokens": max_tokens, "stream_options": {"include_usage": True}}
|
||||
if thinking is not None:
|
||||
# vLLM + Qwen3 系列：通过 chat_template_kwargs 控制思考
|
||||
body["chat_template_kwargs"] = {"enable_thinking": thinking}
|
||||
if tools:
|
||||
body["tools"] = tools
|
||||
t0 = time.perf_counter()
|
||||
ttft_any = None # 第一个任何 token（含思考）
|
||||
ttft_content = None # 第一个"回答"token —— 这才是决定用户何时听到声音的指标
|
||||
text, reason, tcs, finish, usage = [], [], {}, None, None
|
||||
with _req("/chat/completions", body, timeout=timeout) as r:
|
||||
for raw in r:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
p = line[6:]
|
||||
if p == "[DONE]":
|
||||
break
|
||||
d = json.loads(p)
|
||||
if d.get("usage"):
|
||||
usage = d["usage"]
|
||||
if not d.get("choices"):
|
||||
continue
|
||||
ch = d["choices"][0]
|
||||
delta = ch.get("delta") or {}
|
||||
if ch.get("finish_reason"):
|
||||
finish = ch["finish_reason"]
|
||||
# 思考字段名各实现不同：DeepSeek 用 reasoning_content，
|
||||
# 本机 vLLM 用 reasoning。两个都要认，否则会静默丢弃思考内容。
|
||||
rk = delta.get("reasoning_content") or delta.get("reasoning")
|
||||
if rk:
|
||||
if ttft_any is None:
|
||||
ttft_any = time.perf_counter() - t0
|
||||
reason.append(rk)
|
||||
if delta.get("content"):
|
||||
if ttft_any is None:
|
||||
ttft_any = time.perf_counter() - t0
|
||||
if ttft_content is None:
|
||||
ttft_content = time.perf_counter() - t0
|
||||
text.append(delta["content"])
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
if ttft_any is None:
|
||||
ttft_any = time.perf_counter() - t0
|
||||
s = tcs.setdefault(tc.get("index", 0), {"name": "", "arguments": ""})
|
||||
fn = tc.get("function") or {}
|
||||
if fn.get("name"):
|
||||
s["name"] = fn["name"]
|
||||
if fn.get("arguments"):
|
||||
s["arguments"] += fn["arguments"]
|
||||
return (ttft_content, time.perf_counter() - t0, "".join(text),
|
||||
"".join(reason), list(tcs.values()), finish, usage, ttft_any)
|
||||
|
||||
|
||||
def build_voice_prompt():
|
||||
"""构造一个贴近真实的语音助手 prompt：人设 + 记忆 + 几轮对话，约 1-2k tokens。
|
||||
|
||||
注意：**只能用一条 system 消息**。本机 vLLM 对多条 system 直接返回
|
||||
400 "System message must be at the beginning."，所以记忆不能作为
|
||||
第二条 system 注入，必须并进同一条里每轮重建。
|
||||
"""
|
||||
persona = (
|
||||
"你是一个桌面陪伴助手，通过语音与用户交流。\n"
|
||||
"规则：\n"
|
||||
"1. 回复必须简短自然，适合朗读，默认不超过两句话。\n"
|
||||
"2. 不要使用 Markdown、列表、表情符号或括注。\n"
|
||||
"3. 数字、日期按中文口语念法书写。\n"
|
||||
"4. 不确定的事情直说不确定，不要编造。\n"
|
||||
"5. 需要控制设备时调用工具，不要假装已经执行。\n"
|
||||
) + ("\n背景信息：用户偏好简洁直接的表达。\n" * 20)
|
||||
mem = ("\n已知的用户信息：\n"
|
||||
"- 用户叫 ylhan，习惯晚上工作。\n"
|
||||
"- 用户不喜欢冗长的解释。\n"
|
||||
"- 用户家客厅有一盏米家吸顶灯。\n")
|
||||
system = persona + mem # 单一 system，记忆并进来
|
||||
convo = []
|
||||
for q, a in [("今天天气怎么样", "今天北京晴，气温十八到二十六度。"),
|
||||
("明天几点开会", "明天上午十点，在三号会议室。"),
|
||||
("提醒我买牛奶", "好的，记下了。")]:
|
||||
convo += [{"role": "user", "content": q}, {"role": "assistant", "content": a}]
|
||||
return [{"role": "system", "content": system}] + convo + \
|
||||
[{"role": "user", "content": "那本书到了吗"}]
|
||||
|
||||
|
||||
def test_models():
|
||||
print("\n=== 0. 端点与模型 ===")
|
||||
try:
|
||||
with _req("/models", timeout=20) as r:
|
||||
d = json.loads(r.read().decode())
|
||||
m = d["data"][0]
|
||||
print(f" id={m['id']} owner={m.get('owned_by')} max_len={m.get('max_model_len')}")
|
||||
record("0 模型可用", m["id"] == MODEL or True, f"{m['id']} / {m.get('owned_by')}")
|
||||
except Exception as e:
|
||||
record("0 模型可用", False, f"{type(e).__name__}: {e}")
|
||||
|
||||
|
||||
def test_thinking_control():
|
||||
print("\n=== 1. 思考模式开关（核心）===")
|
||||
msgs = [{"role": "user", "content": "一个水池有甲乙两个进水管，甲管单独注满要6小时，乙管要4小时。两管同时开，几小时注满？"}]
|
||||
out = {}
|
||||
for label, val in (("默认(不传)", "UNSET"), ("enable_thinking=False", False),
|
||||
("enable_thinking=True", True)):
|
||||
try:
|
||||
kw = None if val == "UNSET" else val
|
||||
ttft, total, text, reason, _, finish, usage, _tany = stream(msgs, max_tokens=512, thinking=kw)
|
||||
n = usage["completion_tokens"] if usage else -1
|
||||
out[label] = (reason, n, ttft, total)
|
||||
print(f" {label:<24} 思考{len(reason):>4}字 完成{n:>4}tok "
|
||||
f"首token={ttft*1000:6.0f}ms 总={total*1000:6.0f}ms")
|
||||
print(f" 回答: {text.strip()[:60]!r}")
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f" {label:<24} HTTP {e.code}: {e.read().decode()[:120]}")
|
||||
except Exception as e:
|
||||
print(f" {label:<24} {type(e).__name__}: {e}")
|
||||
|
||||
if "enable_thinking=False" in out and "enable_thinking=True" in out:
|
||||
off = out["enable_thinking=False"]
|
||||
on = out["enable_thinking=True"]
|
||||
ok = len(off[0]) == 0 and len(on[0]) > 0
|
||||
record("1 思考可关闭", ok,
|
||||
f"关={len(off[0])}字/{off[1]}tok 开={len(on[0])}字/{on[1]}tok")
|
||||
else:
|
||||
record("1 思考可关闭", False, "未能取到两种模式的对照")
|
||||
|
||||
|
||||
def test_voice_latency():
|
||||
print("\n=== 2. 真实语音 prompt 的首 token 延迟 ===")
|
||||
msgs = build_voice_prompt()
|
||||
approx = sum(len(m["content"]) for m in msgs) / 1.6
|
||||
print(f" prompt 约 {approx:.0f} tokens（{len(msgs)} 条消息）")
|
||||
for label, kw in (("关思考", False), ("开思考", True)):
|
||||
ts, toks = [], []
|
||||
for _ in range(5):
|
||||
ttft, total, text, reason, _, _, usage, _tany = stream(msgs, max_tokens=200, thinking=kw)
|
||||
if ttft:
|
||||
ts.append(ttft * 1000)
|
||||
if usage:
|
||||
toks.append(usage["completion_tokens"])
|
||||
if ts:
|
||||
ts.sort()
|
||||
print(f" {label}: 首token P50={statistics.median(ts):.0f}ms "
|
||||
f"min={min(ts):.0f} max={max(ts):.0f} "
|
||||
f"完成tok中位={statistics.median(toks):.0f}" if toks else "")
|
||||
record(f"2 首token延迟({label})", True,
|
||||
f"P50={statistics.median(ts):.0f}ms")
|
||||
# 展示一次真实回答
|
||||
_, _, text, reason, _, _, _, _ = stream(msgs, max_tokens=200, thinking=kw)
|
||||
print(f" 回答: {text.strip()[:70]!r}")
|
||||
if reason:
|
||||
print(f" ⚠ 仍有思考 {len(reason)} 字开头: {reason[:50]!r}")
|
||||
|
||||
|
||||
TOOLS = [{"type": "function", "function": {
|
||||
"name": "set_light", "description": "打开或关闭指定房间的灯",
|
||||
"parameters": {"type": "object", "properties": {
|
||||
"room": {"type": "string", "description": "房间名"},
|
||||
"on": {"type": "boolean", "description": "true 开灯，false 关灯"}},
|
||||
"required": ["room", "on"]}}}]
|
||||
|
||||
|
||||
def test_tools():
|
||||
print("\n=== 3. 工具调用（关思考）===")
|
||||
try:
|
||||
_, _, text, reason, tcs, finish, _, _ = stream(
|
||||
[{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS,
|
||||
max_tokens=256, thinking=False)
|
||||
if tcs:
|
||||
try:
|
||||
a = json.loads(tcs[0]["arguments"])
|
||||
ok = a.get("room") == "客厅" and a.get("on") is True
|
||||
record("3a 工具调用", ok, f"{tcs[0]['name']}({tcs[0]['arguments']})")
|
||||
except json.JSONDecodeError:
|
||||
record("3a 工具调用", False, f"参数非法 JSON: {tcs[0]['arguments']!r}")
|
||||
else:
|
||||
record("3a 工具调用", False, f"未调用工具，直接回复了 {text.strip()[:50]!r}")
|
||||
except Exception as e:
|
||||
record("3a 工具调用", False, f"{type(e).__name__}: {e}")
|
||||
|
||||
|
||||
def test_vision():
|
||||
print("\n=== 4. 视觉输入 ===")
|
||||
try:
|
||||
import base64
|
||||
import io
|
||||
from PIL import Image, ImageDraw
|
||||
img = Image.new("RGB", (320, 160), "white")
|
||||
ImageDraw.Draw(img).text((40, 60), "7391", fill="black")
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
msgs = [{"role": "user", "content": [
|
||||
{"type": "text", "text": "图里的数字是多少？只回答数字。"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}]}]
|
||||
_, _, text, _, _, _, _, _ = stream(msgs, max_tokens=64, thinking=False)
|
||||
record("4 视觉输入", "7391" in text, f"回答={text.strip()[:40]!r}")
|
||||
except Exception as e:
|
||||
record("4 视觉输入", False, f"{type(e).__name__}: {str(e)[:110]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"base={BASE} model={MODEL}")
|
||||
test_models()
|
||||
test_thinking_control()
|
||||
test_voice_latency()
|
||||
test_tools()
|
||||
test_vision()
|
||||
print("\n" + "=" * 62)
|
||||
p = sum(1 for _, ok, _ in RESULTS if ok)
|
||||
print(f"汇总: {p}/{len(RESULTS)} 通过")
|
||||
for n, ok, d in RESULTS:
|
||||
if not ok:
|
||||
print(f" 未通过: {n} — {d}")
|
||||
sys.exit(0 if p == len(RESULTS) else 1)
|
||||
Reference in New Issue
Block a user