ylhan
35e107c290
从 Obsidian 笔记库剥离为独立仓库。规划文档与 P0 报告留在笔记库， 代码（助手本体、TTS 服务、P0 基准脚本、ESP32 固件）在此。 架构：服务器是助手，本地是客户端 - 助手本体跑在 cadadmin-server，N5095 只做 USB 外设桥与网络桥接 - 起因：实测 N5095 无 AVX2，跑不动可用的语音模型 当前技术栈 - LLM：DeepSeek deepseek-flash（云端，必须关思考） - TTS：Qwen3-TTS 1.7B（服务器 3090，独立 HTTP 服务） - STT：Zipformer 中英双语 int8（CPU，中文识别率未测） 实测：端到端 16/16 通过，首音延迟 1545ms （对比：本地 LLM + vits-zh-ll 是 179ms，但音色不满意——音质与延迟 在此硬件上此消彼长） 设计要点 - MemoryStore 接口按"可能远程"设计（全 async、可超时、可失败） - user_id / session_id / track_id 三者分开，人脸只作候选不自动切身份 - 记忆查询强制带 owner_user_id，无按 id 单独取的口子 - 自动抽取的记忆是"候选"，默认不参与检索 - 异步写记忆绑定创建时的身份，不读"当前用户" - 引擎与传输解耦，只用文件音频即可端到端测试 模型与音频产物不进版本管理（1.1GB），由 download_models.sh 恢复
356 lines
14 KiB
Python
356 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""P0 验证：DeepSeek API 四条路径。
|
||
|
||
对应方案 §5 的 P0 要求：纯文本流式 / 图片输入 / 工具调用 / 取消与超时。
|
||
用标准库直连 HTTP，不依赖 SDK 版本，便于精确计时。
|
||
|
||
用法：
|
||
set -a; . ~/.config/ai_assistant/env; set +a
|
||
python3 verify_deepseek.py
|
||
"""
|
||
|
||
import base64
|
||
import io
|
||
import json
|
||
import os
|
||
import statistics
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
|
||
BASE = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1")
|
||
KEY = os.environ["DEEPSEEK_API_KEY"]
|
||
MODEL = os.environ.get("DEEPSEEK_MODEL", "deepseek-flash")
|
||
|
||
RESULTS = []
|
||
|
||
|
||
def record(name, ok, detail):
|
||
RESULTS.append((name, ok, detail))
|
||
print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}")
|
||
|
||
|
||
def _req(path, body=None, timeout=60, method=None):
|
||
data = None if body is None else json.dumps(body).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
BASE + path,
|
||
data=data,
|
||
method=method or ("GET" if data is None else "POST"),
|
||
headers={
|
||
"Authorization": f"Bearer {KEY}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
)
|
||
return urllib.request.urlopen(req, timeout=timeout)
|
||
|
||
|
||
def chat(messages, thinking="disabled", **kw):
|
||
"""thinking 默认关闭：见 verify 报告——开启时 reasoning_content 必须回传。"""
|
||
body = {"model": MODEL, "messages": messages, "stream": False}
|
||
if thinking:
|
||
body["thinking"] = {"type": thinking}
|
||
body.update(kw)
|
||
with _req("/chat/completions", body) as r:
|
||
return json.loads(r.read().decode("utf-8"))
|
||
|
||
|
||
def chat_stream(messages, tools=None, max_tokens=256, timeout=60, thinking="disabled"):
|
||
"""返回 (ttft, total, text, reasoning, tool_calls, finish)。
|
||
|
||
thinking 默认 disabled：开启时 API 要求把 reasoning_content 原样回传，
|
||
否则多轮/工具回灌一律 400。详见 verify 报告。
|
||
"""
|
||
body = {
|
||
"model": MODEL,
|
||
"messages": messages,
|
||
"stream": True,
|
||
"max_tokens": max_tokens,
|
||
}
|
||
if thinking:
|
||
body["thinking"] = {"type": thinking}
|
||
if tools:
|
||
body["tools"] = tools
|
||
t0 = time.perf_counter()
|
||
ttft = None
|
||
text_parts, reasoning_parts = [], []
|
||
tool_calls = {}
|
||
finish = None
|
||
with _req("/chat/completions", body, timeout=timeout) as r:
|
||
for raw in r:
|
||
line = raw.decode("utf-8", "replace").strip()
|
||
if not line.startswith("data: "):
|
||
continue
|
||
payload = line[6:]
|
||
if payload == "[DONE]":
|
||
break
|
||
d = json.loads(payload)
|
||
if not d.get("choices"):
|
||
continue
|
||
ch = d["choices"][0]
|
||
delta = ch.get("delta") or {}
|
||
if ch.get("finish_reason"):
|
||
finish = ch["finish_reason"]
|
||
if delta.get("content"):
|
||
if ttft is None:
|
||
ttft = time.perf_counter() - t0
|
||
text_parts.append(delta["content"])
|
||
if delta.get("reasoning_content"):
|
||
if ttft is None:
|
||
ttft = time.perf_counter() - t0
|
||
reasoning_parts.append(delta["reasoning_content"])
|
||
for tc in delta.get("tool_calls") or []:
|
||
if ttft is None:
|
||
ttft = time.perf_counter() - t0
|
||
idx = tc.get("index", 0)
|
||
slot = tool_calls.setdefault(idx, {"name": "", "arguments": ""})
|
||
fn = tc.get("function") or {}
|
||
if fn.get("name"):
|
||
slot["name"] = fn["name"]
|
||
if fn.get("arguments"):
|
||
slot["arguments"] += fn["arguments"]
|
||
total = time.perf_counter() - t0
|
||
return ttft, total, "".join(text_parts), "".join(reasoning_parts), list(tool_calls.values()), finish
|
||
|
||
|
||
# ---------------------------------------------------------------- 路径 1：纯文本流式
|
||
def test_text_stream():
|
||
print("\n=== 路径 1：纯文本流式（首 token 延迟）===")
|
||
msgs = [{"role": "user", "content": "用一句话介绍你自己。"}]
|
||
try:
|
||
ttft, total, text, reasoning, tcs, finish = chat_stream(msgs)
|
||
record("1a 流式基本可用", bool(text.strip()),
|
||
f"ttft={ttft*1000:.0f}ms total={total*1000:.0f}ms finish={finish}")
|
||
print(f" 回复: {text.strip()[:60]!r}")
|
||
# 1b 思考内容：两种模式的差异直接决定 TTS 取哪一段
|
||
_, _, txt_on, reason_on, _, _ = chat_stream(msgs, thinking=None)
|
||
print(f" thinking=on → reasoning_content {len(reason_on)} 字，"
|
||
f"content {len(txt_on)} 字")
|
||
print(f" thinking=off → reasoning_content {len(reasoning)} 字，"
|
||
f"content {len(text)} 字")
|
||
if reason_on and not reasoning:
|
||
record("1b 思考内容分离", True,
|
||
"on 模式有 reasoning_content（不可送 TTS）；off 模式无，取 content 即可")
|
||
else:
|
||
record("1b 思考内容分离", False,
|
||
f"两种模式行为异常：on={len(reason_on)}字 off={len(reasoning)}字")
|
||
|
||
# 延迟统计
|
||
N = int(os.environ.get("BENCH_N", "8"))
|
||
ttfts, totals = [], []
|
||
for i in range(N):
|
||
t, tot, *_ = chat_stream(
|
||
[{"role": "user", "content": f"请用一句话回答：{i}+{i} 等于几？"}],
|
||
max_tokens=64,
|
||
)
|
||
if t:
|
||
ttfts.append(t * 1000)
|
||
totals.append(tot * 1000)
|
||
if ttfts:
|
||
ttfts.sort()
|
||
p50 = statistics.median(ttfts)
|
||
p95 = ttfts[min(len(ttfts) - 1, int(len(ttfts) * 0.95))]
|
||
print(f" N={len(ttfts)} 首token: P50={p50:.0f}ms P95={p95:.0f}ms "
|
||
f"min={min(ttfts):.0f} max={max(ttfts):.0f}")
|
||
print(f" 整段耗时: P50={statistics.median(totals):.0f}ms")
|
||
record("1c 首token延迟实测", True, f"P50={p50:.0f}ms P95={p95:.0f}ms")
|
||
except Exception as e:
|
||
record("1 纯文本流式", False, f"{type(e).__name__}: {e}")
|
||
|
||
|
||
# ---------------------------------------------------------------- 路径 2：工具调用
|
||
TOOLS = [{
|
||
"type": "function",
|
||
"function": {
|
||
"name": "set_light",
|
||
"description": "打开或关闭指定房间的灯",
|
||
"parameters": {
|
||
"type": "object",
|
||
"properties": {
|
||
"room": {"type": "string", "description": "房间名"},
|
||
"on": {"type": "boolean", "description": "true 开灯，false 关灯"},
|
||
},
|
||
"required": ["room", "on"],
|
||
},
|
||
},
|
||
}]
|
||
|
||
|
||
def test_tools():
|
||
print("\n=== 路径 2：工具调用 ===")
|
||
try:
|
||
ttft, total, text, reasoning, tcs, finish = chat_stream(
|
||
[{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS
|
||
)
|
||
if tcs:
|
||
args = tcs[0]["arguments"]
|
||
try:
|
||
parsed = json.loads(args)
|
||
ok = parsed.get("room") == "客厅" and parsed.get("on") is True
|
||
record("2a 流式工具调用", ok,
|
||
f"name={tcs[0]['name']} args={args} finish={finish}")
|
||
except json.JSONDecodeError:
|
||
record("2a 流式工具调用", False, f"arguments 非法 JSON: {args!r}")
|
||
else:
|
||
record("2a 流式工具调用", False, f"未产生 tool_call（finish={finish}, text={text[:60]!r}）")
|
||
|
||
# 多轮：把工具结果回灌。必须用真实的 tool_call_id 与真实的 assistant 消息，
|
||
# 不能手工构造——这是 2b 第一版失败的原因（400）。
|
||
r0 = chat([{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS, max_tokens=128)
|
||
asst = r0["choices"][0]["message"]
|
||
call_id = asst["tool_calls"][0]["id"]
|
||
msgs = [
|
||
{"role": "user", "content": "把客厅的灯打开"},
|
||
asst,
|
||
{"role": "tool", "tool_call_id": call_id,
|
||
"content": '{"ok": true, "state": "on"}'},
|
||
]
|
||
r = chat(msgs, tools=TOOLS, max_tokens=128)
|
||
txt = r["choices"][0]["message"].get("content") or ""
|
||
record("2b 工具结果回灌", bool(txt.strip()), f"{txt.strip()[:70]!r}")
|
||
|
||
# 2c 思考模式开启时，手工构造的 assistant 消息应当被拒绝（记录该约束）
|
||
try:
|
||
chat([{"role": "user", "content": "把客厅的灯打开"},
|
||
{"role": "assistant", "content": "",
|
||
"tool_calls": [{"id": "call_1", "type": "function",
|
||
"function": {"name": "set_light",
|
||
"arguments": '{"room":"客厅","on":true}'}}]},
|
||
{"role": "tool", "tool_call_id": "call_1", "content": '{"ok":true}'}],
|
||
tools=TOOLS, thinking=None)
|
||
record("2c 思考模式约束", False, "未复现 400，API 行为可能已变，需重新确认")
|
||
except urllib.error.HTTPError as e:
|
||
body = e.read().decode("utf-8", "replace")
|
||
ok = "reasoning_content" in body
|
||
record("2c 思考模式约束", ok, f"HTTP {e.code}，确认必须回传 reasoning_content")
|
||
except Exception as e:
|
||
record("2 工具调用", False, f"{type(e).__name__}: {e}")
|
||
|
||
|
||
# ---------------------------------------------------------------- 路径 3：图片输入
|
||
def make_test_image():
|
||
"""生成一张含文字的图，用于判断模型是否真的'看见'了。"""
|
||
from PIL import Image, ImageDraw
|
||
img = Image.new("RGB", (320, 160), "white")
|
||
d = ImageDraw.Draw(img)
|
||
d.rectangle([10, 10, 310, 150], outline="black", width=3)
|
||
d.text((40, 60), "7391", fill="black")
|
||
buf = io.BytesIO()
|
||
img.save(buf, format="PNG")
|
||
return base64.b64encode(buf.getvalue()).decode()
|
||
|
||
|
||
def test_vision():
|
||
print("\n=== 路径 3：图片输入（视觉问答前提）===")
|
||
try:
|
||
b64 = make_test_image()
|
||
msgs = [{"role": "user", "content": [
|
||
{"type": "text", "text": "图里的数字是多少？只回答数字。"},
|
||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}},
|
||
]}]
|
||
r = chat(msgs, max_tokens=64)
|
||
txt = (r["choices"][0]["message"].get("content") or "").strip()
|
||
ok = "7391" in txt
|
||
record("3a 图片输入", ok, f"模型回答={txt[:60]!r}（期望含 7391）")
|
||
except urllib.error.HTTPError as e:
|
||
body = e.read().decode("utf-8", "replace")[:300]
|
||
record("3a 图片输入", False, f"HTTP {e.code}: {body}")
|
||
except Exception as e:
|
||
record("3a 图片输入", False, f"{type(e).__name__}: {e}")
|
||
|
||
# 顺便探测是否存在视觉模型
|
||
try:
|
||
with _req("/models", {}, timeout=20) as r:
|
||
_ = r.read()
|
||
print(" （/models 只列出文本模型，见下方摘要）")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
# ---------------------------------------------------------------- 路径 4：取消与超时
|
||
def test_cancel_timeout():
|
||
print("\n=== 路径 4：取消与超时 ===")
|
||
# 4a 客户端主动断开
|
||
try:
|
||
body = {"model": MODEL, "stream": True, "max_tokens": 2000,
|
||
"messages": [{"role": "user", "content": "请详细写一篇 800 字的散文，主题是窗外的雨。"}]}
|
||
req = urllib.request.Request(
|
||
BASE + "/chat/completions",
|
||
data=json.dumps(body).encode(),
|
||
headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"},
|
||
)
|
||
t0 = time.perf_counter()
|
||
r = urllib.request.urlopen(req, timeout=60)
|
||
got = 0
|
||
for raw in r:
|
||
if raw.strip().startswith(b"data: "):
|
||
got += 1
|
||
if got >= 5:
|
||
break
|
||
r.close() # 模拟"用户打断"：拿到几段就断开
|
||
dt = time.perf_counter() - t0
|
||
record("4a 中途断开", True, f"收到 {got} 个 chunk 后断开，耗时 {dt*1000:.0f}ms，无异常")
|
||
except Exception as e:
|
||
record("4a 中途断开", False, f"{type(e).__name__}: {e}")
|
||
|
||
# 4b 超时：非流式 + 长生成，服务端要先算完才回应，2 秒预算必然触发。
|
||
# （流式下首包很快，socket 超时不会触发——这正是"超时"和"打断"要分开测的原因。）
|
||
try:
|
||
t0 = time.perf_counter()
|
||
body = {"model": MODEL, "stream": False, "max_tokens": 3000,
|
||
"thinking": {"type": "disabled"},
|
||
"messages": [{"role": "user", "content": "写一篇 2000 字的散文。"}]}
|
||
req = urllib.request.Request(
|
||
BASE + "/chat/completions",
|
||
data=json.dumps(body).encode(),
|
||
headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"},
|
||
)
|
||
try:
|
||
r = urllib.request.urlopen(req, timeout=2)
|
||
r.read()
|
||
record("4b 超时处理", False, "2 秒超时未触发（响应比预期快）")
|
||
except (TimeoutError, urllib.error.URLError, OSError) as e:
|
||
record("4b 超时处理", True,
|
||
f"2 秒超时正确抛出 {type(e).__name__}，耗时 {(time.perf_counter()-t0)*1000:.0f}ms")
|
||
except Exception as e:
|
||
record("4b 超时处理", False, f"{type(e).__name__}: {e}")
|
||
|
||
# 4c 参数错误是否给出可读错误
|
||
try:
|
||
chat([{"role": "user", "content": "hi"}], model="nonexistent-model-xyz")
|
||
record("4c 错误模型名", False, "不存在的模型名未报错")
|
||
except urllib.error.HTTPError as e:
|
||
record("4c 错误模型名", True, f"HTTP {e.code}（可据此做启动自检）")
|
||
except Exception as e:
|
||
record("4c 错误模型名", False, f"{type(e).__name__}: {e}")
|
||
|
||
|
||
def test_models():
|
||
print("\n=== 附：可用模型（启动自检应调用它）===")
|
||
try:
|
||
with _req("/models", timeout=20) as r: # GET
|
||
data = json.loads(r.read().decode())
|
||
ids = [m["id"] for m in data.get("data", [])]
|
||
print(f" {ids}")
|
||
record("0 模型清单", MODEL in ids,
|
||
f"配置模型 {MODEL!r} {'在' if MODEL in ids else '不在'}清单中")
|
||
except Exception as e:
|
||
record("0 模型清单", False, f"{type(e).__name__}: {e}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
print(f"base_url={BASE} model={MODEL}")
|
||
test_models()
|
||
test_text_stream()
|
||
test_tools()
|
||
test_vision()
|
||
test_cancel_timeout()
|
||
|
||
print("\n" + "=" * 60)
|
||
passed = sum(1 for _, ok, _ in RESULTS if ok)
|
||
print(f"汇总: {passed}/{len(RESULTS)} 通过")
|
||
for name, ok, detail in RESULTS:
|
||
if not ok:
|
||
print(f" 未通过: {name} — {detail}")
|
||
sys.exit(0 if passed == len(RESULTS) else 1)
|