#!/usr/bin/env python3 """P0 验证：DeepSeek API 四条路径。 对应方案 §5 的 P0 要求：纯文本流式 / 图片输入 / 工具调用 / 取消与超时。 用标准库直连 HTTP，不依赖 SDK 版本，便于精确计时。 用法： set -a; . ~/.config/ai_assistant/env; set +a python3 verify_deepseek.py """ import base64 import io import json import os import statistics import sys import time import urllib.error import urllib.request BASE = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") KEY = os.environ["DEEPSEEK_API_KEY"] MODEL = os.environ.get("DEEPSEEK_MODEL", "deepseek-flash") RESULTS = [] def record(name, ok, detail): RESULTS.append((name, ok, detail)) print(f"[{'PASS' if ok else 'FAIL'}] {name}: {detail}") def _req(path, body=None, timeout=60, method=None): data = None if body is None else json.dumps(body).encode("utf-8") req = urllib.request.Request( BASE + path, data=data, method=method or ("GET" if data is None else "POST"), headers={ "Authorization": f"Bearer {KEY}", "Content-Type": "application/json", }, ) return urllib.request.urlopen(req, timeout=timeout) def chat(messages, thinking="disabled", **kw): """thinking 默认关闭：见 verify 报告——开启时 reasoning_content 必须回传。""" body = {"model": MODEL, "messages": messages, "stream": False} if thinking: body["thinking"] = {"type": thinking} body.update(kw) with _req("/chat/completions", body) as r: return json.loads(r.read().decode("utf-8")) def chat_stream(messages, tools=None, max_tokens=256, timeout=60, thinking="disabled"): """返回 (ttft, total, text, reasoning, tool_calls, finish)。 thinking 默认 disabled：开启时 API 要求把 reasoning_content 原样回传， 否则多轮/工具回灌一律 400。详见 verify 报告。 """ body = { "model": MODEL, "messages": messages, "stream": True, "max_tokens": max_tokens, } if thinking: body["thinking"] = {"type": thinking} if tools: body["tools"] = tools t0 = time.perf_counter() ttft = None text_parts, reasoning_parts = [], [] tool_calls = {} finish = None with _req("/chat/completions", body, timeout=timeout) as r: for raw in r: line = raw.decode("utf-8", "replace").strip() if not line.startswith("data: "): continue payload = line[6:] if payload == "[DONE]": break d = json.loads(payload) if not d.get("choices"): continue ch = d["choices"][0] delta = ch.get("delta") or {} if ch.get("finish_reason"): finish = ch["finish_reason"] if delta.get("content"): if ttft is None: ttft = time.perf_counter() - t0 text_parts.append(delta["content"]) if delta.get("reasoning_content"): if ttft is None: ttft = time.perf_counter() - t0 reasoning_parts.append(delta["reasoning_content"]) for tc in delta.get("tool_calls") or []: if ttft is None: ttft = time.perf_counter() - t0 idx = tc.get("index", 0) slot = tool_calls.setdefault(idx, {"name": "", "arguments": ""}) fn = tc.get("function") or {} if fn.get("name"): slot["name"] = fn["name"] if fn.get("arguments"): slot["arguments"] += fn["arguments"] total = time.perf_counter() - t0 return ttft, total, "".join(text_parts), "".join(reasoning_parts), list(tool_calls.values()), finish # ---------------------------------------------------------------- 路径 1：纯文本流式 def test_text_stream(): print("\n=== 路径 1：纯文本流式（首 token 延迟）===") msgs = [{"role": "user", "content": "用一句话介绍你自己。"}] try: ttft, total, text, reasoning, tcs, finish = chat_stream(msgs) record("1a 流式基本可用", bool(text.strip()), f"ttft={ttft*1000:.0f}ms total={total*1000:.0f}ms finish={finish}") print(f" 回复: {text.strip()[:60]!r}") # 1b 思考内容：两种模式的差异直接决定 TTS 取哪一段 _, _, txt_on, reason_on, _, _ = chat_stream(msgs, thinking=None) print(f" thinking=on → reasoning_content {len(reason_on)} 字，" f"content {len(txt_on)} 字") print(f" thinking=off → reasoning_content {len(reasoning)} 字，" f"content {len(text)} 字") if reason_on and not reasoning: record("1b 思考内容分离", True, "on 模式有 reasoning_content（不可送 TTS）；off 模式无，取 content 即可") else: record("1b 思考内容分离", False, f"两种模式行为异常：on={len(reason_on)}字 off={len(reasoning)}字") # 延迟统计 N = int(os.environ.get("BENCH_N", "8")) ttfts, totals = [], [] for i in range(N): t, tot, *_ = chat_stream( [{"role": "user", "content": f"请用一句话回答：{i}+{i} 等于几？"}], max_tokens=64, ) if t: ttfts.append(t * 1000) totals.append(tot * 1000) if ttfts: ttfts.sort() p50 = statistics.median(ttfts) p95 = ttfts[min(len(ttfts) - 1, int(len(ttfts) * 0.95))] print(f" N={len(ttfts)} 首token: P50={p50:.0f}ms P95={p95:.0f}ms " f"min={min(ttfts):.0f} max={max(ttfts):.0f}") print(f" 整段耗时: P50={statistics.median(totals):.0f}ms") record("1c 首token延迟实测", True, f"P50={p50:.0f}ms P95={p95:.0f}ms") except Exception as e: record("1 纯文本流式", False, f"{type(e).__name__}: {e}") # ---------------------------------------------------------------- 路径 2：工具调用 TOOLS = [{ "type": "function", "function": { "name": "set_light", "description": "打开或关闭指定房间的灯", "parameters": { "type": "object", "properties": { "room": {"type": "string", "description": "房间名"}, "on": {"type": "boolean", "description": "true 开灯，false 关灯"}, }, "required": ["room", "on"], }, }, }] def test_tools(): print("\n=== 路径 2：工具调用 ===") try: ttft, total, text, reasoning, tcs, finish = chat_stream( [{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS ) if tcs: args = tcs[0]["arguments"] try: parsed = json.loads(args) ok = parsed.get("room") == "客厅" and parsed.get("on") is True record("2a 流式工具调用", ok, f"name={tcs[0]['name']} args={args} finish={finish}") except json.JSONDecodeError: record("2a 流式工具调用", False, f"arguments 非法 JSON: {args!r}") else: record("2a 流式工具调用", False, f"未产生 tool_call（finish={finish}, text={text[:60]!r}）") # 多轮：把工具结果回灌。必须用真实的 tool_call_id 与真实的 assistant 消息， # 不能手工构造——这是 2b 第一版失败的原因（400）。 r0 = chat([{"role": "user", "content": "把客厅的灯打开"}], tools=TOOLS, max_tokens=128) asst = r0["choices"][0]["message"] call_id = asst["tool_calls"][0]["id"] msgs = [ {"role": "user", "content": "把客厅的灯打开"}, asst, {"role": "tool", "tool_call_id": call_id, "content": '{"ok": true, "state": "on"}'}, ] r = chat(msgs, tools=TOOLS, max_tokens=128) txt = r["choices"][0]["message"].get("content") or "" record("2b 工具结果回灌", bool(txt.strip()), f"{txt.strip()[:70]!r}") # 2c 思考模式开启时，手工构造的 assistant 消息应当被拒绝（记录该约束） try: chat([{"role": "user", "content": "把客厅的灯打开"}, {"role": "assistant", "content": "", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "set_light", "arguments": '{"room":"客厅","on":true}'}}]}, {"role": "tool", "tool_call_id": "call_1", "content": '{"ok":true}'}], tools=TOOLS, thinking=None) record("2c 思考模式约束", False, "未复现 400，API 行为可能已变，需重新确认") except urllib.error.HTTPError as e: body = e.read().decode("utf-8", "replace") ok = "reasoning_content" in body record("2c 思考模式约束", ok, f"HTTP {e.code}，确认必须回传 reasoning_content") except Exception as e: record("2 工具调用", False, f"{type(e).__name__}: {e}") # ---------------------------------------------------------------- 路径 3：图片输入 def make_test_image(): """生成一张含文字的图，用于判断模型是否真的'看见'了。""" from PIL import Image, ImageDraw img = Image.new("RGB", (320, 160), "white") d = ImageDraw.Draw(img) d.rectangle([10, 10, 310, 150], outline="black", width=3) d.text((40, 60), "7391", fill="black") buf = io.BytesIO() img.save(buf, format="PNG") return base64.b64encode(buf.getvalue()).decode() def test_vision(): print("\n=== 路径 3：图片输入（视觉问答前提）===") try: b64 = make_test_image() msgs = [{"role": "user", "content": [ {"type": "text", "text": "图里的数字是多少？只回答数字。"}, {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}, ]}] r = chat(msgs, max_tokens=64) txt = (r["choices"][0]["message"].get("content") or "").strip() ok = "7391" in txt record("3a 图片输入", ok, f"模型回答={txt[:60]!r}（期望含 7391）") except urllib.error.HTTPError as e: body = e.read().decode("utf-8", "replace")[:300] record("3a 图片输入", False, f"HTTP {e.code}: {body}") except Exception as e: record("3a 图片输入", False, f"{type(e).__name__}: {e}") # 顺便探测是否存在视觉模型 try: with _req("/models", {}, timeout=20) as r: _ = r.read() print(" （/models 只列出文本模型，见下方摘要）") except Exception: pass # ---------------------------------------------------------------- 路径 4：取消与超时 def test_cancel_timeout(): print("\n=== 路径 4：取消与超时 ===") # 4a 客户端主动断开 try: body = {"model": MODEL, "stream": True, "max_tokens": 2000, "messages": [{"role": "user", "content": "请详细写一篇 800 字的散文，主题是窗外的雨。"}]} req = urllib.request.Request( BASE + "/chat/completions", data=json.dumps(body).encode(), headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"}, ) t0 = time.perf_counter() r = urllib.request.urlopen(req, timeout=60) got = 0 for raw in r: if raw.strip().startswith(b"data: "): got += 1 if got >= 5: break r.close() # 模拟"用户打断"：拿到几段就断开 dt = time.perf_counter() - t0 record("4a 中途断开", True, f"收到 {got} 个 chunk 后断开，耗时 {dt*1000:.0f}ms，无异常") except Exception as e: record("4a 中途断开", False, f"{type(e).__name__}: {e}") # 4b 超时：非流式 + 长生成，服务端要先算完才回应，2 秒预算必然触发。 # （流式下首包很快，socket 超时不会触发——这正是"超时"和"打断"要分开测的原因。） try: t0 = time.perf_counter() body = {"model": MODEL, "stream": False, "max_tokens": 3000, "thinking": {"type": "disabled"}, "messages": [{"role": "user", "content": "写一篇 2000 字的散文。"}]} req = urllib.request.Request( BASE + "/chat/completions", data=json.dumps(body).encode(), headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"}, ) try: r = urllib.request.urlopen(req, timeout=2) r.read() record("4b 超时处理", False, "2 秒超时未触发（响应比预期快）") except (TimeoutError, urllib.error.URLError, OSError) as e: record("4b 超时处理", True, f"2 秒超时正确抛出 {type(e).__name__}，耗时 {(time.perf_counter()-t0)*1000:.0f}ms") except Exception as e: record("4b 超时处理", False, f"{type(e).__name__}: {e}") # 4c 参数错误是否给出可读错误 try: chat([{"role": "user", "content": "hi"}], model="nonexistent-model-xyz") record("4c 错误模型名", False, "不存在的模型名未报错") except urllib.error.HTTPError as e: record("4c 错误模型名", True, f"HTTP {e.code}（可据此做启动自检）") except Exception as e: record("4c 错误模型名", False, f"{type(e).__name__}: {e}") def test_models(): print("\n=== 附：可用模型（启动自检应调用它）===") try: with _req("/models", timeout=20) as r: # GET data = json.loads(r.read().decode()) ids = [m["id"] for m in data.get("data", [])] print(f" {ids}") record("0 模型清单", MODEL in ids, f"配置模型 {MODEL!r} {'在' if MODEL in ids else '不在'}清单中") except Exception as e: record("0 模型清单", False, f"{type(e).__name__}: {e}") if __name__ == "__main__": print(f"base_url={BASE} model={MODEL}") test_models() test_text_stream() test_tools() test_vision() test_cancel_timeout() print("\n" + "=" * 60) passed = sum(1 for _, ok, _ in RESULTS if ok) print(f"汇总: {passed}/{len(RESULTS)} 通过") for name, ok, detail in RESULTS: if not ok: print(f" 未通过: {name} — {detail}") sys.exit(0 if passed == len(RESULTS) else 1)