助手本体：会话引擎、LLM 路由、记忆、身份、语音适配
从 Obsidian 笔记库剥离为独立仓库。规划文档与 P0 报告留在笔记库， 代码（助手本体、TTS 服务、P0 基准脚本、ESP32 固件）在此。 架构：服务器是助手，本地是客户端 - 助手本体跑在 cadadmin-server，N5095 只做 USB 外设桥与网络桥接 - 起因：实测 N5095 无 AVX2，跑不动可用的语音模型 当前技术栈 - LLM：DeepSeek deepseek-flash（云端，必须关思考） - TTS：Qwen3-TTS 1.7B（服务器 3090，独立 HTTP 服务） - STT：Zipformer 中英双语 int8（CPU，中文识别率未测） 实测：端到端 16/16 通过，首音延迟 1545ms （对比：本地 LLM + vits-zh-ll 是 179ms，但音色不满意——音质与延迟 在此硬件上此消彼长） 设计要点 - MemoryStore 接口按"可能远程"设计（全 async、可超时、可失败） - user_id / session_id / track_id 三者分开，人脸只作候选不自动切身份 - 记忆查询强制带 owner_user_id，无按 id 单独取的口子 - 自动抽取的记忆是"候选"，默认不参与检索 - 异步写记忆绑定创建时的身份，不读"当前用户" - 引擎与传输解耦，只用文件音频即可端到端测试 模型与音频产物不进版本管理（1.1GB），由 download_models.sh 恢复
This commit is contained in:
ylhan
@@ -0,0 +1,116 @@
|
||||
"""LLM 后端选择与降级。
|
||||
|
||||
设计要点：**降级只在"还没吐出任何内容"时发生。**
|
||||
|
||||
一旦已经向调用方 yield 过 chunk，再切后端就会把两段不同来源的文本接在一起，
|
||||
产出一句读不通的回复。那种情况下宁可让本轮失败——方案 §5 要求
|
||||
"网络失败给简短提示，不反复播旧回复"，而不是拼接残句。
|
||||
|
||||
用法：
|
||||
provider = create_provider(AppConfig.load().llm)
|
||||
router = FallbackProvider(primary, fallback)
|
||||
async for chunk in router.stream(messages): ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ..config import LLMConfig
|
||||
from .base import Chunk, ChunkKind, LLMError, LLMProvider, LLMResult, ToolCall
|
||||
from .deepseek import DeepSeekProvider
|
||||
from .local_vllm import LocalVLLMProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_REGISTRY = {
|
||||
"local": LocalVLLMProvider,
|
||||
"deepseek": DeepSeekProvider,
|
||||
}
|
||||
|
||||
|
||||
def create_provider(cfg: LLMConfig) -> LLMProvider:
|
||||
cls = _REGISTRY.get(cfg.name)
|
||||
if cls is None:
|
||||
raise ValueError(f"未知的 LLM 后端 {cfg.name!r}，可选：{sorted(_REGISTRY)}")
|
||||
return cls(cfg)
|
||||
|
||||
|
||||
class FallbackProvider(LLMProvider):
|
||||
"""主后端失败时切备用后端。
|
||||
|
||||
只对**连接层失败**降级（超时、拒连、5xx、模型不可用），
|
||||
不对 4xx 这类"请求本身有问题"降级——那种情况换个后端也一样失败，
|
||||
降级只会掩盖真正的 bug。
|
||||
"""
|
||||
|
||||
supports_vision = True # 只要备用支持就算支持，实际由 _choose 时判断
|
||||
|
||||
def __init__(self, primary: LLMProvider, fallback: LLMProvider | None = None):
|
||||
self.primary = primary
|
||||
self.fallback = fallback
|
||||
self.cfg = primary.cfg
|
||||
self.name = f"{primary.name}+{fallback.name if fallback else 'none'}"
|
||||
self.last_backend: str = primary.name
|
||||
self._session = None
|
||||
self._owns_session = False
|
||||
|
||||
def _build_body(self, *a, **kw): # 实际不会走到，路由在 stream 里
|
||||
raise NotImplementedError
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self.primary.aclose()
|
||||
if self.fallback:
|
||||
await self.fallback.aclose()
|
||||
|
||||
async def stream(self, messages, *, tools=None, max_tokens=None):
|
||||
emitted = False
|
||||
try:
|
||||
async for ch in self.primary.stream(messages, tools=tools, max_tokens=max_tokens):
|
||||
emitted = True
|
||||
self.last_backend = self.primary.name
|
||||
yield ch
|
||||
return
|
||||
except Exception as e: # noqa: BLE001 — 要按类型决定是否降级
|
||||
if emitted:
|
||||
log.error("主后端 %s 在已输出后失败，不再降级：%s", self.primary.name, e)
|
||||
raise
|
||||
if self.fallback is None:
|
||||
raise
|
||||
if not self._should_fallback(e):
|
||||
log.error("主后端 %s 失败且不适合降级：%s", self.primary.name, e)
|
||||
raise
|
||||
log.warning("主后端 %s 失败（%s），切备用 %s",
|
||||
self.primary.name, type(e).__name__, self.fallback.name)
|
||||
|
||||
async for ch in self.fallback.stream(messages, tools=tools, max_tokens=max_tokens):
|
||||
self.last_backend = self.fallback.name
|
||||
yield ch
|
||||
|
||||
@staticmethod
|
||||
def _should_fallback(err: Exception) -> bool:
|
||||
"""连接/超时/服务端错误才降级；请求本身的问题不降级。"""
|
||||
if isinstance(err, LLMError):
|
||||
# LLMError 里带的是 HTTP 状态码，只有 5xx / 429 值得重试到别的后端
|
||||
msg = str(err)
|
||||
for code in ("500", "502", "503", "504", "429"):
|
||||
if f"HTTP {code}" in msg:
|
||||
return True
|
||||
return False
|
||||
# 超时、连接失败、DNS 等
|
||||
return isinstance(err, (TimeoutError, ConnectionError, OSError))
|
||||
|
||||
async def health(self) -> tuple[bool, str]:
|
||||
ok, detail = await self.primary.health()
|
||||
if ok:
|
||||
return True, f"主后端 {self.primary.name}: {detail}"
|
||||
if self.fallback:
|
||||
ok2, d2 = await self.fallback.health()
|
||||
return ok2, f"主后端不可用（{detail}）；备用 {self.fallback.name}: {d2}"
|
||||
return False, detail
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LLMProvider", "LLMError", "Chunk", "ChunkKind", "LLMResult", "ToolCall",
|
||||
"LocalVLLMProvider", "DeepSeekProvider", "FallbackProvider", "create_provider",
|
||||
]
|
||||
+308
@@ -0,0 +1,308 @@
|
||||
"""LLM 抽象层。
|
||||
|
||||
为什么需要这层：本机 vLLM 和 DeepSeek 虽然都是 OpenAI 兼容协议，
|
||||
但有三处实测差异，写死任何一个都会出问题：
|
||||
|
||||
1. 思考内容的字段名
|
||||
本机 vLLM : delta.reasoning
|
||||
DeepSeek : delta.reasoning_content
|
||||
只认一个会**静默丢弃**思考内容——不报错，只是看不见。
|
||||
|
||||
2. 关闭思考的方式
|
||||
本机 vLLM : chat_template_kwargs={"enable_thinking": false}
|
||||
DeepSeek : thinking={"type": "disabled"}
|
||||
|
||||
3. system 消息条数
|
||||
本机 vLLM : 只允许一条，且必须在开头，多条直接 400
|
||||
DeepSeek : 宽松
|
||||
所以统一由本层归一化成单条 system。
|
||||
|
||||
以上均来自实测，见 P0-本地LLM验证报告.md / P0-DeepSeek验证报告.md。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, AsyncIterator, Sequence
|
||||
|
||||
import aiohttp
|
||||
|
||||
from ..config import LLMConfig
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChunkKind(str, Enum):
|
||||
REASONING = "reasoning"
|
||||
CONTENT = "content"
|
||||
TOOL_CALL = "tool_call"
|
||||
DONE = "done"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
kind: ChunkKind
|
||||
text: str = ""
|
||||
tool_index: int = 0
|
||||
tool_id: str = ""
|
||||
tool_name: str = ""
|
||||
tool_arguments: str = ""
|
||||
finish_reason: str | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCall:
|
||||
id: str
|
||||
name: str
|
||||
arguments: str
|
||||
|
||||
def parsed_arguments(self) -> dict[str, Any]:
|
||||
try:
|
||||
return json.loads(self.arguments or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class LLMResult:
|
||||
content: str = ""
|
||||
reasoning: str = ""
|
||||
tool_calls: list[ToolCall] = field(default_factory=list)
|
||||
finish_reason: str | None = None
|
||||
usage: dict[str, Any] | None = None
|
||||
# 计时：首 token 是"思考"还是"回答"要分开，否则会误判延迟。
|
||||
# 实测开思考时 138ms 就出首块，但那是思考内容，用户要听的回答还没开始。
|
||||
first_reasoning_ms: float | None = None
|
||||
first_content_ms: float | None = None
|
||||
total_ms: float = 0.0
|
||||
backend: str = ""
|
||||
|
||||
@property
|
||||
def has_tool_calls(self) -> bool:
|
||||
return bool(self.tool_calls)
|
||||
|
||||
|
||||
class LLMProvider(ABC):
|
||||
"""一个 LLM 后端。
|
||||
|
||||
子类只需实现 `_build_body`（怎么关思考、怎么放参数）；
|
||||
消息归一化、SSE 解析、计时、取消都在基类做，保证各后端行为一致。
|
||||
"""
|
||||
|
||||
#: 是否支持图片输入。本机 vLLM 实例实测不支持（"At most 0 image(s)"）。
|
||||
supports_vision: bool = False
|
||||
|
||||
def __init__(self, cfg: LLMConfig, session: aiohttp.ClientSession | None = None):
|
||||
self.cfg = cfg
|
||||
self.name = cfg.name
|
||||
self._session = session
|
||||
self._owns_session = session is None
|
||||
|
||||
# ------------------------------------------------------------ 供子类覆写
|
||||
|
||||
@abstractmethod
|
||||
def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None,
|
||||
max_tokens: int) -> dict:
|
||||
"""构造请求体。各后端的思考开关差异在这里处理。"""
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {self.cfg.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
@property
|
||||
def endpoint(self) -> str:
|
||||
return self.cfg.base_url.rstrip("/") + "/chat/completions"
|
||||
|
||||
# ------------------------------------------------------------ 消息归一化
|
||||
|
||||
@staticmethod
|
||||
def normalize_messages(messages: Sequence[dict]) -> list[dict]:
|
||||
"""把多条 system 合并成一条并置于开头。
|
||||
|
||||
本机 vLLM 对多条 system 直接 400（实测 "System message must be at the
|
||||
beginning."）。而记忆注入天然想用第二条 system，所以在这里统一抹平。
|
||||
|
||||
顺带这也满足方案 §8 的要求："注入的记忆每轮替换，不是追加"——
|
||||
调用方每轮重新构造 system 内容即可。
|
||||
"""
|
||||
systems, rest = [], []
|
||||
for m in messages:
|
||||
if m.get("role") == "system":
|
||||
c = m.get("content")
|
||||
if isinstance(c, str) and c.strip():
|
||||
systems.append(c.strip())
|
||||
else:
|
||||
rest.append(m)
|
||||
out = []
|
||||
if systems:
|
||||
out.append({"role": "system", "content": "\n\n".join(systems)})
|
||||
out.extend(rest)
|
||||
return out
|
||||
|
||||
# ------------------------------------------------------------ 流式请求
|
||||
|
||||
async def _ensure_session(self) -> aiohttp.ClientSession:
|
||||
if self._session is None or self._session.closed:
|
||||
self._session = aiohttp.ClientSession()
|
||||
self._owns_session = True
|
||||
return self._session
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._owns_session and self._session and not self._session.closed:
|
||||
await self._session.close()
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
messages: Sequence[dict],
|
||||
*,
|
||||
tools: Sequence[dict] | None = None,
|
||||
max_tokens: int | None = None,
|
||||
) -> AsyncIterator[Chunk]:
|
||||
"""流式产出。调用方取消（CancelledError）时连接会随之关闭。
|
||||
|
||||
这就是方案 §5 要求的"打断时取消旧回复生成"的底层能力。
|
||||
"""
|
||||
body = self._build_body(
|
||||
self.normalize_messages(messages),
|
||||
tools=tools,
|
||||
max_tokens=max_tokens or self.cfg.max_tokens,
|
||||
)
|
||||
session = await self._ensure_session()
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
total=self.cfg.timeout_s,
|
||||
sock_connect=5.0,
|
||||
sock_read=self.cfg.first_token_timeout_s,
|
||||
)
|
||||
async with session.post(self.endpoint, json=body,
|
||||
headers=self._headers(), timeout=timeout) as resp:
|
||||
if resp.status != 200:
|
||||
detail = (await resp.text())[:400]
|
||||
raise LLMError(f"{self.name} HTTP {resp.status}: {detail}")
|
||||
async for chunk in self._iter_sse(resp):
|
||||
yield chunk
|
||||
|
||||
@staticmethod
|
||||
async def _iter_sse(resp: aiohttp.ClientResponse) -> AsyncIterator[Chunk]:
|
||||
"""解析 SSE。字段名差异在这里吸收。"""
|
||||
# 工具调用的参数是分片到达的，按 index 累积
|
||||
tool_acc: dict[int, dict] = {}
|
||||
async for raw in resp.content:
|
||||
line = raw.decode("utf-8", "replace").strip()
|
||||
if not line or not line.startswith("data:"):
|
||||
continue
|
||||
payload = line[5:].strip()
|
||||
if payload == "[DONE]":
|
||||
break
|
||||
try:
|
||||
data = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
usage = data.get("usage")
|
||||
choices = data.get("choices") or []
|
||||
if not choices:
|
||||
if usage:
|
||||
yield Chunk(kind=ChunkKind.DONE, usage=usage)
|
||||
continue
|
||||
|
||||
choice = choices[0]
|
||||
delta = choice.get("delta") or {}
|
||||
finish = choice.get("finish_reason")
|
||||
|
||||
# 两个字段名都认。少认一个就会静默丢思考内容。
|
||||
reasoning = delta.get("reasoning_content") or delta.get("reasoning")
|
||||
if reasoning:
|
||||
yield Chunk(kind=ChunkKind.REASONING, text=reasoning)
|
||||
|
||||
content = delta.get("content")
|
||||
if content:
|
||||
yield Chunk(kind=ChunkKind.CONTENT, text=content)
|
||||
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
idx = tc.get("index", 0)
|
||||
slot = tool_acc.setdefault(idx, {"id": "", "name": "", "arguments": ""})
|
||||
if tc.get("id"):
|
||||
slot["id"] = tc["id"]
|
||||
fn = tc.get("function") or {}
|
||||
if fn.get("name"):
|
||||
slot["name"] = fn["name"]
|
||||
if fn.get("arguments"):
|
||||
slot["arguments"] += fn["arguments"]
|
||||
yield Chunk(kind=ChunkKind.TOOL_CALL, tool_index=idx,
|
||||
tool_id=slot["id"], tool_name=slot["name"],
|
||||
tool_arguments=slot["arguments"])
|
||||
|
||||
if finish:
|
||||
yield Chunk(kind=ChunkKind.DONE, finish_reason=finish, usage=usage)
|
||||
# 注意：**不能在这里 return**。开了 stream_options.include_usage 后，
|
||||
# usage 是在 finish_reason 之后的**单独一帧**里到达的
|
||||
# （该帧 choices 为空）。提前返回会把 usage 丢掉。
|
||||
# 循环由 [DONE] 或流结束自然终止。
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
messages: Sequence[dict],
|
||||
*,
|
||||
tools: Sequence[dict] | None = None,
|
||||
max_tokens: int | None = None,
|
||||
) -> LLMResult:
|
||||
"""收完整个流，返回结构化结果。"""
|
||||
t0 = time.perf_counter()
|
||||
res = LLMResult(backend=self.name)
|
||||
parts: list[str] = []
|
||||
reason: list[str] = []
|
||||
acc: dict[int, dict] = {}
|
||||
async for ch in self.stream(messages, tools=tools, max_tokens=max_tokens):
|
||||
if ch.kind == ChunkKind.CONTENT:
|
||||
if res.first_content_ms is None:
|
||||
res.first_content_ms = (time.perf_counter() - t0) * 1000
|
||||
parts.append(ch.text)
|
||||
elif ch.kind == ChunkKind.REASONING:
|
||||
if res.first_reasoning_ms is None:
|
||||
res.first_reasoning_ms = (time.perf_counter() - t0) * 1000
|
||||
reason.append(ch.text)
|
||||
elif ch.kind == ChunkKind.TOOL_CALL:
|
||||
acc[ch.tool_index] = {"id": ch.tool_id, "name": ch.tool_name,
|
||||
"arguments": ch.tool_arguments}
|
||||
elif ch.kind == ChunkKind.DONE:
|
||||
res.finish_reason = ch.finish_reason
|
||||
if ch.usage:
|
||||
res.usage = ch.usage
|
||||
res.content = "".join(parts)
|
||||
res.reasoning = "".join(reason)
|
||||
res.tool_calls = [ToolCall(**v) for _, v in sorted(acc.items())]
|
||||
res.total_ms = (time.perf_counter() - t0) * 1000
|
||||
return res
|
||||
|
||||
async def health(self) -> tuple[bool, str]:
|
||||
"""启动自检：确认模型名有效。方案 §10 要求记录实际版本。"""
|
||||
try:
|
||||
session = await self._ensure_session()
|
||||
url = self.cfg.base_url.rstrip("/") + "/models"
|
||||
async with session.get(url, headers=self._headers(),
|
||||
timeout=aiohttp.ClientTimeout(total=10)) as r:
|
||||
if r.status != 200:
|
||||
return False, f"HTTP {r.status}"
|
||||
data = await r.json()
|
||||
ids = [m.get("id") for m in data.get("data", [])]
|
||||
if self.cfg.model in ids:
|
||||
return True, f"{self.cfg.model} 可用（清单 {len(ids)} 个）"
|
||||
return False, f"模型 {self.cfg.model!r} 不在清单 {ids} 中"
|
||||
except Exception as e: # noqa: BLE001 — 自检不该因异常类型漏判
|
||||
return False, f"{type(e).__name__}: {e}"
|
||||
|
||||
|
||||
class LLMError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["LLMProvider", "LLMError", "Chunk", "ChunkKind", "ToolCall", "LLMResult"]
|
||||
@@ -0,0 +1,47 @@
|
||||
"""DeepSeek 云 API（备用后端）。
|
||||
|
||||
实测数据（P0-DeepSeek验证报告.md）：
|
||||
首 token P50 786ms / P95 1100ms（关思考，N=8）
|
||||
视觉可用——正确读出图中数字
|
||||
模型清单只有 deepseek-flash 与 deepseek-v4-pro
|
||||
|
||||
约束（重要，踩过）：
|
||||
默认开启思考模式，此时 API 要求把上一轮 assistant 消息里的
|
||||
`reasoning_content` 原样回传，否则任何多轮/工具回灌一律 400：
|
||||
"The `reasoning_content` in the thinking mode must be passed back to the API."
|
||||
手工构造 assistant 消息（content=None / content="" / 不带 tools）三种写法
|
||||
实测全部 400，只有回传服务端原始对象才行。
|
||||
所以会话主路径必须显式关闭思考。
|
||||
|
||||
另：本服务思考字段名是 reasoning_content（本机 vLLM 是 reasoning），
|
||||
基类两个都认。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from .base import LLMProvider
|
||||
|
||||
|
||||
class DeepSeekProvider(LLMProvider):
|
||||
supports_vision = True
|
||||
|
||||
def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None,
|
||||
max_tokens: int) -> dict:
|
||||
body: dict = {
|
||||
"model": self.cfg.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"max_tokens": max_tokens,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
# 显式声明思考开关。不传会走默认（开），多轮与工具回灌会 400。
|
||||
body["thinking"] = {"type": "enabled" if self.cfg.thinking else "disabled"}
|
||||
if tools:
|
||||
body["tools"] = list(tools)
|
||||
body["tool_choice"] = "auto"
|
||||
return body
|
||||
|
||||
|
||||
__all__ = ["DeepSeekProvider"]
|
||||
@@ -0,0 +1,46 @@
|
||||
"""本机 Qwen3.8-27B（vLLM，OpenAI 兼容）。
|
||||
|
||||
实测数据（P0-本地LLM验证报告.md）：
|
||||
关思考 首 token 367ms / 完成 10 tokens
|
||||
开思考 首 token 1390ms / 完成 187 tokens，且**回答被 max_tokens 截断**
|
||||
（实测 200 预算下只吐出"我不"两个字）
|
||||
|
||||
开思考不只是慢——思考 token 会占用 max_tokens 预算。
|
||||
语音场景 max_tokens 本来就要设小，所以这不是"慢一点"的取舍，是"说不出话"。
|
||||
默认关闭。
|
||||
|
||||
约束：只接受一条 system 消息，多条返回 400。已在基类归一化。
|
||||
视觉：本实例不支持（"At most 0 image(s) may be provided"），
|
||||
模型目录里没有视觉塔。视觉要走 llama.cpp + mmproj 那条线。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from .base import LLMProvider
|
||||
|
||||
|
||||
class LocalVLLMProvider(LLMProvider):
|
||||
supports_vision = False
|
||||
|
||||
def _build_body(self, messages: list[dict], *, tools: Sequence[dict] | None,
|
||||
max_tokens: int) -> dict:
|
||||
body: dict = {
|
||||
"model": self.cfg.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"max_tokens": max_tokens,
|
||||
"stream_options": {"include_usage": True},
|
||||
}
|
||||
# 关闭思考。这是本机部署实测唯一有效的方式
|
||||
# （服务器上的 ctx_test.sh 也用同一写法）。
|
||||
# 注意：即使关掉，响应里仍可能出现空的 reasoning 字段，属正常。
|
||||
body["chat_template_kwargs"] = {"enable_thinking": self.cfg.thinking}
|
||||
if tools:
|
||||
body["tools"] = list(tools)
|
||||
body["tool_choice"] = "auto"
|
||||
return body
|
||||
|
||||
|
||||
__all__ = ["LocalVLLMProvider"]
|
||||
Reference in New Issue
Block a user