"""LLM 后端选择与降级。 设计要点:**降级只在"还没吐出任何内容"时发生。** 一旦已经向调用方 yield 过 chunk,再切后端就会把两段不同来源的文本接在一起, 产出一句读不通的回复。那种情况下宁可让本轮失败——方案 §5 要求 "网络失败给简短提示,不反复播旧回复",而不是拼接残句。 用法: provider = create_provider(AppConfig.load().llm) router = FallbackProvider(primary, fallback) async for chunk in router.stream(messages): ... """ from __future__ import annotations import logging from ..config import LLMConfig from .base import Chunk, ChunkKind, LLMError, LLMProvider, LLMResult, ToolCall from .deepseek import DeepSeekProvider from .local_vllm import LocalVLLMProvider log = logging.getLogger(__name__) _REGISTRY = { "local": LocalVLLMProvider, "deepseek": DeepSeekProvider, } def create_provider(cfg: LLMConfig) -> LLMProvider: cls = _REGISTRY.get(cfg.name) if cls is None: raise ValueError(f"未知的 LLM 后端 {cfg.name!r},可选:{sorted(_REGISTRY)}") return cls(cfg) class FallbackProvider(LLMProvider): """主后端失败时切备用后端。 只对**连接层失败**降级(超时、拒连、5xx、模型不可用), 不对 4xx 这类"请求本身有问题"降级——那种情况换个后端也一样失败, 降级只会掩盖真正的 bug。 """ supports_vision = True # 只要备用支持就算支持,实际由 _choose 时判断 def __init__(self, primary: LLMProvider, fallback: LLMProvider | None = None): self.primary = primary self.fallback = fallback self.cfg = primary.cfg self.name = f"{primary.name}+{fallback.name if fallback else 'none'}" self.last_backend: str = primary.name self._session = None self._owns_session = False def _build_body(self, *a, **kw): # 实际不会走到,路由在 stream 里 raise NotImplementedError async def aclose(self) -> None: await self.primary.aclose() if self.fallback: await self.fallback.aclose() async def stream(self, messages, *, tools=None, max_tokens=None): emitted = False try: async for ch in self.primary.stream(messages, tools=tools, max_tokens=max_tokens): emitted = True self.last_backend = self.primary.name yield ch return except Exception as e: # noqa: BLE001 — 要按类型决定是否降级 if emitted: log.error("主后端 %s 在已输出后失败,不再降级:%s", self.primary.name, e) raise if self.fallback is None: raise if not self._should_fallback(e): log.error("主后端 %s 失败且不适合降级:%s", self.primary.name, e) raise log.warning("主后端 %s 失败(%s),切备用 %s", self.primary.name, type(e).__name__, self.fallback.name) async for ch in self.fallback.stream(messages, tools=tools, max_tokens=max_tokens): self.last_backend = self.fallback.name yield ch @staticmethod def _should_fallback(err: Exception) -> bool: """连接/超时/服务端错误才降级;请求本身的问题不降级。""" if isinstance(err, LLMError): # LLMError 里带的是 HTTP 状态码,只有 5xx / 429 值得重试到别的后端 msg = str(err) for code in ("500", "502", "503", "504", "429"): if f"HTTP {code}" in msg: return True return False # 超时、连接失败、DNS 等 return isinstance(err, (TimeoutError, ConnectionError, OSError)) async def health(self) -> tuple[bool, str]: ok, detail = await self.primary.health() if ok: return True, f"主后端 {self.primary.name}: {detail}" if self.fallback: ok2, d2 = await self.fallback.health() return ok2, f"主后端不可用({detail});备用 {self.fallback.name}: {d2}" return False, detail __all__ = [ "LLMProvider", "LLMError", "Chunk", "ChunkKind", "LLMResult", "ToolCall", "LocalVLLMProvider", "DeepSeekProvider", "FallbackProvider", "create_provider", ]