Add chat endpoint failover for resilience against unstable LLM APIs

Configure a list of fallback chat endpoints (chatFallbacks); each blank field inherits the primary, so you can swap just the API key, just the model, or the whole vendor. On LLM streaming failure the retry loop advances to the next endpoint, and a failed endpoint enters a cooldown (fallbackCooldownMinutes) so a dead primary is skipped instead of wasting a timeout on every message. Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-09-15 02:56:10 +08:00 · 2026-06-24 22:23:43 +08:00
parent 13329b5fa3
commit 826dc460df
4 changed files with 188 additions and 16 deletions
@@ -42,6 +42,7 @@ import java.time.ZoneOffset
 import java.time.format.DateTimeFormatter
 import kotlin.collections.*
 import kotlin.math.max
+import kotlin.math.min
 import kotlin.math.pow
 import kotlin.math.sign
 import kotlin.time.Duration.Companion.seconds
@@ -683,14 +684,23 @@ object JChatGPT : KotlinPlugin(
                )
            }

+            // 聊天接入点容灾：按健康度排序，主接入点故障冷却时备用接入点会自动排到前面
+            val endpoints = LargeLanguageModels.orderedChatEndpoints()
+            if (endpoints.isEmpty()) throw NullPointerException("OpenAI Token 未设置，无法开始")
+            var endpointIndex = 0
+
            var done: Boolean
            // 至少循环3次
            var retry = max(PluginConfig.retryMax, 3)
            do {
+                // 当前使用的接入点：失败重试时会前移到下一个备用接入点
+                val endpoint = endpoints[min(endpointIndex, endpoints.lastIndex)]
+                // 标记本轮 LLM 流式调用是否成功完成，用于精确区分「LLM失败」与「后续工具失败」
+                var streamingOk = false
                try {
                    val startedAt = OffsetDateTime.now().toEpochSecond().toInt()
                    var lastCacheUsage: ModelService.CacheUsage? = null
-                    val responseFlow = chatCompletions(history) { lastCacheUsage = it }
+                    val responseFlow = chatCompletions(history, endpoint) { lastCacheUsage = it }
                    var responseMessageBuilder: StringBuilder? = null
                    var reasoningContentBuilder: StringBuilder? = null
                    val responseToolCalls = mutableListOf<ToolCall.Function>()
@@ -770,6 +780,9 @@ object JChatGPT : KotlinPlugin(
                        // 捕获token使用量
                        chunk.usage?.let { lastTokenUsage = it }
                    }
+                    // LLM 流式调用成功完成，上报接入点健康（清除冷却）
+                    streamingOk = true
+                    LargeLanguageModels.reportSuccess(endpoint)

                    // 移除思考内容
                    val responseContent = responseMessageBuilder?.replace(thinkRegex, "")?.trim()
@@ -856,11 +869,23 @@ object JChatGPT : KotlinPlugin(
                        }
                    }
                } catch (e: Exception) {
+                    // 仅当 LLM 流式调用本身失败时才上报接入点故障并切换；
+                    // 若流式已成功、异常来自后续工具执行，则保持当前接入点不变
+                    if (!streamingOk) {
+                        LargeLanguageModels.reportFailure(endpoint)
+                        if (endpointIndex < endpoints.lastIndex) {
+                            endpointIndex++
+                            logger.warning("接入点[${endpoint.label}]调用失败，切换备用接入点[${endpoints[endpointIndex].label}]重试", e)
+                        } else {
+                            logger.warning("接入点[${endpoint.label}]调用失败，无更多备用接入点，重试中", e)
+                        }
+                    } else {
+                        logger.warning("调用llm后处理时发生异常，重试中", e)
+                    }
                    if (retry <= 1) {
                        throw e
                    } else {
                        done = false
-                        logger.warning("调用llm时发生异常，重试中", e)
                        // event.subject.sendMessage("出错了...正在重试...")
                    }
                }
@@ -1031,21 +1056,21 @@ object JChatGPT : KotlinPlugin(

    private fun chatCompletions(
        chatMessages: List<ChatMessage>,
+        endpoint: LargeLanguageModels.ChatEndpoint,
        hasTools: Boolean = true,
        onCacheUsage: ((ModelService.CacheUsage) -> Unit)? = null
    ): Flow<ChatCompletionChunk> {
-        val llm = LargeLanguageModels.chat ?: throw NullPointerException("OpenAI Token 未设置，无法开始")
        val availableTools = if (hasTools) {
            myTools.filter { it.isEnabled }.map { it.tool }
        } else null
        val request = ChatCompletionRequest(
-            model = ModelId(PluginConfig.chatModel),
-            temperature = PluginConfig.chatTemperature,
+            model = ModelId(endpoint.model),
+            temperature = endpoint.temperature,
            messages = chatMessages,
            tools = availableTools,
        )
-        logger.info("API Requesting... Model=${PluginConfig.chatModel}")
-        return llm.chatCompletions(request, onCacheUsage)
+        logger.info("API Requesting... Model=${endpoint.model} [${endpoint.label}]")
+        return endpoint.service.chatCompletions(request, onCacheUsage)
    }

    private fun getNameCard(member: Member): String {
@@ -14,9 +14,28 @@ object LargeLanguageModels {
        private set

    /**
-     * 聊天助手
+     * 一个聊天接入点：封装了请求服务、模型名与温度。
+     * 主接入点为列表第 0 项，其余为备用接入点，用于容灾切换。
     */
-    var chat: ModelService? = null
+    data class ChatEndpoint(
+        val service: ModelService,
+        val model: String,
+        val temperature: Double?,
+        /** 唯一标识，用于健康状态跟踪与日志 */
+        val label: String,
+    )
+
+    /**
+     * 聊天接入点列表：index 0 为主接入点，其余按配置顺序为备用接入点。
+     */
+    var chatEndpoints: List<ChatEndpoint> = emptyList()
+        private set
+
+    /**
+     * 主聊天接入点服务（向后兼容旧用法）。
+     */
+    val chat: ModelService?
+        get() = chatEndpoints.firstOrNull()?.service

    /**
     * 推理模型
@@ -28,6 +47,40 @@ object LargeLanguageModels {
     */
    var visual: ModelService? = null

+    /**
+     * 接入点健康状态：记录各接入点的冷却截止时间戳（毫秒）。
+     * 失败的接入点进入冷却，期间在 [orderedChatEndpoints] 中被排到队尾，
+     * 避免每条消息都先卡在故障接入点上白白等一次超时。
+     */
+    private val cooldownUntil = HashMap<String, Long>()
+
+    /** 上报某接入点调用失败，使其进入冷却。 */
+    fun reportFailure(endpoint: ChatEndpoint) {
+        val minutes = PluginConfig.fallbackCooldownMinutes
+        // 只有存在备用接入点时冷却才有意义；否则没有可切换的目标，标记冷却反而无益
+        if (minutes > 0 && chatEndpoints.size > 1) {
+            cooldownUntil[endpoint.label] = System.currentTimeMillis() + minutes * 60_000L
+        }
+    }
+
+    /** 上报某接入点调用成功，清除其冷却。 */
+    fun reportSuccess(endpoint: ChatEndpoint) {
+        cooldownUntil.remove(endpoint.label)
+    }
+
+    /**
+     * 返回按健康度排序的接入点：未冷却的保持配置原顺序在前，冷却中的排到后面
+     * （冷却中再按剩余冷却时间升序，优先重试快恢复的）。排序稳定，主接入点健康时始终最先。
+     */
+    fun orderedChatEndpoints(): List<ChatEndpoint> {
+        if (chatEndpoints.size <= 1) return chatEndpoints
+        val now = System.currentTimeMillis()
+        return chatEndpoints.sortedBy { ep ->
+            val until = cooldownUntil[ep.label] ?: 0L
+            if (until > now) until else 0L
+        }
+    }
+
    private val json = Json {
        isLenient = true
        ignoreUnknownKeys = true
@@ -46,16 +99,50 @@ object LargeLanguageModels {
        val timeout = PluginConfig.timeout.milliseconds
        val firstChunkTimeout = PluginConfig.firstChunkTimeout.milliseconds

-        // 初始化聊天模型
+        // 初始化聊天接入点（主 + 备用），并重置健康状态
+        cooldownUntil.clear()
+        val endpoints = mutableListOf<ChatEndpoint>()
        if (PluginConfig.openAiApi.isNotBlank() && PluginConfig.openAiToken.isNotBlank()) {
-            chat = ModelService(
-                baseUrl = PluginConfig.openAiApi,
-                token = PluginConfig.openAiToken,
-                timeout = timeout,
-                firstChunkTimeout = firstChunkTimeout,
-                extraBody = parseExtraBody(PluginConfig.chatModelExtraBody)
+            endpoints.add(
+                ChatEndpoint(
+                    service = ModelService(
+                        baseUrl = PluginConfig.openAiApi,
+                        token = PluginConfig.openAiToken,
+                        timeout = timeout,
+                        firstChunkTimeout = firstChunkTimeout,
+                        extraBody = parseExtraBody(PluginConfig.chatModelExtraBody)
+                    ),
+                    model = PluginConfig.chatModel,
+                    temperature = PluginConfig.chatTemperature,
+                    label = "primary",
+                )
            )
+
+            // 备用接入点：留空字段继承主接入点配置
+            PluginConfig.chatFallbacks.forEachIndexed { i, fb ->
+                val api = fb.api.ifBlank { PluginConfig.openAiApi }
+                val token = fb.token.ifBlank { PluginConfig.openAiToken }
+                val model = fb.model.ifBlank { PluginConfig.chatModel }
+                val extraBody = fb.extraBody.ifBlank { PluginConfig.chatModelExtraBody }
+                if (api.isNotBlank() && token.isNotBlank()) {
+                    endpoints.add(
+                        ChatEndpoint(
+                            service = ModelService(
+                                baseUrl = api,
+                                token = token,
+                                timeout = timeout,
+                                firstChunkTimeout = firstChunkTimeout,
+                                extraBody = parseExtraBody(extraBody)
+                            ),
+                            model = model,
+                            temperature = PluginConfig.chatTemperature,
+                            label = "fallback$i:$model",
+                        )
+                    )
+                }
+            }
        }
+        chatEndpoints = endpoints

        // 初始化推理模型
        if (PluginConfig.reasoningModelApi.isNotBlank() && PluginConfig.reasoningModelToken.isNotBlank()) {
@@ -1,9 +1,22 @@
 package top.jie65535.mirai

+import kotlinx.serialization.Serializable
 import net.mamoe.mirai.console.data.AutoSavePluginConfig
 import net.mamoe.mirai.console.data.ValueDescription
 import net.mamoe.mirai.console.data.value

+/**
+ * 聊天模型备用接入点。用于主接入点（openAiApi/openAiToken/chatModel）连续失败时容灾切换。
+ * 任一字段留空则继承主接入点对应配置，因此可只换 API KEY、只换模型、或整体换一个服务商。
+ */
+@Serializable
+data class ChatFallbackEndpoint(
+    val api: String = "",
+    val token: String = "",
+    val model: String = "",
+    val extraBody: String = "",
+)
+
 object PluginConfig : AutoSavePluginConfig("Config") {
    @ValueDescription("主人QQ，AI可以通过工具向主人发起请求，会等待一段时间")
    val ownerId: Long by value()
@@ -41,6 +54,12 @@ object PluginConfig : AutoSavePluginConfig("Config") {
    @ValueDescription("聊天模型额外请求体JSON，会合并到请求体中。例如DeepSeek关闭思维: {\"thinking\": {\"type\": \"disabled\"}}")
    val chatModelExtraBody: String by value("")

+    @ValueDescription("聊天模型备用接入点列表（容灾）。主接入点连续失败时按顺序切换；每项留空的字段会继承主接入点，例如只换API KEY就只填token，只换模型就只填model")
+    val chatFallbacks: List<ChatFallbackEndpoint> by value()
+
+    @ValueDescription("备用接入点冷却时间（分钟）。某接入点失败后在此时间内会被排到重试队尾，避免每条消息都先卡在故障接入点上。设为0禁用，默认5分钟")
+    val fallbackCooldownMinutes: Long by value(5L)
+
    @ValueDescription("推理模型额外请求体JSON，会合并到请求体中。例如DeepSeek启用思维: {\"thinking\": {\"type\": \"enabled\"}}")
    val reasoningModelExtraBody: String by value("")