tools: improve visual image handling

2026-09-15 02:56:10 +08:00 · 2026-07-26 14:43:34 +08:00
parent 7b5a83ba9c
commit a4f5bad322
6 changed files with 994 additions and 24 deletions
@@ -2,30 +2,44 @@ package top.jie65535.mirai.tools

 import com.aallam.openai.api.chat.ChatCompletionRequest
 import com.aallam.openai.api.chat.ChatMessage
+import com.aallam.openai.api.chat.ContentPart
 import com.aallam.openai.api.chat.ImagePart
 import com.aallam.openai.api.chat.TextPart
 import com.aallam.openai.api.chat.Tool
 import com.aallam.openai.api.core.Parameters
 import com.aallam.openai.api.model.ModelId
+import io.ktor.client.plugins.ClientRequestException
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.add
+import kotlinx.serialization.json.jsonArray
 import kotlinx.serialization.json.jsonPrimitive
 import kotlinx.serialization.json.put
 import kotlinx.serialization.json.putJsonArray
 import kotlinx.serialization.json.putJsonObject
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
+import top.jie65535.mirai.JChatGPT
 import top.jie65535.mirai.LargeLanguageModels
 import top.jie65535.mirai.PluginConfig
+import java.net.URI

 class VisualAgent : BaseAgent(
    tool = Tool.function(
        name = "imageRecognition",
-        description = "可通过调用视觉模型来识别图片内容。备注：该方法成本较高，非必要尽量不要调用。",
+        description = "可通过调用视觉模型识别一张或多张图片，并进行比较、关联或顺序理解。备注：该方法成本较高，非必要尽量不要调用。",
        parameters = Parameters.buildJsonObject {
            put("type", "object")
            putJsonObject("properties") {
-                putJsonObject("image_url") {
-                    put("type", "string")
-                    put("description", "图片地址")
+                putJsonObject("image_urls") {
+                    put("type", "array")
+                    put("description", "图片地址数组，按用户消息中的出现顺序传入")
+                    put("minItems", 1)
+                    put("maxItems", MAX_SOURCE_IMAGES)
+                    putJsonObject("items") {
+                        put("type", "string")
+                    }
                }
                putJsonObject("prompt") {
                    put("type", "string")
@@ -33,12 +47,15 @@ class VisualAgent : BaseAgent(
                }
            }
            putJsonArray("required") {
-                add("image_url")
+                add("image_urls")
                add("prompt")
            }
        }
    )
 ) {
+    private val imageResolver = VisualImageResolver()
+    private val concurrencyLimiter = Semaphore(VISUAL_MAX_CONCURRENCY)
+
    override val loadingMessage: String
        get() = "识别中..."

@@ -48,29 +65,129 @@ class VisualAgent : BaseAgent(
    override suspend fun execute(args: JsonObject?): String {
        requireNotNull(args)
        val llm = LargeLanguageModels.visual ?: return "未配置llm，无法进行识别。"
-        val imageUrl = args.getValue("image_url").jsonPrimitive.content
+        val imageUrls = args["image_urls"]?.jsonArray
+            ?.map { it.jsonPrimitive.content }
+            ?.filter { it.isNotBlank() }
+            ?.ifEmpty { null }
+            ?: throw IllegalArgumentException("至少需要提供一张图片")
+        require(imageUrls.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" }
        val prompt = args.getValue("prompt").jsonPrimitive.content

-        val answerContent = StringBuilder()
-        llm.chatCompletions(ChatCompletionRequest(
-            model = ModelId(PluginConfig.visualModel),
-            messages = listOf(
-                ChatMessage.System("You are a helpful assistant."),
-                ChatMessage.User(
-                    content = listOf(
-                        ImagePart(imageUrl),
-                        TextPart(prompt)
+        return concurrencyLimiter.withPermit {
+            val imageGroups = imageUrls.mapIndexed { index, imageUrl ->
+                if (PluginConfig.visualImageBase64Enabled) {
+                    val resolved = imageResolver.resolve(imageUrl)
+                    val host = runCatching { URI(imageUrl).host }.getOrNull() ?: "unknown"
+                    val mimeTypes = resolved.images.map { it.mimeType }.distinct().joinToString()
+                    JChatGPT.logger.info(
+                        "视觉图片已本地化: source=${index + 1}/${imageUrls.size}, host=$host, " +
+                                "parts=${resolved.images.size}, mime=$mimeTypes, " +
+                                "sourceBytes=${resolved.sourceSize}, payloadChars=${resolved.payloadSize}, " +
+                                "transcoded=${resolved.transcoded}"
                    )
-                )
-            )
-        )).collect {
-            if (it.choices.isNotEmpty()) {
-                val delta = it.choices[0].delta ?: return@collect
-                if (!delta.content.isNullOrEmpty()) {
-                    answerContent.append(delta.content)
+                    PreparedImageGroup(
+                        inputs = resolved.images.map { it.dataUrl },
+                        orderHint = resolved.orderHint,
+                        payloadSize = resolved.payloadSize,
+                    )
+                } else {
+                    PreparedImageGroup(inputs = listOf(imageUrl), orderHint = null, payloadSize = 0)
                }
            }
+            val modelImageCount = imageGroups.sumOf { it.inputs.size }
+            val totalPayloadSize = imageGroups.sumOf { it.payloadSize }
+            require(modelImageCount <= MAX_MODEL_IMAGES) {
+                "图片及长图切片共 $modelImageCount 张，超过单次工程限制 $MAX_MODEL_IMAGES 张"
+            }
+            require(totalPayloadSize <= MAX_TOTAL_PAYLOAD_CHARS) {
+                "图片 Base64 总大小超过 ${MAX_TOTAL_PAYLOAD_CHARS / 1_000_000}MB 工程限制"
+            }
+            val messageContent = buildMessageContent(imageGroups, prompt)
+
+            val maxAttempts = PluginConfig.visualRetryMax.coerceIn(1, 3)
+            var lastError: Throwable? = null
+            repeat(maxAttempts) { attempt ->
+                try {
+                    val answerContent = StringBuilder()
+                    llm.chatCompletions(
+                        ChatCompletionRequest(
+                            model = ModelId(PluginConfig.visualModel),
+                            messages = listOf(
+                                ChatMessage.User(
+                                    content = messageContent
+                                )
+                            )
+                        )
+                    ).collect {
+                        if (it.choices.isNotEmpty()) {
+                            val delta = it.choices[0].delta ?: return@collect
+                            if (!delta.content.isNullOrEmpty()) {
+                                answerContent.append(delta.content)
+                            }
+                        }
+                    }
+
+                    if (answerContent.isNotEmpty()) {
+                        return@withPermit answerContent.toString()
+                    }
+                    throw IllegalStateException("识图异常，结果为空")
+                } catch (e: CancellationException) {
+                    throw e
+                } catch (e: Throwable) {
+                    if (!isRetryable(e)) throw e
+                    lastError = e
+                    if (attempt + 1 < maxAttempts) {
+                        JChatGPT.logger.warning(
+                            "视觉模型调用失败，将进行第 ${attempt + 2}/$maxAttempts 次尝试",
+                            e
+                        )
+                        delay(RETRY_BASE_DELAY_MILLIS * (attempt + 1L))
+                    }
+                }
+            }
+
+            throw lastError ?: IllegalStateException("视觉模型调用失败")
        }
-        return answerContent.toString().ifEmpty { "识图异常，结果为空" }
    }
-}
+
+    companion object {
+        private const val VISUAL_MAX_CONCURRENCY = 2
+        private const val RETRY_BASE_DELAY_MILLIS = 800L
+        private const val MAX_SOURCE_IMAGES = 16
+        private const val MAX_MODEL_IMAGES = 32
+        private const val MAX_TOTAL_PAYLOAD_CHARS = 48_000_000
+
+        private data class PreparedImageGroup(
+            val inputs: List<String>,
+            val orderHint: String?,
+            val payloadSize: Int,
+        )
+
+        private fun buildMessageContent(groups: List<PreparedImageGroup>, prompt: String): List<ContentPart> {
+            if (groups.size == 1 && groups[0].inputs.size == 1) {
+                return listOf(ImagePart(groups[0].inputs[0]), TextPart(prompt))
+            }
+
+            return buildList {
+                groups.forEachIndexed { groupIndex, group ->
+                    add(
+                        TextPart(
+                            "用户图片 ${groupIndex + 1}/${groups.size}" +
+                                    if (group.inputs.size > 1) "，已切分为 ${group.inputs.size} 张连续切片：" else "："
+                        )
+                    )
+                    group.inputs.forEach { add(ImagePart(it)) }
+                    group.orderHint?.let { add(TextPart(it)) }
+                }
+                add(TextPart("请结合以上所有用户图片回答：$prompt"))
+            }
+        }
+
+        private fun isRetryable(error: Throwable): Boolean {
+            if (error is ClientRequestException) {
+                return error.response.status.value in setOf(408, 409, 425, 429)
+            }
+            return true
+        }
+    }
+}