tools: resolve images by short references

This commit is contained in:
2026-07-27 14:01:12 +08:00
parent a4f5bad322
commit c328a798f7
7 changed files with 187 additions and 50 deletions
+29
View File
@@ -0,0 +1,29 @@
package top.jie65535.mirai
/**
* 会话内图片短索引:向 LLM 暴露递增整数,内部保留从原消息图片取得的精确 URL。
* 同一 imageId 重复出现在上下文中时复用原编号,并用最新取得的 URL 刷新映射。
*/
internal class ImageIndex {
private val imageUrlByIndex = LinkedHashMap<Int, String>()
private val indexByImageId = HashMap<String, Int>()
private var counter = 0
@Synchronized
fun add(imageId: String, imageUrl: String): Int {
require(imageId.isNotBlank()) { "图片ID不能为空" }
require(imageUrl.isNotBlank()) { "图片URL不能为空" }
indexByImageId[imageId]?.let { index ->
imageUrlByIndex[index] = imageUrl
return index
}
val index = ++counter
imageUrlByIndex[index] = imageUrl
indexByImageId[imageId] = index
return index
}
@Synchronized
fun getUrl(index: Int): String? = imageUrlByIndex[index]
}
+46 -26
View File
@@ -144,7 +144,8 @@ object JChatGPT : KotlinPlugin(
private data class ConversationCache( private data class ConversationCache(
val history: MutableList<ChatMessage>, val history: MutableList<ChatMessage>,
val lastActivityAt: Int, val lastActivityAt: Int,
val replyIndex: ReplyIndex val replyIndex: ReplyIndex,
val imageIndex: ImageIndex,
) { ) {
fun isExpired(ttlSeconds: Int): Boolean { fun isExpired(ttlSeconds: Int): Boolean {
return OffsetDateTime.now().toEpochSecond().toInt() - lastActivityAt > ttlSeconds return OffsetDateTime.now().toEpochSecond().toInt() - lastActivityAt > ttlSeconds
@@ -182,10 +183,21 @@ object JChatGPT : KotlinPlugin(
/** 各会话的回复索引,startChat 开始时重建,结束时清理 */ /** 各会话的回复索引,startChat 开始时重建,结束时清理 */
private val replyIndexMap = ConcurrentMap<Long, ReplyIndex>() private val replyIndexMap = ConcurrentMap<Long, ReplyIndex>()
/** 各会话的图片索引,生命周期与回复索引、对话缓存一致。 */
private val imageIndexMap = ConcurrentMap<Long, ImageIndex>()
/** 供发言工具按编号查找被引用的历史消息 */ /** 供发言工具按编号查找被引用的历史消息 */
internal fun lookupReplyTarget(subjectId: Long, index: Int): MessageRecord? = internal fun lookupReplyTarget(subjectId: Long, index: Int): MessageRecord? =
replyIndexMap[subjectId]?.get(index) replyIndexMap[subjectId]?.get(index)
/** 将从原消息图片取得的精确 URL 登记为短编号,供历史搜索等工具追加图片引用。 */
internal fun registerImage(subjectId: Long, imageId: String, imageUrl: String): Int? =
imageIndexMap[subjectId]?.add(imageId, imageUrl)
/** 按会话内短编号获取原消息解析出的 URL,避免根据 imageId 二次构造和查询。 */
internal fun lookupImageUrl(subjectId: Long, index: Int): String? =
imageIndexMap[subjectId]?.getUrl(index)
private val shortTimeFormatter = DateTimeFormatter.ofPattern("HH:mm") private val shortTimeFormatter = DateTimeFormatter.ofPattern("HH:mm")
.withZone(ZoneOffset.systemDefault()) .withZone(ZoneOffset.systemDefault())
@@ -339,8 +351,9 @@ object JChatGPT : KotlinPlugin(
* @return 如果未获取到则返回空字符串 * @return 如果未获取到则返回空字符串
*/ */
private fun getHistory(event: MessageEvent): String { private fun getHistory(event: MessageEvent): String {
val imageIndex = imageIndexMap.getOrPut(event.subject.id) { ImageIndex() }
if (!includeHistory) { if (!includeHistory) {
return event.message.content return formatRecordContent(event.message, event.subject, imageIndex)
} }
val now = OffsetDateTime.now() val now = OffsetDateTime.now()
// 一段时间内的消息 // 一段时间内的消息
@@ -378,6 +391,7 @@ object JChatGPT : KotlinPlugin(
var lastTime = 0L var lastTime = 0L
// 本轮回复索引,逐条登记消息编号供 [n] 引用 // 本轮回复索引,逐条登记消息编号供 [n] 引用
val replyIndex = replyIndexMap.getOrPut(event.subject.id) { ReplyIndex() } val replyIndex = replyIndexMap.getOrPut(event.subject.id) { ReplyIndex() }
val imageIndex = imageIndexMap.getOrPut(event.subject.id) { ImageIndex() }
if (event is GroupMessageEvent) { if (event is GroupMessageEvent) {
if (PluginConfig.enableFavorabilitySystem) { if (PluginConfig.enableFavorabilitySystem) {
val knownUsers = history.asSequence() val knownUsers = history.asSequence()
@@ -404,12 +418,12 @@ object JChatGPT : KotlinPlugin(
} }
} }
historyText.appendLine("## 近期群消息(更早已隐藏,行首[n]为消息编号,可用于引用回复") historyText.appendLine("## 近期群消息(更早已隐藏,行首[n]为消息编号;正文[图片n]/[表情包n]中的n为识图或图片编辑编号")
for (record in history) { for (record in history) {
// 同一人发言不要反复出现这人的名字,减少上下文 // 同一人发言不要反复出现这人的名字,减少上下文
val showSender = lastId != record.fromId val showSender = lastId != record.fromId
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
appendGroupMessageRecord(historyText, record, event, replyIndex, showSender, showTime) appendGroupMessageRecord(historyText, record, event, replyIndex, imageIndex, showSender, showTime)
lastId = record.fromId lastId = record.fromId
lastTime = record.time.toLong() lastTime = record.time.toLong()
} }
@@ -428,12 +442,12 @@ object JChatGPT : KotlinPlugin(
} }
} }
historyText.appendLine("## 近期对话(更早已隐藏,行首[n]为消息编号,可用于引用回复") historyText.appendLine("## 近期对话(更早已隐藏,行首[n]为消息编号;正文[图片n]/[表情包n]中的n为识图或图片编辑编号")
for (record in history) { for (record in history) {
// 同一人发言不要反复出现这人的名字,减少上下文 // 同一人发言不要反复出现这人的名字,减少上下文
val showSender = lastId != record.fromId val showSender = lastId != record.fromId
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
appendMessageRecord(historyText, record, event, replyIndex, showSender, showTime) appendMessageRecord(historyText, record, event, replyIndex, imageIndex, showSender, showTime)
lastId = record.fromId lastId = record.fromId
lastTime = record.time.toLong() lastTime = record.time.toLong()
} }
@@ -448,11 +462,12 @@ object JChatGPT : KotlinPlugin(
* @param record 群消息记录 * @param record 群消息记录
* @param event 群消息事件 * @param event 群消息事件
*/ */
fun appendGroupMessageRecord( private fun appendGroupMessageRecord(
historyText: StringBuilder, historyText: StringBuilder,
record: MessageRecord, record: MessageRecord,
event: GroupMessageEvent, event: GroupMessageEvent,
replyIndex: ReplyIndex, replyIndex: ReplyIndex,
imageIndex: ImageIndex,
showSender: Boolean, showSender: Boolean,
showTime: Boolean, showTime: Boolean,
) { ) {
@@ -481,10 +496,10 @@ object JChatGPT : KotlinPlugin(
// 引用:用编号指针替代内联原文,避免被误认为是本人发言 // 引用:用编号指针替代内联原文,避免被误认为是本人发言
recordMessage[QuoteReply.Key]?.let { recordMessage[QuoteReply.Key]?.let {
appendQuoteMarker(historyText, it, event.subject, replyIndex) appendQuoteMarker(historyText, it, event.subject, replyIndex, imageIndex)
} }
historyText.appendLine(formatRecordContent(recordMessage, event.subject)) historyText.appendLine(formatRecordContent(recordMessage, event.subject, imageIndex))
} }
/** /**
@@ -494,7 +509,8 @@ object JChatGPT : KotlinPlugin(
sb: StringBuilder, sb: StringBuilder,
quote: QuoteReply, quote: QuoteReply,
contact: Contact, contact: Contact,
replyIndex: ReplyIndex replyIndex: ReplyIndex,
imageIndex: ImageIndex,
) { ) {
val srcIds = quote.source.ids.joinToString(",") val srcIds = quote.source.ids.joinToString(",")
val idx = replyIndex.indexOfIds(srcIds) val idx = replyIndex.indexOfIds(srcIds)
@@ -507,7 +523,7 @@ object JChatGPT : KotlinPlugin(
quote.source.fromId.toString() quote.source.fromId.toString()
} }
val snippet = quote.source.originalMessage val snippet = quote.source.originalMessage
.joinToString("", transform = ::singleMessageToText) .joinToString("") { singleMessageToText(it, imageIndex) }
.replace("\n", " ") .replace("\n", " ")
.let { if (it.length > 20) it.take(20) + "" else it } .let { if (it.length > 20) it.take(20) + "" else it }
sb.append("↩(").append(author).append(":\"").append(snippet).append("\") ") sb.append("↩(").append(author).append(":\"").append(snippet).append("\") ")
@@ -517,13 +533,13 @@ object JChatGPT : KotlinPlugin(
/** /**
* 序列化消息正文(剔除引用/源元数据,@显示为名称,转发折叠)。 * 序列化消息正文(剔除引用/源元数据,@显示为名称,转发折叠)。
*/ */
private fun formatRecordContent(chain: MessageChain, contact: Contact): String = private fun formatRecordContent(chain: MessageChain, contact: Contact, imageIndex: ImageIndex): String =
chain.asSequence() chain.asSequence()
.filterNot { it is QuoteReply || it is MessageSource } .filterNot { it is QuoteReply || it is MessageSource }
.joinToString("") { .joinToString("") {
when (it) { when (it) {
is At -> if (contact is Group) it.getDisplay(contact) else it.content is At -> if (contact is Group) it.getDisplay(contact) else it.content
else -> singleMessageToText(it) else -> singleMessageToText(it, imageIndex)
} }
} }
@@ -542,11 +558,12 @@ object JChatGPT : KotlinPlugin(
* @param record 消息记录 * @param record 消息记录
* @param event 消息事件 * @param event 消息事件
*/ */
fun appendMessageRecord( private fun appendMessageRecord(
historyText: StringBuilder, historyText: StringBuilder,
record: MessageRecord, record: MessageRecord,
event: MessageEvent, event: MessageEvent,
replyIndex: ReplyIndex, replyIndex: ReplyIndex,
imageIndex: ImageIndex,
showSender: Boolean, showSender: Boolean,
showTime: Boolean, showTime: Boolean,
) { ) {
@@ -573,24 +590,23 @@ object JChatGPT : KotlinPlugin(
} }
recordMessage[QuoteReply.Key]?.let { recordMessage[QuoteReply.Key]?.let {
appendQuoteMarker(historyText, it, event.subject, replyIndex) appendQuoteMarker(historyText, it, event.subject, replyIndex, imageIndex)
} }
historyText.appendLine(formatRecordContent(recordMessage, event.subject)) historyText.appendLine(formatRecordContent(recordMessage, event.subject, imageIndex))
} }
private fun singleMessageToText(it: SingleMessage): String { private fun singleMessageToText(it: SingleMessage, imageIndex: ImageIndex): String {
return when (it) { return when (it) {
// 完整展开合并转发内容,便于 LLM 阅读分析转发的对话(依赖大上下文+缓存,不做截断) // 完整展开合并转发内容,便于 LLM 阅读分析转发的对话(依赖大上下文+缓存,不做截断)
is ForwardMessage -> formatForward(it, 1) is ForwardMessage -> formatForward(it, 1, imageIndex)
// 图片格式化 // 图片格式化
is Image -> { is Image -> {
try { try {
val imageUrl = runBlocking { val imageUrl = runBlocking { it.queryUrl() }
it.queryUrl() val index = imageIndex.add(it.imageId, imageUrl)
} "[${if (it.isEmoji) "表情包" else "图片"}$index]"
"![${if (it.isEmoji) "表情包" else "图片"}]($imageUrl)"
} catch (e: Throwable) { } catch (e: Throwable) {
logger.warning("图片地址获取失败", e) logger.warning("图片地址获取失败", e)
it.content it.content
@@ -605,7 +621,7 @@ object JChatGPT : KotlinPlugin(
* 递归展开合并转发消息,用 Markdown 引用块表示:每加深一层嵌套多一个 `>`>、>>、>>>…)。 * 递归展开合并转发消息,用 Markdown 引用块表示:每加深一层嵌套多一个 `>`>、>>、>>>…)。
* @param depth 当前嵌套层级,从 1 开始 * @param depth 当前嵌套层级,从 1 开始
*/ */
private fun formatForward(forward: ForwardMessage, depth: Int): String = buildString { private fun formatForward(forward: ForwardMessage, depth: Int, imageIndex: ImageIndex): String = buildString {
val quote = ">".repeat(depth) + " " val quote = ">".repeat(depth) + " "
append("[转发消息·").append(forward.nodeList.size).append("") append("[转发消息·").append(forward.nodeList.size).append("")
if (forward.title.isNotEmpty()) append(':').append(forward.title) if (forward.title.isNotEmpty()) append(':').append(forward.title)
@@ -618,10 +634,10 @@ object JChatGPT : KotlinPlugin(
node.messageChain.forEach { sub -> node.messageChain.forEach { sub ->
if (sub is ForwardMessage) { if (sub is ForwardMessage) {
// 嵌套转发:层级加深,自带更深的 `>` 前缀,无需再次缩进 // 嵌套转发:层级加深,自带更深的 `>` 前缀,无需再次缩进
append(formatForward(sub, depth + 1)) append(formatForward(sub, depth + 1, imageIndex))
} else { } else {
// 其它内容:多行正文对齐到当前引用层级 // 其它内容:多行正文对齐到当前引用层级
append(singleMessageToText(sub).replace("\n", "\n$quote")) append(singleMessageToText(sub, imageIndex).replace("\n", "\n$quote"))
} }
} }
} }
@@ -658,7 +674,9 @@ object JChatGPT : KotlinPlugin(
// 回复索引与对话上下文同寿命:复用缓存时沿用旧索引,保证 LLM 看到的 [n] 编号连续不串号; // 回复索引与对话上下文同寿命:复用缓存时沿用旧索引,保证 LLM 看到的 [n] 编号连续不串号;
// 否则新建(供 sendSingleMessage 的 replyTo 按编号引用历史消息) // 否则新建(供 sendSingleMessage 的 replyTo 按编号引用历史消息)
val replyIndex = if (reuseCache) cache!!.replyIndex else ReplyIndex() val replyIndex = if (reuseCache) cache!!.replyIndex else ReplyIndex()
val imageIndex = if (reuseCache) cache!!.imageIndex else ImageIndex()
replyIndexMap[subjectId] = replyIndex replyIndexMap[subjectId] = replyIndex
imageIndexMap[subjectId] = imageIndex
val history = if (reuseCache) { val history = if (reuseCache) {
// 缓存有效,复用历史 // 缓存有效,复用历史
logger.info("使用缓存的对话上下文,包含 ${cache!!.history.size} 条互动消息") logger.info("使用缓存的对话上下文,包含 ${cache!!.history.size} 条互动消息")
@@ -868,7 +886,8 @@ object JChatGPT : KotlinPlugin(
contextCache[subjectId] = ConversationCache( contextCache[subjectId] = ConversationCache(
history = history, history = history,
lastActivityAt = startedAt, lastActivityAt = startedAt,
replyIndex = replyIndex replyIndex = replyIndex,
imageIndex = imageIndex,
) )
logger.debug("已保存对话上下文到缓存") logger.debug("已保存对话上下文到缓存")
} }
@@ -901,6 +920,7 @@ object JChatGPT : KotlinPlugin(
} finally { } finally {
// 清理本轮回复索引 // 清理本轮回复索引
replyIndexMap.remove(event.subject.id) replyIndexMap.remove(event.subject.id)
imageIndexMap.remove(event.subject.id)
// 一段时间后才允许再次提问,防止高频对话 // 一段时间后才允许再次提问,防止高频对话
launch { launch {
delay(500.milliseconds) delay(500.milliseconds)
+15 -7
View File
@@ -13,31 +13,34 @@ import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.add import kotlinx.serialization.json.add
import kotlinx.serialization.json.addJsonObject import kotlinx.serialization.json.addJsonObject
import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.int
import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig import top.jie65535.mirai.PluginConfig
class ImageAgent : BaseAgent( class ImageAgent : BaseAgent(
tool = Tool.function( tool = Tool.function(
name = "imageAgent", name = "imageAgent",
description = "调用千问图像模型生成或编辑图片。不传 image_urls 即纯文生图;" + description = "调用千问图像模型生成或编辑图片。不传 image_indices 即纯文生图;" +
"传 1~3 张图片可进行编辑、修改或多图融合。" + "传 1~3 张图片可进行编辑、修改或多图融合。" +
"备注:该方法成本较高,非必要尽量不要调用。" + "备注:该方法成本较高,非必要尽量不要调用。" +
"编辑图片前无需识别图片内容,模型自己会理解图片内容。", "编辑图片前无需识别图片内容,模型自己会理解图片内容。",
parameters = Parameters.buildJsonObject { parameters = Parameters.buildJsonObject {
put("type", "object") put("type", "object")
putJsonObject("properties") { putJsonObject("properties") {
putJsonObject("image_urls") { putJsonObject("image_indices") {
put("type", "array") put("type", "array")
putJsonObject("items") { putJsonObject("items") {
put("type", "string") put("type", "integer")
put("minimum", 1)
} }
put("description", "参考图片地址列表,可传 0~3 张。" + put("description", "用户消息中[图片n]或[表情包n]标记的参考图片编号,可传 0~3 张。" +
"不传或为空即纯文生图;传 1 张为编辑;多张为融合,输出比例与最后一张对齐。") "不传或为空即纯文生图;传 1 张为编辑;多张为融合,输出比例与最后一张对齐。")
} }
putJsonObject("prompt") { putJsonObject("prompt") {
@@ -61,12 +64,17 @@ class ImageAgent : BaseAgent(
override val loadingMessage: String override val loadingMessage: String
get() = "作图中..." get() = "作图中..."
override suspend fun execute(args: JsonObject?): String { override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
requireNotNull(args) requireNotNull(args)
val prompt = args.getValue("prompt").jsonPrimitive.content val prompt = args.getValue("prompt").jsonPrimitive.content
val imageUrls = args["image_urls"]?.jsonArray val imageIndices = args["image_indices"]?.jsonArray
?.map { it.jsonPrimitive.content } ?.map { it.jsonPrimitive.int }
?: emptyList() ?: emptyList()
require(imageIndices.size <= 3) { "参考图片最多只能传3张" }
val imageUrls = imageIndices.map { imageIndex ->
JChatGPT.lookupImageUrl(event.subject.id, imageIndex)
?: throw IllegalArgumentException("图片编号[$imageIndex]不存在或已失效")
}
val response = httpClient.post(API_URL) { val response = httpClient.post(API_URL) {
contentType(ContentType("application", "json")) contentType(ContentType("application", "json"))
+6 -4
View File
@@ -172,19 +172,21 @@ class SearchChatHistory : BaseAgent(
.append("") .append("")
} }
for (msg in record.toMessageChain()) { for (msg in record.toMessageChain()) {
sb.append(singleMessageToText(msg)) sb.append(singleMessageToText(msg, event.subject.id))
} }
sb.appendLine() sb.appendLine()
lastFromId = record.fromId lastFromId = record.fromId
} }
} }
private suspend fun singleMessageToText(msg: SingleMessage): String { private suspend fun singleMessageToText(msg: SingleMessage, subjectId: Long): String {
return when (msg) { return when (msg) {
is Image -> { is Image -> {
try { try {
val url = msg.queryUrl() val imageUrl = msg.queryUrl()
"![${if (msg.isEmoji) "表情包" else "图片"}]($url)" val index = JChatGPT.registerImage(subjectId, msg.imageId, imageUrl)
?: return msg.content
"[${if (msg.isEmoji) "表情包" else "图片"}$index]"
} catch (_: Throwable) { } catch (_: Throwable) {
msg.content msg.content
} }
+24 -11
View File
@@ -11,6 +11,7 @@ import com.aallam.openai.api.model.ModelId
import io.ktor.client.plugins.ClientRequestException import io.ktor.client.plugins.ClientRequestException
import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.add import kotlinx.serialization.json.add
import kotlinx.serialization.json.int
import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put import kotlinx.serialization.json.put
@@ -20,6 +21,7 @@ import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.sync.withPermit
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.LargeLanguageModels import top.jie65535.mirai.LargeLanguageModels
import top.jie65535.mirai.PluginConfig import top.jie65535.mirai.PluginConfig
@@ -32,13 +34,14 @@ class VisualAgent : BaseAgent(
parameters = Parameters.buildJsonObject { parameters = Parameters.buildJsonObject {
put("type", "object") put("type", "object")
putJsonObject("properties") { putJsonObject("properties") {
putJsonObject("image_urls") { putJsonObject("image_indices") {
put("type", "array") put("type", "array")
put("description", "图片地址数组,按用户消息中的出现顺序传入") put("description", "用户消息中[图片n]或[表情包n]标记的图片编号数组,按需要理解的顺序传入")
put("minItems", 1) put("minItems", 1)
put("maxItems", MAX_SOURCE_IMAGES) put("maxItems", MAX_SOURCE_IMAGES)
putJsonObject("items") { putJsonObject("items") {
put("type", "string") put("type", "integer")
put("minimum", 1)
} }
} }
putJsonObject("prompt") { putJsonObject("prompt") {
@@ -47,7 +50,7 @@ class VisualAgent : BaseAgent(
} }
} }
putJsonArray("required") { putJsonArray("required") {
add("image_urls") add("image_indices")
add("prompt") add("prompt")
} }
} }
@@ -62,25 +65,35 @@ class VisualAgent : BaseAgent(
override val isEnabled: Boolean override val isEnabled: Boolean
get() = LargeLanguageModels.visual != null get() = LargeLanguageModels.visual != null
override suspend fun execute(args: JsonObject?): String { override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
requireNotNull(args) requireNotNull(args)
val llm = LargeLanguageModels.visual ?: return "未配置llm,无法进行识别。" val llm = LargeLanguageModels.visual ?: return "未配置llm,无法进行识别。"
val imageUrls = args["image_urls"]?.jsonArray val imageIndices = args["image_indices"]?.jsonArray
?.map { it.jsonPrimitive.content } ?.map { it.jsonPrimitive.int }
?.filter { it.isNotBlank() }
?.ifEmpty { null } ?.ifEmpty { null }
?: throw IllegalArgumentException("至少需要提供一张图片") ?: throw IllegalArgumentException("至少需要提供一张图片")
require(imageUrls.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" } require(imageIndices.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" }
val imageUrls = imageIndices.map { imageIndex ->
JChatGPT.lookupImageUrl(event.subject.id, imageIndex)
?: throw IllegalArgumentException("图片编号[$imageIndex]不存在或已失效")
}
val prompt = args.getValue("prompt").jsonPrimitive.content val prompt = args.getValue("prompt").jsonPrimitive.content
return concurrencyLimiter.withPermit { return concurrencyLimiter.withPermit {
val imageGroups = imageUrls.mapIndexed { index, imageUrl -> val imageGroups = imageUrls.mapIndexed { index, imageUrl ->
if (PluginConfig.visualImageBase64Enabled) { if (PluginConfig.visualImageBase64Enabled) {
val resolved = imageResolver.resolve(imageUrl)
val host = runCatching { URI(imageUrl).host }.getOrNull() ?: "unknown" val host = runCatching { URI(imageUrl).host }.getOrNull() ?: "unknown"
val resolved = try {
imageResolver.resolve(imageUrl)
} catch (e: Throwable) {
JChatGPT.logger.error(
"视觉图片下载失败: image=${imageIndices[index]}, url=$imageUrl"
)
throw e
}
val mimeTypes = resolved.images.map { it.mimeType }.distinct().joinToString() val mimeTypes = resolved.images.map { it.mimeType }.distinct().joinToString()
JChatGPT.logger.info( JChatGPT.logger.info(
"视觉图片已本地化: source=${index + 1}/${imageUrls.size}, host=$host, " + "视觉图片已本地化: image=${imageIndices[index]}, source=${index + 1}/${imageUrls.size}, host=$host, " +
"parts=${resolved.images.size}, mime=$mimeTypes, " + "parts=${resolved.images.size}, mime=$mimeTypes, " +
"sourceBytes=${resolved.sourceSize}, payloadChars=${resolved.payloadSize}, " + "sourceBytes=${resolved.sourceSize}, payloadChars=${resolved.payloadSize}, " +
"transcoded=${resolved.transcoded}" "transcoded=${resolved.transcoded}"
+35 -2
View File
@@ -108,8 +108,15 @@ internal class VisualImageResolver {
} }
if (!response.status.isSuccess()) { if (!response.status.isSuccess()) {
response.bodyAsChannel().cancel() val errorBody = readErrorBody(response.bodyAsChannel())
throw IllegalArgumentException("图片下载失败:HTTP ${response.status.value}") val errorNumber = response.headers["X-ErrNo"]
throw IllegalArgumentException(
buildString {
append("图片下载失败:HTTP ").append(response.status.value)
if (!errorNumber.isNullOrBlank()) append("X-ErrNo=").append(errorNumber)
if (errorBody.isNotBlank()) append(",响应=").append(errorBody)
}
)
} }
val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull()
@@ -190,6 +197,31 @@ internal class VisualImageResolver {
} }
} }
private suspend fun readErrorBody(channel: io.ktor.utils.io.ByteReadChannel): String {
val output = ByteArrayOutputStream()
val buffer = ByteArray(DOWNLOAD_BUFFER_SIZE)
var total = 0
try {
while (total < MAX_ERROR_RESPONSE_BYTES) {
val count = channel.readAvailable(
buffer,
0,
min(buffer.size, MAX_ERROR_RESPONSE_BYTES - total)
)
if (count < 0) break
if (count == 0) continue
output.write(buffer, 0, count)
total += count
}
} finally {
channel.cancel()
}
return output.toByteArray()
.toString(Charsets.UTF_8)
.replace(Regex("[\\r\\n]+"), " ")
.trim()
}
private fun buildPayload(bytes: ByteArray, mimeType: String): ImagePayload { private fun buildPayload(bytes: ByteArray, mimeType: String): ImagePayload {
val encoded = Base64.getEncoder().encodeToString(bytes) val encoded = Base64.getEncoder().encodeToString(bytes)
val dataUrl = "data:$mimeType;base64,$encoded" val dataUrl = "data:$mimeType;base64,$encoded"
@@ -612,6 +644,7 @@ internal class VisualImageResolver {
private const val LONG_IMAGE_MAX_OVERLAP = 256 private const val LONG_IMAGE_MAX_OVERLAP = 256
private const val MAX_LONG_IMAGE_PARTS = 16 private const val MAX_LONG_IMAGE_PARTS = 16
private const val MAX_COMPRESSION_ROUNDS = 6 private const val MAX_COMPRESSION_ROUNDS = 6
private const val MAX_ERROR_RESPONSE_BYTES = 4096
private const val DOWNSCALE_FACTOR = 0.82 private const val DOWNSCALE_FACTOR = 0.82
private const val USER_AGENT = "JChatGPT/1.13 image-fetcher" private const val USER_AGENT = "JChatGPT/1.13 image-fetcher"
private val JPEG_QUALITIES = floatArrayOf(0.90f, 0.82f, 0.74f, 0.66f) private val JPEG_QUALITIES = floatArrayOf(0.90f, 0.82f, 0.74f, 0.66f)
+32
View File
@@ -0,0 +1,32 @@
package top.jie65535.mirai
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
class ImageIndexTest {
@Test
fun `assigns short sequential indices`() {
val index = ImageIndex()
assertEquals(1, index.add("image-a", "https://example.com/a"))
assertEquals(2, index.add("image-b", "https://example.com/b"))
assertEquals("https://example.com/a", index.getUrl(1))
assertEquals("https://example.com/b", index.getUrl(2))
}
@Test
fun `reuses index for repeated image id`() {
val index = ImageIndex()
assertEquals(1, index.add("image-a", "https://example.com/a-old"))
assertEquals(1, index.add("image-a", "https://example.com/a-new"))
assertEquals(2, index.add("image-b", "https://example.com/b"))
assertEquals("https://example.com/a-new", index.getUrl(1))
}
@Test
fun `returns null for unknown index`() {
assertNull(ImageIndex().getUrl(1))
}
}