jie65535/JChatGPT
1
0
Fork 0
mirror of https://github.com/jie65535/JChatGPT.git synced 2026-09-15 02:56:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

tools: resolve images by short references

Browse Source
This commit is contained in:
jie65535
2026-07-27 14:01:12 +08:00
parent a4f5bad322
commit c328a798f7
7 changed files with 187 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/kotlin/ImageIndex.kt
+29
View File
@@ -0,0 +1,29 @@
package top.jie65535.mirai
/**
* 会话内图片短索引：向 LLM 暴露递增整数，内部保留从原消息图片取得的精确 URL。
* 同一 imageId 重复出现在上下文中时复用原编号，并用最新取得的 URL 刷新映射。
*/
internal class ImageIndex {
private val imageUrlByIndex = LinkedHashMap<Int, String>()
private val indexByImageId = HashMap<String, Int>()
private var counter = 0
@Synchronized
fun add(imageId: String, imageUrl: String): Int {
require(imageId.isNotBlank()) { "图片ID不能为空" }
require(imageUrl.isNotBlank()) { "图片URL不能为空" }
indexByImageId[imageId]?.let { index ->
imageUrlByIndex[index] = imageUrl
return index
}
val index = ++counter
imageUrlByIndex[index] = imageUrl
indexByImageId[imageId] = index
return index
}
@Synchronized
fun getUrl(index: Int): String? = imageUrlByIndex[index]
}
src/main/kotlin/JChatGPT.kt
+46 -26
View File
@@ -144,7 +144,8 @@ object JChatGPT : KotlinPlugin(
private data class ConversationCache(
val history: MutableList<ChatMessage>,
val lastActivityAt: Int,
val replyIndex: ReplyIndex
val replyIndex: ReplyIndex,
val imageIndex: ImageIndex,
) {
fun isExpired(ttlSeconds: Int): Boolean {
return OffsetDateTime.now().toEpochSecond().toInt() - lastActivityAt > ttlSeconds
@@ -182,10 +183,21 @@ object JChatGPT : KotlinPlugin(
/** 各会话的回复索引，startChat 开始时重建，结束时清理 */
private val replyIndexMap = ConcurrentMap<Long, ReplyIndex>()
/** 各会话的图片索引，生命周期与回复索引、对话缓存一致。 */
private val imageIndexMap = ConcurrentMap<Long, ImageIndex>()
/** 供发言工具按编号查找被引用的历史消息 */
internal fun lookupReplyTarget(subjectId: Long, index: Int): MessageRecord? =
replyIndexMap[subjectId]?.get(index)
/** 将从原消息图片取得的精确 URL 登记为短编号，供历史搜索等工具追加图片引用。 */
internal fun registerImage(subjectId: Long, imageId: String, imageUrl: String): Int? =
imageIndexMap[subjectId]?.add(imageId, imageUrl)
/** 按会话内短编号获取原消息解析出的 URL，避免根据 imageId 二次构造和查询。 */
internal fun lookupImageUrl(subjectId: Long, index: Int): String? =
imageIndexMap[subjectId]?.getUrl(index)
private val shortTimeFormatter = DateTimeFormatter.ofPattern("HH:mm")
.withZone(ZoneOffset.systemDefault())
@@ -339,8 +351,9 @@ object JChatGPT : KotlinPlugin(
* @return 如果未获取到则返回空字符串
*/
private fun getHistory(event: MessageEvent): String {
val imageIndex = imageIndexMap.getOrPut(event.subject.id) { ImageIndex() }
if (!includeHistory) {
return event.message.content
return formatRecordContent(event.message, event.subject, imageIndex)
}
val now = OffsetDateTime.now()
// 一段时间内的消息
@@ -378,6 +391,7 @@ object JChatGPT : KotlinPlugin(
var lastTime = 0L
// 本轮回复索引，逐条登记消息编号供 [n] 引用
val replyIndex = replyIndexMap.getOrPut(event.subject.id) { ReplyIndex() }
val imageIndex = imageIndexMap.getOrPut(event.subject.id) { ImageIndex() }
if (event is GroupMessageEvent) {
if (PluginConfig.enableFavorabilitySystem) {
val knownUsers = history.asSequence()
@@ -404,12 +418,12 @@ object JChatGPT : KotlinPlugin(
}
}
historyText.appendLine("## 近期群消息（更早已隐藏，行首[n]为消息编号，可用于引用回复）")
historyText.appendLine("## 近期群消息（更早已隐藏，行首[n]为消息编号；正文[图片n]/[表情包n]中的n为识图或图片编辑编号）")
for (record in history) {
// 同一人发言不要反复出现这人的名字，减少上下文
val showSender = lastId != record.fromId
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
appendGroupMessageRecord(historyText, record, event, replyIndex, showSender, showTime)
appendGroupMessageRecord(historyText, record, event, replyIndex, imageIndex, showSender, showTime)
lastId = record.fromId
lastTime = record.time.toLong()
}
@@ -428,12 +442,12 @@ object JChatGPT : KotlinPlugin(
}
}
historyText.appendLine("## 近期对话（更早已隐藏，行首[n]为消息编号，可用于引用回复）")
historyText.appendLine("## 近期对话（更早已隐藏，行首[n]为消息编号；正文[图片n]/[表情包n]中的n为识图或图片编辑编号）")
for (record in history) {
// 同一人发言不要反复出现这人的名字，减少上下文
val showSender = lastId != record.fromId
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
appendMessageRecord(historyText, record, event, replyIndex, showSender, showTime)
appendMessageRecord(historyText, record, event, replyIndex, imageIndex, showSender, showTime)
lastId = record.fromId
lastTime = record.time.toLong()
}
@@ -448,11 +462,12 @@ object JChatGPT : KotlinPlugin(
* @param record 群消息记录
* @param event 群消息事件
*/
fun appendGroupMessageRecord(
private fun appendGroupMessageRecord(
historyText: StringBuilder,
record: MessageRecord,
event: GroupMessageEvent,
replyIndex: ReplyIndex,
imageIndex: ImageIndex,
showSender: Boolean,
showTime: Boolean,
) {
@@ -481,10 +496,10 @@ object JChatGPT : KotlinPlugin(
// 引用：用编号指针替代内联原文，避免被误认为是本人发言
recordMessage[QuoteReply.Key]?.let {
appendQuoteMarker(historyText, it, event.subject, replyIndex)
appendQuoteMarker(historyText, it, event.subject, replyIndex, imageIndex)
}
historyText.appendLine(formatRecordContent(recordMessage, event.subject))
historyText.appendLine(formatRecordContent(recordMessage, event.subject, imageIndex))
}
/**
@@ -494,7 +509,8 @@ object JChatGPT : KotlinPlugin(
sb: StringBuilder,
quote: QuoteReply,
contact: Contact,
replyIndex: ReplyIndex
replyIndex: ReplyIndex,
imageIndex: ImageIndex,
) {
val srcIds = quote.source.ids.joinToString(",")
val idx = replyIndex.indexOfIds(srcIds)
@@ -507,7 +523,7 @@ object JChatGPT : KotlinPlugin(
quote.source.fromId.toString()
}
val snippet = quote.source.originalMessage
.joinToString("", transform = ::singleMessageToText)
.joinToString("") { singleMessageToText(it, imageIndex) }
.replace("\n", " ")
.let { if (it.length > 20) it.take(20) + "…" else it }
sb.append("↩(").append(author).append(":\"").append(snippet).append("\") ")
@@ -517,13 +533,13 @@ object JChatGPT : KotlinPlugin(
/**
* 序列化消息正文（剔除引用/源元数据，@显示为名称，转发折叠）。
*/
private fun formatRecordContent(chain: MessageChain, contact: Contact): String =
private fun formatRecordContent(chain: MessageChain, contact: Contact, imageIndex: ImageIndex): String =
chain.asSequence()
.filterNot { it is QuoteReply || it is MessageSource }
.joinToString("") {
when (it) {
is At -> if (contact is Group) it.getDisplay(contact) else it.content
else -> singleMessageToText(it)
else -> singleMessageToText(it, imageIndex)
}
}
@@ -542,11 +558,12 @@ object JChatGPT : KotlinPlugin(
* @param record 消息记录
* @param event 消息事件
*/
fun appendMessageRecord(
private fun appendMessageRecord(
historyText: StringBuilder,
record: MessageRecord,
event: MessageEvent,
replyIndex: ReplyIndex,
imageIndex: ImageIndex,
showSender: Boolean,
showTime: Boolean,
) {
@@ -573,24 +590,23 @@ object JChatGPT : KotlinPlugin(
}
recordMessage[QuoteReply.Key]?.let {
appendQuoteMarker(historyText, it, event.subject, replyIndex)
appendQuoteMarker(historyText, it, event.subject, replyIndex, imageIndex)
}
historyText.appendLine(formatRecordContent(recordMessage, event.subject))
historyText.appendLine(formatRecordContent(recordMessage, event.subject, imageIndex))
}
private fun singleMessageToText(it: SingleMessage): String {
private fun singleMessageToText(it: SingleMessage, imageIndex: ImageIndex): String {
return when (it) {
// 完整展开合并转发内容，便于 LLM 阅读分析转发的对话（依赖大上下文+缓存，不做截断）
is ForwardMessage -> formatForward(it, 1)
is ForwardMessage -> formatForward(it, 1, imageIndex)
// 图片格式化
is Image -> {
try {
val imageUrl = runBlocking {
it.queryUrl()
}
"![${if (it.isEmoji) "表情包" else "图片"}]($imageUrl)"
val imageUrl = runBlocking { it.queryUrl() }
val index = imageIndex.add(it.imageId, imageUrl)
"[${if (it.isEmoji) "表情包" else "图片"}$index]"
} catch (e: Throwable) {
logger.warning("图片地址获取失败", e)
it.content
@@ -605,7 +621,7 @@ object JChatGPT : KotlinPlugin(
* 递归展开合并转发消息，用 Markdown 引用块表示：每加深一层嵌套多一个 `>`（>、>>、>>>…）。
* @param depth 当前嵌套层级，从 1 开始
*/
private fun formatForward(forward: ForwardMessage, depth: Int): String = buildString {
private fun formatForward(forward: ForwardMessage, depth: Int, imageIndex: ImageIndex): String = buildString {
val quote = ">".repeat(depth) + " "
append("[转发消息·").append(forward.nodeList.size).append("条")
if (forward.title.isNotEmpty()) append(':').append(forward.title)
@@ -618,10 +634,10 @@ object JChatGPT : KotlinPlugin(
node.messageChain.forEach { sub ->
if (sub is ForwardMessage) {
// 嵌套转发：层级加深，自带更深的 `>` 前缀，无需再次缩进
append(formatForward(sub, depth + 1))
append(formatForward(sub, depth + 1, imageIndex))
} else {
// 其它内容：多行正文对齐到当前引用层级
append(singleMessageToText(sub).replace("\n", "\n$quote"))
append(singleMessageToText(sub, imageIndex).replace("\n", "\n$quote"))
}
}
}
@@ -658,7 +674,9 @@ object JChatGPT : KotlinPlugin(
// 回复索引与对话上下文同寿命：复用缓存时沿用旧索引，保证 LLM 看到的 [n] 编号连续不串号；
// 否则新建（供 sendSingleMessage 的 replyTo 按编号引用历史消息）
val replyIndex = if (reuseCache) cache!!.replyIndex else ReplyIndex()
val imageIndex = if (reuseCache) cache!!.imageIndex else ImageIndex()
replyIndexMap[subjectId] = replyIndex
imageIndexMap[subjectId] = imageIndex
val history = if (reuseCache) {
// 缓存有效，复用历史
logger.info("使用缓存的对话上下文，包含 ${cache!!.history.size} 条互动消息")
@@ -868,7 +886,8 @@ object JChatGPT : KotlinPlugin(
contextCache[subjectId] = ConversationCache(
history = history,
lastActivityAt = startedAt,
replyIndex = replyIndex
replyIndex = replyIndex,
imageIndex = imageIndex,
)
logger.debug("已保存对话上下文到缓存")
}
@@ -901,6 +920,7 @@ object JChatGPT : KotlinPlugin(
} finally {
// 清理本轮回复索引
replyIndexMap.remove(event.subject.id)
imageIndexMap.remove(event.subject.id)
// 一段时间后才允许再次提问，防止高频对话
launch {
delay(500.milliseconds)
src/main/kotlin/tools/ImageAgent.kt
+15 -7
View File
@@ -13,31 +13,34 @@ import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.add
import kotlinx.serialization.json.addJsonObject
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.int
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
class ImageAgent : BaseAgent(
tool = Tool.function(
name = "imageAgent",
description = "调用千问图像模型生成或编辑图片。不传 image_urls 即纯文生图；" +
description = "调用千问图像模型生成或编辑图片。不传 image_indices 即纯文生图；" +
"传 1~3 张图片可进行编辑、修改或多图融合。" +
"备注：该方法成本较高，非必要尽量不要调用。" +
"编辑图片前无需识别图片内容，模型自己会理解图片内容。",
parameters = Parameters.buildJsonObject {
put("type", "object")
putJsonObject("properties") {
putJsonObject("image_urls") {
putJsonObject("image_indices") {
put("type", "array")
putJsonObject("items") {
put("type", "string")
put("type", "integer")
put("minimum", 1)
}
put("description", "参考图片地址列表，可传 0~3 张。" +
put("description", "用户消息中[图片n]或[表情包n]标记的参考图片编号，可传 0~3 张。" +
"不传或为空即纯文生图；传 1 张为编辑；多张为融合，输出比例与最后一张对齐。")
}
putJsonObject("prompt") {
@@ -61,12 +64,17 @@ class ImageAgent : BaseAgent(
override val loadingMessage: String
get() = "作图中..."
override suspend fun execute(args: JsonObject?): String {
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
requireNotNull(args)
val prompt = args.getValue("prompt").jsonPrimitive.content
val imageUrls = args["image_urls"]?.jsonArray
?.map { it.jsonPrimitive.content }
val imageIndices = args["image_indices"]?.jsonArray
?.map { it.jsonPrimitive.int }
?: emptyList()
require(imageIndices.size <= 3) { "参考图片最多只能传3张" }
val imageUrls = imageIndices.map { imageIndex ->
JChatGPT.lookupImageUrl(event.subject.id, imageIndex)
?: throw IllegalArgumentException("图片编号[$imageIndex]不存在或已失效")
}
val response = httpClient.post(API_URL) {
contentType(ContentType("application", "json"))
src/main/kotlin/tools/SearchChatHistory.kt
+6 -4
View File
@@ -172,19 +172,21 @@ class SearchChatHistory : BaseAgent(
.append("：")
}
for (msg in record.toMessageChain()) {
sb.append(singleMessageToText(msg))
sb.append(singleMessageToText(msg, event.subject.id))
}
sb.appendLine()
lastFromId = record.fromId
}
}
private suspend fun singleMessageToText(msg: SingleMessage): String {
private suspend fun singleMessageToText(msg: SingleMessage, subjectId: Long): String {
return when (msg) {
is Image -> {
try {
val url = msg.queryUrl()
"![${if (msg.isEmoji) "表情包" else "图片"}]($url)"
val imageUrl = msg.queryUrl()
val index = JChatGPT.registerImage(subjectId, msg.imageId, imageUrl)
?: return msg.content
"[${if (msg.isEmoji) "表情包" else "图片"}$index]"
} catch (_: Throwable) {
msg.content
}
src/main/kotlin/tools/VisualAgent.kt
+24 -11
View File
@@ -11,6 +11,7 @@ import com.aallam.openai.api.model.ModelId
import io.ktor.client.plugins.ClientRequestException
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.add
import kotlinx.serialization.json.int
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
@@ -20,6 +21,7 @@ import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.delay
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.LargeLanguageModels
import top.jie65535.mirai.PluginConfig
@@ -32,13 +34,14 @@ class VisualAgent : BaseAgent(
parameters = Parameters.buildJsonObject {
put("type", "object")
putJsonObject("properties") {
putJsonObject("image_urls") {
putJsonObject("image_indices") {
put("type", "array")
put("description", "图片地址数组，按用户消息中的出现顺序传入")
put("description", "用户消息中[图片n]或[表情包n]标记的图片编号数组，按需要理解的顺序传入")
put("minItems", 1)
put("maxItems", MAX_SOURCE_IMAGES)
putJsonObject("items") {
put("type", "string")
put("type", "integer")
put("minimum", 1)
}
}
putJsonObject("prompt") {
@@ -47,7 +50,7 @@ class VisualAgent : BaseAgent(
}
}
putJsonArray("required") {
add("image_urls")
add("image_indices")
add("prompt")
}
}
@@ -62,25 +65,35 @@ class VisualAgent : BaseAgent(
override val isEnabled: Boolean
get() = LargeLanguageModels.visual != null
override suspend fun execute(args: JsonObject?): String {
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
requireNotNull(args)
val llm = LargeLanguageModels.visual ?: return "未配置llm，无法进行识别。"
val imageUrls = args["image_urls"]?.jsonArray
?.map { it.jsonPrimitive.content }
?.filter { it.isNotBlank() }
val imageIndices = args["image_indices"]?.jsonArray
?.map { it.jsonPrimitive.int }
?.ifEmpty { null }
?: throw IllegalArgumentException("至少需要提供一张图片")
require(imageUrls.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" }
require(imageIndices.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" }
val imageUrls = imageIndices.map { imageIndex ->
JChatGPT.lookupImageUrl(event.subject.id, imageIndex)
?: throw IllegalArgumentException("图片编号[$imageIndex]不存在或已失效")
}
val prompt = args.getValue("prompt").jsonPrimitive.content
return concurrencyLimiter.withPermit {
val imageGroups = imageUrls.mapIndexed { index, imageUrl ->
if (PluginConfig.visualImageBase64Enabled) {
val resolved = imageResolver.resolve(imageUrl)
val host = runCatching { URI(imageUrl).host }.getOrNull() ?: "unknown"
val resolved = try {
imageResolver.resolve(imageUrl)
} catch (e: Throwable) {
JChatGPT.logger.error(
"视觉图片下载失败: image=${imageIndices[index]}, url=$imageUrl"
)
throw e
}
val mimeTypes = resolved.images.map { it.mimeType }.distinct().joinToString()
JChatGPT.logger.info(
"视觉图片已本地化: source=${index + 1}/${imageUrls.size}, host=$host, " +
"视觉图片已本地化: image=${imageIndices[index]}, source=${index + 1}/${imageUrls.size}, host=$host, " +
"parts=${resolved.images.size}, mime=$mimeTypes, " +
"sourceBytes=${resolved.sourceSize}, payloadChars=${resolved.payloadSize}, " +
"transcoded=${resolved.transcoded}"
src/main/kotlin/tools/VisualImageResolver.kt
+35 -2
View File
@@ -108,8 +108,15 @@ internal class VisualImageResolver {
}
if (!response.status.isSuccess()) {
response.bodyAsChannel().cancel()
throw IllegalArgumentException("图片下载失败：HTTP ${response.status.value}")
val errorBody = readErrorBody(response.bodyAsChannel())
val errorNumber = response.headers["X-ErrNo"]
throw IllegalArgumentException(
buildString {
append("图片下载失败：HTTP ").append(response.status.value)
if (!errorNumber.isNullOrBlank()) append("，X-ErrNo=").append(errorNumber)
if (errorBody.isNotBlank()) append("，响应=").append(errorBody)
}
)
}
val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull()
@@ -190,6 +197,31 @@ internal class VisualImageResolver {
}
}
private suspend fun readErrorBody(channel: io.ktor.utils.io.ByteReadChannel): String {
val output = ByteArrayOutputStream()
val buffer = ByteArray(DOWNLOAD_BUFFER_SIZE)
var total = 0
try {
while (total < MAX_ERROR_RESPONSE_BYTES) {
val count = channel.readAvailable(
buffer,
0,
min(buffer.size, MAX_ERROR_RESPONSE_BYTES - total)
)
if (count < 0) break
if (count == 0) continue
output.write(buffer, 0, count)
total += count
}
} finally {
channel.cancel()
}
return output.toByteArray()
.toString(Charsets.UTF_8)
.replace(Regex("[\\r\\n]+"), " ")
.trim()
}
private fun buildPayload(bytes: ByteArray, mimeType: String): ImagePayload {
val encoded = Base64.getEncoder().encodeToString(bytes)
val dataUrl = "data:$mimeType;base64,$encoded"
@@ -612,6 +644,7 @@ internal class VisualImageResolver {
private const val LONG_IMAGE_MAX_OVERLAP = 256
private const val MAX_LONG_IMAGE_PARTS = 16
private const val MAX_COMPRESSION_ROUNDS = 6
private const val MAX_ERROR_RESPONSE_BYTES = 4096
private const val DOWNSCALE_FACTOR = 0.82
private const val USER_AGENT = "JChatGPT/1.13 image-fetcher"
private val JPEG_QUALITIES = floatArrayOf(0.90f, 0.82f, 0.74f, 0.66f)