mirror of
https://github.com/jie65535/JChatGPT.git
synced 2026-09-22 03:06:10 +08:00
tools: resolve images by short references
This commit is contained in:
@@ -0,0 +1,29 @@
|
|||||||
|
package top.jie65535.mirai
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 会话内图片短索引:向 LLM 暴露递增整数,内部保留从原消息图片取得的精确 URL。
|
||||||
|
* 同一 imageId 重复出现在上下文中时复用原编号,并用最新取得的 URL 刷新映射。
|
||||||
|
*/
|
||||||
|
internal class ImageIndex {
|
||||||
|
private val imageUrlByIndex = LinkedHashMap<Int, String>()
|
||||||
|
private val indexByImageId = HashMap<String, Int>()
|
||||||
|
private var counter = 0
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
fun add(imageId: String, imageUrl: String): Int {
|
||||||
|
require(imageId.isNotBlank()) { "图片ID不能为空" }
|
||||||
|
require(imageUrl.isNotBlank()) { "图片URL不能为空" }
|
||||||
|
indexByImageId[imageId]?.let { index ->
|
||||||
|
imageUrlByIndex[index] = imageUrl
|
||||||
|
return index
|
||||||
|
}
|
||||||
|
|
||||||
|
val index = ++counter
|
||||||
|
imageUrlByIndex[index] = imageUrl
|
||||||
|
indexByImageId[imageId] = index
|
||||||
|
return index
|
||||||
|
}
|
||||||
|
|
||||||
|
@Synchronized
|
||||||
|
fun getUrl(index: Int): String? = imageUrlByIndex[index]
|
||||||
|
}
|
||||||
+46
-26
@@ -144,7 +144,8 @@ object JChatGPT : KotlinPlugin(
|
|||||||
private data class ConversationCache(
|
private data class ConversationCache(
|
||||||
val history: MutableList<ChatMessage>,
|
val history: MutableList<ChatMessage>,
|
||||||
val lastActivityAt: Int,
|
val lastActivityAt: Int,
|
||||||
val replyIndex: ReplyIndex
|
val replyIndex: ReplyIndex,
|
||||||
|
val imageIndex: ImageIndex,
|
||||||
) {
|
) {
|
||||||
fun isExpired(ttlSeconds: Int): Boolean {
|
fun isExpired(ttlSeconds: Int): Boolean {
|
||||||
return OffsetDateTime.now().toEpochSecond().toInt() - lastActivityAt > ttlSeconds
|
return OffsetDateTime.now().toEpochSecond().toInt() - lastActivityAt > ttlSeconds
|
||||||
@@ -182,10 +183,21 @@ object JChatGPT : KotlinPlugin(
|
|||||||
/** 各会话的回复索引,startChat 开始时重建,结束时清理 */
|
/** 各会话的回复索引,startChat 开始时重建,结束时清理 */
|
||||||
private val replyIndexMap = ConcurrentMap<Long, ReplyIndex>()
|
private val replyIndexMap = ConcurrentMap<Long, ReplyIndex>()
|
||||||
|
|
||||||
|
/** 各会话的图片索引,生命周期与回复索引、对话缓存一致。 */
|
||||||
|
private val imageIndexMap = ConcurrentMap<Long, ImageIndex>()
|
||||||
|
|
||||||
/** 供发言工具按编号查找被引用的历史消息 */
|
/** 供发言工具按编号查找被引用的历史消息 */
|
||||||
internal fun lookupReplyTarget(subjectId: Long, index: Int): MessageRecord? =
|
internal fun lookupReplyTarget(subjectId: Long, index: Int): MessageRecord? =
|
||||||
replyIndexMap[subjectId]?.get(index)
|
replyIndexMap[subjectId]?.get(index)
|
||||||
|
|
||||||
|
/** 将从原消息图片取得的精确 URL 登记为短编号,供历史搜索等工具追加图片引用。 */
|
||||||
|
internal fun registerImage(subjectId: Long, imageId: String, imageUrl: String): Int? =
|
||||||
|
imageIndexMap[subjectId]?.add(imageId, imageUrl)
|
||||||
|
|
||||||
|
/** 按会话内短编号获取原消息解析出的 URL,避免根据 imageId 二次构造和查询。 */
|
||||||
|
internal fun lookupImageUrl(subjectId: Long, index: Int): String? =
|
||||||
|
imageIndexMap[subjectId]?.getUrl(index)
|
||||||
|
|
||||||
private val shortTimeFormatter = DateTimeFormatter.ofPattern("HH:mm")
|
private val shortTimeFormatter = DateTimeFormatter.ofPattern("HH:mm")
|
||||||
.withZone(ZoneOffset.systemDefault())
|
.withZone(ZoneOffset.systemDefault())
|
||||||
|
|
||||||
@@ -339,8 +351,9 @@ object JChatGPT : KotlinPlugin(
|
|||||||
* @return 如果未获取到则返回空字符串
|
* @return 如果未获取到则返回空字符串
|
||||||
*/
|
*/
|
||||||
private fun getHistory(event: MessageEvent): String {
|
private fun getHistory(event: MessageEvent): String {
|
||||||
|
val imageIndex = imageIndexMap.getOrPut(event.subject.id) { ImageIndex() }
|
||||||
if (!includeHistory) {
|
if (!includeHistory) {
|
||||||
return event.message.content
|
return formatRecordContent(event.message, event.subject, imageIndex)
|
||||||
}
|
}
|
||||||
val now = OffsetDateTime.now()
|
val now = OffsetDateTime.now()
|
||||||
// 一段时间内的消息
|
// 一段时间内的消息
|
||||||
@@ -378,6 +391,7 @@ object JChatGPT : KotlinPlugin(
|
|||||||
var lastTime = 0L
|
var lastTime = 0L
|
||||||
// 本轮回复索引,逐条登记消息编号供 [n] 引用
|
// 本轮回复索引,逐条登记消息编号供 [n] 引用
|
||||||
val replyIndex = replyIndexMap.getOrPut(event.subject.id) { ReplyIndex() }
|
val replyIndex = replyIndexMap.getOrPut(event.subject.id) { ReplyIndex() }
|
||||||
|
val imageIndex = imageIndexMap.getOrPut(event.subject.id) { ImageIndex() }
|
||||||
if (event is GroupMessageEvent) {
|
if (event is GroupMessageEvent) {
|
||||||
if (PluginConfig.enableFavorabilitySystem) {
|
if (PluginConfig.enableFavorabilitySystem) {
|
||||||
val knownUsers = history.asSequence()
|
val knownUsers = history.asSequence()
|
||||||
@@ -404,12 +418,12 @@ object JChatGPT : KotlinPlugin(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
historyText.appendLine("## 近期群消息(更早已隐藏,行首[n]为消息编号,可用于引用回复)")
|
historyText.appendLine("## 近期群消息(更早已隐藏,行首[n]为消息编号;正文[图片n]/[表情包n]中的n为识图或图片编辑编号)")
|
||||||
for (record in history) {
|
for (record in history) {
|
||||||
// 同一人发言不要反复出现这人的名字,减少上下文
|
// 同一人发言不要反复出现这人的名字,减少上下文
|
||||||
val showSender = lastId != record.fromId
|
val showSender = lastId != record.fromId
|
||||||
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
|
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
|
||||||
appendGroupMessageRecord(historyText, record, event, replyIndex, showSender, showTime)
|
appendGroupMessageRecord(historyText, record, event, replyIndex, imageIndex, showSender, showTime)
|
||||||
lastId = record.fromId
|
lastId = record.fromId
|
||||||
lastTime = record.time.toLong()
|
lastTime = record.time.toLong()
|
||||||
}
|
}
|
||||||
@@ -428,12 +442,12 @@ object JChatGPT : KotlinPlugin(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
historyText.appendLine("## 近期对话(更早已隐藏,行首[n]为消息编号,可用于引用回复)")
|
historyText.appendLine("## 近期对话(更早已隐藏,行首[n]为消息编号;正文[图片n]/[表情包n]中的n为识图或图片编辑编号)")
|
||||||
for (record in history) {
|
for (record in history) {
|
||||||
// 同一人发言不要反复出现这人的名字,减少上下文
|
// 同一人发言不要反复出现这人的名字,减少上下文
|
||||||
val showSender = lastId != record.fromId
|
val showSender = lastId != record.fromId
|
||||||
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
|
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
|
||||||
appendMessageRecord(historyText, record, event, replyIndex, showSender, showTime)
|
appendMessageRecord(historyText, record, event, replyIndex, imageIndex, showSender, showTime)
|
||||||
lastId = record.fromId
|
lastId = record.fromId
|
||||||
lastTime = record.time.toLong()
|
lastTime = record.time.toLong()
|
||||||
}
|
}
|
||||||
@@ -448,11 +462,12 @@ object JChatGPT : KotlinPlugin(
|
|||||||
* @param record 群消息记录
|
* @param record 群消息记录
|
||||||
* @param event 群消息事件
|
* @param event 群消息事件
|
||||||
*/
|
*/
|
||||||
fun appendGroupMessageRecord(
|
private fun appendGroupMessageRecord(
|
||||||
historyText: StringBuilder,
|
historyText: StringBuilder,
|
||||||
record: MessageRecord,
|
record: MessageRecord,
|
||||||
event: GroupMessageEvent,
|
event: GroupMessageEvent,
|
||||||
replyIndex: ReplyIndex,
|
replyIndex: ReplyIndex,
|
||||||
|
imageIndex: ImageIndex,
|
||||||
showSender: Boolean,
|
showSender: Boolean,
|
||||||
showTime: Boolean,
|
showTime: Boolean,
|
||||||
) {
|
) {
|
||||||
@@ -481,10 +496,10 @@ object JChatGPT : KotlinPlugin(
|
|||||||
|
|
||||||
// 引用:用编号指针替代内联原文,避免被误认为是本人发言
|
// 引用:用编号指针替代内联原文,避免被误认为是本人发言
|
||||||
recordMessage[QuoteReply.Key]?.let {
|
recordMessage[QuoteReply.Key]?.let {
|
||||||
appendQuoteMarker(historyText, it, event.subject, replyIndex)
|
appendQuoteMarker(historyText, it, event.subject, replyIndex, imageIndex)
|
||||||
}
|
}
|
||||||
|
|
||||||
historyText.appendLine(formatRecordContent(recordMessage, event.subject))
|
historyText.appendLine(formatRecordContent(recordMessage, event.subject, imageIndex))
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -494,7 +509,8 @@ object JChatGPT : KotlinPlugin(
|
|||||||
sb: StringBuilder,
|
sb: StringBuilder,
|
||||||
quote: QuoteReply,
|
quote: QuoteReply,
|
||||||
contact: Contact,
|
contact: Contact,
|
||||||
replyIndex: ReplyIndex
|
replyIndex: ReplyIndex,
|
||||||
|
imageIndex: ImageIndex,
|
||||||
) {
|
) {
|
||||||
val srcIds = quote.source.ids.joinToString(",")
|
val srcIds = quote.source.ids.joinToString(",")
|
||||||
val idx = replyIndex.indexOfIds(srcIds)
|
val idx = replyIndex.indexOfIds(srcIds)
|
||||||
@@ -507,7 +523,7 @@ object JChatGPT : KotlinPlugin(
|
|||||||
quote.source.fromId.toString()
|
quote.source.fromId.toString()
|
||||||
}
|
}
|
||||||
val snippet = quote.source.originalMessage
|
val snippet = quote.source.originalMessage
|
||||||
.joinToString("", transform = ::singleMessageToText)
|
.joinToString("") { singleMessageToText(it, imageIndex) }
|
||||||
.replace("\n", " ")
|
.replace("\n", " ")
|
||||||
.let { if (it.length > 20) it.take(20) + "…" else it }
|
.let { if (it.length > 20) it.take(20) + "…" else it }
|
||||||
sb.append("↩(").append(author).append(":\"").append(snippet).append("\") ")
|
sb.append("↩(").append(author).append(":\"").append(snippet).append("\") ")
|
||||||
@@ -517,13 +533,13 @@ object JChatGPT : KotlinPlugin(
|
|||||||
/**
|
/**
|
||||||
* 序列化消息正文(剔除引用/源元数据,@显示为名称,转发折叠)。
|
* 序列化消息正文(剔除引用/源元数据,@显示为名称,转发折叠)。
|
||||||
*/
|
*/
|
||||||
private fun formatRecordContent(chain: MessageChain, contact: Contact): String =
|
private fun formatRecordContent(chain: MessageChain, contact: Contact, imageIndex: ImageIndex): String =
|
||||||
chain.asSequence()
|
chain.asSequence()
|
||||||
.filterNot { it is QuoteReply || it is MessageSource }
|
.filterNot { it is QuoteReply || it is MessageSource }
|
||||||
.joinToString("") {
|
.joinToString("") {
|
||||||
when (it) {
|
when (it) {
|
||||||
is At -> if (contact is Group) it.getDisplay(contact) else it.content
|
is At -> if (contact is Group) it.getDisplay(contact) else it.content
|
||||||
else -> singleMessageToText(it)
|
else -> singleMessageToText(it, imageIndex)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -542,11 +558,12 @@ object JChatGPT : KotlinPlugin(
|
|||||||
* @param record 消息记录
|
* @param record 消息记录
|
||||||
* @param event 消息事件
|
* @param event 消息事件
|
||||||
*/
|
*/
|
||||||
fun appendMessageRecord(
|
private fun appendMessageRecord(
|
||||||
historyText: StringBuilder,
|
historyText: StringBuilder,
|
||||||
record: MessageRecord,
|
record: MessageRecord,
|
||||||
event: MessageEvent,
|
event: MessageEvent,
|
||||||
replyIndex: ReplyIndex,
|
replyIndex: ReplyIndex,
|
||||||
|
imageIndex: ImageIndex,
|
||||||
showSender: Boolean,
|
showSender: Boolean,
|
||||||
showTime: Boolean,
|
showTime: Boolean,
|
||||||
) {
|
) {
|
||||||
@@ -573,24 +590,23 @@ object JChatGPT : KotlinPlugin(
|
|||||||
}
|
}
|
||||||
|
|
||||||
recordMessage[QuoteReply.Key]?.let {
|
recordMessage[QuoteReply.Key]?.let {
|
||||||
appendQuoteMarker(historyText, it, event.subject, replyIndex)
|
appendQuoteMarker(historyText, it, event.subject, replyIndex, imageIndex)
|
||||||
}
|
}
|
||||||
|
|
||||||
historyText.appendLine(formatRecordContent(recordMessage, event.subject))
|
historyText.appendLine(formatRecordContent(recordMessage, event.subject, imageIndex))
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun singleMessageToText(it: SingleMessage): String {
|
private fun singleMessageToText(it: SingleMessage, imageIndex: ImageIndex): String {
|
||||||
return when (it) {
|
return when (it) {
|
||||||
// 完整展开合并转发内容,便于 LLM 阅读分析转发的对话(依赖大上下文+缓存,不做截断)
|
// 完整展开合并转发内容,便于 LLM 阅读分析转发的对话(依赖大上下文+缓存,不做截断)
|
||||||
is ForwardMessage -> formatForward(it, 1)
|
is ForwardMessage -> formatForward(it, 1, imageIndex)
|
||||||
|
|
||||||
// 图片格式化
|
// 图片格式化
|
||||||
is Image -> {
|
is Image -> {
|
||||||
try {
|
try {
|
||||||
val imageUrl = runBlocking {
|
val imageUrl = runBlocking { it.queryUrl() }
|
||||||
it.queryUrl()
|
val index = imageIndex.add(it.imageId, imageUrl)
|
||||||
}
|
"[${if (it.isEmoji) "表情包" else "图片"}$index]"
|
||||||
""
|
|
||||||
} catch (e: Throwable) {
|
} catch (e: Throwable) {
|
||||||
logger.warning("图片地址获取失败", e)
|
logger.warning("图片地址获取失败", e)
|
||||||
it.content
|
it.content
|
||||||
@@ -605,7 +621,7 @@ object JChatGPT : KotlinPlugin(
|
|||||||
* 递归展开合并转发消息,用 Markdown 引用块表示:每加深一层嵌套多一个 `>`(>、>>、>>>…)。
|
* 递归展开合并转发消息,用 Markdown 引用块表示:每加深一层嵌套多一个 `>`(>、>>、>>>…)。
|
||||||
* @param depth 当前嵌套层级,从 1 开始
|
* @param depth 当前嵌套层级,从 1 开始
|
||||||
*/
|
*/
|
||||||
private fun formatForward(forward: ForwardMessage, depth: Int): String = buildString {
|
private fun formatForward(forward: ForwardMessage, depth: Int, imageIndex: ImageIndex): String = buildString {
|
||||||
val quote = ">".repeat(depth) + " "
|
val quote = ">".repeat(depth) + " "
|
||||||
append("[转发消息·").append(forward.nodeList.size).append("条")
|
append("[转发消息·").append(forward.nodeList.size).append("条")
|
||||||
if (forward.title.isNotEmpty()) append(':').append(forward.title)
|
if (forward.title.isNotEmpty()) append(':').append(forward.title)
|
||||||
@@ -618,10 +634,10 @@ object JChatGPT : KotlinPlugin(
|
|||||||
node.messageChain.forEach { sub ->
|
node.messageChain.forEach { sub ->
|
||||||
if (sub is ForwardMessage) {
|
if (sub is ForwardMessage) {
|
||||||
// 嵌套转发:层级加深,自带更深的 `>` 前缀,无需再次缩进
|
// 嵌套转发:层级加深,自带更深的 `>` 前缀,无需再次缩进
|
||||||
append(formatForward(sub, depth + 1))
|
append(formatForward(sub, depth + 1, imageIndex))
|
||||||
} else {
|
} else {
|
||||||
// 其它内容:多行正文对齐到当前引用层级
|
// 其它内容:多行正文对齐到当前引用层级
|
||||||
append(singleMessageToText(sub).replace("\n", "\n$quote"))
|
append(singleMessageToText(sub, imageIndex).replace("\n", "\n$quote"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -658,7 +674,9 @@ object JChatGPT : KotlinPlugin(
|
|||||||
// 回复索引与对话上下文同寿命:复用缓存时沿用旧索引,保证 LLM 看到的 [n] 编号连续不串号;
|
// 回复索引与对话上下文同寿命:复用缓存时沿用旧索引,保证 LLM 看到的 [n] 编号连续不串号;
|
||||||
// 否则新建(供 sendSingleMessage 的 replyTo 按编号引用历史消息)
|
// 否则新建(供 sendSingleMessage 的 replyTo 按编号引用历史消息)
|
||||||
val replyIndex = if (reuseCache) cache!!.replyIndex else ReplyIndex()
|
val replyIndex = if (reuseCache) cache!!.replyIndex else ReplyIndex()
|
||||||
|
val imageIndex = if (reuseCache) cache!!.imageIndex else ImageIndex()
|
||||||
replyIndexMap[subjectId] = replyIndex
|
replyIndexMap[subjectId] = replyIndex
|
||||||
|
imageIndexMap[subjectId] = imageIndex
|
||||||
val history = if (reuseCache) {
|
val history = if (reuseCache) {
|
||||||
// 缓存有效,复用历史
|
// 缓存有效,复用历史
|
||||||
logger.info("使用缓存的对话上下文,包含 ${cache!!.history.size} 条互动消息")
|
logger.info("使用缓存的对话上下文,包含 ${cache!!.history.size} 条互动消息")
|
||||||
@@ -868,7 +886,8 @@ object JChatGPT : KotlinPlugin(
|
|||||||
contextCache[subjectId] = ConversationCache(
|
contextCache[subjectId] = ConversationCache(
|
||||||
history = history,
|
history = history,
|
||||||
lastActivityAt = startedAt,
|
lastActivityAt = startedAt,
|
||||||
replyIndex = replyIndex
|
replyIndex = replyIndex,
|
||||||
|
imageIndex = imageIndex,
|
||||||
)
|
)
|
||||||
logger.debug("已保存对话上下文到缓存")
|
logger.debug("已保存对话上下文到缓存")
|
||||||
}
|
}
|
||||||
@@ -901,6 +920,7 @@ object JChatGPT : KotlinPlugin(
|
|||||||
} finally {
|
} finally {
|
||||||
// 清理本轮回复索引
|
// 清理本轮回复索引
|
||||||
replyIndexMap.remove(event.subject.id)
|
replyIndexMap.remove(event.subject.id)
|
||||||
|
imageIndexMap.remove(event.subject.id)
|
||||||
// 一段时间后才允许再次提问,防止高频对话
|
// 一段时间后才允许再次提问,防止高频对话
|
||||||
launch {
|
launch {
|
||||||
delay(500.milliseconds)
|
delay(500.milliseconds)
|
||||||
|
|||||||
@@ -13,31 +13,34 @@ import kotlinx.serialization.json.JsonObject
|
|||||||
import kotlinx.serialization.json.add
|
import kotlinx.serialization.json.add
|
||||||
import kotlinx.serialization.json.addJsonObject
|
import kotlinx.serialization.json.addJsonObject
|
||||||
import kotlinx.serialization.json.buildJsonObject
|
import kotlinx.serialization.json.buildJsonObject
|
||||||
|
import kotlinx.serialization.json.int
|
||||||
import kotlinx.serialization.json.jsonArray
|
import kotlinx.serialization.json.jsonArray
|
||||||
import kotlinx.serialization.json.jsonObject
|
import kotlinx.serialization.json.jsonObject
|
||||||
import kotlinx.serialization.json.jsonPrimitive
|
import kotlinx.serialization.json.jsonPrimitive
|
||||||
import kotlinx.serialization.json.put
|
import kotlinx.serialization.json.put
|
||||||
import kotlinx.serialization.json.putJsonArray
|
import kotlinx.serialization.json.putJsonArray
|
||||||
import kotlinx.serialization.json.putJsonObject
|
import kotlinx.serialization.json.putJsonObject
|
||||||
|
import net.mamoe.mirai.event.events.MessageEvent
|
||||||
import top.jie65535.mirai.JChatGPT
|
import top.jie65535.mirai.JChatGPT
|
||||||
import top.jie65535.mirai.PluginConfig
|
import top.jie65535.mirai.PluginConfig
|
||||||
|
|
||||||
class ImageAgent : BaseAgent(
|
class ImageAgent : BaseAgent(
|
||||||
tool = Tool.function(
|
tool = Tool.function(
|
||||||
name = "imageAgent",
|
name = "imageAgent",
|
||||||
description = "调用千问图像模型生成或编辑图片。不传 image_urls 即纯文生图;" +
|
description = "调用千问图像模型生成或编辑图片。不传 image_indices 即纯文生图;" +
|
||||||
"传 1~3 张图片可进行编辑、修改或多图融合。" +
|
"传 1~3 张图片可进行编辑、修改或多图融合。" +
|
||||||
"备注:该方法成本较高,非必要尽量不要调用。" +
|
"备注:该方法成本较高,非必要尽量不要调用。" +
|
||||||
"编辑图片前无需识别图片内容,模型自己会理解图片内容。",
|
"编辑图片前无需识别图片内容,模型自己会理解图片内容。",
|
||||||
parameters = Parameters.buildJsonObject {
|
parameters = Parameters.buildJsonObject {
|
||||||
put("type", "object")
|
put("type", "object")
|
||||||
putJsonObject("properties") {
|
putJsonObject("properties") {
|
||||||
putJsonObject("image_urls") {
|
putJsonObject("image_indices") {
|
||||||
put("type", "array")
|
put("type", "array")
|
||||||
putJsonObject("items") {
|
putJsonObject("items") {
|
||||||
put("type", "string")
|
put("type", "integer")
|
||||||
|
put("minimum", 1)
|
||||||
}
|
}
|
||||||
put("description", "参考图片地址列表,可传 0~3 张。" +
|
put("description", "用户消息中[图片n]或[表情包n]标记的参考图片编号,可传 0~3 张。" +
|
||||||
"不传或为空即纯文生图;传 1 张为编辑;多张为融合,输出比例与最后一张对齐。")
|
"不传或为空即纯文生图;传 1 张为编辑;多张为融合,输出比例与最后一张对齐。")
|
||||||
}
|
}
|
||||||
putJsonObject("prompt") {
|
putJsonObject("prompt") {
|
||||||
@@ -61,12 +64,17 @@ class ImageAgent : BaseAgent(
|
|||||||
override val loadingMessage: String
|
override val loadingMessage: String
|
||||||
get() = "作图中..."
|
get() = "作图中..."
|
||||||
|
|
||||||
override suspend fun execute(args: JsonObject?): String {
|
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
|
||||||
requireNotNull(args)
|
requireNotNull(args)
|
||||||
val prompt = args.getValue("prompt").jsonPrimitive.content
|
val prompt = args.getValue("prompt").jsonPrimitive.content
|
||||||
val imageUrls = args["image_urls"]?.jsonArray
|
val imageIndices = args["image_indices"]?.jsonArray
|
||||||
?.map { it.jsonPrimitive.content }
|
?.map { it.jsonPrimitive.int }
|
||||||
?: emptyList()
|
?: emptyList()
|
||||||
|
require(imageIndices.size <= 3) { "参考图片最多只能传3张" }
|
||||||
|
val imageUrls = imageIndices.map { imageIndex ->
|
||||||
|
JChatGPT.lookupImageUrl(event.subject.id, imageIndex)
|
||||||
|
?: throw IllegalArgumentException("图片编号[$imageIndex]不存在或已失效")
|
||||||
|
}
|
||||||
|
|
||||||
val response = httpClient.post(API_URL) {
|
val response = httpClient.post(API_URL) {
|
||||||
contentType(ContentType("application", "json"))
|
contentType(ContentType("application", "json"))
|
||||||
|
|||||||
@@ -172,19 +172,21 @@ class SearchChatHistory : BaseAgent(
|
|||||||
.append(":")
|
.append(":")
|
||||||
}
|
}
|
||||||
for (msg in record.toMessageChain()) {
|
for (msg in record.toMessageChain()) {
|
||||||
sb.append(singleMessageToText(msg))
|
sb.append(singleMessageToText(msg, event.subject.id))
|
||||||
}
|
}
|
||||||
sb.appendLine()
|
sb.appendLine()
|
||||||
lastFromId = record.fromId
|
lastFromId = record.fromId
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun singleMessageToText(msg: SingleMessage): String {
|
private suspend fun singleMessageToText(msg: SingleMessage, subjectId: Long): String {
|
||||||
return when (msg) {
|
return when (msg) {
|
||||||
is Image -> {
|
is Image -> {
|
||||||
try {
|
try {
|
||||||
val url = msg.queryUrl()
|
val imageUrl = msg.queryUrl()
|
||||||
""
|
val index = JChatGPT.registerImage(subjectId, msg.imageId, imageUrl)
|
||||||
|
?: return msg.content
|
||||||
|
"[${if (msg.isEmoji) "表情包" else "图片"}$index]"
|
||||||
} catch (_: Throwable) {
|
} catch (_: Throwable) {
|
||||||
msg.content
|
msg.content
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import com.aallam.openai.api.model.ModelId
|
|||||||
import io.ktor.client.plugins.ClientRequestException
|
import io.ktor.client.plugins.ClientRequestException
|
||||||
import kotlinx.serialization.json.JsonObject
|
import kotlinx.serialization.json.JsonObject
|
||||||
import kotlinx.serialization.json.add
|
import kotlinx.serialization.json.add
|
||||||
|
import kotlinx.serialization.json.int
|
||||||
import kotlinx.serialization.json.jsonArray
|
import kotlinx.serialization.json.jsonArray
|
||||||
import kotlinx.serialization.json.jsonPrimitive
|
import kotlinx.serialization.json.jsonPrimitive
|
||||||
import kotlinx.serialization.json.put
|
import kotlinx.serialization.json.put
|
||||||
@@ -20,6 +21,7 @@ import kotlinx.coroutines.CancellationException
|
|||||||
import kotlinx.coroutines.delay
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.sync.Semaphore
|
import kotlinx.coroutines.sync.Semaphore
|
||||||
import kotlinx.coroutines.sync.withPermit
|
import kotlinx.coroutines.sync.withPermit
|
||||||
|
import net.mamoe.mirai.event.events.MessageEvent
|
||||||
import top.jie65535.mirai.JChatGPT
|
import top.jie65535.mirai.JChatGPT
|
||||||
import top.jie65535.mirai.LargeLanguageModels
|
import top.jie65535.mirai.LargeLanguageModels
|
||||||
import top.jie65535.mirai.PluginConfig
|
import top.jie65535.mirai.PluginConfig
|
||||||
@@ -32,13 +34,14 @@ class VisualAgent : BaseAgent(
|
|||||||
parameters = Parameters.buildJsonObject {
|
parameters = Parameters.buildJsonObject {
|
||||||
put("type", "object")
|
put("type", "object")
|
||||||
putJsonObject("properties") {
|
putJsonObject("properties") {
|
||||||
putJsonObject("image_urls") {
|
putJsonObject("image_indices") {
|
||||||
put("type", "array")
|
put("type", "array")
|
||||||
put("description", "图片地址数组,按用户消息中的出现顺序传入")
|
put("description", "用户消息中[图片n]或[表情包n]标记的图片编号数组,按需要理解的顺序传入")
|
||||||
put("minItems", 1)
|
put("minItems", 1)
|
||||||
put("maxItems", MAX_SOURCE_IMAGES)
|
put("maxItems", MAX_SOURCE_IMAGES)
|
||||||
putJsonObject("items") {
|
putJsonObject("items") {
|
||||||
put("type", "string")
|
put("type", "integer")
|
||||||
|
put("minimum", 1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
putJsonObject("prompt") {
|
putJsonObject("prompt") {
|
||||||
@@ -47,7 +50,7 @@ class VisualAgent : BaseAgent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
putJsonArray("required") {
|
putJsonArray("required") {
|
||||||
add("image_urls")
|
add("image_indices")
|
||||||
add("prompt")
|
add("prompt")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -62,25 +65,35 @@ class VisualAgent : BaseAgent(
|
|||||||
override val isEnabled: Boolean
|
override val isEnabled: Boolean
|
||||||
get() = LargeLanguageModels.visual != null
|
get() = LargeLanguageModels.visual != null
|
||||||
|
|
||||||
override suspend fun execute(args: JsonObject?): String {
|
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
|
||||||
requireNotNull(args)
|
requireNotNull(args)
|
||||||
val llm = LargeLanguageModels.visual ?: return "未配置llm,无法进行识别。"
|
val llm = LargeLanguageModels.visual ?: return "未配置llm,无法进行识别。"
|
||||||
val imageUrls = args["image_urls"]?.jsonArray
|
val imageIndices = args["image_indices"]?.jsonArray
|
||||||
?.map { it.jsonPrimitive.content }
|
?.map { it.jsonPrimitive.int }
|
||||||
?.filter { it.isNotBlank() }
|
|
||||||
?.ifEmpty { null }
|
?.ifEmpty { null }
|
||||||
?: throw IllegalArgumentException("至少需要提供一张图片")
|
?: throw IllegalArgumentException("至少需要提供一张图片")
|
||||||
require(imageUrls.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" }
|
require(imageIndices.size <= MAX_SOURCE_IMAGES) { "单次最多处理 $MAX_SOURCE_IMAGES 张用户图片" }
|
||||||
|
val imageUrls = imageIndices.map { imageIndex ->
|
||||||
|
JChatGPT.lookupImageUrl(event.subject.id, imageIndex)
|
||||||
|
?: throw IllegalArgumentException("图片编号[$imageIndex]不存在或已失效")
|
||||||
|
}
|
||||||
val prompt = args.getValue("prompt").jsonPrimitive.content
|
val prompt = args.getValue("prompt").jsonPrimitive.content
|
||||||
|
|
||||||
return concurrencyLimiter.withPermit {
|
return concurrencyLimiter.withPermit {
|
||||||
val imageGroups = imageUrls.mapIndexed { index, imageUrl ->
|
val imageGroups = imageUrls.mapIndexed { index, imageUrl ->
|
||||||
if (PluginConfig.visualImageBase64Enabled) {
|
if (PluginConfig.visualImageBase64Enabled) {
|
||||||
val resolved = imageResolver.resolve(imageUrl)
|
|
||||||
val host = runCatching { URI(imageUrl).host }.getOrNull() ?: "unknown"
|
val host = runCatching { URI(imageUrl).host }.getOrNull() ?: "unknown"
|
||||||
|
val resolved = try {
|
||||||
|
imageResolver.resolve(imageUrl)
|
||||||
|
} catch (e: Throwable) {
|
||||||
|
JChatGPT.logger.error(
|
||||||
|
"视觉图片下载失败: image=${imageIndices[index]}, url=$imageUrl"
|
||||||
|
)
|
||||||
|
throw e
|
||||||
|
}
|
||||||
val mimeTypes = resolved.images.map { it.mimeType }.distinct().joinToString()
|
val mimeTypes = resolved.images.map { it.mimeType }.distinct().joinToString()
|
||||||
JChatGPT.logger.info(
|
JChatGPT.logger.info(
|
||||||
"视觉图片已本地化: source=${index + 1}/${imageUrls.size}, host=$host, " +
|
"视觉图片已本地化: image=${imageIndices[index]}, source=${index + 1}/${imageUrls.size}, host=$host, " +
|
||||||
"parts=${resolved.images.size}, mime=$mimeTypes, " +
|
"parts=${resolved.images.size}, mime=$mimeTypes, " +
|
||||||
"sourceBytes=${resolved.sourceSize}, payloadChars=${resolved.payloadSize}, " +
|
"sourceBytes=${resolved.sourceSize}, payloadChars=${resolved.payloadSize}, " +
|
||||||
"transcoded=${resolved.transcoded}"
|
"transcoded=${resolved.transcoded}"
|
||||||
|
|||||||
@@ -108,8 +108,15 @@ internal class VisualImageResolver {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!response.status.isSuccess()) {
|
if (!response.status.isSuccess()) {
|
||||||
response.bodyAsChannel().cancel()
|
val errorBody = readErrorBody(response.bodyAsChannel())
|
||||||
throw IllegalArgumentException("图片下载失败:HTTP ${response.status.value}")
|
val errorNumber = response.headers["X-ErrNo"]
|
||||||
|
throw IllegalArgumentException(
|
||||||
|
buildString {
|
||||||
|
append("图片下载失败:HTTP ").append(response.status.value)
|
||||||
|
if (!errorNumber.isNullOrBlank()) append(",X-ErrNo=").append(errorNumber)
|
||||||
|
if (errorBody.isNotBlank()) append(",响应=").append(errorBody)
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull()
|
val declaredLength = response.headers[HttpHeaders.ContentLength]?.toLongOrNull()
|
||||||
@@ -190,6 +197,31 @@ internal class VisualImageResolver {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private suspend fun readErrorBody(channel: io.ktor.utils.io.ByteReadChannel): String {
|
||||||
|
val output = ByteArrayOutputStream()
|
||||||
|
val buffer = ByteArray(DOWNLOAD_BUFFER_SIZE)
|
||||||
|
var total = 0
|
||||||
|
try {
|
||||||
|
while (total < MAX_ERROR_RESPONSE_BYTES) {
|
||||||
|
val count = channel.readAvailable(
|
||||||
|
buffer,
|
||||||
|
0,
|
||||||
|
min(buffer.size, MAX_ERROR_RESPONSE_BYTES - total)
|
||||||
|
)
|
||||||
|
if (count < 0) break
|
||||||
|
if (count == 0) continue
|
||||||
|
output.write(buffer, 0, count)
|
||||||
|
total += count
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
channel.cancel()
|
||||||
|
}
|
||||||
|
return output.toByteArray()
|
||||||
|
.toString(Charsets.UTF_8)
|
||||||
|
.replace(Regex("[\\r\\n]+"), " ")
|
||||||
|
.trim()
|
||||||
|
}
|
||||||
|
|
||||||
private fun buildPayload(bytes: ByteArray, mimeType: String): ImagePayload {
|
private fun buildPayload(bytes: ByteArray, mimeType: String): ImagePayload {
|
||||||
val encoded = Base64.getEncoder().encodeToString(bytes)
|
val encoded = Base64.getEncoder().encodeToString(bytes)
|
||||||
val dataUrl = "data:$mimeType;base64,$encoded"
|
val dataUrl = "data:$mimeType;base64,$encoded"
|
||||||
@@ -612,6 +644,7 @@ internal class VisualImageResolver {
|
|||||||
private const val LONG_IMAGE_MAX_OVERLAP = 256
|
private const val LONG_IMAGE_MAX_OVERLAP = 256
|
||||||
private const val MAX_LONG_IMAGE_PARTS = 16
|
private const val MAX_LONG_IMAGE_PARTS = 16
|
||||||
private const val MAX_COMPRESSION_ROUNDS = 6
|
private const val MAX_COMPRESSION_ROUNDS = 6
|
||||||
|
private const val MAX_ERROR_RESPONSE_BYTES = 4096
|
||||||
private const val DOWNSCALE_FACTOR = 0.82
|
private const val DOWNSCALE_FACTOR = 0.82
|
||||||
private const val USER_AGENT = "JChatGPT/1.13 image-fetcher"
|
private const val USER_AGENT = "JChatGPT/1.13 image-fetcher"
|
||||||
private val JPEG_QUALITIES = floatArrayOf(0.90f, 0.82f, 0.74f, 0.66f)
|
private val JPEG_QUALITIES = floatArrayOf(0.90f, 0.82f, 0.74f, 0.66f)
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package top.jie65535.mirai
|
||||||
|
|
||||||
|
import kotlin.test.Test
|
||||||
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertNull
|
||||||
|
|
||||||
|
class ImageIndexTest {
|
||||||
|
@Test
|
||||||
|
fun `assigns short sequential indices`() {
|
||||||
|
val index = ImageIndex()
|
||||||
|
|
||||||
|
assertEquals(1, index.add("image-a", "https://example.com/a"))
|
||||||
|
assertEquals(2, index.add("image-b", "https://example.com/b"))
|
||||||
|
assertEquals("https://example.com/a", index.getUrl(1))
|
||||||
|
assertEquals("https://example.com/b", index.getUrl(2))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `reuses index for repeated image id`() {
|
||||||
|
val index = ImageIndex()
|
||||||
|
|
||||||
|
assertEquals(1, index.add("image-a", "https://example.com/a-old"))
|
||||||
|
assertEquals(1, index.add("image-a", "https://example.com/a-new"))
|
||||||
|
assertEquals(2, index.add("image-b", "https://example.com/b"))
|
||||||
|
assertEquals("https://example.com/a-new", index.getUrl(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `returns null for unknown index`() {
|
||||||
|
assertNull(ImageIndex().getUrl(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user