profile: add progressive user profiles

Add cache-expiry profile maintenance and contextual injection, reorganize runtime code by responsibility, remove automatic favorability decay, and prepare version 1.15.0.
This commit is contained in:
2026-08-02 21:45:34 +08:00
parent 2ed39e9fe8
commit 23299b2ae7
59 changed files with 4799 additions and 1340 deletions
File diff suppressed because it is too large Load Diff
@@ -1,5 +1,7 @@
package top.jie65535.mirai
package top.jie65535.mirai.command
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.launch
import net.mamoe.mirai.console.command.CommandSender
import net.mamoe.mirai.console.command.CompositeCommand
import net.mamoe.mirai.console.permission.PermissionService.Companion.cancel
@@ -9,8 +11,23 @@ import net.mamoe.mirai.contact.Contact
import net.mamoe.mirai.contact.Group
import net.mamoe.mirai.contact.Member
import net.mamoe.mirai.contact.User
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.JChatGPT.reload
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.PluginData
import top.jie65535.mirai.data.SkillStore
import top.jie65535.mirai.data.TokenUsageStore
import top.jie65535.mirai.llm.LargeLanguageModels
import top.jie65535.mirai.profile.ProfileAnalysisReport
import top.jie65535.mirai.profile.ProfileAutoMaintenance
import top.jie65535.mirai.profile.ProfilePromptStore
import top.jie65535.mirai.profile.UserProfileAnalysisService
import top.jie65535.mirai.profile.UserProfileSnapshot
import top.jie65535.mirai.profile.UserProfileStore
import java.time.Instant
import java.time.LocalDate
import java.time.ZoneId
import java.time.format.DateTimeFormatter
object PluginCommands : CompositeCommand(
JChatGPT, "jgpt", description = "J OpenAI ChatGPT"
@@ -21,10 +38,53 @@ object PluginCommands : CompositeCommand(
PluginConfig.reload()
PluginData.reload()
LargeLanguageModels.reload()
ProfilePromptStore.reload()
if (!PluginConfig.profileEnabled || !PluginConfig.profileAutoUpdateEnabled) {
ProfileAutoMaintenance.clear()
}
SkillStore.reload()
sendMessage("OK")
}
@SubCommand
suspend fun CommandSender.profileAnalyze(userId: Long, batches: Int = 1) {
require(batches in 1..50) { "batches 必须在 1 到 50 之间" }
sendMessage("已启动用户 $userId 的画像分析,本次最多推进 $batches 个批次。")
JChatGPT.launch {
try {
val report = UserProfileAnalysisService.analyze(userId, batches) { progress ->
JChatGPT.logger.info(
"PROFILE_BATCH user=$userId batch=${progress.batchIndex}/$batches " +
"range=${progress.startTime}-${progress.endTime} " +
"messages=${progress.messageCount} operations=${progress.operationCount} " +
"tokens=${progress.usage.promptTokens}/${progress.usage.completionTokens} " +
"cached=${progress.usage.cachedTokens}"
)
}
when {
report.alreadyRunning -> sendMessage("用户 $userId 已有画像分析任务在运行。")
report.profile == null -> sendMessage("聊天记录中没有找到用户 $userId 的群聊发言。")
else -> sendMessage(formatProfileReport(report))
}
} catch (cause: CancellationException) {
throw cause
} catch (cause: Exception) {
JChatGPT.logger.error("用户 $userId 画像分析失败", cause)
sendMessage("用户 $userId 画像分析失败:${cause.message ?: cause::class.simpleName}")
}
}
}
@SubCommand
suspend fun CommandSender.profileShow(userId: Long) {
if (!UserProfileStore.isAvailable) {
sendMessage("用户画像数据库不可用。")
return
}
val profile = UserProfileStore.load(userId)
sendMessage(profile?.let(::formatProfile) ?: "用户 $userId 尚无画像。")
}
@SubCommand
suspend fun CommandSender.skills() {
val all = SkillStore.all
@@ -65,24 +125,6 @@ object PluginCommands : CompositeCommand(
sendMessage("OK")
}
@SubCommand
suspend fun CommandSender.setFavor(user: User, value: Int) {
// 限制好感度值在-100到100之间
val clampedValue = value.coerceIn(-100, 100)
// 获取当前的好感度信息
val currentInfo = PluginData.userFavorability[user.id] ?: FavorabilityInfo(user.id)
// 创建新的好感度信息,保持原因和印象不变
val newInfo = currentInfo.copy(value = clampedValue)
PluginData.userFavorability[user.id] = newInfo
sendMessage("OK")
}
@SubCommand
suspend fun CommandSender.clearFavor() {
PluginData.userFavorability.clear()
sendMessage("OK")
}
@SubCommand
suspend fun CommandSender.clearContextCache() {
JChatGPT.clearContextCache()
@@ -223,7 +265,49 @@ object PluginCommands : CompositeCommand(
private fun validateDays(days: Int) {
require(days > 0) { "days must be positive: $days" }
}
private fun formatProfileReport(report: ProfileAnalysisReport): String = buildString {
appendLine(
"画像分析完成:${report.processedBatches} 批,${report.processedMessages} 条上下文消息," +
"${report.appliedOperations} 项变更"
)
appendLine(
"Token:输入 ${formatNumber(report.usage.promptTokens)},输出 " +
"${formatNumber(report.usage.completionTokens)},缓存命中 " +
formatNumber(report.usage.cachedTokens)
)
appendLine("状态:${if (report.caughtUp) "已追平当前快照" else "可继续推进"}")
append(formatProfile(checkNotNull(report.profile)))
}.trim()
private fun formatProfile(profile: UserProfileSnapshot): String = buildString {
appendLine("用户 ${profile.userId} · 画像 v${profile.version}")
if (profile.cursorTime <= 0) {
appendLine("历史回顾:尚未开始(当前画像来自自动会话归纳)")
} else {
appendLine("历史回顾覆盖至 ${formatProfileTime(profile.cursorTime)}")
}
appendLine("摘要:${profile.summary.ifBlank { "(暂无)" }}")
if (profile.items.isEmpty()) {
append("条目:(暂无)")
} else {
appendLine("条目:")
profile.items.forEach { item ->
append("- [").append(item.category.name.lowercase()).append('/')
.append(item.confidence.name.lowercase()).append("] ")
.append(item.content)
item.relatedUserId?.let { append("(关联用户 ").append(it).append('') }
appendLine()
}
}
}.trim()
private fun formatProfileTime(epochSecond: Int): String =
PROFILE_TIME_FORMATTER.format(Instant.ofEpochSecond(epochSecond.toLong()))
}
// 常量定义
private const val TOP_LIMIT = 5
private const val TOP_LIMIT = 5
private val PROFILE_TIME_FORMATTER: DateTimeFormatter = DateTimeFormatter
.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.systemDefault())
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.config
import kotlinx.serialization.Serializable
import net.mamoe.mirai.console.data.AutoSavePluginConfig
@@ -57,6 +57,78 @@ object PluginConfig : AutoSavePluginConfig("Config") {
@ValueDescription("聊天模型备用接入点列表(容灾)。主接入点连续失败时按顺序切换;每项留空的字段会继承主接入点,例如只换API KEY就只填token,只换模型就只填model")
val chatFallbacks: List<ChatFallbackEndpoint> by value()
@ValueDescription("是否启用实验性的历史用户画像分析")
val profileEnabled: Boolean by value(true)
@ValueDescription("画像分析模型API。留空时继承聊天模型API")
val profileModelApi: String by value("")
@ValueDescription("画像分析模型Token。留空时继承聊天模型Token")
val profileModelToken: String by value("")
@ValueDescription("画像分析模型。留空时继承聊天模型")
val profileModel: String by value("")
@ValueDescription("画像分析模型温度。留空时继承聊天模型温度")
val profileModelTemperature: Double? by value(null)
@ValueDescription("画像分析模型额外请求体JSON。留空时继承聊天模型额外请求体")
val profileModelExtraBody: String by value("")
@ValueDescription("画像分析提示词文件路径,相对于插件配置目录")
val profilePromptFile: String by value("ProfilePrompt.md")
@ValueDescription("闭合群聊多人画像提示词文件路径,相对于插件配置目录")
val profileConversationPromptFile: String by value("ProfileConversationPrompt.md")
@ValueDescription("画像分析使用的聊天记录SQLite路径。留空时使用插件自己的chat-history.sqlite;本地实验可填写历史库绝对路径")
val profileHistoryDatabasePath: String by value("")
@ValueDescription("每个画像分析批次最多读取目标用户多少条消息")
val profileBatchTargetMessages: Int by value(120)
@ValueDescription("每个画像分析批次最多包含多少个离散对话片段;同一秒的消息仍会一起处理")
val profileBatchMaxEpisodes: Int by value(16)
@ValueDescription("目标用户相邻发言超过多少分钟时划分为新的对话片段")
val profileEpisodeGapMinutes: Int by value(60)
@ValueDescription("每个对话片段最多附带多少条前置上下文")
val profileContextBeforeMessages: Int by value(30)
@ValueDescription("每个对话片段最多附带多少条后续上下文")
val profileContextAfterMessages: Int by value(30)
@ValueDescription("画像分析单批次最多保留多少条非目标用户上下文消息")
val profileContextCoreMessages: Int by value(300)
@ValueDescription("画像分析中单条消息最多保留的字符数")
val profileMaxMessageChars: Int by value(1000)
@ValueDescription("画像模型响应无效时的最大重试次数,取值0~3")
val profileRetryMax: Int by value(2)
@ValueDescription("单条画像结论的最大字符数")
val profileSummaryMaxLength: Int by value(500)
@ValueDescription("是否在群聊会话结束后静默自动维护相关用户画像")
val profileAutoUpdateEnabled: Boolean by value(true)
@ValueDescription("自动画像分析一次闭合会话最多读取多少条最近消息")
val profileAutoConversationMessageLimit: Int by value(150)
@ValueDescription("用户在会话中至少包含多少个本人文本字符才调用画像模型")
val profileAutoMinAuthoredTextChars: Int by value(20)
@ValueDescription("是否在普通群聊上下文中自动注入相关用户的画像摘要")
val profileAutoInjectEnabled: Boolean by value(true)
@ValueDescription("一次普通对话最多自动注入多少名相关用户的画像摘要")
val profileAutoInjectMaxUsers: Int by value(4)
@ValueDescription("普通对话中每名用户的画像摘要最多注入多少字符")
val profileAutoInjectSummaryMaxChars: Int by value(300)
@ValueDescription("备用接入点冷却时间(分钟)。某接入点失败后在此时间内会被排到重试队尾,避免每条消息都先卡在故障接入点上。设为0禁用,默认5分钟")
val fallbackCooldownMinutes: Long by value(5L)
@@ -129,6 +201,9 @@ object PluginConfig : AutoSavePluginConfig("Config") {
@ValueDescription("推理模型首块响应超时时间,单位毫秒,默认90秒。推理模型出首块前常有思考预热,比对话慢,故单独放宽")
val reasoningFirstChunkTimeout: Long by value(90000L)
@ValueDescription("画像分析模型首块响应超时时间,单位毫秒,默认180秒")
val profileFirstChunkTimeout: Long by value(180000L)
@Deprecated("使用外部文件而不是在配置文件内保存提示词")
@ValueDescription("系统提示词,该字段已弃用,使用提示词文件而不是在这里修改")
var prompt: String by value("你是一个乐于助人的助手")
@@ -172,9 +247,6 @@ object PluginConfig : AutoSavePluginConfig("Config") {
@ValueDescription("是否启用好感度系统")
val enableFavorabilitySystem by value(true)
@ValueDescription("好感度每日基础偏移速度(点/天)")
val favorabilityBaseShiftSpeed by value(2.0)
@ValueDescription("表情包路径,配置后会加载目录下的文件名,提示词中需要用{meme}来插入上下文")
val memeDir: String by value("")
@@ -0,0 +1,466 @@
package top.jie65535.mirai.conversation
import com.aallam.openai.api.chat.ChatMessage
import kotlinx.coroutines.runBlocking
import net.mamoe.mirai.contact.Contact
import net.mamoe.mirai.contact.Group
import net.mamoe.mirai.contact.Member
import net.mamoe.mirai.contact.MemberPermission.ADMINISTRATOR
import net.mamoe.mirai.contact.MemberPermission.MEMBER
import net.mamoe.mirai.contact.MemberPermission.OWNER
import net.mamoe.mirai.contact.nameCardOrNick
import net.mamoe.mirai.event.events.GroupMessageEvent
import net.mamoe.mirai.event.events.MessageEvent
import net.mamoe.mirai.message.data.At
import net.mamoe.mirai.message.data.ForwardMessage
import net.mamoe.mirai.message.data.Image
import net.mamoe.mirai.message.data.Image.Key.queryUrl
import net.mamoe.mirai.message.data.Message
import net.mamoe.mirai.message.data.MessageChain
import net.mamoe.mirai.message.data.MessageSource
import net.mamoe.mirai.message.data.PlainText
import net.mamoe.mirai.message.data.QuoteReply
import net.mamoe.mirai.message.data.SingleMessage
import net.mamoe.mirai.message.data.buildMessageChain
import net.mamoe.mirai.message.data.content
import net.mamoe.mirai.message.data.ids
import net.mamoe.mirai.message.data.source
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.ChatHistoryStore
import top.jie65535.mirai.data.ChatMessageRecord
import top.jie65535.mirai.data.PluginData
import top.jie65535.mirai.data.SkillStore
import top.jie65535.mirai.llm.LargeLanguageModels
import top.jie65535.mirai.media.ImageIndex
import top.jie65535.mirai.profile.UserProfileContextRenderer
import top.jie65535.mirai.profile.UserProfileStore
import util.LunarDateUtil
import java.io.File
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import java.time.format.DateTimeFormatter
internal data class ConversationCache(
val history: MutableList<ChatMessage>,
val lastActivityAt: Int,
val replyIndex: ReplyIndex,
val imageIndex: ImageIndex,
) {
fun isExpired(ttlSeconds: Int): Boolean =
OffsetDateTime.now().toEpochSecond().toInt() - lastActivityAt > ttlSeconds
}
internal class ReplyIndex {
private val byIndex = LinkedHashMap<Int, ChatMessageRecord>()
private val indexByIds = HashMap<String, Int>()
private var counter = 0
fun add(record: ChatMessageRecord): Int {
record.ids?.let { ids -> indexByIds[ids]?.let { return it } }
val index = ++counter
byIndex[index] = record
record.ids?.let { indexByIds[it] = index }
return index
}
fun get(index: Int): ChatMessageRecord? = byIndex[index]
fun indexOfIds(ids: String): Int? = indexByIds[ids]
}
internal object ConversationContext {
private val contextCache = mutableMapOf<Long, ConversationCache>()
private val replyIndexes = mutableMapOf<Long, ReplyIndex>()
private val imageIndexes = mutableMapOf<Long, ImageIndex>()
private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy年MM月dd E HH:mm:ss")
private val shortTimeFormatter = DateTimeFormatter.ofPattern("HH:mm")
.withZone(ZoneOffset.systemDefault())
private var memePrompt: String? = null
fun clearCache() {
synchronized(contextCache) { contextCache.clear() }
}
fun clearAll() {
synchronized(contextCache) { contextCache.clear() }
synchronized(replyIndexes) { replyIndexes.clear() }
synchronized(imageIndexes) { imageIndexes.clear() }
}
fun cache(subjectId: Long): ConversationCache? = synchronized(contextCache) {
contextCache[subjectId]
}
fun saveCache(subjectId: Long, cache: ConversationCache) {
synchronized(contextCache) { contextCache[subjectId] = cache }
}
fun activateReplyIndex(subjectId: Long, cached: ReplyIndex?): ReplyIndex =
(cached ?: ReplyIndex()).also { index ->
synchronized(replyIndexes) { replyIndexes[subjectId] = index }
}
fun activateImageIndex(subjectId: Long, cached: ImageIndex?): ImageIndex =
(cached ?: ImageIndex()).also { index ->
synchronized(imageIndexes) { imageIndexes[subjectId] = index }
}
fun releaseActiveIndexes(subjectId: Long) {
synchronized(replyIndexes) { replyIndexes.remove(subjectId) }
synchronized(imageIndexes) { imageIndexes.remove(subjectId) }
}
fun lookupReplyTarget(subjectId: Long, index: Int): ChatMessageRecord? =
synchronized(replyIndexes) { replyIndexes[subjectId]?.get(index) }
fun registerImage(subjectId: Long, imageId: String, imageUrl: String): Int? =
synchronized(imageIndexes) { imageIndexes[subjectId]?.add(imageId, imageUrl) }
fun lookupImageUrl(subjectId: Long, index: Int): String? =
synchronized(imageIndexes) { imageIndexes[subjectId]?.getUrl(index) }
fun getSystemPrompt(event: MessageEvent): String {
val now = OffsetDateTime.now()
val prompt = StringBuilder(LargeLanguageModels.systemPrompt)
fun replace(target: String, replacement: () -> String) {
val index = prompt.indexOf(target)
if (index != -1) prompt.replace(index, index + target.length, replacement())
}
replace("{time}") {
val solarTime = dateTimeFormatter.format(now)
"$solarTime\n农历${LunarDateUtil.getFormattedLunarAndHoliday(now)}"
}
replace("{subject}") {
if (event is GroupMessageEvent) {
"\"${event.subject.name}\" 群聊中,你在本群的名片是:${getNameCard(event.subject.botAsMember)}"
} else {
"\"${event.senderName}\" 私聊中"
}
}
replace("{memory}") {
PluginData.contactMemory[event.subject.id].orEmpty().ifEmpty { "暂无相关记忆" }
}
replace("{skills}") {
if (PluginConfig.skillsEnabled) SkillStore.buildIndexPrompt() else "暂无技能"
}
replace("{meme}") { buildMemePrompt() }
return prompt.toString()
}
fun getHistory(event: MessageEvent): String {
val imageIndex = activeImageIndex(event.subject.id)
if (!JChatGPT.includeHistory) {
return formatRecordContent(event.message, event.subject, imageIndex)
}
val beforeTimestamp = OffsetDateTime.now()
.minusMinutes(PluginConfig.historyWindowMin.toLong())
.toEpochSecond()
.toInt()
return getAfterHistory(beforeTimestamp, event)
}
fun getAfterHistory(time: Int, event: MessageEvent): String {
if (!JChatGPT.includeHistory) return ""
val history = try {
ChatHistoryStore.query(
contact = event.subject,
start = time,
end = OffsetDateTime.now().toEpochSecond().toInt(),
limit = PluginConfig.historyMessageLimit,
).sortedBy { it.time }.toMutableList()
} catch (cause: Throwable) {
JChatGPT.logger.warning("查询 SQLite 消息历史失败", cause)
mutableListOf()
}
val messageIds = event.message.ids.joinToString(",")
if (history.none { it.ids == messageIds }) {
history += ChatMessageRecord.fromSuccess(event.message.source, event.message)
}
val result = StringBuilder()
var lastUserId = 0L
var lastTime = 0L
val replyIndex = activeReplyIndex(event.subject.id)
val imageIndex = activeImageIndex(event.subject.id)
if (event is GroupMessageEvent) {
appendUserProfileContext(result, history, event)
result.appendLine("## 近期群消息(更早已隐藏,行首[n]为消息编号;正文[图片n]/[表情包n]中的n为识图或图片编辑编号)")
history.forEach { record ->
val showSender = lastUserId != record.fromId
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
appendGroupMessageRecord(result, record, event, replyIndex, imageIndex, showSender, showTime)
lastUserId = record.fromId
lastTime = record.time.toLong()
}
} else {
appendPrivateFavorabilityContext(result, event)
result.appendLine("## 近期对话(更早已隐藏,行首[n]为消息编号;正文[图片n]/[表情包n]中的n为识图或图片编辑编号)")
history.forEach { record ->
val showSender = lastUserId != record.fromId
val showTime = showSender || record.time.toLong() - lastTime > CONTINUATION_TIME_GAP_SECONDS
appendMessageRecord(result, record, event, replyIndex, imageIndex, showSender, showTime)
lastUserId = record.fromId
lastTime = record.time.toLong()
}
}
return result.toString()
}
fun toMessage(contact: Contact, content: String): Message {
if (content.isEmpty()) return PlainText("...")
if (content.length < 3) return PlainText(content)
val chunks = mutableListOf<MessageChunk>()
REGEX_AT_QQ.findAll(content).forEach { match ->
val qq = match.groups[1]?.value?.toLongOrNull()
if (qq != null && contact is Group) {
contact[qq]?.let { chunks += MessageChunk(match.range, At(it)) }
}
}
REGEX_IMAGE.findAll(content).forEach { match ->
chunks += MessageChunk(match.range, Image(match.groupValues[2]))
}
return buildMessageChain {
var index = 0
chunks.sortedBy { it.range.first }.forEach { (range, message) ->
if (index < range.first) append(content, index, range.first)
append(message)
index = range.last + 1
}
if (index < content.length) append(content, index, content.length)
}
}
private fun activeReplyIndex(subjectId: Long): ReplyIndex = synchronized(replyIndexes) {
replyIndexes.getOrPut(subjectId) { ReplyIndex() }
}
private fun activeImageIndex(subjectId: Long): ImageIndex = synchronized(imageIndexes) {
imageIndexes.getOrPut(subjectId) { ImageIndex() }
}
private fun buildMemePrompt(): String {
memePrompt?.let { return it }
if (PluginConfig.memeDir.isEmpty()) return ""
return buildString {
val directory = File(PluginConfig.memeDir)
if (!directory.isDirectory) {
append("配置的meme路径不存在!")
return@buildString
}
append("memes文件夹地址为:").appendLine(PluginConfig.memeDir)
val memes = directory.list().orEmpty()
if (memes.isEmpty()) {
append("暂无表情包~")
} else {
memes.forEach { append("- ").appendLine(it) }
appendLine()
append("表情包示例:![").append(memes[0]).append("](")
.append(File(directory, memes[0]).absoluteFile).appendLine(")")
}
}.also { memePrompt = it }
}
private fun appendUserProfileContext(
target: StringBuilder,
history: List<ChatMessageRecord>,
event: GroupMessageEvent,
) {
if (!PluginConfig.profileAutoInjectEnabled && !PluginConfig.enableFavorabilitySystem) return
val candidateIds = buildList {
add(event.sender.id)
history.asReversed().forEach { add(it.fromId) }
}.asSequence()
.filter { it != event.bot.id }
.distinct()
.take(PluginConfig.profileAutoInjectMaxUsers.coerceIn(1, 10))
.toList()
val profiles = if (PluginConfig.profileEnabled && PluginConfig.profileAutoInjectEnabled &&
UserProfileStore.isAvailable
) {
candidateIds.mapNotNull { userId ->
runCatching { UserProfileStore.load(userId) }
.onFailure { JChatGPT.logger.warning("读取用户画像失败: user=$userId", it) }
.getOrNull()
}
} else emptyList()
val favorability = if (PluginConfig.enableFavorabilitySystem) {
candidateIds.mapNotNull { id -> PluginData.userFavorability[id]?.let { id to it } }.toMap()
} else emptyMap()
val names = candidateIds.associateWith { id -> event.group[id]?.nameCardOrNick ?: id.toString() }
target.append(
UserProfileContextRenderer.render(
profiles = profiles,
favorabilityByUserId = favorability,
displayNames = names,
activeUserIds = candidateIds.toSet(),
summaryMaxChars = PluginConfig.profileAutoInjectSummaryMaxChars,
)
)
}
private fun appendPrivateFavorabilityContext(target: StringBuilder, event: MessageEvent) {
if (!PluginConfig.enableFavorabilitySystem) return
val info = PluginData.userFavorability[event.sender.id] ?: return
if (info.name.isEmpty() && info.tags.isEmpty() && info.impression.isEmpty()) return
val displayName = info.name.ifEmpty { event.senderName }
target.appendLine("【你认识的对方】")
.append("- ").append(displayName).append("(${event.sender.id})")
.append(" 好感度${if (info.value >= 0) "+" else ""}${info.value}")
if (info.tags.isNotEmpty()) target.append(" [${info.tags.joinToString(", ")}]")
if (info.impression.isNotEmpty()) target.append(" ${info.impression}")
target.appendLine().appendLine()
}
private fun appendGroupMessageRecord(
target: StringBuilder,
record: ChatMessageRecord,
event: GroupMessageEvent,
replyIndex: ReplyIndex,
imageIndex: ImageIndex,
showSender: Boolean,
showTime: Boolean,
) {
val chain = record.toMessageChain()
target.append('[').append(replyIndex.add(record)).append("] ")
if (showSender) {
if (event.bot.id == record.fromId) {
target.append("**你** ").append(getNameCard(event.subject.botAsMember))
} else {
target.append(getNameCard(event.subject, record.fromId))
}
target.append(' ').append(shortTime(record.time)).append(' ')
} else {
target.append("")
if (showTime) target.append(shortTime(record.time)).append(' ')
}
chain[QuoteReply.Key]?.let { appendQuoteMarker(target, it, event.subject, replyIndex, imageIndex) }
target.appendLine(formatRecordContent(chain, event.subject, imageIndex))
}
private fun appendMessageRecord(
target: StringBuilder,
record: ChatMessageRecord,
event: MessageEvent,
replyIndex: ReplyIndex,
imageIndex: ImageIndex,
showSender: Boolean,
showTime: Boolean,
) {
val chain = record.toMessageChain()
target.append('[').append(replyIndex.add(record)).append("] ")
if (showSender) {
if (event.bot.id == record.fromId) target.append("**你** ").append(event.bot.nameCardOrNick)
else target.append(event.senderName)
target.append(' ').append(shortTime(record.time)).append(' ')
} else {
target.append("")
if (showTime) target.append(shortTime(record.time)).append(' ')
}
chain[QuoteReply.Key]?.let { appendQuoteMarker(target, it, event.subject, replyIndex, imageIndex) }
target.appendLine(formatRecordContent(chain, event.subject, imageIndex))
}
private fun appendQuoteMarker(
target: StringBuilder,
quote: QuoteReply,
contact: Contact,
replyIndex: ReplyIndex,
imageIndex: ImageIndex,
) {
replyIndex.indexOfIds(quote.source.ids.joinToString(","))?.let { index ->
target.append("↩[").append(index).append("] ")
return
}
val author = if (contact is Group) {
contact[quote.source.fromId]?.nameCardOrNick ?: "未知(${quote.source.fromId})"
} else quote.source.fromId.toString()
val snippet = quote.source.originalMessage
.joinToString("") { singleMessageToText(it, imageIndex) }
.replace("\n", " ")
.let { if (it.length > 20) it.take(20) + "" else it }
target.append("↩(").append(author).append(":\"").append(snippet).append("\") ")
}
private fun formatRecordContent(
chain: MessageChain,
contact: Contact,
imageIndex: ImageIndex,
): String = chain.asSequence()
.filterNot { it is QuoteReply || it is MessageSource }
.joinToString("") { message ->
when (message) {
is At -> if (contact is Group) message.getDisplay(contact) else message.content
else -> singleMessageToText(message, imageIndex)
}
}
private fun singleMessageToText(message: SingleMessage, imageIndex: ImageIndex): String = when (message) {
is ForwardMessage -> formatForward(message, 1, imageIndex)
is Image -> try {
val url = runBlocking { message.queryUrl() }
val index = imageIndex.add(message.imageId, url)
"[${if (message.isEmoji) "表情包" else "图片"}$index]"
} catch (cause: Throwable) {
JChatGPT.logger.warning("图片地址获取失败", cause)
message.content
}
else -> message.content
}
private fun formatForward(forward: ForwardMessage, depth: Int, imageIndex: ImageIndex): String = buildString {
val quote = ">".repeat(depth) + " "
append("[转发消息·").append(forward.nodeList.size).append("")
if (forward.title.isNotEmpty()) append(':').append(forward.title)
append(']')
forward.nodeList.forEach { node ->
append('\n').append(quote).append(node.senderName).append(' ')
.append(shortTimeFormatter.format(Instant.ofEpochSecond(node.time.toLong())))
.append(": ")
node.messageChain.forEach { child ->
if (child is ForwardMessage) append(formatForward(child, depth + 1, imageIndex))
else append(singleMessageToText(child, imageIndex).replace("\n", "\n$quote"))
}
}
}
private fun getNameCard(group: Group, qq: Long): String =
group[qq]?.let(::getNameCard) ?: "未知群员($qq)"
private fun getNameCard(member: Member): String {
val result = StringBuilder("")
try {
result.append("lv").append(member.active.temperature).append(' ')
} catch (cause: Throwable) {
JChatGPT.logger.warning("获取群活跃等级失败", cause)
}
result.append(
when (member.permission) {
OWNER -> "群主"
ADMINISTRATOR -> "管理员"
MEMBER -> "群员"
}
)
try {
if (member.specialTitle.isNotEmpty()) result.append(" 头衔\"").append(member.specialTitle).append('"')
else if (member.temperatureTitle.isNotEmpty()) result.append(' ').append(member.temperatureTitle)
} catch (cause: Throwable) {
JChatGPT.logger.warning("获取群头衔失败", cause)
}
return result.append("\t\"").append(member.nameCardOrNick)
.append("\"\t(qq=").append(member.id).append(')').toString()
}
private fun shortTime(epochSecond: Int): String =
shortTimeFormatter.format(Instant.ofEpochSecond(epochSecond.toLong()))
private data class MessageChunk(val range: IntRange, val content: Message)
private const val CONTINUATION_TIME_GAP_SECONDS = 60L
private val REGEX_AT_QQ = Regex("""@(\d{5,12})""")
private val REGEX_IMAGE = Regex("""!\[(.*?)]\(([^\s"']+).*?\)""")
}
@@ -0,0 +1,344 @@
package top.jie65535.mirai.conversation
import com.aallam.openai.api.chat.ChatCompletionChunk
import com.aallam.openai.api.chat.ChatCompletionRequest
import com.aallam.openai.api.chat.ChatMessage
import com.aallam.openai.api.chat.ChatRole
import com.aallam.openai.api.chat.ToolCall
import com.aallam.openai.api.core.Usage
import com.aallam.openai.api.model.ModelId
import io.ktor.util.collections.ConcurrentSet
import kotlinx.coroutines.Deferred
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.launch
import net.mamoe.mirai.event.events.GroupMessageEvent
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.TokenUsageStore
import top.jie65535.mirai.llm.LargeLanguageModels
import top.jie65535.mirai.llm.ModelService
import top.jie65535.mirai.profile.ProfileAutoMaintenance
import top.jie65535.mirai.tools.AdjustUserFavorabilityAgent
import top.jie65535.mirai.tools.BaseAgent
import top.jie65535.mirai.tools.DeleteSkill
import top.jie65535.mirai.tools.GroupManageAgent
import top.jie65535.mirai.tools.ImageAgent
import top.jie65535.mirai.tools.LoadSkill
import top.jie65535.mirai.tools.MemoryAppend
import top.jie65535.mirai.tools.MemoryReplace
import top.jie65535.mirai.tools.ReasoningAgent
import top.jie65535.mirai.tools.RequestOwner
import top.jie65535.mirai.tools.RunCode
import top.jie65535.mirai.tools.SaveSkill
import top.jie65535.mirai.tools.SearchChatHistory
import top.jie65535.mirai.tools.SendCompositeMessage
import top.jie65535.mirai.tools.SendLaTeXExpression
import top.jie65535.mirai.tools.SendSingleMessageAgent
import top.jie65535.mirai.tools.SendVoiceMessage
import top.jie65535.mirai.tools.StopLoopAgent
import top.jie65535.mirai.tools.VisitWeb
import top.jie65535.mirai.tools.VisualAgent
import top.jie65535.mirai.tools.WeatherService
import top.jie65535.mirai.tools.WebSearch
import java.time.OffsetDateTime
import java.time.format.DateTimeFormatter
import kotlin.math.max
import kotlin.math.min
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
internal object ConversationEngine {
private val activeRequests = ConcurrentSet<Long>()
private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy年MM月dd E HH:mm:ss")
private val thinkRegex = Regex("<think>[\\s\\S]*?</think>")
private val tools: List<BaseAgent> = listOf(
SendSingleMessageAgent(),
SendCompositeMessage(),
SendVoiceMessage(),
SendLaTeXExpression(),
StopLoopAgent(),
MemoryAppend(),
MemoryReplace(),
LoadSkill(),
SaveSkill(),
DeleteSkill(),
SearchChatHistory(),
WebSearch(),
VisitWeb(),
RunCode(),
ReasoningAgent(),
VisualAgent(),
ImageAgent(),
WeatherService(),
AdjustUserFavorabilityAgent(),
RequestOwner(),
GroupManageAgent(),
)
fun clear() {
activeRequests.clear()
}
suspend fun start(event: MessageEvent) {
val subjectId = event.subject.id
if (!activeRequests.add(subjectId)) {
JChatGPT.logger.warning("The current Contact is busy!")
return
}
try {
val cache = ConversationContext.cache(subjectId)
val reuseCache = PluginConfig.enableContextCache && cache != null &&
!cache.isExpired(PluginConfig.contextCacheTimeoutMinutes * 60)
val replyIndex = ConversationContext.activateReplyIndex(
subjectId,
cache?.replyIndex?.takeIf { reuseCache },
)
val imageIndex = ConversationContext.activateImageIndex(
subjectId,
cache?.imageIndex?.takeIf { reuseCache },
)
val history = if (reuseCache) {
JChatGPT.logger.info("使用缓存的对话上下文,包含 ${cache.history.size} 条互动消息")
cache.history
} else mutableListOf()
if (history.isEmpty() || cache == null) {
val prompt = ConversationContext.getSystemPrompt(event)
if (PluginConfig.logPrompt) JChatGPT.logger.info("Prompt: $prompt")
history += ChatMessage(ChatRole.System, prompt)
val historyText = ConversationContext.getHistory(event)
JChatGPT.logger.info("注入聊天记录:\n$historyText")
history += ChatMessage.User(historyText)
} else {
val newMessages = ConversationContext.getAfterHistory(cache.lastActivityAt, event)
JChatGPT.logger.info("补充聊天记录:\n$newMessages")
history += ChatMessage.User("## 以下是上次对话结束至今的新消息\n\n$newMessages")
}
val endpoints = LargeLanguageModels.orderedChatEndpoints()
if (endpoints.isEmpty()) error("OpenAI Token 未设置,无法开始")
var endpointIndex = 0
var done: Boolean
var retry = max(PluginConfig.retryMax, 3)
do {
val endpoint = endpoints[min(endpointIndex, endpoints.lastIndex)]
var streamingOk = false
try {
val startedAt = OffsetDateTime.now().toEpochSecond().toInt()
var lastCacheUsage: ModelService.CacheUsage? = null
val responseFlow = chatCompletions(history, endpoint) { lastCacheUsage = it }
var responseContent: StringBuilder? = null
var reasoningContent: StringBuilder? = null
val responseToolCalls = mutableListOf<ToolCall.Function>()
val toolCallTasks = mutableListOf<Deferred<ChatMessage>>()
var lastTokenUsage: Usage? = null
responseFlow.collect { chunk ->
val delta = chunk.choices[0].delta ?: return@collect
delta.reasoningContent?.let { content ->
if (reasoningContent == null) reasoningContent = StringBuilder(content)
else reasoningContent.append(content)
}
delta.content?.let { content ->
if (responseContent == null) responseContent = StringBuilder(content)
else responseContent.append(content)
}
delta.toolCalls?.forEach { toolCallChunk ->
val index = toolCallChunk.index
val function = toolCallChunk.function
if (index >= responseToolCalls.size) {
responseToolCalls.lastOrNull()?.let { toolCall ->
toolCallTasks += JChatGPT.async {
toolCall.toResultMessage(event)
}
}
val id = toolCallChunk.id
if (id != null && function != null) {
responseToolCalls += ToolCall.Function(id, function)
}
} else if (function != null) {
val current = responseToolCalls[index]
var updated = current.function
function.nameOrNull?.let { name ->
updated = updated.copy(nameOrNull = updated.nameOrNull.orEmpty() + name)
}
function.argumentsOrNull?.let { arguments ->
updated = updated.copy(
argumentsOrNull = updated.argumentsOrNull.orEmpty() + arguments
)
}
responseToolCalls[index] = current.copy(function = updated)
}
}
chunk.usage?.let { lastTokenUsage = it }
}
streamingOk = true
LargeLanguageModels.reportSuccess(endpoint)
val answer = responseContent?.replace(thinkRegex, "")?.trim()
JChatGPT.logger.info("LLM Response: $answer")
history += ChatMessage(
role = ChatRole.Assistant,
content = answer,
toolCalls = responseToolCalls.ifEmpty { null },
reasoningContent = if (responseToolCalls.isNotEmpty()) reasoningContent?.toString() else null,
)
recordUsage(event, lastTokenUsage, lastCacheUsage)
if (responseToolCalls.size > toolCallTasks.size) {
val finalToolResult = responseToolCalls.last().toResultMessage(event)
if (toolCallTasks.isNotEmpty()) history += toolCallTasks.awaitAll()
history += finalToolResult
done = responseToolCalls.any { it.function.name == "endConversation" }
} else {
done = true
}
if (!done) {
history += ChatMessage.User(buildContinuationPrompt(retry, startedAt, event))
} else {
if (PluginConfig.enableContextCache) {
ConversationContext.saveCache(
subjectId,
ConversationCache(history, startedAt, replyIndex, imageIndex),
)
JChatGPT.logger.debug("已保存对话上下文到缓存")
}
if (event is GroupMessageEvent) {
ProfileAutoMaintenance.recordCompletedConversation(event, startedAt)
}
}
} catch (cause: Exception) {
if (!streamingOk) {
LargeLanguageModels.reportFailure(endpoint)
if (endpointIndex < endpoints.lastIndex) {
endpointIndex++
JChatGPT.logger.warning(
"接入点[${endpoint.label}]调用失败,切换备用接入点[${endpoints[endpointIndex].label}]重试",
cause,
)
} else {
JChatGPT.logger.warning("接入点[${endpoint.label}]调用失败,无更多备用接入点,重试中", cause)
}
} else {
JChatGPT.logger.warning("调用llm后处理时发生异常,重试中", cause)
}
if (retry <= 1) throw cause
done = false
}
} while (!done && 0 < --retry)
} catch (cause: Throwable) {
JChatGPT.logger.warning(cause)
event.subject.sendMessage("很抱歉,发生异常,请稍后重试")
} finally {
ConversationContext.releaseActiveIndexes(subjectId)
JChatGPT.launch {
delay(500.milliseconds)
activeRequests.remove(subjectId)
}
}
}
private fun chatCompletions(
history: List<ChatMessage>,
endpoint: LargeLanguageModels.ChatEndpoint,
onCacheUsage: ((ModelService.CacheUsage) -> Unit)? = null,
): Flow<ChatCompletionChunk> {
val availableTools = tools.filter { it.isEnabled }.map { it.tool }
val request = ChatCompletionRequest(
model = ModelId(endpoint.model),
temperature = endpoint.temperature,
messages = history,
tools = availableTools,
)
JChatGPT.logger.info("API Requesting... Model=${endpoint.model} [${endpoint.label}]")
return endpoint.service.chatCompletions(request, onCacheUsage)
}
private suspend fun ToolCall.Function.toResultMessage(event: MessageEvent): ChatMessage = ChatMessage(
role = ChatRole.Tool,
toolCallId = id,
name = function.name,
content = execute(event),
)
private suspend fun ToolCall.Function.execute(event: MessageEvent): String {
val agent = tools.find { it.tool.function.name == function.name }
?: return "Function ${function.name} not found"
val receipt = if (PluginConfig.showToolCallingMessage && agent.loadingMessage.isNotEmpty()) {
event.subject.sendMessage(agent.loadingMessage)
} else null
val result = try {
val arguments = function.argumentsAsJsonOrNull()
JChatGPT.logger.info("Calling ${function.name}($arguments)")
agent.execute(arguments, event)
} catch (cause: Throwable) {
JChatGPT.logger.error("Failed to call ${function.name}", cause)
"工具调用失败,请尝试自行回答用户,或如实告知。\n异常信息:${cause.message}"
}
JChatGPT.logger.info("Result=\"$result\"")
val truncated = truncateToolOutput(result)
if (truncated.length != result.length) {
JChatGPT.logger.warning(
"工具 ${function.name} 返回内容过长,已从 ${result.length} 字符截断至 ${truncated.length} 字符"
)
}
if (receipt != null) {
JChatGPT.launch {
delay(3.seconds)
try {
receipt.recall()
} catch (cause: Throwable) {
JChatGPT.logger.error(
"消息撤回失败,调试信息:source.internalIds=${receipt.source.internalIds.joinToString()} " +
"source.ids=${receipt.source.ids.joinToString()}",
cause,
)
}
}
}
return truncated
}
private fun buildContinuationPrompt(retry: Int, startedAt: Int, event: MessageEvent): String = buildString {
appendLine("## 系统提示")
append("本次运行最多还剩").append(retry - 1).appendLine("轮。")
appendLine("如果要多次发言,可以一次性调用多次发言工具。")
appendLine("如果没有什么要做的,可以提前结束。")
appendLine("当前时间:${dateTimeFormatter.format(OffsetDateTime.now())}")
val messages = ConversationContext.getAfterHistory(startedAt, event)
if (messages.isNotEmpty()) append("## 以下是上次运行至今的新消息\n\n$messages")
}
private fun recordUsage(
event: MessageEvent,
usage: Usage?,
cacheUsage: ModelService.CacheUsage?,
) {
usage ?: return
val group = (event as? GroupMessageEvent)?.group
TokenUsageStore.record(
timestamp = OffsetDateTime.now().toEpochSecond(),
userId = event.sender.id,
userNickname = event.senderName,
groupId = group?.id,
groupName = group?.name,
promptTokens = usage.promptTokens ?: 0,
completionTokens = usage.completionTokens ?: 0,
totalTokens = usage.totalTokens ?: 0,
cachedTokens = cacheUsage?.hitTokens ?: 0,
)
}
private fun truncateToolOutput(content: String): String {
val maxLength = PluginConfig.maxToolOutputLength
return if (content.length <= maxLength) content
else content.take(maxLength) + "\n\n[系统提示:因内容过长,部分内容已被省略]"
}
}
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.data
import net.mamoe.mirai.Bot
import net.mamoe.mirai.contact.Contact
@@ -34,14 +34,22 @@ object ChatHistoryStore {
private var initialized = false
private lateinit var databaseFile: File
private var writeConnection: Connection? = null
private var warningLogger: ((String, Throwable?) -> Unit)? = null
val isAvailable: Boolean
get() = initialized
fun init(dataFolder: File) {
val databaseFileOrNull: File?
get() = if (initialized) databaseFile else null
fun init(
dataFolder: File,
onWarning: (String, Throwable?) -> Unit = { _, _ -> },
) {
synchronized(lifecycleLock) {
if (initialized) return
warningLogger = onWarning
Class.forName("org.sqlite.JDBC")
dataFolder.mkdirs()
databaseFile = dataFolder.resolve("chat-history.sqlite")
@@ -68,11 +76,12 @@ object ChatHistoryStore {
connection.createStatement().use { statement ->
statement.execute("PRAGMA wal_checkpoint(TRUNCATE)")
}
}.onFailure { JChatGPT.logger.warning("SQLite WAL checkpoint 失败", it) }
}.onFailure { warn("SQLite WAL checkpoint 失败", it) }
connection.close()
}
writeConnection = null
initialized = false
warningLogger = null
}
}
}
@@ -139,7 +148,7 @@ object ChatHistoryStore {
statement.setInt(8, event.messageTime)
val updated = statement.executeUpdate()
if (updated == 0) {
JChatGPT.logger.warning(
warn(
"未在 SQLite 中找到撤回消息: bot=${event.bot.id}, " +
"author=${event.authorId}, target=$targetId, ids=$messageIds, " +
"internalIds=$messageInternalIds, " +
@@ -365,4 +374,8 @@ object ChatHistoryStore {
recalled = getInt("recalled"),
)
}
private fun warn(message: String, cause: Throwable? = null) {
runCatching { warningLogger?.invoke(message, cause) }
}
}
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.data
import kotlinx.serialization.SerializationException
import net.mamoe.mirai.Mirai
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.data
import kotlinx.serialization.Serializable
import net.mamoe.mirai.console.data.AutoSavePluginData
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.data
import java.io.File
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.data
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.json.Json
@@ -1,8 +1,10 @@
package top.jie65535.mirai
package top.jie65535.mirai.llm
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonObject
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import kotlin.time.Duration.Companion.milliseconds
object LargeLanguageModels {
@@ -25,6 +27,12 @@ object LargeLanguageModels {
val label: String,
)
data class ProfileEndpoint(
val service: ModelService,
val model: String,
val temperature: Double?,
)
/**
* 聊天接入点列表index 0 为主接入点其余按配置顺序为备用接入点
*/
@@ -47,6 +55,10 @@ object LargeLanguageModels {
*/
var visual: ModelService? = null
/** 历史用户画像分析模型。 */
var profile: ProfileEndpoint? = null
private set
/**
* 接入点健康状态记录各接入点的冷却截止时间戳毫秒
* 失败的接入点进入冷却期间在 [orderedChatEndpoints] 中被排到队尾
@@ -144,6 +156,28 @@ object LargeLanguageModels {
}
chatEndpoints = endpoints
profile = null
if (PluginConfig.profileEnabled) {
val api = PluginConfig.profileModelApi.ifBlank { PluginConfig.openAiApi }
val token = PluginConfig.profileModelToken.ifBlank { PluginConfig.openAiToken }
val model = PluginConfig.profileModel.ifBlank { PluginConfig.chatModel }
val extraBody = PluginConfig.profileModelExtraBody.ifBlank { PluginConfig.chatModelExtraBody }
if (api.isNotBlank() && token.isNotBlank() && model.isNotBlank()) {
val profileFirstChunk = PluginConfig.profileFirstChunkTimeout.milliseconds
profile = ProfileEndpoint(
service = ModelService(
baseUrl = api,
token = token,
timeout = maxOf(timeout, profileFirstChunk),
firstChunkTimeout = profileFirstChunk,
extraBody = parseExtraBody(extraBody),
),
model = model,
temperature = PluginConfig.profileModelTemperature ?: PluginConfig.chatTemperature,
)
}
}
// 初始化推理模型
if (PluginConfig.reasoningModelApi.isNotBlank() && PluginConfig.reasoningModelToken.isNotBlank()) {
// 推理模型出首块前常有思考预热,比对话慢,使用单独放宽的首块超时;
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.llm
import com.aallam.openai.api.chat.ChatCompletionChunk
import com.aallam.openai.api.chat.ChatCompletionRequest
@@ -48,18 +48,19 @@ class ModelService(
explicitNulls = false
}
/**
* 一次响应的缓存命中用量DeepSeek usage 顶层返回的非标准字段
* openai-kotlin Usage 类不含这些字段必须从原始 JSON 抠出来
*/
/** openai-kotlin 的 Usage 尚未暴露缓存明细,因此从原始 JSON 读取。 */
data class CacheUsage(val hitTokens: Int, val missTokens: Int)
/** 从原始 data 行(已去掉 "data: " 前缀)解析缓存命中用量;无相关字段返回 null。 */
private fun extractCacheUsage(rawJson: String): CacheUsage? {
internal fun extractCacheUsage(rawJson: String): CacheUsage? {
return try {
val usage = json.parseToJsonElement(rawJson).jsonObject["usage"]?.jsonObject ?: return null
val promptTokens = usage["prompt_tokens"]?.jsonPrimitive?.intOrNull
val promptDetails = usage["prompt_tokens_details"]?.jsonObject
val hit = usage["prompt_cache_hit_tokens"]?.jsonPrimitive?.intOrNull
?: promptDetails?.get("cached_tokens")?.jsonPrimitive?.intOrNull
val miss = usage["prompt_cache_miss_tokens"]?.jsonPrimitive?.intOrNull
?: if (hit != null && promptTokens != null) (promptTokens - hit).coerceAtLeast(0) else null
if (hit == null && miss == null) null else CacheUsage(hit ?: 0, miss ?: 0)
} catch (_: Exception) {
null
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.media
/**
* 会话内图片短索引 LLM 暴露递增整数内部保留从原消息图片取得的精确 URL
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.media
import org.scilab.forge.jlatexmath.TeXConstants
import org.scilab.forge.jlatexmath.TeXFormula
@@ -29,4 +29,4 @@ object LaTeXConverter {
ImageIO.write(image, format, stream)
return stream.toByteArray()
}
}
}
@@ -0,0 +1,46 @@
package top.jie65535.mirai.profile
object ConversationProfileReducer {
fun reduce(
profiles: Map<Long, UserProfileSnapshot>,
batch: ConversationProfileBatch,
eligibleUserIds: Set<Long>,
response: ConversationProfileModelResponse,
model: String,
promptVersion: String,
summaryMaxLength: Int,
): List<ProfileReduction> {
val responsesByUserId = response.users.associate { userResponse ->
val userId = batch.aliasToUserId[userResponse.userAlias]
?: throw IllegalArgumentException("模型返回了不存在的用户别名 ${userResponse.userAlias}")
require(userId in eligibleUserIds) {
"模型返回了非候选用户别名 ${userResponse.userAlias}"
}
require(userResponse.operations.size <= MAX_OPERATIONS_PER_USER) {
"模型为 ${userResponse.userAlias} 返回了超过 $MAX_OPERATIONS_PER_USER 项画像操作"
}
userId to userResponse
}
require(responsesByUserId.size == response.users.size) {
"模型对同一用户返回了多组结果"
}
return eligibleUserIds.sorted().map { userId ->
val userResponse = responsesByUserId[userId]
UserProfileReducer.reduce(
current = checkNotNull(profiles[userId]) { "缺少用户 $userId 的当前画像" },
batch = batch.forUser(userId),
response = ProfileModelResponse(
operations = userResponse?.operations.orEmpty(),
summary = userResponse?.summary.orEmpty(),
),
model = model,
promptVersion = promptVersion,
summaryMaxLength = summaryMaxLength,
advanceBackfillCursor = false,
)
}
}
private const val MAX_OPERATIONS_PER_USER = 4
}
@@ -0,0 +1,98 @@
package top.jie65535.mirai.profile
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import net.mamoe.mirai.event.events.GroupMessageEvent
import net.mamoe.mirai.message.data.source
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.ChatHistoryStore
import java.util.concurrent.atomic.AtomicLong
import kotlin.time.Duration.Companion.seconds
object ProfileAutoMaintenance {
private data class PendingConversation(
val generation: Long,
val groupId: Long,
val botId: Long,
val startTime: Int,
val endTime: Int,
val lastActivityAt: Int,
)
private val lock = Any()
private val generation = AtomicLong()
private val pending = mutableMapOf<Long, PendingConversation>()
fun recordCompletedConversation(event: GroupMessageEvent, lastActivityAt: Int) {
if (!PluginConfig.profileEnabled || !PluginConfig.profileAutoUpdateEnabled) return
if (!UserProfileStore.isAvailable || !ChatHistoryStore.isAvailable) return
val subjectId = event.subject.id
val now = currentEpochSecond()
val currentGeneration = generation.incrementAndGet()
val triggerTime = event.message.source.time
val initialStart = (triggerTime.toLong() - PluginConfig.historyWindowMin.coerceAtLeast(0) * 60L)
.coerceAtLeast(0)
.toInt()
val conversation = synchronized(lock) {
val previous = pending[subjectId]
PendingConversation(
generation = currentGeneration,
groupId = event.group.id,
botId = event.bot.id,
startTime = minOf(previous?.startTime ?: initialStart, initialStart),
endTime = maxOf(previous?.endTime ?: now.safeNextSecond(), now.safeNextSecond()),
lastActivityAt = maxOf(previous?.lastActivityAt ?: lastActivityAt, lastActivityAt),
).also { pending[subjectId] = it }
}
val ttlSeconds = PluginConfig.contextCacheTimeoutMinutes.coerceAtLeast(1) * 60
JChatGPT.launch {
val remainingSeconds = conversation.lastActivityAt.toLong() + ttlSeconds - currentEpochSecond()
if (remainingSeconds > 0) delay(remainingSeconds.seconds)
val closed = synchronized(lock) {
pending[subjectId]
?.takeIf { it.generation == conversation.generation }
?.copy(endTime = currentEpochSecond().safeNextSecond())
?.also { pending.remove(subjectId) }
} ?: return@launch
process(closed)
}
}
fun clear() {
synchronized(lock) { pending.clear() }
}
private suspend fun process(conversation: PendingConversation) {
try {
val report = UserProfileAnalysisService.analyzeConversation(
botId = conversation.botId,
groupId = conversation.groupId,
startTime = conversation.startTime,
endTime = conversation.endTime,
minAuthoredTextChars = PluginConfig.profileAutoMinAuthoredTextChars,
) ?: return
JChatGPT.logger.info(
"PROFILE_AUTO group=${conversation.groupId} users=${report.analyzedUsers} " +
"messages=${report.processedMessages} operations=${report.appliedOperations} " +
"tokens=${report.usage.promptTokens}/${report.usage.completionTokens}"
)
} catch (cause: CancellationException) {
throw cause
} catch (cause: Exception) {
JChatGPT.logger.warning(
"自动画像维护失败: group=${conversation.groupId}, " +
"range=[${conversation.startTime}, ${conversation.endTime})",
cause,
)
}
}
private fun currentEpochSecond(): Int =
(System.currentTimeMillis() / 1000L).coerceAtMost(Int.MAX_VALUE.toLong()).toInt()
private fun Int.safeNextSecond(): Int = if (this == Int.MAX_VALUE) this else this + 1
}
@@ -0,0 +1,475 @@
package top.jie65535.mirai.profile
import net.mamoe.mirai.message.data.MessageSourceKind
import org.sqlite.SQLiteConfig
import top.jie65535.mirai.data.ChatMessageRecord
import java.io.File
import java.security.MessageDigest
import java.sql.Connection
import java.sql.DriverManager
import java.sql.ResultSet
class ProfileHistoryReader(private val databaseFile: File) {
data class TimeBounds(val startTime: Int, val endTime: Int)
private data class Episode(
val index: Int,
val groupId: Long,
val targetMessages: List<ChatMessageRecord>,
)
init {
require(databaseFile.isFile) { "聊天历史数据库不存在: ${databaseFile.absolutePath}" }
Class.forName("org.sqlite.JDBC")
}
fun findUserTimeBounds(userId: Long): TimeBounds? = openReadConnection().use { connection ->
connection.prepareStatement(
"""
SELECT MIN(time) AS min_time, MAX(time) AS max_time
FROM message_record
WHERE kind = ? AND recalled = 0 AND from_id = ?
""".trimIndent()
).use { statement ->
statement.setInt(1, MessageSourceKind.GROUP.ordinal)
statement.setLong(2, userId)
statement.executeQuery().use { results ->
if (!results.next()) return@use null
val start = results.getInt("min_time")
if (results.wasNull()) return@use null
val max = results.getInt("max_time")
TimeBounds(start, max.safeNextSecond())
}
}
}
fun loadNextBatch(
userId: Long,
startTime: Int,
snapshotEndTime: Int,
targetMessageLimit: Int,
maxEpisodes: Int,
episodeGapSeconds: Int,
contextBeforeMessages: Int,
contextAfterMessages: Int,
contextCoreMessages: Int,
maxMessageChars: Int,
): ProfileHistoryBatch? {
require(startTime <= snapshotEndTime) { "startTime must not be after snapshotEndTime" }
return openReadConnection().use { connection ->
val firstPage = queryTargetMessages(
connection,
userId,
startTime,
snapshotEndTime,
targetMessageLimit.coerceAtLeast(1),
)
if (firstPage.isEmpty()) return@use null
val pageEndTime = firstPage.last().time.safeNextSecond()
val pageTargetMessages = queryTargetMessages(
connection,
userId,
startTime,
pageEndTime,
Int.MAX_VALUE,
)
val pageEpisodes = buildEpisodes(pageTargetMessages, episodeGapSeconds.coerceAtLeast(0))
val initialEpisodes = pageEpisodes.take(maxEpisodes.coerceAtLeast(1))
val lastSelectedTime = initialEpisodes.last().targetMessages.maxOf { it.time }
val episodes = pageEpisodes.takeWhile { episode ->
episode.targetMessages.first().time <= lastSelectedTime
}
val targetMessages = episodes.flatMap(Episode::targetMessages)
val endTime = targetMessages.maxOf { it.time }.safeNextSecond()
val recordsByFingerprint = linkedMapOf<String, Pair<Int, ChatMessageRecord>>()
val perEpisodeCoreLimit = (contextCoreMessages.coerceAtLeast(1) / episodes.size)
.coerceAtLeast(1)
episodes.forEach { episode ->
val firstTargetTime = episode.targetMessages.minOf { it.time }
val lastTargetTime = episode.targetMessages.maxOf { it.time }
val records = buildList {
addAll(queryContextBefore(connection, episode.groupId, firstTargetTime, contextBeforeMessages))
addAll(
queryContextCore(
connection,
episode.groupId,
firstTargetTime,
lastTargetTime.safeNextSecond(),
perEpisodeCoreLimit,
)
)
addAll(queryContextAfter(connection, episode.groupId, lastTargetTime, contextAfterMessages))
addAll(episode.targetMessages)
}
records.forEach { record ->
recordsByFingerprint.putIfAbsent(record.fingerprint(), episode.index to record)
}
}
val targetFingerprints = targetMessages.mapTo(hashSetOf()) { it.fingerprint() }
val targetRecords = recordsByFingerprint.values.filter { it.second.fingerprint() in targetFingerprints }
val contextRecords = recordsByFingerprint.values
.asSequence()
.filter { it.second.fingerprint() !in targetFingerprints }
.sortedWith(
compareBy<Pair<Int, ChatMessageRecord>>(
{ candidate -> contextDistance(candidate.second, targetMessages) },
{ it.second.time },
{ it.second.targetId },
{ it.second.fromId },
)
)
.take(contextCoreMessages.coerceAtLeast(0))
.toList()
val records = (targetRecords + contextRecords).sortedWith(
compareBy<Pair<Int, ChatMessageRecord>>(
{ it.second.time },
{ it.first },
{ it.second.targetId },
{ it.second.fromId },
{ it.second.code },
)
)
createBatch(userId, startTime, endTime, records, maxMessageChars)
}
}
fun loadConversationBatch(
botId: Long,
groupId: Long,
startTime: Int,
endTime: Int,
messageLimit: Int,
maxMessageChars: Int,
): ConversationProfileBatch? {
require(startTime < endTime) { "startTime must be before endTime" }
return openReadConnection().use { connection ->
val records = queryConversationMessages(
connection = connection,
botId = botId,
groupId = groupId,
startTime = startTime,
endTime = endTime,
limit = messageLimit.coerceAtLeast(1),
)
if (records.isEmpty()) return@use null
createConversationBatch(botId, groupId, startTime, endTime, records, maxMessageChars)
}
}
private fun queryTargetMessages(
connection: Connection,
userId: Long,
startTime: Int,
endTime: Int,
limit: Int,
): List<ChatMessageRecord> {
val sql = buildString {
append(
"""
SELECT id, bot_id, from_id, target_id, ids, internal_ids,
time, kind, code, recalled
FROM message_record
WHERE kind = ? AND recalled = 0 AND from_id = ?
AND time >= ? AND time < ?
ORDER BY time ASC
""".trimIndent()
)
if (limit != Int.MAX_VALUE) append(" LIMIT ?")
}
return connection.prepareStatement(sql).use { statement ->
statement.setInt(1, MessageSourceKind.GROUP.ordinal)
statement.setLong(2, userId)
statement.setInt(3, startTime)
statement.setInt(4, endTime)
if (limit != Int.MAX_VALUE) statement.setInt(5, limit)
statement.executeQuery().use(::readRecords)
}
}
private fun queryConversationMessages(
connection: Connection,
botId: Long,
groupId: Long,
startTime: Int,
endTime: Int,
limit: Int,
): List<ChatMessageRecord> {
return connection.prepareStatement(
"""
SELECT id, bot_id, from_id, target_id, ids, internal_ids,
time, kind, code, recalled
FROM message_record
WHERE bot_id = ? AND target_id = ? AND kind = ? AND recalled = 0
AND time >= ? AND time < ?
ORDER BY time DESC, id DESC
LIMIT ?
""".trimIndent()
).use { statement ->
statement.setLong(1, botId)
statement.setLong(2, groupId)
statement.setInt(3, MessageSourceKind.GROUP.ordinal)
statement.setInt(4, startTime)
statement.setInt(5, endTime)
statement.setInt(6, limit)
statement.executeQuery().use(::readRecords).asReversed()
}
}
private fun createConversationBatch(
botId: Long,
groupId: Long,
startTime: Int,
endTime: Int,
records: List<ChatMessageRecord>,
maxMessageChars: Int,
): ConversationProfileBatch {
val participantIds = buildSet {
add(botId)
records.forEach { record ->
add(record.fromId)
addAll(ProfileMessageRenderer.referencedUserIds(record))
}
}
val aliases = buildMap {
put(botId, "BOT")
participantIds.asSequence()
.filter { it != botId }
.sorted()
.forEachIndexed { index, participantId -> put(participantId, "U${index + 1}") }
}
val promptMessages = records.mapIndexed { index, record ->
ProfilePromptMessage(
record = record,
text = ProfileMessageRenderer.render(record, aliases, maxMessageChars.coerceAtLeast(80)),
evidenceRef = index + 1,
episodeIndex = 1,
)
}
return ConversationProfileBatch(
botId = botId,
groupId = groupId,
startTime = startTime,
endTime = endTime,
messages = promptMessages,
aliases = aliases,
inputHash = calculateInputHash(promptMessages),
)
}
private fun createBatch(
userId: Long,
startTime: Int,
endTime: Int,
records: List<Pair<Int, ChatMessageRecord>>,
maxMessageChars: Int,
): ProfileHistoryBatch {
val participantIds = buildSet {
add(userId)
records.forEach { (_, record) ->
add(record.fromId)
addAll(ProfileMessageRenderer.referencedUserIds(record))
}
}
val aliases = buildMap {
put(userId, "TARGET")
participantIds.asSequence()
.filter { it != userId }
.sorted()
.forEachIndexed { index, participantId -> put(participantId, "U${index + 1}") }
}
var evidenceRef = 0
val promptMessages = records.map { (episodeIndex, record) ->
val ref = if (record.time >= startTime && record.time < endTime) ++evidenceRef else null
ProfilePromptMessage(
record = record,
text = ProfileMessageRenderer.render(record, aliases, maxMessageChars.coerceAtLeast(80)),
evidenceRef = ref,
episodeIndex = episodeIndex,
)
}
return ProfileHistoryBatch(
userId = userId,
startTime = startTime,
endTime = endTime,
messages = promptMessages,
aliases = aliases,
inputHash = calculateInputHash(promptMessages),
)
}
private fun contextDistance(
record: ChatMessageRecord,
targetMessages: List<ChatMessageRecord>,
): Long = targetMessages.asSequence()
.filter { it.targetId == record.targetId }
.minOfOrNull { target -> kotlin.math.abs(target.time.toLong() - record.time.toLong()) }
?: Long.MAX_VALUE
private fun buildEpisodes(
targetMessages: List<ChatMessageRecord>,
gapSeconds: Int,
): List<Episode> {
val episodes = mutableListOf<Episode>()
var current = mutableListOf<ChatMessageRecord>()
var currentGroup = 0L
var lastTime = 0
fun flush() {
if (current.isNotEmpty()) {
episodes += Episode(episodes.size + 1, currentGroup, current.toList())
current = mutableListOf()
}
}
targetMessages.forEach { message ->
if (current.isNotEmpty() &&
(message.targetId != currentGroup || message.time - lastTime > gapSeconds)
) {
flush()
}
if (current.isEmpty()) currentGroup = message.targetId
current += message
lastTime = message.time
}
flush()
return episodes
}
private fun queryContextBefore(
connection: Connection,
groupId: Long,
beforeTime: Int,
limit: Int,
): List<ChatMessageRecord> {
if (limit <= 0) return emptyList()
return queryGroupContext(
connection,
"target_id = ? AND kind = ? AND recalled = 0 AND time < ? ORDER BY time DESC LIMIT ?",
groupId,
beforeTime,
limit,
).asReversed()
}
private fun queryContextAfter(
connection: Connection,
groupId: Long,
afterTime: Int,
limit: Int,
): List<ChatMessageRecord> {
if (limit <= 0) return emptyList()
return queryGroupContext(
connection,
"target_id = ? AND kind = ? AND recalled = 0 AND time > ? ORDER BY time ASC LIMIT ?",
groupId,
afterTime,
limit,
)
}
private fun queryContextCore(
connection: Connection,
groupId: Long,
startTime: Int,
endTime: Int,
limit: Int,
): List<ChatMessageRecord> {
if (limit <= 0) return emptyList()
return connection.prepareStatement(
"""
SELECT id, bot_id, from_id, target_id, ids, internal_ids,
time, kind, code, recalled
FROM message_record
WHERE target_id = ? AND kind = ? AND recalled = 0
AND time >= ? AND time < ?
ORDER BY time ASC
LIMIT ?
""".trimIndent()
).use { statement ->
statement.setLong(1, groupId)
statement.setInt(2, MessageSourceKind.GROUP.ordinal)
statement.setInt(3, startTime)
statement.setInt(4, endTime)
statement.setInt(5, limit)
statement.executeQuery().use(::readRecords)
}
}
private fun queryGroupContext(
connection: Connection,
predicate: String,
groupId: Long,
boundaryTime: Int,
limit: Int,
): List<ChatMessageRecord> = connection.prepareStatement(
"""
SELECT id, bot_id, from_id, target_id, ids, internal_ids,
time, kind, code, recalled
FROM message_record
WHERE $predicate
""".trimIndent()
).use { statement ->
statement.setLong(1, groupId)
statement.setInt(2, MessageSourceKind.GROUP.ordinal)
statement.setInt(3, boundaryTime)
statement.setInt(4, limit)
statement.executeQuery().use(::readRecords)
}
private fun readRecords(results: ResultSet): List<ChatMessageRecord> = buildList {
while (results.next()) add(results.toRecord())
}
private fun ResultSet.toRecord(): ChatMessageRecord {
val kind = MessageSourceKind.values().getOrNull(getInt("kind"))
?: throw IllegalStateException("未知的消息类型")
return ChatMessageRecord(
id = getLong("id"),
botId = getLong("bot_id"),
fromId = getLong("from_id"),
targetId = getLong("target_id"),
ids = getString("ids"),
internalIds = getString("internal_ids"),
time = getInt("time"),
kind = kind,
code = getString("code"),
recalled = getInt("recalled"),
)
}
private fun openReadConnection(): Connection {
val config = SQLiteConfig().apply {
setReadOnly(true)
setBusyTimeout(30_000)
}
return DriverManager.getConnection(
"jdbc:sqlite:${databaseFile.absolutePath}",
config.toProperties(),
).also { connection ->
connection.createStatement().use { statement ->
statement.execute("PRAGMA query_only=ON")
}
}
}
private fun ChatMessageRecord.fingerprint(): String =
"$botId|$fromId|$targetId|$time|${kind.ordinal}|$code".sha256()
private fun calculateInputHash(messages: List<ProfilePromptMessage>): String = buildString {
messages.forEach { message ->
append(message.episodeIndex).append('|')
append(message.evidenceRef ?: 0).append('|')
append(message.record.fingerprint()).append('\n')
}
}.sha256()
private fun String.sha256(): String = MessageDigest.getInstance("SHA-256")
.digest(toByteArray(Charsets.UTF_8))
.joinToString("") { byte -> "%02x".format(byte) }
private fun Int.safeNextSecond(): Int = if (this == Int.MAX_VALUE) this else this + 1
}
@@ -0,0 +1,177 @@
package top.jie65535.mirai.profile
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.booleanOrNull
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.longOrNull
import net.mamoe.mirai.message.data.At
import net.mamoe.mirai.message.data.ForwardMessage
import net.mamoe.mirai.message.data.Image
import net.mamoe.mirai.message.data.MessageChain
import net.mamoe.mirai.message.data.MessageSource
import net.mamoe.mirai.message.data.PlainText
import net.mamoe.mirai.message.data.QuoteReply
import net.mamoe.mirai.message.data.SingleMessage
import net.mamoe.mirai.message.data.content
import top.jie65535.mirai.data.ChatMessageRecord
object ProfileMessageRenderer {
private val json = Json { ignoreUnknownKeys = true }
fun render(record: ChatMessageRecord, aliases: Map<Long, String>, maxChars: Int): String {
val content = runCatching {
renderJsonCode(record.code, aliases)
}.recoverCatching {
renderChain(record.toMessageChain(), aliases)
}.getOrElse {
"[消息内容解析失败]"
}.replace(Regex("[\\r\\n]+"), " ").trim()
if (content.length <= maxChars) return content.ifEmpty { "[无文本消息]" }
return content.take(maxChars).trimEnd() + "...[截断]"
}
fun referencedUserIds(record: ChatMessageRecord): Set<Long> = runCatching {
referencedUserIdsFromJson(record.code)
}.recoverCatching {
buildSet {
record.toMessageChain().forEach { message ->
when (message) {
is At -> add(message.target)
is QuoteReply -> add(message.source.fromId)
}
}
}
}.getOrDefault(emptySet())
fun authoredTextLength(record: ChatMessageRecord): Int = runCatching {
val messages = json.parseToJsonElement(record.code) as? JsonArray
?: throw IllegalArgumentException("消息记录不是 JSON array")
messages.sumOf { element ->
val message = element as? JsonObject
if (message?.string("type") == "PlainText") message.string("content").orEmpty().trim().length else 0
}
}.recoverCatching {
record.toMessageChain().filterIsInstance<PlainText>().sumOf { it.content.trim().length }
}.getOrDefault(0)
private fun renderJsonCode(code: String, aliases: Map<Long, String>): String {
val messages = json.parseToJsonElement(code) as? JsonArray
?: throw IllegalArgumentException("消息记录不是 JSON array")
return renderJsonMessages(messages, aliases)
}
private fun renderJsonMessages(messages: JsonArray, aliases: Map<Long, String>): String =
messages.joinToString("") { element ->
val message = element as? JsonObject ?: return@joinToString ""
when (val type = message.string("type")) {
"PlainText" -> message.string("content").orEmpty()
"At" -> message.long("target")?.let { target ->
"@${aliases[target] ?: "用户"}"
}.orEmpty()
"AtAll" -> "@全体成员"
"Image", "FlashImage" -> if (message.boolean("isEmoji") == true) "[表情包]" else "[图片]"
"QuoteReply" -> renderJsonQuote(message, aliases)
"ForwardMessage" -> renderJsonForward(message, aliases)
"MessageOrigin", "ShowImageFlag" -> ""
"Face", "MarketFace", "VipFace" -> "[表情]"
"Audio" -> "[语音]"
"FileMessage" -> "[文件${message.string("name")?.let { ": $it" }.orEmpty()}]"
"LightApp", "SimpleServiceMessage", "MusicShare" -> "[卡片消息]"
"PokeMessage" -> "[戳一戳]"
null -> ""
else -> message.string("content") ?: "[$type]"
}
}
private fun renderJsonQuote(message: JsonObject, aliases: Map<Long, String>): String {
val source = message["source"] as? JsonObject ?: return "[引用消息]"
val authorId = source.long("fromId")
val author = authorId?.let { aliases[it] } ?: "其他用户"
val original = (source["originalMessage"] as? JsonArray)
?.let { renderJsonMessages(it, aliases) }
.orEmpty()
.replace(Regex("[\\r\\n]+"), " ")
.take(160)
return "[引用 $author: $original]"
}
private fun renderJsonForward(message: JsonObject, aliases: Map<Long, String>): String = buildString {
append("[转发消息]")
val nodes = message["nodeList"] as? JsonArray ?: return@buildString
nodes.take(20).forEach { element ->
val node = element as? JsonObject ?: return@forEach
val sender = node.string("senderName") ?: "未知用户"
val chain = node["messageChain"] as? JsonArray
append(' ').append(sender).append(": ")
append(chain?.let { renderJsonMessages(it, aliases) }.orEmpty().take(200))
}
if (nodes.size > 20) append(" ...[转发内容截断]")
}
private fun referencedUserIdsFromJson(code: String): Set<Long> {
val messages = json.parseToJsonElement(code) as? JsonArray
?: throw IllegalArgumentException("消息记录不是 JSON array")
return buildSet { collectReferencedUserIds(messages, this) }
}
private fun collectReferencedUserIds(messages: JsonArray, destination: MutableSet<Long>) {
messages.forEach { element ->
val message = element as? JsonObject ?: return@forEach
when (message.string("type")) {
"At" -> message.long("target")?.let(destination::add)
"QuoteReply" -> {
val source = message["source"] as? JsonObject ?: return@forEach
source.long("fromId")?.let(destination::add)
(source["originalMessage"] as? JsonArray)?.let {
collectReferencedUserIds(it, destination)
}
}
"ForwardMessage" -> (message["nodeList"] as? JsonArray)?.forEach nodeLoop@ { nodeElement ->
val node = nodeElement as? JsonObject ?: return@nodeLoop
(node["messageChain"] as? JsonArray)?.let {
collectReferencedUserIds(it, destination)
}
}
}
}
}
private fun JsonObject.string(key: String): String? =
(get(key) as? JsonPrimitive)?.contentOrNull
private fun JsonObject.long(key: String): Long? =
(get(key) as? JsonPrimitive)?.longOrNull
private fun JsonObject.boolean(key: String): Boolean? =
(get(key) as? JsonPrimitive)?.booleanOrNull
private fun renderChain(chain: MessageChain, aliases: Map<Long, String>): String =
chain.joinToString("") { message -> renderSingle(message, aliases) }
private fun renderSingle(message: SingleMessage, aliases: Map<Long, String>): String = when (message) {
is MessageSource -> ""
is PlainText -> message.content
is At -> "@${aliases[message.target] ?: "用户"}"
is Image -> if (message.isEmoji) "[表情包]" else "[图片]"
is QuoteReply -> {
val author = aliases[message.source.fromId] ?: "其他用户"
val quoted = renderChain(message.source.originalMessage, aliases)
.replace(Regex("[\\r\\n]+"), " ")
.take(160)
"[引用 $author: $quoted]"
}
is ForwardMessage -> buildString {
append("[转发消息]")
message.nodeList.take(20).forEach { node ->
append(" ").append(node.senderName).append(": ")
append(renderChain(node.messageChain, aliases).replace(Regex("[\\r\\n]+"), " ").take(200))
}
if (message.nodeList.size > 20) append(" ...[转发内容截断]")
}
else -> message.content
}
}
@@ -0,0 +1,140 @@
package top.jie65535.mirai.profile
import com.aallam.openai.api.chat.ChatCompletionRequest
import com.aallam.openai.api.chat.ChatMessage
import com.aallam.openai.api.chat.ChatResponseFormat
import com.aallam.openai.api.chat.StreamOptions
import com.aallam.openai.api.core.Usage
import com.aallam.openai.api.model.ModelId
import kotlinx.serialization.SerializationException
import kotlinx.serialization.json.Json
import top.jie65535.mirai.llm.LargeLanguageModels
import top.jie65535.mirai.llm.ModelService
interface ProfileModel {
val modelName: String
suspend fun analyze(
profile: UserProfileSnapshot,
batch: ProfileHistoryBatch,
): ProfileModelResult
}
interface ConversationProfileModel {
val modelName: String
suspend fun analyzeConversation(
profiles: Map<Long, UserProfileSnapshot>,
batch: ConversationProfileBatch,
eligibleUserIds: Set<Long>,
): ConversationProfileModelResult
}
class ProfileModelClient(
private val endpoint: LargeLanguageModels.ProfileEndpoint,
) : ProfileModel, ConversationProfileModel {
private val json = Json {
ignoreUnknownKeys = false
explicitNulls = false
}
override val modelName: String
get() = endpoint.model
override suspend fun analyze(
profile: UserProfileSnapshot,
batch: ProfileHistoryBatch,
): ProfileModelResult {
val content = StringBuilder()
var lastUsage: Usage? = null
var cacheUsage: ModelService.CacheUsage? = null
endpoint.service.chatCompletions(
ChatCompletionRequest(
model = ModelId(endpoint.model),
temperature = endpoint.temperature,
responseFormat = ChatResponseFormat.JsonObject,
streamOptions = StreamOptions(includeUsage = true),
messages = listOf(
ChatMessage.System(ProfilePromptStore.systemPrompt),
ChatMessage.User(ProfilePromptStore.buildUserPrompt(profile, batch)),
),
)
) { cacheUsage = it }.collect { chunk ->
chunk.choices.firstOrNull()?.delta?.content?.let(content::append)
chunk.usage?.let { lastUsage = it }
}
val raw = content.toString().replace(THINK_REGEX, "").trim()
val response = parseResponse(raw)
return ProfileModelResult(
response = response,
rawResponse = raw,
usage = ProfileTokenUsage(
promptTokens = lastUsage?.promptTokens ?: 0,
completionTokens = lastUsage?.completionTokens ?: 0,
cachedTokens = cacheUsage?.hitTokens ?: 0,
),
)
}
override suspend fun analyzeConversation(
profiles: Map<Long, UserProfileSnapshot>,
batch: ConversationProfileBatch,
eligibleUserIds: Set<Long>,
): ConversationProfileModelResult {
val content = StringBuilder()
var lastUsage: Usage? = null
var cacheUsage: ModelService.CacheUsage? = null
endpoint.service.chatCompletions(
ChatCompletionRequest(
model = ModelId(endpoint.model),
temperature = endpoint.temperature,
responseFormat = ChatResponseFormat.JsonObject,
streamOptions = StreamOptions(includeUsage = true),
messages = listOf(
ChatMessage.System(ProfilePromptStore.conversationSystemPrompt),
ChatMessage.User(
ProfilePromptStore.buildConversationUserPrompt(profiles, batch, eligibleUserIds)
),
),
)
) { cacheUsage = it }.collect { chunk ->
chunk.choices.firstOrNull()?.delta?.content?.let(content::append)
chunk.usage?.let { lastUsage = it }
}
val raw = content.toString().replace(THINK_REGEX, "").trim()
return ConversationProfileModelResult(
response = parseObject(raw),
rawResponse = raw,
usage = ProfileTokenUsage(
promptTokens = lastUsage?.promptTokens ?: 0,
completionTokens = lastUsage?.completionTokens ?: 0,
cachedTokens = cacheUsage?.hitTokens ?: 0,
),
)
}
private fun parseResponse(raw: String): ProfileModelResponse {
return parseObject(raw)
}
private inline fun <reified T> parseObject(raw: String): T {
val unfenced = raw
.removePrefix("```json").removePrefix("```")
.removeSuffix("```").trim()
val objectText = if (unfenced.startsWith('{') && unfenced.endsWith('}')) {
unfenced
} else {
val start = unfenced.indexOf('{')
val end = unfenced.lastIndexOf('}')
if (start < 0 || end <= start) throw SerializationException("模型响应中没有完整 JSON object")
unfenced.substring(start, end + 1)
}
return json.decodeFromString(objectText)
}
companion object {
private val THINK_REGEX = Regex("<think>[\\s\\S]*?</think>")
}
}
@@ -0,0 +1,251 @@
package top.jie65535.mirai.profile
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import java.time.Instant
import java.time.ZoneId
import java.time.format.DateTimeFormatter
object ProfilePromptStore {
const val PROMPT_VERSION = "profile-v3.3"
private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneId.systemDefault())
var systemPrompt: String = DEFAULT_SYSTEM_PROMPT
private set
var conversationSystemPrompt: String = DEFAULT_CONVERSATION_SYSTEM_PROMPT
private set
fun reload() {
systemPrompt = loadPrompt(
configuredPath = PluginConfig.profilePromptFile,
defaultPrompt = DEFAULT_SYSTEM_PROMPT,
label = "定向画像",
)
conversationSystemPrompt = loadPrompt(
configuredPath = PluginConfig.profileConversationPromptFile,
defaultPrompt = DEFAULT_CONVERSATION_SYSTEM_PROMPT,
label = "多人会话画像",
)
}
private fun loadPrompt(configuredPath: String, defaultPrompt: String, label: String): String {
if (configuredPath.isBlank()) return defaultPrompt
val file = JChatGPT.resolveConfigFile(configuredPath)
return try {
when {
file.exists() && file.readText().isNotBlank() -> file.readText()
else -> defaultPrompt.also { default ->
file.parentFile?.mkdirs()
file.writeText(default)
}
}
} catch (cause: Throwable) {
JChatGPT.logger.warning("载入${label}提示词失败,暂时使用内置提示词", cause)
defaultPrompt
}
}
fun buildUserPrompt(profile: UserProfileSnapshot, batch: ProfileHistoryBatch): String = buildString {
appendLine("## 目标")
appendLine("目标用户别名: TARGET")
appendLine("本批时间范围: [${formatTime(batch.startTime)}, ${formatTime(batch.endTime)})")
appendLine("别名只在本批有效,不要输出 QQ 号或数据库消息 ID。")
appendLine()
appendLine("## 当前画像(可修正状态,不是事实证据)")
if (profile.items.isEmpty()) {
appendLine("(尚无画像条目)")
} else {
profile.items.forEach { item ->
append("[P:").append(item.id).append("] ")
append(item.category.wireName()).append(" | ")
append(item.confidence.wireName()).append(" | ")
append(item.content)
item.relatedUserId?.let { related ->
append(" | related=").append(batch.aliases[related] ?: "历史用户")
}
append(" | ").append(formatTime(item.firstSeenAt))
append(" ~ ").append(formatTime(item.lastConfirmedAt))
appendLine()
}
}
appendLine("当前短摘要: ${profile.summary.ifBlank { "(空)" }}")
appendLine()
appendLine("## 本批参与者别名")
batch.aliases.entries.sortedBy { it.value }.forEach { (_, alias) ->
appendLine("- $alias")
}
appendLine()
appendLine("## 带上下文的原始群聊")
var currentEpisode = -1
batch.messages.forEach { message ->
if (message.episodeIndex != currentEpisode) {
currentEpisode = message.episodeIndex
appendLine()
appendLine("### 对话片段 $currentEpisode / 群 ${message.record.targetId}")
}
val marker = message.evidenceRef?.let { "[e:$it]" } ?: "[context]"
val time = formatTime(message.record.time)
val alias = batch.aliases[message.record.fromId] ?: "其他用户"
append(marker).append('[').append(time).append(']')
.append('[').append(alias).append("] ")
.appendLine(message.text)
}
}
fun buildConversationUserPrompt(
profiles: Map<Long, UserProfileSnapshot>,
batch: ConversationProfileBatch,
eligibleUserIds: Set<Long>,
): String = buildString {
appendLine("## 任务")
appendLine("分析这一段已经闭合的群聊,一次性更新所有出现可靠长期信息的候选用户画像。")
appendLine("会话时间范围: [${formatTime(batch.startTime)}, ${formatTime(batch.endTime)})")
appendLine("别名只在本批有效,不要输出 QQ 号或数据库消息 ID。")
appendLine()
appendLine("## 候选用户及当前画像")
eligibleUserIds.sortedBy { batch.aliases[it] }.forEach { userId ->
val alias = batch.aliases[userId] ?: return@forEach
val profile = profiles[userId]
appendLine("### $alias")
if (profile == null || profile.items.isEmpty()) {
appendLine("(尚无画像条目)")
} else {
profile.items.forEach { item ->
append("[P:").append(item.id).append("] ")
append(item.category.wireName()).append(" | ")
append(item.confidence.wireName()).append(" | ")
append(item.content)
item.relatedUserId?.let { related ->
append(" | related=").append(batch.aliases[related] ?: "历史用户")
}
append(" | ").append(formatTime(item.firstSeenAt))
append(" ~ ").append(formatTime(item.lastConfirmedAt))
appendLine()
}
}
appendLine("当前短摘要: ${profile?.summary?.ifBlank { "(空)" } ?: ""}")
}
appendLine()
appendLine("## 本批保留的闭合群聊消息")
batch.messages.forEach { message ->
val marker = message.evidenceRef?.let { "[e:$it]" } ?: "[context]"
val time = formatTime(message.record.time)
val alias = batch.aliases[message.record.fromId] ?: "其他用户"
append(marker).append('[').append(time).append(']')
.append('[').append(alias).append("] ")
.appendLine(message.text)
}
}
private fun formatTime(epochSecond: Int): String =
dateTimeFormatter.format(Instant.ofEpochSecond(epochSecond.toLong()))
private fun ProfileCategory.wireName(): String = name.lowercase()
private fun ProfileConfidence.wireName(): String = name.lowercase()
private const val DEFAULT_SYSTEM_PROMPT = """你是保守、严谨的群聊人物画像归纳器。
你会收到一个目标用户的当前画像,以及一个带完整发言者、时间、回复引用和相邻消息的原始群聊批次。
你的任务是判断本批信息是否应当 ADD、UPDATE、CONFIRM 或 DELETE 长期画像条目;没有可靠变化时返回空 operations。
画像只描述:
- notable_fact:本人明确披露且半年后仍有助于认识此人的事实
- interest:持续关注或参与的领域
- expertise_signal:反复表现出的具体知识或解决问题能力,不授予专家头衔
- thinking_style:分析、判断和解决问题的方式
- expression_style:稳定的措辞和表达方式
- social_mode:一般群聊参与和互动方式
- preference:本人明确表达的长期偏好
- relationship_note:与某个具体用户反复出现的互动模式
严格原则:
1. 当前画像只是可修正状态,不是证据。所有操作必须引用本批 [e:n]。
2. 每个操作至少引用一条 TARGET 自己的发言。其他人的消息只能帮助理解上下文和关系。
3. 引用原文的作者不是回复者;不要把被引用者的话归给回复者。
4. 不从玩笑、反讽、夸张、角色扮演、图片、外部新闻或别人的自述推断 TARGET 的事实。
5. 一次技术回答或同一话题中的连续补充只算一个语境。新增 expertise_signal、thinking_style、expression_style、social_mode 或 relationship_note,必须由至少两个独立对话片段中的一致表现支持;单个片段最多用于 CONFIRM 已有条目。本人明确自述的 notable_fact 和 preference 不受此限制。
6. 每个条目只表达一个主题。禁止把不同时间、不同领域的内容拼成一个所谓稳定特点。
7. 禁止使用“精通、专家、导师、领袖、天才、极强、全栈、核心成员”等拔高表述。
8. ADD 不填写 item_idUPDATE、CONFIRM、DELETE 必须填写当前画像中的 item_id。
9. relationship_note 必须填写本批存在的 related_user_alias,只描述互动方式,不推断现实亲疏。
10. 新证据与旧画像无关时不要勉强更新。未输出的旧条目由程序自动保留。
11. DELETE 仅用于新证据明确证明旧条目归因错误或已被本人否定,不能因为本批没提到就删除。
12. summary 是更新后的日常短摘要,必须自然、克制,不写证据编号、时间、QQ 号、进度或内部条目 ID。
只输出一个 JSON object,不要 Markdown 代码围栏,不要解释。格式:
{
"operations": [
{
"action": "ADD|UPDATE|CONFIRM|DELETE",
"item_id": "UPDATE/CONFIRM/DELETE 时填写,ADD 为 null",
"category": "notable_fact|interest|expertise_signal|thinking_style|expression_style|social_mode|preference|relationship_note",
"content": "ADD/UPDATE 时填写的单一、克制结论;其余操作可为 null",
"confidence": "low|medium|high",
"related_user_alias": "仅 relationship_note 填写,否则 null",
"evidence_refs": [1, 2]
}
],
"summary": "更新后的短摘要;没有画像时可以为空"
}
"""
private const val DEFAULT_CONVERSATION_SYSTEM_PROMPT = """你是保守、严谨的群聊人物画像归纳器。
你会收到一段已经闭合的真实群聊、候选用户别名,以及他们各自的当前画像。你的任务是一次性判断这段对话是否足以 ADD、UPDATE、CONFIRM 或 DELETE 各候选用户的长期画像条目。没有可靠变化的用户不要输出,每名用户最多输出 4 个真正有长期价值的操作。
画像只描述:
- notable_fact:本人明确披露且半年后仍有助于认识此人的事实
- interest:持续关注或参与的领域
- expertise_signal:反复表现出的具体知识或解决问题能力,不授予专家头衔
- thinking_style:分析、判断和解决问题的方式
- expression_style:稳定的措辞和表达方式
- social_mode:一般群聊参与和互动方式
- preference:本人明确表达的长期偏好
- relationship_note:与某个具体用户反复出现的互动模式
严格原则:
1. 当前画像只是可修正状态,不是事实证据。所有操作必须引用本批 [e:n]。
2. 每个用户操作至少引用一条该 user_alias 本人说出的消息。其他人的消息只能帮助理解上下文和关系。
3. 引用原文的作者不是回复者;不要把被引用者的话归给回复者。
4. 不从玩笑、反讽、夸张、角色扮演、图片、外部新闻或别人的自述推断某人的事实。
5. 一次明确自述可以支持 notable_fact 或 preference。新增 expertise_signal、thinking_style、expression_style、social_mode 或 relationship_note,必须在本会话中有多条分离的本人证据;证据不足时宁可不写。
6. 每个条目只表达一个主题,禁止把不同人的特点或不同领域拼接在一起。
7. 禁止使用“精通、专家、导师、领袖、天才、极强、全栈、核心成员”等拔高表述。
8. ADD 不填写 item_idUPDATE、CONFIRM、DELETE 必须填写该用户当前画像中的 item_id,不能引用其他用户的条目。
9. relationship_note 必须填写本批存在的 related_user_alias,只描述互动方式,不推断现实亲疏。
10. 未输出的用户和旧条目由程序自动保留。DELETE 仅用于新证据明确证明旧条目归因错误或已被本人否定。
11. summary 是该用户更新后的日常短摘要,必须自然、克制,不写证据编号、时间、QQ 号、进度或内部 ID。
12. 不生成或修改好感度、代号、主观印象和标签;这些属于另一套 Bot 关系状态。
13. 本批只是一段会话。除非当前画像已有同类条目且本批在确认它,否则不得使用“长期、持续、一贯、总是、通常”等跨时间措辞;只能描述本批确实支持的事实、关注点或表现。
14. 对尚无同类旧条目的用户,thinking_style、expression_style、social_mode、expertise_signal 和 relationship_note 必须有多个彼此分离的本人证据才可新增,并保持 low 或 medium 可信度;同一问答链中的连续补充不算多次独立表现。
只输出一个 JSON object,不要 Markdown 代码围栏,不要解释。格式:
{
"users": [
{
"user_alias": "U1",
"operations": [
{
"action": "ADD|UPDATE|CONFIRM|DELETE",
"item_id": "UPDATE/CONFIRM/DELETE 时填写,ADD 为 null",
"category": "notable_fact|interest|expertise_signal|thinking_style|expression_style|social_mode|preference|relationship_note",
"content": "ADD/UPDATE 时填写的单一、克制结论;其余操作可为 null",
"confidence": "low|medium|high",
"related_user_alias": "仅 relationship_note 填写,否则 null",
"evidence_refs": [1, 2]
}
],
"summary": "该用户更新后的短摘要"
}
]
}
"""
}
@@ -0,0 +1,303 @@
package top.jie65535.mirai.profile
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import kotlinx.coroutines.withContext
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.ChatHistoryStore
import top.jie65535.mirai.llm.LargeLanguageModels
import java.io.File
import java.util.concurrent.ConcurrentHashMap
object UserProfileAnalysisService {
private val runningUsers = ConcurrentHashMap.newKeySet<Long>()
private val concurrencyLimiter = Semaphore(1)
suspend fun analyze(
userId: Long,
maxBatches: Int,
onProgress: suspend (ProfileAnalysisProgress) -> Unit = {},
): ProfileAnalysisReport {
require(userId > 0) { "userId 必须是正数" }
require(maxBatches > 0) { "maxBatches 必须是正数" }
check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" }
check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
if (!runningUsers.add(userId)) {
return ProfileAnalysisReport(
userId = userId,
processedBatches = 0,
processedMessages = 0,
appliedOperations = 0,
usage = ProfileTokenUsage(),
profile = UserProfileStore.load(userId),
caughtUp = false,
alreadyRunning = true,
)
}
try {
return concurrencyLimiter.withPermit {
analyzeExclusive(userId, maxBatches, onProgress)
}
} finally {
runningUsers.remove(userId)
}
}
private suspend fun analyzeExclusive(
userId: Long,
maxBatches: Int,
onProgress: suspend (ProfileAnalysisProgress) -> Unit,
): ProfileAnalysisReport {
val endpoint = checkNotNull(LargeLanguageModels.profile) {
"画像分析模型未配置,请设置 profileModelApi/profileModelToken,或配置可继承的聊天模型接入点"
}
val model: ProfileModel = ProfileModelClient(endpoint)
val historyFile = resolveHistoryFile()
val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(historyFile) }
val bounds = withContext(Dispatchers.IO) { reader.findUserTimeBounds(userId) }
?: return emptyReport(userId)
var profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) }
?: UserProfileSnapshot(
userId = userId,
cursorTime = bounds.startTime,
snapshotEndTime = bounds.endTime,
)
if (profile.cursorTime >= profile.snapshotEndTime && bounds.endTime > profile.snapshotEndTime) {
profile = profile.copy(snapshotEndTime = bounds.endTime)
}
var processedBatches = 0
var processedMessages = 0
var appliedOperations = 0
var totalUsage = ProfileTokenUsage()
var caughtUp = false
while (processedBatches < maxBatches) {
val batch = withContext(Dispatchers.IO) {
reader.loadNextBatch(
userId = userId,
startTime = profile.cursorTime,
snapshotEndTime = profile.snapshotEndTime,
targetMessageLimit = PluginConfig.profileBatchTargetMessages.coerceAtLeast(1),
maxEpisodes = PluginConfig.profileBatchMaxEpisodes.coerceAtLeast(1),
episodeGapSeconds = PluginConfig.profileEpisodeGapMinutes.coerceAtLeast(0) * 60,
contextBeforeMessages = PluginConfig.profileContextBeforeMessages.coerceAtLeast(0),
contextAfterMessages = PluginConfig.profileContextAfterMessages.coerceAtLeast(0),
contextCoreMessages = PluginConfig.profileContextCoreMessages.coerceAtLeast(1),
maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80),
)
}
if (batch == null) {
caughtUp = true
break
}
val (result, reduction) = analyzeWithRetry(model, profile, batch)
withContext(Dispatchers.IO) {
UserProfileStore.commit(
reduction = reduction,
batch = batch,
usage = result.usage,
source = ProfileRevisionSource.BACKFILL,
)
}
profile = reduction.profile
processedBatches++
processedMessages += batch.messages.size
appliedOperations += reduction.operations.size
totalUsage += result.usage
onProgress(
ProfileAnalysisProgress(
batchIndex = processedBatches,
startTime = batch.startTime,
endTime = batch.endTime,
messageCount = batch.messages.size,
operationCount = reduction.operations.size,
usage = result.usage,
)
)
}
if (!caughtUp && profile.cursorTime >= profile.snapshotEndTime) caughtUp = true
return ProfileAnalysisReport(
userId = userId,
processedBatches = processedBatches,
processedMessages = processedMessages,
appliedOperations = appliedOperations,
usage = totalUsage,
profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) } ?: profile,
caughtUp = caughtUp,
)
}
suspend fun analyzeConversation(
botId: Long,
groupId: Long,
startTime: Int,
endTime: Int,
minAuthoredTextChars: Int,
): ConversationProfileAnalysisReport? {
require(startTime < endTime) { "startTime must be before endTime" }
check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" }
check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
return concurrencyLimiter.withPermit {
val endpoint = checkNotNull(LargeLanguageModels.profile) { "画像分析模型未配置" }
val model: ConversationProfileModel = ProfileModelClient(endpoint)
val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(resolveLiveHistoryFile()) }
val batch = withContext(Dispatchers.IO) {
reader.loadConversationBatch(
botId = botId,
groupId = groupId,
startTime = startTime,
endTime = endTime,
messageLimit = PluginConfig.profileAutoConversationMessageLimit.coerceIn(20, 500),
maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80),
)
} ?: return@withPermit null
val eligibleUserIds = batch.authoredTextCharsByUser
.filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) }
.keys
if (eligibleUserIds.isEmpty()) return@withPermit null
if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) {
return@withPermit null
}
val profiles = withContext(Dispatchers.IO) {
eligibleUserIds.associateWith { userId ->
UserProfileStore.load(userId) ?: UserProfileSnapshot(
userId = userId,
cursorTime = 0,
snapshotEndTime = 0,
)
}
}
val (result, reductions) = analyzeConversationWithRetry(
model = model,
profiles = profiles,
batch = batch,
eligibleUserIds = eligibleUserIds,
)
withContext(Dispatchers.IO) {
UserProfileStore.commitConversation(
reductions = reductions.map { reduction -> reduction to batch.forUser(reduction.profile.userId) },
usage = result.usage,
)
}
ConversationProfileAnalysisReport(
analyzedUsers = eligibleUserIds.size,
processedMessages = batch.messages.size,
appliedOperations = reductions.sumOf { it.operations.size },
usage = result.usage,
)
}
}
private suspend fun analyzeConversationWithRetry(
model: ConversationProfileModel,
profiles: Map<Long, UserProfileSnapshot>,
batch: ConversationProfileBatch,
eligibleUserIds: Set<Long>,
): Pair<ConversationProfileModelResult, List<ProfileReduction>> {
val attempts = PluginConfig.profileRetryMax.coerceIn(0, 3) + 1
var lastFailure: Throwable? = null
repeat(attempts) { attempt ->
try {
val result = model.analyzeConversation(profiles, batch, eligibleUserIds)
val reductions = ConversationProfileReducer.reduce(
profiles = profiles,
batch = batch,
eligibleUserIds = eligibleUserIds,
response = result.response,
model = model.modelName,
promptVersion = ProfilePromptStore.PROMPT_VERSION,
summaryMaxLength = PluginConfig.profileSummaryMaxLength.coerceAtLeast(100),
)
return result to reductions
} catch (cause: Exception) {
if (cause is CancellationException) throw cause
lastFailure = cause
JChatGPT.logger.warning(
"${batch.groupId} 会话画像 [${batch.startTime}, ${batch.endTime}) " +
"${attempt + 1}/$attempts 次分析失败",
cause,
)
}
}
throw IllegalStateException(
"会话画像 [${batch.startTime}, ${batch.endTime}) 连续 $attempts 次分析失败,未提交任何结果",
lastFailure,
)
}
private suspend fun analyzeWithRetry(
model: ProfileModel,
profile: UserProfileSnapshot,
batch: ProfileHistoryBatch,
advanceBackfillCursor: Boolean = true,
): Pair<ProfileModelResult, ProfileReduction> {
val attempts = PluginConfig.profileRetryMax.coerceIn(0, 3) + 1
var lastFailure: Throwable? = null
repeat(attempts) { attempt ->
try {
val result = model.analyze(profile, batch)
val reduction = UserProfileReducer.reduce(
current = profile,
batch = batch,
response = result.response,
model = model.modelName,
promptVersion = ProfilePromptStore.PROMPT_VERSION,
summaryMaxLength = PluginConfig.profileSummaryMaxLength.coerceAtLeast(100),
advanceBackfillCursor = advanceBackfillCursor,
)
return result to reduction
} catch (cause: Exception) {
if (cause is CancellationException) throw cause
lastFailure = cause
JChatGPT.logger.warning(
"用户 ${batch.userId} 画像批次 [${batch.startTime}, ${batch.endTime}) " +
"${attempt + 1}/$attempts 次分析失败",
cause,
)
}
}
throw IllegalStateException(
"画像批次 [${batch.startTime}, ${batch.endTime}) 连续 $attempts 次分析失败,水位线未推进",
lastFailure,
)
}
private fun resolveHistoryFile(): File {
val configured = PluginConfig.profileHistoryDatabasePath.trim()
return if (configured.isNotEmpty()) {
File(configured).absoluteFile
} else {
checkNotNull(ChatHistoryStore.databaseFileOrNull) { "聊天记录数据库不可用" }
}
}
private fun resolveLiveHistoryFile(): File =
checkNotNull(ChatHistoryStore.databaseFileOrNull) { "聊天记录数据库不可用" }
private fun emptyReport(userId: Long) = ProfileAnalysisReport(
userId = userId,
processedBatches = 0,
processedMessages = 0,
appliedOperations = 0,
usage = ProfileTokenUsage(),
profile = null,
caughtUp = true,
)
private operator fun ProfileTokenUsage.plus(other: ProfileTokenUsage) = ProfileTokenUsage(
promptTokens = promptTokens + other.promptTokens,
completionTokens = completionTokens + other.completionTokens,
cachedTokens = cachedTokens + other.cachedTokens,
)
}
@@ -0,0 +1,62 @@
package top.jie65535.mirai.profile
import top.jie65535.mirai.data.FavorabilityInfo
object UserProfileContextRenderer {
fun render(
profiles: List<UserProfileSnapshot>,
favorabilityByUserId: Map<Long, FavorabilityInfo>,
displayNames: Map<Long, String>,
activeUserIds: Set<Long>,
summaryMaxChars: Int,
): String {
val profilesByUserId = profiles
.filter { it.reliable && it.summary.isNotBlank() }
.associateBy { it.userId }
val userIds = activeUserIds.filter { userId ->
userId in profilesByUserId || favorabilityByUserId[userId]?.hasVisibleContext() == true
}
if (userIds.isEmpty()) return ""
val maxChars = summaryMaxChars.coerceAtLeast(50)
return buildString {
appendLine("## 你对相关群友的认识")
appendLine("好感度、代号和主观印象代表你的关系状态;长期认识来自可修正的历史归纳。仅在当前话题相关时自然运用,不要逐条复述或提及信息来源。")
userIds.forEach { userId ->
val profile = profilesByUserId[userId]
val favorability = favorabilityByUserId[userId]
val name = favorability?.name.orEmpty().ifBlank {
displayNames[userId].orEmpty().ifBlank { userId.toString() }
}
append("- ").append(name).append('(').append(userId).append(')')
favorability?.takeIf { it.hasVisibleContext() }?.let { info ->
append(" | 好感度").append(if (info.value >= 0) "+" else "").append(info.value)
if (info.tags.isNotEmpty()) append(" | 标签:").append(info.tags.joinToString(""))
if (info.impression.isNotBlank()) append(" | 主观印象:").append(info.impression.normalized())
}
profile?.let {
append(" | 长期认识:").append(it.summary.normalized().take(maxChars))
}
profile?.items?.asSequence()
?.filter { item ->
item.category == ProfileCategory.RELATIONSHIP_NOTE &&
item.relatedUserId != null && item.relatedUserId in activeUserIds
}
?.take(2)
?.forEach { item ->
val relatedId = checkNotNull(item.relatedUserId)
val relatedName = displayNames[relatedId].orEmpty().ifBlank { relatedId.toString() }
append(";与").append(relatedName).append("").append(item.content.normalized())
}
appendLine()
}
appendLine()
}
}
private fun FavorabilityInfo.hasVisibleContext(): Boolean =
value != 0 || name.isNotBlank() || tags.isNotEmpty() || impression.isNotBlank()
private fun String.normalized(): String = trim().replace(Regex("\\s+"), " ")
}
@@ -0,0 +1,232 @@
package top.jie65535.mirai.profile
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import top.jie65535.mirai.data.ChatMessageRecord
@Serializable
enum class ProfileCategory {
@SerialName("notable_fact")
NOTABLE_FACT,
@SerialName("interest")
INTEREST,
@SerialName("expertise_signal")
EXPERTISE_SIGNAL,
@SerialName("thinking_style")
THINKING_STYLE,
@SerialName("expression_style")
EXPRESSION_STYLE,
@SerialName("social_mode")
SOCIAL_MODE,
@SerialName("preference")
PREFERENCE,
@SerialName("relationship_note")
RELATIONSHIP_NOTE,
}
@Serializable
enum class ProfileConfidence {
@SerialName("low")
LOW,
@SerialName("medium")
MEDIUM,
@SerialName("high")
HIGH,
}
@Serializable
enum class ProfileOperationAction {
ADD,
UPDATE,
CONFIRM,
DELETE,
}
enum class ProfileRevisionSource {
BACKFILL,
CONVERSATION,
}
data class UserProfileItem(
val id: String,
val category: ProfileCategory,
val content: String,
val confidence: ProfileConfidence,
val relatedUserId: Long? = null,
val firstSeenAt: Int,
val lastConfirmedAt: Int,
)
data class UserProfileSnapshot(
val userId: Long,
val summary: String = "",
val version: Int = 0,
val cursorTime: Int,
val snapshotEndTime: Int,
val reliable: Boolean = false,
val model: String = "",
val promptVersion: String = "",
val updatedAt: Long = 0,
val items: List<UserProfileItem> = emptyList(),
)
@Serializable
data class ProfileModelOperation(
val action: ProfileOperationAction,
@SerialName("item_id")
val itemId: String? = null,
val category: ProfileCategory? = null,
val content: String? = null,
val confidence: ProfileConfidence? = null,
@SerialName("related_user_alias")
val relatedUserAlias: String? = null,
@SerialName("evidence_refs")
val evidenceRefs: List<Int> = emptyList(),
)
@Serializable
data class ProfileModelResponse(
val operations: List<ProfileModelOperation> = emptyList(),
val summary: String = "",
)
data class ProfilePromptMessage(
val record: ChatMessageRecord,
val text: String,
val evidenceRef: Int? = null,
val episodeIndex: Int,
)
data class ProfileHistoryBatch(
val userId: Long,
val startTime: Int,
val endTime: Int,
val messages: List<ProfilePromptMessage>,
val aliases: Map<Long, String>,
val inputHash: String,
) {
val evidenceByRef: Map<Int, ProfilePromptMessage> = messages
.mapNotNull { message -> message.evidenceRef?.let { it to message } }
.toMap()
val aliasToUserId: Map<String, Long> = aliases.entries.associate { (userId, alias) -> alias to userId }
val targetMessageCount: Int = messages.count { it.record.fromId == userId && it.evidenceRef != null }
val targetAuthoredTextChars: Int = messages.asSequence()
.filter { it.record.fromId == userId && it.evidenceRef != null }
.sumOf { ProfileMessageRenderer.authoredTextLength(it.record) }
}
data class ProfileTokenUsage(
val promptTokens: Int = 0,
val completionTokens: Int = 0,
val cachedTokens: Int = 0,
)
data class ProfileModelResult(
val response: ProfileModelResponse,
val rawResponse: String,
val usage: ProfileTokenUsage,
)
@Serializable
data class ConversationProfileUserResponse(
@SerialName("user_alias")
val userAlias: String,
val operations: List<ProfileModelOperation> = emptyList(),
val summary: String = "",
)
@Serializable
data class ConversationProfileModelResponse(
val users: List<ConversationProfileUserResponse> = emptyList(),
)
data class ConversationProfileModelResult(
val response: ConversationProfileModelResponse,
val rawResponse: String,
val usage: ProfileTokenUsage,
)
data class ConversationProfileBatch(
val botId: Long,
val groupId: Long,
val startTime: Int,
val endTime: Int,
val messages: List<ProfilePromptMessage>,
val aliases: Map<Long, String>,
val inputHash: String,
) {
val evidenceByRef: Map<Int, ProfilePromptMessage> = messages
.mapNotNull { message -> message.evidenceRef?.let { it to message } }
.toMap()
val aliasToUserId: Map<String, Long> = aliases.entries.associate { (userId, alias) -> alias to userId }
val authoredTextCharsByUser: Map<Long, Int> = messages.asSequence()
.filter { it.evidenceRef != null && it.record.fromId != botId }
.groupBy { it.record.fromId }
.mapValues { (_, authored) -> authored.sumOf { ProfileMessageRenderer.authoredTextLength(it.record) } }
fun forUser(userId: Long): ProfileHistoryBatch = ProfileHistoryBatch(
userId = userId,
startTime = startTime,
endTime = endTime,
messages = messages,
aliases = aliases,
inputHash = inputHash,
)
}
@Serializable
data class AppliedProfileOperation(
val action: ProfileOperationAction,
val itemId: String,
val category: ProfileCategory,
val content: String,
val confidence: ProfileConfidence,
val relatedUserId: Long?,
val evidenceRefs: List<Int>,
)
data class ProfileReduction(
val profile: UserProfileSnapshot,
val operations: List<AppliedProfileOperation>,
)
data class ProfileAnalysisProgress(
val batchIndex: Int,
val startTime: Int,
val endTime: Int,
val messageCount: Int,
val operationCount: Int,
val usage: ProfileTokenUsage,
)
data class ConversationProfileAnalysisReport(
val analyzedUsers: Int,
val processedMessages: Int,
val appliedOperations: Int,
val usage: ProfileTokenUsage,
)
data class ProfileAnalysisReport(
val userId: Long,
val processedBatches: Int,
val processedMessages: Int,
val appliedOperations: Int,
val usage: ProfileTokenUsage,
val profile: UserProfileSnapshot?,
val caughtUp: Boolean,
val alreadyRunning: Boolean = false,
)
@@ -0,0 +1,192 @@
package top.jie65535.mirai.profile
import java.util.UUID
object UserProfileReducer {
private val overclaimPattern = Regex("深厚|扎实|精通|专家|导师|领袖|天才|极强|全栈|核心成员|公认")
fun reduce(
current: UserProfileSnapshot,
batch: ProfileHistoryBatch,
response: ProfileModelResponse,
model: String,
promptVersion: String,
summaryMaxLength: Int,
advanceBackfillCursor: Boolean = true,
): ProfileReduction {
require(response.summary.length <= summaryMaxLength) {
"画像摘要超过 ${summaryMaxLength} 字符"
}
val items = current.items.associateBy { it.id }.toMutableMap()
val applied = mutableListOf<AppliedProfileOperation>()
response.operations.forEachIndexed { index, operation ->
val evidence = operation.evidenceRefs.distinct().map { ref ->
batch.evidenceByRef[ref]
?: throw IllegalArgumentException("operations[$index] 引用了不存在的证据 e:$ref")
}
require(evidence.isNotEmpty()) { "operations[$index] 缺少证据" }
require(evidence.any { it.record.fromId == batch.userId }) {
"operations[$index] 没有目标用户自己的发言证据"
}
val evidenceTime = evidence
.asSequence()
.filter { it.record.fromId == batch.userId }
.maxOf { it.record.time }
val firstEvidenceTime = evidence
.asSequence()
.filter { it.record.fromId == batch.userId }
.minOf { it.record.time }
when (operation.action) {
ProfileOperationAction.ADD -> {
require(operation.itemId.isNullOrBlank()) {
"operations[$index] ADD 不能指定 item_id"
}
val category = requireNotNull(operation.category) {
"operations[$index] ADD 缺少 category"
}
val content = validateContent(index, operation.content)
val confidence = requireNotNull(operation.confidence) {
"operations[$index] ADD 缺少 confidence"
}
val relatedUserId = resolveRelatedUser(index, category, operation.relatedUserAlias, batch)
val duplicate = items.values.any {
it.category == category && normalize(it.content) == normalize(content)
}
if (duplicate) return@forEachIndexed
val item = UserProfileItem(
id = UUID.randomUUID().toString(),
category = category,
content = content,
confidence = confidence,
relatedUserId = relatedUserId,
firstSeenAt = firstEvidenceTime,
lastConfirmedAt = evidenceTime,
)
items[item.id] = item
applied += item.toApplied(operation.action, operation.evidenceRefs)
}
ProfileOperationAction.UPDATE -> {
val old = requireExistingItem(index, operation, items)
val category = operation.category ?: old.category
val content = validateContent(index, operation.content)
val confidence = operation.confidence ?: old.confidence
val relatedUserId = resolveRelatedUser(
index,
category,
operation.relatedUserAlias,
batch,
fallback = old.relatedUserId,
)
val updated = old.copy(
category = category,
content = content,
confidence = confidence,
relatedUserId = relatedUserId,
lastConfirmedAt = evidenceTime,
)
items[old.id] = updated
applied += updated.toApplied(operation.action, operation.evidenceRefs)
}
ProfileOperationAction.CONFIRM -> {
val old = requireExistingItem(index, operation, items)
val updated = old.copy(
confidence = operation.confidence ?: old.confidence,
lastConfirmedAt = evidenceTime,
)
items[old.id] = updated
applied += updated.toApplied(operation.action, operation.evidenceRefs)
}
ProfileOperationAction.DELETE -> {
val old = requireExistingItem(index, operation, items)
items.remove(old.id)
applied += old.toApplied(operation.action, operation.evidenceRefs)
}
}
}
val summary = if (applied.isEmpty()) {
current.summary
} else {
response.summary.trim().ifEmpty { current.summary }
}
val changed = applied.isNotEmpty() || summary != current.summary
val profile = current.copy(
summary = summary,
version = current.version + if (changed) 1 else 0,
cursorTime = if (advanceBackfillCursor) batch.endTime else current.cursorTime,
reliable = items.isNotEmpty(),
model = model,
promptVersion = promptVersion,
updatedAt = System.currentTimeMillis(),
items = items.values.sortedWith(
compareBy<UserProfileItem>({ it.category.ordinal }, { it.firstSeenAt }, { it.id })
),
)
return ProfileReduction(profile, applied)
}
private fun validateContent(index: Int, raw: String?): String {
val content = raw?.trim().orEmpty()
require(content.isNotEmpty()) { "operations[$index] 缺少 content" }
require(content.length <= 120) { "operations[$index].content 超过 120 字符" }
require(!overclaimPattern.containsMatchIn(content)) {
"operations[$index].content 包含夸张身份或能力判断"
}
return content
}
private fun requireExistingItem(
index: Int,
operation: ProfileModelOperation,
items: Map<String, UserProfileItem>,
): UserProfileItem {
val itemId = operation.itemId?.takeIf { it.isNotBlank() }
?: throw IllegalArgumentException("operations[$index] ${operation.action} 缺少 item_id")
return items[itemId]
?: throw IllegalArgumentException("operations[$index] 指向不存在的画像条目 $itemId")
}
private fun resolveRelatedUser(
index: Int,
category: ProfileCategory,
alias: String?,
batch: ProfileHistoryBatch,
fallback: Long? = null,
): Long? {
if (category != ProfileCategory.RELATIONSHIP_NOTE) return null
if (alias.isNullOrBlank()) {
return fallback ?: throw IllegalArgumentException(
"operations[$index] relationship_note 缺少 related_user_alias"
)
}
val related = batch.aliasToUserId[alias]
?: throw IllegalArgumentException("operations[$index] 的关联用户别名不存在: $alias")
require(related != batch.userId) { "operations[$index] 不能建立指向自己的关系条目" }
return related
}
private fun UserProfileItem.toApplied(
action: ProfileOperationAction,
evidenceRefs: List<Int>,
) = AppliedProfileOperation(
action = action,
itemId = id,
category = category,
content = content,
confidence = confidence,
relatedUserId = relatedUserId,
evidenceRefs = evidenceRefs.distinct(),
)
private fun normalize(value: String): String = value
.lowercase()
.replace(Regex("[\\s\\p{Punct},。;、!?()【】‘’“”]+"), "")
}
+473
View File
@@ -0,0 +1,473 @@
package top.jie65535.mirai.profile
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.json.Json
import java.io.File
import java.sql.Connection
import java.sql.DriverManager
import java.sql.ResultSet
object UserProfileStore {
private const val SCHEMA_VERSION = 2
private const val BUSY_TIMEOUT_MS = 30_000
private val lifecycleLock = Any()
private val writeLock = Any()
private val json = Json { encodeDefaults = true }
private val operationListSerializer = ListSerializer(AppliedProfileOperation.serializer())
@Volatile
private var initialized = false
private lateinit var databaseFile: File
private var writeConnection: Connection? = null
val isAvailable: Boolean
get() = initialized
fun init(dataFolder: File) {
synchronized(lifecycleLock) {
if (initialized) return
Class.forName("org.sqlite.JDBC")
dataFolder.mkdirs()
databaseFile = dataFolder.resolve("user-profile.sqlite")
val connection = openConnection()
try {
configureWriteConnection(connection)
createSchema(connection)
writeConnection = connection
initialized = true
} catch (cause: Throwable) {
connection.close()
throw cause
}
}
}
fun close() {
synchronized(lifecycleLock) {
if (!initialized) return
synchronized(writeLock) {
writeConnection?.let { connection ->
runCatching {
connection.createStatement().use { statement ->
statement.execute("PRAGMA wal_checkpoint(TRUNCATE)")
}
}
connection.close()
}
writeConnection = null
initialized = false
}
}
}
fun load(userId: Long): UserProfileSnapshot? {
check(initialized) { "用户画像数据库尚未初始化" }
return openReadConnection().use { connection ->
val profileRow = connection.prepareStatement(
"""
SELECT user_id, summary, version, cursor_time, snapshot_end_time,
reliable, model, prompt_version, updated_at
FROM user_profile
WHERE user_id = ?
""".trimIndent()
).use { statement ->
statement.setLong(1, userId)
statement.executeQuery().use { results ->
if (results.next()) results.toProfileWithoutItems() else null
}
} ?: return@use null
val items = connection.prepareStatement(
"""
SELECT item_id, category, content, confidence, related_user_id,
first_seen_at, last_confirmed_at
FROM profile_item
WHERE user_id = ?
ORDER BY category, first_seen_at, item_id
""".trimIndent()
).use { statement ->
statement.setLong(1, userId)
statement.executeQuery().use { results ->
buildList {
while (results.next()) add(results.toProfileItem())
}
}
}
profileRow.copy(items = items)
}
}
fun commit(
reduction: ProfileReduction,
batch: ProfileHistoryBatch,
usage: ProfileTokenUsage,
source: ProfileRevisionSource = ProfileRevisionSource.BACKFILL,
) = commitAll(listOf(reduction to batch), usage, source)
fun commitConversation(
reductions: List<Pair<ProfileReduction, ProfileHistoryBatch>>,
usage: ProfileTokenUsage,
) = commitAll(reductions, usage, ProfileRevisionSource.CONVERSATION)
fun isConversationProcessed(inputHash: String): Boolean {
check(initialized) { "用户画像数据库尚未初始化" }
return openReadConnection().use { connection ->
connection.prepareStatement(
"SELECT 1 FROM profile_revision WHERE source = 'conversation' AND input_hash = ? LIMIT 1"
).use { statement ->
statement.setString(1, inputHash)
statement.executeQuery().use { it.next() }
}
}
}
private fun commitAll(
entries: List<Pair<ProfileReduction, ProfileHistoryBatch>>,
usage: ProfileTokenUsage,
source: ProfileRevisionSource,
) {
check(initialized) { "用户画像数据库尚未初始化" }
if (entries.isEmpty()) return
withWriteConnection { connection ->
val oldAutoCommit = connection.autoCommit
connection.autoCommit = false
try {
entries.forEachIndexed { index, (reduction, batch) ->
persist(
connection = connection,
reduction = reduction,
batch = batch,
usage = if (index == 0) usage else ProfileTokenUsage(),
source = source,
)
}
connection.commit()
} catch (cause: Throwable) {
connection.rollback()
throw cause
} finally {
connection.autoCommit = oldAutoCommit
}
}
}
private fun persist(
connection: Connection,
reduction: ProfileReduction,
batch: ProfileHistoryBatch,
usage: ProfileTokenUsage,
source: ProfileRevisionSource,
) {
val profile = reduction.profile
connection.prepareStatement(
"""
INSERT INTO user_profile(
user_id, summary, version, cursor_time, snapshot_end_time,
reliable, model, prompt_version, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(user_id) DO UPDATE SET
summary = excluded.summary,
version = excluded.version,
cursor_time = excluded.cursor_time,
snapshot_end_time = excluded.snapshot_end_time,
reliable = excluded.reliable,
model = excluded.model,
prompt_version = excluded.prompt_version,
updated_at = excluded.updated_at
""".trimIndent()
).use { statement ->
statement.setLong(1, profile.userId)
statement.setString(2, profile.summary)
statement.setInt(3, profile.version)
statement.setInt(4, profile.cursorTime)
statement.setInt(5, profile.snapshotEndTime)
statement.setInt(6, if (profile.reliable) 1 else 0)
statement.setString(7, profile.model)
statement.setString(8, profile.promptVersion)
statement.setLong(9, profile.updatedAt)
statement.executeUpdate()
}
connection.prepareStatement("DELETE FROM profile_item WHERE user_id = ?").use { statement ->
statement.setLong(1, profile.userId)
statement.executeUpdate()
}
connection.prepareStatement(
"""
INSERT INTO profile_item(
item_id, user_id, category, content, confidence, related_user_id,
first_seen_at, last_confirmed_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
profile.items.forEach { item ->
statement.setString(1, item.id)
statement.setLong(2, profile.userId)
statement.setString(3, item.category.toStorageValue())
statement.setString(4, item.content)
statement.setString(5, item.confidence.toStorageValue())
if (item.relatedUserId == null) statement.setNull(6, java.sql.Types.BIGINT)
else statement.setLong(6, item.relatedUserId)
statement.setInt(7, item.firstSeenAt)
statement.setInt(8, item.lastConfirmedAt)
statement.addBatch()
}
statement.executeBatch()
}
connection.prepareStatement(
"""
INSERT INTO profile_support(
item_id, user_id, group_ids, start_time, end_time,
input_hash, action, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
reduction.operations.forEach { operation ->
val evidence = operation.evidenceRefs.mapNotNull(batch.evidenceByRef::get)
val startTime = evidence.minOf { it.record.time }
val endTime = evidence.maxOf { it.record.time }.safeNextSecond()
val groupIds = evidence.map { it.record.targetId }.distinct().sorted().joinToString(",")
statement.setString(1, operation.itemId)
statement.setLong(2, profile.userId)
statement.setString(3, groupIds)
statement.setInt(4, startTime)
statement.setInt(5, endTime)
statement.setString(6, batch.inputHash)
statement.setString(7, operation.action.name)
statement.setLong(8, System.currentTimeMillis())
statement.addBatch()
}
statement.executeBatch()
}
connection.prepareStatement(
"""
INSERT INTO profile_revision(
user_id, profile_version, source, start_time, end_time, input_hash,
operations_json, summary, model, prompt_version,
prompt_tokens, completion_tokens, cached_tokens, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
statement.setLong(1, profile.userId)
statement.setInt(2, profile.version)
statement.setString(3, source.name.lowercase())
statement.setInt(4, batch.startTime)
statement.setInt(5, batch.endTime)
statement.setString(6, batch.inputHash)
statement.setString(7, json.encodeToString(operationListSerializer, reduction.operations))
statement.setString(8, profile.summary)
statement.setString(9, profile.model)
statement.setString(10, profile.promptVersion)
statement.setInt(11, usage.promptTokens)
statement.setInt(12, usage.completionTokens)
statement.setInt(13, usage.cachedTokens)
statement.setLong(14, System.currentTimeMillis())
statement.executeUpdate()
}
}
fun lastRevisionAt(userId: Long, source: ProfileRevisionSource): Long? {
check(initialized) { "用户画像数据库尚未初始化" }
return openReadConnection().use { connection ->
connection.prepareStatement(
"SELECT MAX(created_at) FROM profile_revision WHERE user_id = ? AND source = ?"
).use { statement ->
statement.setLong(1, userId)
statement.setString(2, source.name.lowercase())
statement.executeQuery().use { results ->
if (!results.next()) return@use null
results.getLong(1).let { if (results.wasNull()) null else it }
}
}
}
}
private fun createSchema(connection: Connection) {
val oldAutoCommit = connection.autoCommit
connection.autoCommit = false
try {
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS user_profile(
user_id INTEGER PRIMARY KEY,
summary TEXT NOT NULL,
version INTEGER NOT NULL,
cursor_time INTEGER NOT NULL,
snapshot_end_time INTEGER NOT NULL,
reliable INTEGER NOT NULL,
model TEXT NOT NULL,
prompt_version TEXT NOT NULL,
updated_at INTEGER NOT NULL
)
""".trimIndent()
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS profile_item(
item_id TEXT PRIMARY KEY,
user_id INTEGER NOT NULL,
category TEXT NOT NULL,
content TEXT NOT NULL,
confidence TEXT NOT NULL,
related_user_id INTEGER,
first_seen_at INTEGER NOT NULL,
last_confirmed_at INTEGER NOT NULL
)
""".trimIndent()
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_profile_item_user ON profile_item(user_id, category)"
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS profile_support(
id INTEGER PRIMARY KEY AUTOINCREMENT,
item_id TEXT NOT NULL,
user_id INTEGER NOT NULL,
group_ids TEXT NOT NULL,
start_time INTEGER NOT NULL,
end_time INTEGER NOT NULL,
input_hash TEXT NOT NULL,
action TEXT NOT NULL,
created_at INTEGER NOT NULL
)
""".trimIndent()
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_profile_support_user ON profile_support(user_id, start_time)"
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS profile_revision(
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
profile_version INTEGER NOT NULL,
source TEXT NOT NULL DEFAULT 'backfill',
start_time INTEGER NOT NULL,
end_time INTEGER NOT NULL,
input_hash TEXT NOT NULL,
operations_json TEXT NOT NULL,
summary TEXT NOT NULL,
model TEXT NOT NULL,
prompt_version TEXT NOT NULL,
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
cached_tokens INTEGER NOT NULL,
created_at INTEGER NOT NULL,
UNIQUE(user_id, start_time, end_time, input_hash)
)
""".trimIndent()
)
ensureColumn(
connection = connection,
table = "profile_revision",
column = "source",
definition = "TEXT NOT NULL DEFAULT 'backfill'",
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS user_profile_meta(
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)
""".trimIndent()
)
}
connection.prepareStatement(
"INSERT INTO user_profile_meta(key, value) VALUES ('schema_version', ?) " +
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
).use { statement ->
statement.setString(1, SCHEMA_VERSION.toString())
statement.executeUpdate()
}
connection.commit()
} catch (cause: Throwable) {
connection.rollback()
throw cause
} finally {
connection.autoCommit = oldAutoCommit
}
}
private fun configureWriteConnection(connection: Connection) {
connection.createStatement().use { statement ->
statement.execute("PRAGMA journal_mode=WAL")
statement.execute("PRAGMA synchronous=NORMAL")
statement.execute("PRAGMA busy_timeout=$BUSY_TIMEOUT_MS")
statement.execute("PRAGMA wal_autocheckpoint=1000")
}
}
private fun ensureColumn(
connection: Connection,
table: String,
column: String,
definition: String,
) {
val exists = connection.createStatement().use { statement ->
statement.executeQuery("PRAGMA table_info($table)").use { results ->
var found = false
while (results.next()) {
if (results.getString("name") == column) found = true
}
found
}
}
if (!exists) {
connection.createStatement().use { statement ->
statement.executeUpdate("ALTER TABLE $table ADD COLUMN $column $definition")
}
}
}
private fun openConnection(): Connection =
DriverManager.getConnection("jdbc:sqlite:${databaseFile.absolutePath}")
private fun openReadConnection(): Connection = openConnection().also { connection ->
connection.createStatement().use { statement ->
statement.execute("PRAGMA busy_timeout=$BUSY_TIMEOUT_MS")
statement.execute("PRAGMA query_only=ON")
}
}
private fun withWriteConnection(block: (Connection) -> Unit) {
synchronized(writeLock) {
val connection = writeConnection?.takeUnless(Connection::isClosed)
?: openConnection().also {
configureWriteConnection(it)
writeConnection = it
}
block(connection)
}
}
private fun ResultSet.toProfileWithoutItems() = UserProfileSnapshot(
userId = getLong("user_id"),
summary = getString("summary"),
version = getInt("version"),
cursorTime = getInt("cursor_time"),
snapshotEndTime = getInt("snapshot_end_time"),
reliable = getInt("reliable") != 0,
model = getString("model"),
promptVersion = getString("prompt_version"),
updatedAt = getLong("updated_at"),
)
private fun ResultSet.toProfileItem() = UserProfileItem(
id = getString("item_id"),
category = ProfileCategory.valueOf(getString("category").uppercase()),
content = getString("content"),
confidence = ProfileConfidence.valueOf(getString("confidence").uppercase()),
relatedUserId = getLong("related_user_id").let { if (wasNull()) null else it },
firstSeenAt = getInt("first_seen_at"),
lastConfirmedAt = getInt("last_confirmed_at"),
)
private fun ProfileCategory.toStorageValue(): String = name.lowercase()
private fun ProfileConfidence.toStorageValue(): String = name.lowercase()
private fun Int.safeNextSecond(): Int = if (this == Int.MAX_VALUE) this else this + 1
}
@@ -15,8 +15,8 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginData
import top.jie65535.mirai.FavorabilityInfo
import top.jie65535.mirai.data.PluginData
import top.jie65535.mirai.data.FavorabilityInfo
import java.time.OffsetDateTime
import java.time.format.DateTimeFormatter
+2 -2
View File
@@ -10,8 +10,8 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.SkillStore
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.SkillStore
/**
* 删除一个过时或失效的技能。
+2 -2
View File
@@ -13,7 +13,7 @@ import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.contact.MemberPermission
import net.mamoe.mirai.event.events.GroupMessageEvent
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import kotlin.time.Duration.Companion.seconds
class GroupManageAgent : BaseAgent(
@@ -65,4 +65,4 @@ class GroupManageAgent : BaseAgent(
member.mute(duration.coerceIn(1, 10) * 60)
return "已禁言目标"
}
}
}
+1 -1
View File
@@ -22,7 +22,7 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
class ImageAgent : BaseAgent(
tool = Tool.function(
+2 -2
View File
@@ -10,8 +10,8 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.SkillStore
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.SkillStore
/**
* 按需加载某个技能的正文进上下文。技能索引(name+简介)常驻系统提示词,
+3 -3
View File
@@ -10,8 +10,8 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.PluginData
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.PluginData
class MemoryAppend : BaseAgent(
tool = Tool.function(
@@ -42,4 +42,4 @@ class MemoryAppend : BaseAgent(
PluginData.appendContactMemory(contactId, memoryText)
return "OK"
}
}
}
+3 -3
View File
@@ -10,8 +10,8 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.PluginData
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.PluginData
class MemoryReplace : BaseAgent(
tool = Tool.function(
@@ -48,4 +48,4 @@ class MemoryReplace : BaseAgent(
PluginData.replaceContactMemory(contactId, oldMemoryText, newMemoryText)
return "OK"
}
}
}
+3 -3
View File
@@ -6,8 +6,8 @@ import com.aallam.openai.api.chat.Tool
import com.aallam.openai.api.core.Parameters
import com.aallam.openai.api.model.ModelId
import kotlinx.serialization.json.*
import top.jie65535.mirai.LargeLanguageModels
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.llm.LargeLanguageModels
class ReasoningAgent : BaseAgent(
tool = Tool.function(
@@ -63,4 +63,4 @@ class ReasoningAgent : BaseAgent(
else -> "推理出错,结果为空"
}
}
}
}
+2 -2
View File
@@ -16,7 +16,7 @@ import net.mamoe.mirai.event.events.MessageEvent
import net.mamoe.mirai.event.nextEvent
import net.mamoe.mirai.message.data.content
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import kotlin.collections.getValue
class RequestOwner : BaseAgent(
@@ -54,4 +54,4 @@ class RequestOwner : BaseAgent(
JChatGPT.logger.info("主人回复:$response")
return response
}
}
}
+2 -2
View File
@@ -6,7 +6,7 @@ import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
import kotlinx.serialization.json.*
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
class RunCode : BaseAgent(
tool = Tool.function(
@@ -105,4 +105,4 @@ class RunCode : BaseAgent(
}
return response.bodyAsText()
}
}
}
+2 -2
View File
@@ -10,8 +10,8 @@ import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.SkillStore
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.SkillStore
/**
* 新增或整篇覆盖一个技能(全局,跨群共享)。
+3 -3
View File
@@ -12,9 +12,9 @@ import net.mamoe.mirai.message.data.Image.Key.queryUrl
import net.mamoe.mirai.message.data.SingleMessage
import net.mamoe.mirai.message.data.content
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.ChatHistoryStore
import top.jie65535.mirai.ChatMessageRecord
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.ChatHistoryStore
import top.jie65535.mirai.data.ChatMessageRecord
import java.time.Instant
import java.time.LocalDateTime
import java.time.OffsetDateTime
@@ -11,7 +11,7 @@ import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import net.mamoe.mirai.message.data.buildForwardMessage
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import kotlin.collections.getValue
class SendCompositeMessage : BaseAgent(
@@ -47,4 +47,4 @@ class SendCompositeMessage : BaseAgent(
)
return "OK"
}
}
}
+2 -2
View File
@@ -4,7 +4,7 @@ import com.aallam.openai.api.chat.Tool
import com.aallam.openai.api.core.Parameters
import kotlinx.serialization.json.*
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.LaTeXConverter
import top.jie65535.mirai.media.LaTeXConverter
import net.mamoe.mirai.utils.ExternalResource.Companion.toExternalResource
class SendLaTeXExpression : BaseAgent(
@@ -43,4 +43,4 @@ class SendLaTeXExpression : BaseAgent(
return "处理LaTeX表达式时发生异常: ${ex.message}"
}
}
}
}
+2 -2
View File
@@ -10,7 +10,7 @@ import net.mamoe.mirai.contact.AudioSupported
import net.mamoe.mirai.event.events.MessageEvent
import net.mamoe.mirai.utils.ExternalResource.Companion.toExternalResource
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import java.io.File
import java.util.concurrent.TimeUnit
import kotlin.time.measureTime
@@ -146,4 +146,4 @@ class SendVoiceMessage : BaseAgent(
JChatGPT.logger.info("转换音频耗时 $convertDuration")
}
}
}
+2 -2
View File
@@ -7,7 +7,7 @@ import io.ktor.client.statement.*
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.serialization.json.*
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
class VisitWeb : BaseAgent(
tool = Tool.function(
@@ -70,4 +70,4 @@ class VisitWeb : BaseAgent(
"Error fetching \"$url\": ${e.message}"
}
}
}
}
+2 -2
View File
@@ -23,8 +23,8 @@ import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.LargeLanguageModels
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.llm.LargeLanguageModels
import java.net.URI
class VisualAgent : BaseAgent(
+1 -1
View File
@@ -14,7 +14,7 @@ import net.i2p.crypto.eddsa.EdDSAEngine
import net.i2p.crypto.eddsa.EdDSAPrivateKey
import net.i2p.crypto.eddsa.spec.EdDSANamedCurveTable
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
import java.nio.charset.StandardCharsets
import java.security.MessageDigest
import java.security.spec.PKCS8EncodedKeySpec
+2 -2
View File
@@ -10,7 +10,7 @@ import kotlinx.coroutines.awaitAll
import kotlinx.serialization.json.*
import org.apache.commons.text.StringEscapeUtils
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.PluginConfig
import top.jie65535.mirai.config.PluginConfig
class WebSearch : BaseAgent(
tool = Tool.function(
@@ -114,4 +114,4 @@ class WebSearch : BaseAgent(
"Failed to search \"$q\": ${e.message}"
}
}
}
}
@@ -0,0 +1,35 @@
package top.jie65535.mirai.conversation
import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatMessageRecord
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
class ConversationContextTest {
@Test
fun replyIndexKeepsStableNumbersForStoredMessageIds() {
val index = ReplyIndex()
val first = record(ids = "10,20", time = 100)
val duplicate = record(ids = "10,20", time = 101)
val withoutIds = record(ids = null, time = 102)
assertEquals(1, index.add(first))
assertEquals(1, index.add(duplicate))
assertEquals(2, index.add(withoutIds))
assertEquals(first, index.get(1))
assertEquals(1, index.indexOfIds("10,20"))
assertNull(index.indexOfIds("missing"))
}
private fun record(ids: String?, time: Int) = ChatMessageRecord(
botId = 1,
fromId = 2,
targetId = 3,
ids = ids,
internalIds = null,
time = time,
kind = MessageSourceKind.GROUP,
code = "[]",
)
}
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.data
import java.nio.file.Files
import java.sql.DriverManager
+38
View File
@@ -0,0 +1,38 @@
package top.jie65535.mirai.llm
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.time.Duration.Companion.seconds
class ModelServiceTest {
private val service = ModelService(
baseUrl = "http://localhost/",
token = "test",
timeout = 1.seconds,
firstChunkTimeout = 1.seconds,
)
@Test
fun readsStandardCachedPromptTokens() {
val usage = service.extractCacheUsage(
"""{"usage":{"prompt_tokens":100,"prompt_tokens_details":{"cached_tokens":75}}}"""
)
assertEquals(ModelService.CacheUsage(hitTokens = 75, missTokens = 25), usage)
}
@Test
fun readsDeepSeekCacheTokens() {
val usage = service.extractCacheUsage(
"""{"usage":{"prompt_cache_hit_tokens":80,"prompt_cache_miss_tokens":20}}"""
)
assertEquals(ModelService.CacheUsage(hitTokens = 80, missTokens = 20), usage)
}
@Test
fun ignoresResponsesWithoutCacheDetails() {
assertNull(service.extractCacheUsage("""{"usage":{"prompt_tokens":100}}"""))
}
}
@@ -1,4 +1,4 @@
package top.jie65535.mirai
package top.jie65535.mirai.media
import kotlin.test.Test
import kotlin.test.assertEquals
@@ -0,0 +1,119 @@
package top.jie65535.mirai.profile
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import top.jie65535.mirai.llm.LargeLanguageModels
import top.jie65535.mirai.llm.ModelService
import java.io.File
import kotlin.test.Test
import kotlin.time.Duration.Companion.seconds
class ConversationProfileLiveExperiment {
@Test
fun analyzeConfiguredHistoricalConversation() = runBlocking {
val apiKey = System.getenv("PROFILE_EXPERIMENT_API_KEY") ?: return@runBlocking
val source = File(checkNotNull(System.getenv("PROFILE_EXPERIMENT_SOURCE")))
val botId = checkNotNull(System.getenv("PROFILE_EXPERIMENT_BOT_ID")).toLong()
val groupId = checkNotNull(System.getenv("PROFILE_EXPERIMENT_GROUP_ID")).toLong()
val startTime = checkNotNull(System.getenv("PROFILE_EXPERIMENT_START")).toInt()
val endTime = checkNotNull(System.getenv("PROFILE_EXPERIMENT_END")).toInt()
val output = File(checkNotNull(System.getenv("PROFILE_EXPERIMENT_OUTPUT")))
val modelName = checkNotNull(System.getenv("PROFILE_EXPERIMENT_MODEL"))
val baseUrl = checkNotNull(System.getenv("PROFILE_EXPERIMENT_API"))
val messageLimit = System.getenv("PROFILE_EXPERIMENT_MESSAGE_LIMIT")?.toInt() ?: 150
val firstChunkTimeout = System.getenv("PROFILE_EXPERIMENT_FIRST_CHUNK_TIMEOUT")?.toInt() ?: 240
val batch = checkNotNull(
ProfileHistoryReader(source).loadConversationBatch(
botId = botId,
groupId = groupId,
startTime = startTime,
endTime = endTime,
messageLimit = messageLimit,
maxMessageChars = 1_000,
)
)
val eligible = batch.authoredTextCharsByUser.filterValues { it >= 20 }.keys
check(eligible.isNotEmpty()) { "所选会话没有达到文本门槛的参与者" }
val profiles = eligible.associateWith { userId ->
UserProfileSnapshot(userId = userId, cursorTime = 0, snapshotEndTime = 0)
}
val service = ModelService(
baseUrl = baseUrl,
token = apiKey,
timeout = maxOf(180, firstChunkTimeout).seconds,
firstChunkTimeout = firstChunkTimeout.seconds,
)
try {
val client = ProfileModelClient(
LargeLanguageModels.ProfileEndpoint(service, modelName, temperature = 0.1)
)
val result = client.analyzeConversation(profiles, batch, eligible)
val reductions = ConversationProfileReducer.reduce(
profiles = profiles,
batch = batch,
eligibleUserIds = eligible,
response = result.response,
model = modelName,
promptVersion = ProfilePromptStore.PROMPT_VERSION,
summaryMaxLength = 500,
)
val names = batch.aliases.mapValues { (userId, alias) -> "$alias/$userId" }
val injection = UserProfileContextRenderer.render(
profiles = reductions.map { it.profile },
favorabilityByUserId = emptyMap(),
displayNames = names,
activeUserIds = eligible,
summaryMaxChars = 300,
)
output.parentFile?.mkdirs()
output.writeText(
buildString {
appendLine("# 多人会话画像真实数据实验")
appendLine()
appendLine("- group: $groupId")
appendLine("- range: [$startTime, $endTime)")
appendLine("- messages: ${batch.messages.size}")
appendLine("- eligible_users: ${eligible.size}")
appendLine("- operations: ${reductions.sumOf { it.operations.size }}")
appendLine("- tokens: ${result.usage.promptTokens}/${result.usage.completionTokens}, cached=${result.usage.cachedTokens}")
appendLine()
appendLine("## 模型结构化输出")
appendLine()
appendLine("```json")
appendLine(prettyJson.encodeToString(result.response))
appendLine("```")
appendLine()
appendLine("## 证据核对")
reductions.flatMap { reduction ->
reduction.operations.map { operation -> reduction.profile.userId to operation }
}.forEach { (userId, operation) ->
appendLine()
appendLine("- ${batch.aliases[userId]}/$userId: ${operation.action} ${operation.category} ${operation.content}")
operation.evidenceRefs.forEach { ref ->
val evidence = checkNotNull(batch.evidenceByRef[ref])
append(" - [e:").append(ref).append("][")
.append(batch.aliases[evidence.record.fromId]).append("] ")
.appendLine(evidence.text.replace('\n', ' '))
}
}
appendLine()
appendLine("## 实际注入文本")
appendLine()
appendLine("```")
append(injection)
appendLine("```")
},
Charsets.UTF_8,
)
} finally {
service.httpClient.close()
}
}
companion object {
private val prettyJson = Json { prettyPrint = true }
}
}
@@ -0,0 +1,223 @@
package top.jie65535.mirai.profile
import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatMessageRecord
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
class ConversationProfileReducerTest {
@Test
fun reducesMultipleUsersFromOneConversation() {
val batch = batch()
val profiles = USERS.associateWith(::emptyProfile)
val reductions = ConversationProfileReducer.reduce(
profiles = profiles,
batch = batch,
eligibleUserIds = USERS,
response = ConversationProfileModelResponse(
users = listOf(
ConversationProfileUserResponse(
userAlias = "U1",
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.NOTABLE_FACT,
content = "日常使用 Kotlin 开发",
confidence = ProfileConfidence.MEDIUM,
evidenceRefs = listOf(1),
)
),
summary = "日常使用 Kotlin 开发。",
),
ConversationProfileUserResponse(
userAlias = "U2",
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.INTEREST,
content = "关注本地大模型",
confidence = ProfileConfidence.MEDIUM,
evidenceRefs = listOf(2),
)
),
summary = "关注本地大模型。",
),
)
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
assertEquals(2, reductions.size)
assertEquals("日常使用 Kotlin 开发", reductions.single { it.profile.userId == USER_A }.profile.items.single().content)
assertEquals("关注本地大模型", reductions.single { it.profile.userId == USER_B }.profile.items.single().content)
assertTrue(reductions.all { it.profile.cursorTime == 0 })
}
@Test
fun rejectsEvidenceAuthoredByAnotherUser() {
val failure = assertFailsWith<IllegalArgumentException> {
ConversationProfileReducer.reduce(
profiles = USERS.associateWith(::emptyProfile),
batch = batch(),
eligibleUserIds = USERS,
response = ConversationProfileModelResponse(
users = listOf(
ConversationProfileUserResponse(
userAlias = "U1",
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.NOTABLE_FACT,
content = "关注本地大模型",
confidence = ProfileConfidence.MEDIUM,
evidenceRefs = listOf(2),
)
),
summary = "关注本地大模型。",
)
)
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
}
assertTrue(failure.message.orEmpty().contains("目标用户"))
}
@Test
fun confirmsAnExistingItemInsteadOfCreatingAnIndependentProfile() {
val oldItem = UserProfileItem(
id = "existing-item",
category = ProfileCategory.INTEREST,
content = "关注 Kotlin 开发",
confidence = ProfileConfidence.LOW,
firstSeenAt = 80,
lastConfirmedAt = 80,
)
val existing = emptyProfile(USER_A).copy(
summary = "关注 Kotlin 开发。",
version = 1,
reliable = true,
items = listOf(oldItem),
)
val profiles = mapOf(USER_A to existing, USER_B to emptyProfile(USER_B))
val reductions = ConversationProfileReducer.reduce(
profiles = profiles,
batch = batch(),
eligibleUserIds = USERS,
response = ConversationProfileModelResponse(
users = listOf(
ConversationProfileUserResponse(
userAlias = "U1",
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.CONFIRM,
itemId = oldItem.id,
confidence = ProfileConfidence.MEDIUM,
evidenceRefs = listOf(1),
)
),
summary = "关注 Kotlin 开发。",
)
)
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
val updated = reductions.single { it.profile.userId == USER_A }.profile
assertEquals(2, updated.version)
assertEquals("existing-item", updated.items.single().id)
assertEquals(ProfileConfidence.MEDIUM, updated.items.single().confidence)
assertTrue(
ProfilePromptStore.buildConversationUserPrompt(profiles, batch(), USERS)
.contains("[P:existing-item]")
)
}
@Test
fun rejectsMoreThanFourOperationsForOneUser() {
val operations = (1..5).map { index ->
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.NOTABLE_FACT,
content = "事实 $index",
confidence = ProfileConfidence.LOW,
evidenceRefs = listOf(1),
)
}
val failure = assertFailsWith<IllegalArgumentException> {
ConversationProfileReducer.reduce(
profiles = USERS.associateWith(::emptyProfile),
batch = batch(),
eligibleUserIds = USERS,
response = ConversationProfileModelResponse(
users = listOf(
ConversationProfileUserResponse(
userAlias = "U1",
operations = operations,
summary = "包含过多事实。",
)
)
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
}
assertTrue(failure.message.orEmpty().contains("超过 4 项"))
}
private fun batch() = ConversationProfileBatch(
botId = BOT,
groupId = 10,
startTime = 100,
endTime = 200,
messages = listOf(
message(1, USER_A, "我平时用 Kotlin 写项目"),
message(2, USER_B, "我最近一直在研究本地大模型"),
),
aliases = mapOf(BOT to "BOT", USER_A to "U1", USER_B to "U2"),
inputHash = "conversation-hash",
)
private fun emptyProfile(userId: Long) = UserProfileSnapshot(
userId = userId,
cursorTime = 0,
snapshotEndTime = 0,
)
private fun message(ref: Int, fromId: Long, text: String) = ProfilePromptMessage(
record = ChatMessageRecord(
botId = BOT,
fromId = fromId,
targetId = 10,
ids = null,
internalIds = null,
time = 120 + ref,
kind = MessageSourceKind.GROUP,
code = text,
),
text = text,
evidenceRef = ref,
episodeIndex = 1,
)
companion object {
private const val BOT = 1L
private const val USER_A = 100L
private const val USER_B = 200L
private val USERS = setOf(USER_A, USER_B)
}
}
@@ -0,0 +1,122 @@
package top.jie65535.mirai.profile
import net.mamoe.mirai.message.data.MessageSourceKind
import java.nio.file.Files
import java.sql.DriverManager
import kotlin.io.path.absolutePathString
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
class ProfileHistoryReaderTest {
@Test
fun keepsEqualTimestampTargetMessagesTogetherAndLoadsGroupContext() {
val directory = Files.createTempDirectory("jchatgpt-profile-history-test-")
val database = directory.resolve("history.sqlite")
try {
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE TABLE message_record(
id INTEGER PRIMARY KEY AUTOINCREMENT,
bot_id INTEGER NOT NULL,
from_id INTEGER NOT NULL,
target_id INTEGER NOT NULL,
ids TEXT,
internal_ids TEXT,
time INTEGER NOT NULL,
kind INTEGER NOT NULL,
code TEXT NOT NULL,
recalled INTEGER NOT NULL DEFAULT 0
)
""".trimIndent()
)
}
connection.prepareStatement(
"INSERT INTO message_record(" +
"bot_id, from_id, target_id, time, kind, code, recalled" +
") VALUES (1, ?, ?, ?, ?, ?, 0)"
).use { statement ->
fun insert(fromId: Long, groupId: Long, time: Int, text: String) {
statement.setLong(1, fromId)
statement.setLong(2, groupId)
statement.setInt(3, time)
statement.setInt(4, MessageSourceKind.GROUP.ordinal)
statement.setString(5, """[{"type":"PlainText","content":"$text"}]""")
statement.executeUpdate()
}
insert(TARGET, 10, 100, "目标发言一")
insert(OTHER, 10, 110, "用于理解语境的回复")
insert(TARGET, 10, 130, "目标发言二")
insert(TARGET, 20, 130, "同一秒的另一群发言")
insert(OTHER, 20, 140, "后续上下文")
insert(TARGET, 10, 200, "下一批目标发言")
}
}
val reader = ProfileHistoryReader(database.toFile())
val bounds = assertNotNull(reader.findUserTimeBounds(TARGET))
assertEquals(100, bounds.startTime)
assertEquals(201, bounds.endTime)
val first = assertNotNull(
reader.loadNextBatch(
userId = TARGET,
startTime = bounds.startTime,
snapshotEndTime = bounds.endTime,
targetMessageLimit = 2,
maxEpisodes = 1,
episodeGapSeconds = 60,
contextBeforeMessages = 2,
contextAfterMessages = 2,
contextCoreMessages = 20,
maxMessageChars = 200,
)
)
assertEquals(131, first.endTime)
assertEquals(3, first.evidenceByRef.values.count { it.record.fromId == TARGET })
assertTrue(first.messages.any { it.record.fromId == OTHER })
assertEquals("TARGET", first.aliases[TARGET])
val second = assertNotNull(
reader.loadNextBatch(
userId = TARGET,
startTime = first.endTime,
snapshotEndTime = bounds.endTime,
targetMessageLimit = 2,
maxEpisodes = 10,
episodeGapSeconds = 60,
contextBeforeMessages = 2,
contextAfterMessages = 2,
contextCoreMessages = 20,
maxMessageChars = 200,
)
)
assertEquals(201, second.endTime)
assertEquals(1, second.evidenceByRef.values.count { it.record.fromId == TARGET })
val conversation = assertNotNull(
reader.loadConversationBatch(
botId = 1,
groupId = 10,
startTime = 90,
endTime = 150,
messageLimit = 10,
maxMessageChars = 200,
)
)
assertTrue(conversation.authoredTextCharsByUser.getValue(TARGET) > 0)
assertTrue(conversation.messages.all { it.record.targetId == 10L })
assertTrue(conversation.messages.all { it.record.time in 90 until 150 })
} finally {
directory.toFile().deleteRecursively()
}
}
companion object {
private const val TARGET = 100L
private const val OTHER = 200L
}
}
@@ -0,0 +1,46 @@
package top.jie65535.mirai.profile
import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatMessageRecord
import kotlin.test.Test
import kotlin.test.assertEquals
class ProfileMessageRendererTest {
@Test
fun rendersStoredJsonWithoutMiraiRuntime() {
val record = record(
"""
[
{"type":"QuoteReply","source":{"fromId":200,"originalMessage":[{"type":"PlainText","content":"我在公明工作"}]}},
{"type":"At","target":200},
{"type":"PlainText","content":" 这说的是你,不是我"},
{"type":"Image","isEmoji":false}
]
""".trimIndent()
)
assertEquals(
"[引用 U1: 我在公明工作]@U1 这说的是你,不是我[图片]",
ProfileMessageRenderer.render(record, mapOf(100L to "TARGET", 200L to "U1"), 500),
)
assertEquals(setOf(200L), ProfileMessageRenderer.referencedUserIds(record))
}
@Test
fun truncatesLongMessagesAfterRendering() {
val record = record("""[{"type":"PlainText","content":"1234567890"}]""")
assertEquals("12345678...[截断]", ProfileMessageRenderer.render(record, emptyMap(), 8))
}
private fun record(code: String) = ChatMessageRecord(
botId = 1,
fromId = 100,
targetId = 10,
ids = null,
internalIds = null,
time = 100,
kind = MessageSourceKind.GROUP,
code = code,
)
}
@@ -0,0 +1,91 @@
package top.jie65535.mirai.profile
import top.jie65535.mirai.data.FavorabilityInfo
import kotlin.test.Test
import kotlin.test.assertContains
import kotlin.test.assertFalse
class UserProfileContextRendererTest {
@Test
fun rendersCompactSummariesAndOnlyRelationshipsInsideCurrentConversation() {
val profile = UserProfileSnapshot(
userId = 100,
summary = "长期关注 Kotlin 开发,也经常讨论大模型应用。",
cursorTime = 1,
snapshotEndTime = 2,
reliable = true,
items = listOf(
relationship("visible", 200, "经常互相讨论技术方案"),
relationship("hidden", 300, "曾共同讨论游戏"),
),
)
val rendered = UserProfileContextRenderer.render(
profiles = listOf(profile),
favorabilityByUserId = mapOf(
100L to FavorabilityInfo(
userId = 100,
value = 12,
name = "小明代号",
tags = listOf("老群友"),
impression = "聊天很直接",
)
),
displayNames = mapOf(100L to "小明", 200L to "小王", 300L to "小李"),
activeUserIds = setOf(100, 200),
summaryMaxChars = 50,
)
assertContains(rendered, "小明代号(100)")
assertContains(rendered, "好感度+12")
assertContains(rendered, "长期认识:长期关注 Kotlin 开发")
assertContains(rendered, "与小王:经常互相讨论技术方案")
assertFalse(rendered.contains("小李"))
assertFalse(rendered.contains("曾共同讨论游戏"))
}
@Test
fun keepsFavorabilityVisibleWithoutAHistoricalProfile() {
val rendered = UserProfileContextRenderer.render(
profiles = emptyList(),
favorabilityByUserId = mapOf(
100L to FavorabilityInfo(userId = 100, value = -8, impression = "偶尔喜欢抬杠")
),
displayNames = mapOf(100L to "小明"),
activeUserIds = setOf(100),
summaryMaxChars = 300,
)
assertContains(rendered, "小明(100)")
assertContains(rendered, "好感度-8")
assertContains(rendered, "主观印象:偶尔喜欢抬杠")
assertFalse(rendered.contains("长期认识:"))
}
@Test
fun keepsImpressionVisibleWhenFavorabilityIsZero() {
val rendered = UserProfileContextRenderer.render(
profiles = emptyList(),
favorabilityByUserId = mapOf(
100L to FavorabilityInfo(userId = 100, value = 0, impression = "长期活跃的老群友")
),
displayNames = mapOf(100L to "小明"),
activeUserIds = setOf(100),
summaryMaxChars = 300,
)
assertContains(rendered, "小明(100)")
assertContains(rendered, "好感度+0")
assertContains(rendered, "主观印象:长期活跃的老群友")
}
private fun relationship(id: String, relatedUserId: Long, content: String) = UserProfileItem(
id = id,
category = ProfileCategory.RELATIONSHIP_NOTE,
content = content,
confidence = ProfileConfidence.MEDIUM,
relatedUserId = relatedUserId,
firstSeenAt = 1,
lastConfirmedAt = 1,
)
}
@@ -0,0 +1,158 @@
package top.jie65535.mirai.profile
import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatMessageRecord
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
class UserProfileReducerTest {
@Test
fun addsItemFromTargetAuthoredEvidence() {
val batch = batchOf(
message(ref = 1, fromId = TARGET, text = "我平时会写 Kotlin"),
message(ref = 2, fromId = OTHER, text = "确实"),
)
val current = emptyProfile()
val reduction = UserProfileReducer.reduce(
current = current,
batch = batch,
response = ProfileModelResponse(
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.INTEREST,
content = "持续关注 Kotlin 开发",
confidence = ProfileConfidence.MEDIUM,
evidenceRefs = listOf(1, 2),
)
),
summary = "关注 Kotlin 开发。",
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
assertEquals(1, reduction.profile.items.size)
assertEquals("持续关注 Kotlin 开发", reduction.profile.items.single().content)
assertEquals(batch.endTime, reduction.profile.cursorTime)
assertEquals("关注 Kotlin 开发。", reduction.profile.summary)
}
@Test
fun rejectsItemSupportedOnlyByAnotherUser() {
val batch = batchOf(message(ref = 1, fromId = OTHER, text = "我平时会写 Kotlin"))
val failure = assertFailsWith<IllegalArgumentException> {
UserProfileReducer.reduce(
current = emptyProfile(),
batch = batch,
response = ProfileModelResponse(
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.NOTABLE_FACT,
content = "从事 Kotlin 开发",
confidence = ProfileConfidence.HIGH,
evidenceRefs = listOf(1),
)
),
summary = "从事 Kotlin 开发。",
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
}
assertTrue(failure.message.orEmpty().contains("目标用户"))
}
@Test
fun ignoresSummaryRewriteWhenThereIsNoValidOperation() {
val batch = batchOf(message(ref = 1, fromId = TARGET, text = "今天天气不错"))
val current = emptyProfile().copy(summary = "原摘要")
val reduction = UserProfileReducer.reduce(
current = current,
batch = batch,
response = ProfileModelResponse(summary = "凭空出现的新摘要"),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
assertEquals("原摘要", reduction.profile.summary)
assertEquals(0, reduction.profile.version)
assertEquals(batch.endTime, reduction.profile.cursorTime)
}
@Test
fun conversationUpdateDoesNotAdvanceHistoricalCursor() {
val batch = batchOf(message(ref = 1, fromId = TARGET, text = "我平时会写 Kotlin"))
val current = emptyProfile()
val reduction = UserProfileReducer.reduce(
current = current,
batch = batch,
response = ProfileModelResponse(
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.INTEREST,
content = "持续关注 Kotlin 开发",
confidence = ProfileConfidence.MEDIUM,
evidenceRefs = listOf(1),
)
),
summary = "关注 Kotlin 开发。",
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
advanceBackfillCursor = false,
)
assertEquals(current.cursorTime, reduction.profile.cursorTime)
assertEquals(1, reduction.profile.version)
}
private fun emptyProfile() = UserProfileSnapshot(
userId = TARGET,
cursorTime = 100,
snapshotEndTime = 1_000,
)
private fun batchOf(vararg messages: ProfilePromptMessage) = ProfileHistoryBatch(
userId = TARGET,
startTime = 100,
endTime = 200,
messages = messages.toList(),
aliases = mapOf(TARGET to "TARGET", OTHER to "U1"),
inputHash = "hash",
)
private fun message(ref: Int, fromId: Long, text: String) = ProfilePromptMessage(
record = ChatMessageRecord(
botId = 1,
fromId = fromId,
targetId = 10,
ids = null,
internalIds = null,
time = 120 + ref,
kind = MessageSourceKind.GROUP,
code = text,
),
text = text,
evidenceRef = ref,
episodeIndex = 1,
)
companion object {
private const val TARGET = 100L
private const val OTHER = 200L
}
}
@@ -0,0 +1,234 @@
package top.jie65535.mirai.profile
import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatMessageRecord
import java.nio.file.Files
import java.sql.DriverManager
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
class UserProfileStoreTest {
@Test
fun commitsProfileItemsRevisionAndWaterline() {
val directory = Files.createTempDirectory("jchatgpt-profile-test-")
try {
UserProfileStore.init(directory.toFile())
val batch = ProfileHistoryBatch(
userId = 100,
startTime = 100,
endTime = 200,
messages = listOf(
ProfilePromptMessage(
record = ChatMessageRecord(
botId = 1,
fromId = 100,
targetId = 300,
ids = null,
internalIds = null,
time = 150,
kind = MessageSourceKind.GROUP,
code = "message",
),
text = "message",
evidenceRef = 1,
episodeIndex = 1,
)
),
aliases = mapOf(100L to "TARGET"),
inputHash = "input-hash",
)
val item = UserProfileItem(
id = "item-1",
category = ProfileCategory.INTEREST,
content = "关注 Kotlin 开发",
confidence = ProfileConfidence.MEDIUM,
firstSeenAt = 150,
lastConfirmedAt = 150,
)
val profile = UserProfileSnapshot(
userId = 100,
summary = "关注 Kotlin 开发。",
version = 1,
cursorTime = 200,
snapshotEndTime = 1_000,
reliable = true,
model = "test-model",
promptVersion = "test-prompt",
updatedAt = 123,
items = listOf(item),
)
UserProfileStore.commit(
reduction = ProfileReduction(
profile = profile,
operations = listOf(
AppliedProfileOperation(
action = ProfileOperationAction.ADD,
itemId = item.id,
category = item.category,
content = item.content,
confidence = item.confidence,
relatedUserId = null,
evidenceRefs = listOf(1),
)
),
),
batch = batch,
usage = ProfileTokenUsage(100, 20, 50),
source = ProfileRevisionSource.CONVERSATION,
)
val loaded = assertNotNull(UserProfileStore.load(100))
assertEquals(200, loaded.cursorTime)
assertEquals(1_000, loaded.snapshotEndTime)
assertEquals("关注 Kotlin 开发。", loaded.summary)
assertEquals(item, loaded.items.single())
assertTrue(loaded.reliable)
assertNotNull(UserProfileStore.lastRevisionAt(100, ProfileRevisionSource.CONVERSATION))
assertTrue(UserProfileStore.isConversationProcessed("input-hash"))
} finally {
UserProfileStore.close()
directory.toFile().deleteRecursively()
}
}
@Test
fun migratesRevisionSourceColumnWithoutRebuildingExistingDatabase() {
val directory = Files.createTempDirectory("jchatgpt-profile-migration-test-")
val database = directory.resolve("user-profile.sqlite")
try {
DriverManager.getConnection("jdbc:sqlite:${database.toAbsolutePath()}").use { connection ->
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE TABLE profile_revision(
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
profile_version INTEGER NOT NULL,
start_time INTEGER NOT NULL,
end_time INTEGER NOT NULL,
input_hash TEXT NOT NULL,
operations_json TEXT NOT NULL,
summary TEXT NOT NULL,
model TEXT NOT NULL,
prompt_version TEXT NOT NULL,
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
cached_tokens INTEGER NOT NULL,
created_at INTEGER NOT NULL,
UNIQUE(user_id, start_time, end_time, input_hash)
)
""".trimIndent()
)
}
}
UserProfileStore.init(directory.toFile())
UserProfileStore.close()
DriverManager.getConnection("jdbc:sqlite:${database.toAbsolutePath()}").use { connection ->
val columns = connection.createStatement().use { statement ->
statement.executeQuery("PRAGMA table_info(profile_revision)").use { results ->
buildSet {
while (results.next()) add(results.getString("name"))
}
}
}
assertTrue("source" in columns)
val version = connection.createStatement().use { statement ->
statement.executeQuery(
"SELECT value FROM user_profile_meta WHERE key = 'schema_version'"
).use { results ->
assertTrue(results.next())
results.getString(1)
}
}
assertEquals("2", version)
}
} finally {
UserProfileStore.close()
directory.toFile().deleteRecursively()
}
}
@Test
fun commitsAllConversationProfilesInOneTransactionAndCountsUsageOnce() {
val directory = Files.createTempDirectory("jchatgpt-profile-conversation-commit-test-")
try {
UserProfileStore.init(directory.toFile())
val entries = listOf(100L, 200L).map { userId ->
val batch = batchFor(userId, "shared-conversation-hash")
val profile = UserProfileSnapshot(
userId = userId,
summary = "用户 $userId 的测试摘要",
version = 1,
cursorTime = 0,
snapshotEndTime = 0,
reliable = true,
model = "test-model",
promptVersion = "test-prompt",
items = listOf(itemFor("item-$userId")),
)
ProfileReduction(profile, emptyList()) to batch
}
UserProfileStore.commitConversation(entries, ProfileTokenUsage(100, 20, 50))
assertNotNull(UserProfileStore.load(100))
assertNotNull(UserProfileStore.load(200))
assertTrue(UserProfileStore.isConversationProcessed("shared-conversation-hash"))
val database = directory.resolve("user-profile.sqlite")
DriverManager.getConnection("jdbc:sqlite:${database.toAbsolutePath()}").use { connection ->
connection.createStatement().use { statement ->
statement.executeQuery(
"SELECT COUNT(*), SUM(prompt_tokens), SUM(completion_tokens) " +
"FROM profile_revision WHERE input_hash = 'shared-conversation-hash'"
).use { results ->
assertTrue(results.next())
assertEquals(2, results.getInt(1))
assertEquals(100, results.getInt(2))
assertEquals(20, results.getInt(3))
}
}
}
} finally {
UserProfileStore.close()
directory.toFile().deleteRecursively()
}
}
private fun batchFor(userId: Long, inputHash: String) = ProfileHistoryBatch(
userId = userId,
startTime = 100,
endTime = 200,
messages = listOf(
ProfilePromptMessage(
record = ChatMessageRecord(
botId = 1,
fromId = userId,
targetId = 300,
ids = null,
internalIds = null,
time = 150,
kind = MessageSourceKind.GROUP,
code = "message",
),
text = "message",
evidenceRef = 1,
episodeIndex = 1,
)
),
aliases = mapOf(userId to "TARGET"),
inputHash = inputHash,
)
private fun itemFor(id: String) = UserProfileItem(
id = id,
category = ProfileCategory.INTEREST,
content = "关注 Kotlin 开发",
confidence = ProfileConfidence.MEDIUM,
firstSeenAt = 150,
lastConfirmedAt = 150,
)
}