From f102264a55209544467c692acc34ef28b8aae4d3 Mon Sep 17 00:00:00 2001 From: jie65535 Date: Mon, 3 Aug 2026 02:53:59 +0800 Subject: [PATCH] profile: harden batch analysis and compaction --- README.md | 17 +- src/main/kotlin/JChatGPT.kt | 2 - src/main/kotlin/command/PluginCommands.kt | 118 ++++++- src/main/kotlin/config/PluginConfig.kt | 6 - .../profile/ConversationProfileReducer.kt | 26 +- .../EvidenceReferenceListSerializer.kt | 33 ++ .../kotlin/profile/ProfileAutoMaintenance.kt | 1 + .../kotlin/profile/ProfileCompactionModels.kt | 82 +++++ .../kotlin/profile/ProfileContentRules.kt | 27 ++ .../kotlin/profile/ProfileHistoryReader.kt | 88 ++++++ .../kotlin/profile/ProfileItemReferences.kt | 25 ++ src/main/kotlin/profile/ProfileModelClient.kt | 63 +++- .../kotlin/profile/ProfilePersistentText.kt | 65 ++++ src/main/kotlin/profile/ProfilePromptStore.kt | 172 ++++++---- .../profile/UserProfileAnalysisService.kt | 274 ++++++++++++++++ .../kotlin/profile/UserProfileCompactor.kt | 154 +++++++++ .../profile/UserProfileContextRenderer.kt | 19 +- src/main/kotlin/profile/UserProfileModels.kt | 44 ++- src/main/kotlin/profile/UserProfileReducer.kt | 242 +++++++------- src/main/kotlin/profile/UserProfileStore.kt | 170 ++++++++-- .../profile/ConversationProfileReducerTest.kt | 209 +++++++++---- .../profile/ProfileHistoryReaderTest.kt | 30 ++ .../ProfileModelResponseParsingTest.kt | 40 +++ .../profile/UserProfileAnalysisServiceTest.kt | 18 +- .../profile/UserProfileCompactorTest.kt | 295 ++++++++++++++++++ .../profile/UserProfileContextRendererTest.kt | 10 +- .../kotlin/profile/UserProfileReducerTest.kt | 224 +++++++++++-- .../kotlin/profile/UserProfileStoreTest.kt | 124 +++++++- 28 files changed, 2232 insertions(+), 346 deletions(-) create mode 100644 src/main/kotlin/profile/EvidenceReferenceListSerializer.kt create mode 100644 src/main/kotlin/profile/ProfileCompactionModels.kt create mode 100644 src/main/kotlin/profile/ProfileContentRules.kt create mode 100644 src/main/kotlin/profile/ProfileItemReferences.kt create mode 100644 src/main/kotlin/profile/ProfilePersistentText.kt create mode 100644 src/main/kotlin/profile/UserProfileCompactor.kt create mode 100644 src/test/kotlin/profile/ProfileModelResponseParsingTest.kt create mode 100644 src/test/kotlin/profile/UserProfileCompactorTest.kt diff --git a/README.md b/README.md index b8f122a..36f1192 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,10 @@ AI 可以自动调用多种工具来完成复杂任务： ### 渐进式历史画像（实验） - 日常使用无需画像命令：群聊缓存会话闭合后，一次模型调用会静默归纳其中所有有实质发言的参与者 -- `/jgpt profileAnalyze [batches]` - 诊断或验收时手动推进指定用户画像，默认1批、最多50批 +- `/jgpt profileAnalyze [batches]` - 手动推进指定用户画像，默认1批 +- `/jgpt profileAnalyzeGroup [batches]` - 按群历史批量推进参与者画像，默认1批 - `/jgpt profileShow ` - 诊断或验收时查看已经提交的完整画像和覆盖时间 +- `/jgpt profileCompact ` - 独立反思并压缩指定用户的重复、过细或低价值画像条目 ## 配置文件 @@ -101,9 +103,6 @@ profileModelToken: '' profileModel: '' profileModelTemperature: null profileModelExtraBody: '' -# 定向历史画像和闭合会话多人画像的提示词文件，相对于插件配置目录 -profilePromptFile: 'ProfilePrompt.md' -profileConversationPromptFile: 'ProfileConversationPrompt.md' # 留空使用插件自己的聊天库；本地实验可填写外部SQLite历史库的绝对路径 profileHistoryDatabasePath: '' # 每批目标用户消息数、片段软间隔及上下文限制 @@ -213,9 +212,9 @@ searchHistoryMaxRecords: 5000 本人有效文本达到门槛的账号列为候选，然后用一次模型调用同时比较所有候选人的当前画像并返回按用户分组的 `ADD / UPDATE / CONFIRM / DELETE` 操作。没有可靠变化的参与者仍会被标记为已检查，但不会生成空洞画像。 -程序逐条验证每项操作至少引用了对应用户本人的发言；任何用户别名、条目 ID、证据归因或摘要长度非法，整段 -会话都不会部分提交。全部候选人的画像和本次覆盖记录在一个 SQLite 事务中落库，失败不会阻塞 Bot 回复。 -这一阶段不自动扫描未触发群聊，也不自动回填旧历史；`profileAnalyze` 只保留给离线验收和诊断。 +模型只使用单次请求内有效的临时编号，不接触画像条目的内部 UUID；不合规建议会被跳过，不阻断其他有效更新。 +临时用户别名不会写入最终画像。`profileCompact` 可独立清理重复或低价值条目，且不推进历史水位线。 +旧历史可通过 `profileAnalyze` 或 `profileAnalyzeGroup` 手动分批推进。 下一次正常群聊会自动携带触发者和最近发言者的认识。现有好感度、Bot 代号、标签和主观印象会与证据驱动的 长期画像按同一个人合并渲染，但两套数据仍独立保存，自动画像不会修改好感度。 @@ -230,9 +229,7 @@ profileModelToken: '在部署环境中填写，不要提交到Git' profileModel: '兼容chat/completions的模型名' ``` -执行 `/jgpt reload` 后即可自动运行。`ProfileConversationPrompt.md` 会在首次启用时生成，可直接迭代多人归纳 -规则。`profileAnalyze` 和 `profileShow` 仍可用于人工验收，但不承担日常维护职责。外部历史库始终以只读 -方式打开，程序不会为了画像实验修改备份或创建索引。 +执行 `/jgpt reload` 后即可自动运行。画像提示词由插件内置；相关命令仅用于人工验收。外部历史库始终只读。 ### 和风天气 diff --git a/src/main/kotlin/JChatGPT.kt b/src/main/kotlin/JChatGPT.kt index bc6696f..54c7094 100644 --- a/src/main/kotlin/JChatGPT.kt +++ b/src/main/kotlin/JChatGPT.kt @@ -32,7 +32,6 @@ import top.jie65535.mirai.data.SkillStore import top.jie65535.mirai.data.TokenUsageStore import top.jie65535.mirai.llm.LargeLanguageModels import top.jie65535.mirai.profile.ProfileAutoMaintenance -import top.jie65535.mirai.profile.ProfilePromptStore import top.jie65535.mirai.profile.UserProfileStore import kotlin.random.Random @@ -72,7 +71,6 @@ object JChatGPT : KotlinPlugin( .onFailure { logger.error("初始化用户画像数据库失败，画像分析将暂时禁用", it) } LargeLanguageModels.reload() - ProfilePromptStore.reload() PluginCommands.register() keyword = PluginConfig.callKeyword.takeIf(String::isNotEmpty)?.let(::Regex) diff --git a/src/main/kotlin/command/PluginCommands.kt b/src/main/kotlin/command/PluginCommands.kt index 1f695c9..6667c0b 100644 --- a/src/main/kotlin/command/PluginCommands.kt +++ b/src/main/kotlin/command/PluginCommands.kt @@ -18,9 +18,12 @@ import top.jie65535.mirai.data.PluginData import top.jie65535.mirai.data.SkillStore import top.jie65535.mirai.data.TokenUsageStore import top.jie65535.mirai.llm.LargeLanguageModels +import top.jie65535.mirai.profile.GroupProfileAnalysisReport import top.jie65535.mirai.profile.ProfileAnalysisReport import top.jie65535.mirai.profile.ProfileAutoMaintenance -import top.jie65535.mirai.profile.ProfilePromptStore +import top.jie65535.mirai.profile.ProfileCategory +import top.jie65535.mirai.profile.ProfileCompactionReport +import top.jie65535.mirai.profile.ProfilePersistentText import top.jie65535.mirai.profile.UserProfileAnalysisService import top.jie65535.mirai.profile.UserProfileSnapshot import top.jie65535.mirai.profile.UserProfileStore @@ -38,7 +41,6 @@ object PluginCommands : CompositeCommand( PluginConfig.reload() PluginData.reload() LargeLanguageModels.reload() - ProfilePromptStore.reload() if (!PluginConfig.profileEnabled || !PluginConfig.profileAutoUpdateEnabled) { ProfileAutoMaintenance.clear() } @@ -48,7 +50,7 @@ object PluginCommands : CompositeCommand( @SubCommand suspend fun CommandSender.profileAnalyze(userId: Long, batches: Int = 1) { - require(batches in 1..50) { "batches 必须在 1 到 50 之间" } + require(batches > 0) { "batches 必须是正数" } sendMessage("已启动用户 $userId 的画像分析，本次最多推进 $batches 个批次。") JChatGPT.launch { try { @@ -57,6 +59,7 @@ object PluginCommands : CompositeCommand( "PROFILE_BATCH user=$userId batch=${progress.batchIndex}/$batches " + "range=${progress.startTime}-${progress.endTime} " + "messages=${progress.messageCount} operations=${progress.operationCount} " + + "skipped=${progress.skippedOperationCount} " + "tokens=${progress.usage.promptTokens}/${progress.usage.completionTokens} " + "cached=${progress.usage.cachedTokens}" ) @@ -75,6 +78,36 @@ object PluginCommands : CompositeCommand( } } + @SubCommand + suspend fun CommandSender.profileAnalyzeGroup(groupId: Long, batches: Int = 1) { + require(batches > 0) { "batches 必须是正数" } + sendMessage("已启动群 $groupId 的批量画像分析，本次最多推进 $batches 个批次。") + JChatGPT.launch { + try { + val report = UserProfileAnalysisService.analyzeGroup(groupId, batches) { progress -> + JChatGPT.logger.info( + "PROFILE_GROUP_BATCH group=$groupId batch=${progress.batchIndex}/$batches " + + "range=${progress.startTime}-${progress.endTime} " + + "messages=${progress.messageCount} users=${progress.analyzedUsers} " + + "operations=${progress.appliedOperations} skipped=${progress.skippedOperations} " + + "tokens=${progress.usage.promptTokens}/${progress.usage.completionTokens} " + + "cached=${progress.usage.cachedTokens}" + ) + } + when { + report.alreadyRunning -> sendMessage("群 $groupId 已有画像分析任务在运行。") + report.botId == null -> sendMessage("聊天记录中没有找到群 $groupId 的消息。") + else -> sendMessage(formatGroupProfileReport(report)) + } + } catch (cause: CancellationException) { + throw cause + } catch (cause: Exception) { + JChatGPT.logger.error("群 $groupId 批量画像分析失败", cause) + sendMessage("群 $groupId 批量画像分析失败：${cause.message ?: cause::class.simpleName}") + } + } + } + @SubCommand suspend fun CommandSender.profileShow(userId: Long) { if (!UserProfileStore.isAvailable) { @@ -85,6 +118,22 @@ object PluginCommands : CompositeCommand( sendMessage(profile?.let(::formatProfile) ?: "用户 $userId 尚无画像。") } + @SubCommand + suspend fun CommandSender.profileCompact(userId: Long) { + sendMessage("已启动用户 $userId 的画像压缩反思。") + JChatGPT.launch { + try { + val report = UserProfileAnalysisService.compact(userId) + sendMessage(formatProfileCompactionReport(report)) + } catch (cause: CancellationException) { + throw cause + } catch (cause: Exception) { + JChatGPT.logger.error("用户 $userId 画像压缩失败", cause) + sendMessage("用户 $userId 画像压缩失败：${cause.message ?: cause::class.simpleName}") + } + } + } + @SubCommand suspend fun CommandSender.skills() { val all = SkillStore.all @@ -269,7 +318,7 @@ object PluginCommands : CompositeCommand( private fun formatProfileReport(report: ProfileAnalysisReport): String = buildString { appendLine( "画像分析完成：${report.processedBatches} 批，${report.processedMessages} 条上下文消息，" + - "${report.appliedOperations} 项变更" + "${report.appliedOperations} 项变更，跳过 ${report.skippedOperations} 项无效建议" ) appendLine( "Token：输入 ${formatNumber(report.usage.promptTokens)}，输出 " + @@ -280,6 +329,36 @@ object PluginCommands : CompositeCommand( append(formatProfile(checkNotNull(report.profile))) }.trim() + private fun formatGroupProfileReport(report: GroupProfileAnalysisReport): String = buildString { + appendLine( + "群 ${report.groupId} 画像分析完成：${report.processedBatches} 批，" + + "${report.processedMessages} 条消息，${report.analyzedUsers} 用户人次，" + + "${report.appliedOperations} 项变更，跳过 ${report.skippedOperations} 项无效建议" + ) + appendLine( + "Token：输入 ${formatNumber(report.usage.promptTokens)}，输出 " + + "${formatNumber(report.usage.completionTokens)}，缓存命中 " + + formatNumber(report.usage.cachedTokens) + ) + appendLine("群历史覆盖至 ${formatProfileTime(report.cursorTime)}") + append("状态：${if (report.caughtUp) "已追平当前快照" else "可继续推进"}") + }.trim() + + private fun formatProfileCompactionReport(report: ProfileCompactionReport): String = buildString { + appendLine( + "画像压缩完成：${report.beforeItems} -> ${report.afterItems} 条，" + + "合并 ${report.mergedGroups} 组，改写 ${report.rewrittenItems} 条，删除 ${report.deletedItems} 条，" + + "摘要${if (report.summaryChanged) "已重写" else "未变"}，" + + "跳过 ${report.skippedOperations} 项不安全建议" + ) + appendLine( + "Token：输入 ${formatNumber(report.usage.promptTokens)}，输出 " + + "${formatNumber(report.usage.completionTokens)}，缓存命中 " + + formatNumber(report.usage.cachedTokens) + ) + append(formatProfile(report.profile)) + }.trim() + private fun formatProfile(profile: UserProfileSnapshot): String = buildString { appendLine("用户 ${profile.userId} · 画像 v${profile.version}") if (profile.cursorTime <= 0) { @@ -287,16 +366,33 @@ object PluginCommands : CompositeCommand( } else { appendLine("历史回顾覆盖至 ${formatProfileTime(profile.cursorTime)}") } - appendLine("摘要：${profile.summary.ifBlank { "（暂无）" }}") + val summary = ProfilePersistentText.summaryForDisplay(profile.summary) + appendLine("摘要：${summary.ifBlank { "（暂无）" }}") if (profile.items.isEmpty()) { append("条目：（暂无）") } else { appendLine("条目：") profile.items.forEach { item -> append("- [").append(item.category.name.lowercase()).append('/') - .append(item.confidence.name.lowercase()).append("] ") - .append(item.content) - item.relatedUserId?.let { append("（关联用户 ").append(it).append('）') } + .append(item.confidence.name.lowercase()).append(" · 记录于 ") + .append(formatProfileDate(item.firstSeenAt)) + if (item.lastConfirmedAt != item.firstSeenAt) { + append("，确认至 ").append(formatProfileDate(item.lastConfirmedAt)) + } + append("] ") + val relatedUserId = item.relatedUserId + if (item.category == ProfileCategory.RELATIONSHIP_NOTE && relatedUserId != null) { + val relatedName = PluginData.userFavorability[relatedUserId]?.name.orEmpty() + if (relatedName.isBlank()) { + append("与用户 ").append(relatedUserId).append("：") + } else { + append('与').append(relatedName).append('（').append(relatedUserId).append("）：") + } + } + append(ProfilePersistentText.itemForDisplay( + item.content, + relationship = item.category == ProfileCategory.RELATIONSHIP_NOTE, + )) appendLine() } } @@ -304,6 +400,9 @@ object PluginCommands : CompositeCommand( private fun formatProfileTime(epochSecond: Int): String = PROFILE_TIME_FORMATTER.format(Instant.ofEpochSecond(epochSecond.toLong())) + + private fun formatProfileDate(epochSecond: Int): String = + PROFILE_DATE_FORMATTER.format(Instant.ofEpochSecond(epochSecond.toLong())) } // 常量定义 @@ -311,3 +410,6 @@ private const val TOP_LIMIT = 5 private val PROFILE_TIME_FORMATTER: DateTimeFormatter = DateTimeFormatter .ofPattern("yyyy-MM-dd HH:mm:ss") .withZone(ZoneId.systemDefault()) +private val PROFILE_DATE_FORMATTER: DateTimeFormatter = DateTimeFormatter + .ofPattern("yyyy-MM-dd") + .withZone(ZoneId.systemDefault()) diff --git a/src/main/kotlin/config/PluginConfig.kt b/src/main/kotlin/config/PluginConfig.kt index be07ad9..3966146 100644 --- a/src/main/kotlin/config/PluginConfig.kt +++ b/src/main/kotlin/config/PluginConfig.kt @@ -75,12 +75,6 @@ object PluginConfig : AutoSavePluginConfig("Config") { @ValueDescription("画像分析模型额外请求体JSON。留空时继承聊天模型额外请求体") val profileModelExtraBody: String by value("") - @ValueDescription("画像分析提示词文件路径，相对于插件配置目录") - val profilePromptFile: String by value("ProfilePrompt.md") - - @ValueDescription("闭合群聊多人画像提示词文件路径，相对于插件配置目录") - val profileConversationPromptFile: String by value("ProfileConversationPrompt.md") - @ValueDescription("画像分析使用的聊天记录SQLite路径。留空时使用插件自己的chat-history.sqlite；本地实验可填写历史库绝对路径") val profileHistoryDatabasePath: String by value("") diff --git a/src/main/kotlin/profile/ConversationProfileReducer.kt b/src/main/kotlin/profile/ConversationProfileReducer.kt index a410828..52e52a5 100644 --- a/src/main/kotlin/profile/ConversationProfileReducer.kt +++ b/src/main/kotlin/profile/ConversationProfileReducer.kt @@ -10,29 +10,21 @@ object ConversationProfileReducer { promptVersion: String, summaryMaxLength: Int, ): List { - val responsesByUserId = response.users.associate { userResponse -> - val userId = batch.aliasToUserId[userResponse.userAlias] - ?: throw IllegalArgumentException("模型返回了不存在的用户别名 ${userResponse.userAlias}") - require(userId in eligibleUserIds) { - "模型返回了非候选用户别名 ${userResponse.userAlias}" - } - require(userResponse.operations.size <= MAX_OPERATIONS_PER_USER) { - "模型为 ${userResponse.userAlias} 返回了超过 $MAX_OPERATIONS_PER_USER 项画像操作" - } - userId to userResponse - } - require(responsesByUserId.size == response.users.size) { - "模型对同一用户返回了多组结果" + val responsesByUserId = linkedMapOf>() + response.users.forEach { userResponse -> + val userId = batch.aliasToUserId[userResponse.userAlias] ?: return@forEach + if (userId !in eligibleUserIds) return@forEach + responsesByUserId.getOrPut(userId, ::mutableListOf) += userResponse } return eligibleUserIds.sorted().map { userId -> - val userResponse = responsesByUserId[userId] + val userResponses = responsesByUserId[userId].orEmpty() UserProfileReducer.reduce( current = checkNotNull(profiles[userId]) { "缺少用户 $userId 的当前画像" }, batch = batch.forUser(userId), response = ProfileModelResponse( - operations = userResponse?.operations.orEmpty(), - summary = userResponse?.summary.orEmpty(), + operations = userResponses.flatMap(ConversationProfileUserResponse::operations).distinct(), + summary = userResponses.lastOrNull { it.summary.isNotBlank() }?.summary.orEmpty(), ), model = model, promptVersion = promptVersion, @@ -41,6 +33,4 @@ object ConversationProfileReducer { ) } } - - private const val MAX_OPERATIONS_PER_USER = 4 } diff --git a/src/main/kotlin/profile/EvidenceReferenceListSerializer.kt b/src/main/kotlin/profile/EvidenceReferenceListSerializer.kt new file mode 100644 index 0000000..493c76d --- /dev/null +++ b/src/main/kotlin/profile/EvidenceReferenceListSerializer.kt @@ -0,0 +1,33 @@ +package top.jie65535.mirai.profile + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonDecoder +import kotlinx.serialization.json.JsonPrimitive + +object EvidenceReferenceListSerializer : KSerializer> { + private val delegate = ListSerializer(Int.serializer()) + + override val descriptor: SerialDescriptor = delegate.descriptor + + override fun deserialize(decoder: Decoder): List { + val jsonDecoder = decoder as? JsonDecoder ?: return delegate.deserialize(decoder) + val array = jsonDecoder.decodeJsonElement() as? JsonArray ?: return listOf(INVALID_REFERENCE) + return array.map { element -> + val raw = (element as? JsonPrimitive)?.content?.trim() ?: return@map INVALID_REFERENCE + val numeric = if (raw.startsWith("e:", ignoreCase = true)) raw.substring(2) else raw + numeric.toIntOrNull()?.takeIf { it > 0 } ?: INVALID_REFERENCE + } + } + + override fun serialize(encoder: Encoder, value: List) { + delegate.serialize(encoder, value) + } + + private const val INVALID_REFERENCE = 0 +} diff --git a/src/main/kotlin/profile/ProfileAutoMaintenance.kt b/src/main/kotlin/profile/ProfileAutoMaintenance.kt index c6a9971..cf68b68 100644 --- a/src/main/kotlin/profile/ProfileAutoMaintenance.kt +++ b/src/main/kotlin/profile/ProfileAutoMaintenance.kt @@ -78,6 +78,7 @@ object ProfileAutoMaintenance { JChatGPT.logger.info( "PROFILE_AUTO group=${conversation.groupId} users=${report.analyzedUsers} " + "messages=${report.processedMessages} operations=${report.appliedOperations} " + + "skipped=${report.skippedOperations} " + "tokens=${report.usage.promptTokens}/${report.usage.completionTokens}" ) } catch (cause: CancellationException) { diff --git a/src/main/kotlin/profile/ProfileCompactionModels.kt b/src/main/kotlin/profile/ProfileCompactionModels.kt new file mode 100644 index 0000000..5058817 --- /dev/null +++ b/src/main/kotlin/profile/ProfileCompactionModels.kt @@ -0,0 +1,82 @@ +package top.jie65535.mirai.profile + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +enum class ProfileCompactionDeleteReason { + @SerialName("one_off") + ONE_OFF, + + @SerialName("over_specific") + OVER_SPECIFIC, + + @SerialName("transient") + TRANSIENT, + + @SerialName("not_profile") + NOT_PROFILE, +} + +@Serializable +data class ProfileCompactionMerge( + @SerialName("item_refs") + val itemRefs: List, + val content: String, +) + +@Serializable +data class ProfileCompactionRewrite( + @SerialName("item_ref") + val itemRef: String, + val content: String, +) + +@Serializable +data class ProfileCompactionDelete( + @SerialName("item_ref") + val itemRef: String, + val reason: ProfileCompactionDeleteReason, +) + +@Serializable +data class ProfileCompactionResponse( + val merges: List = emptyList(), + val rewrites: List = emptyList(), + val deletes: List = emptyList(), + val summary: String = "", +) + +data class ProfileItemSupportStats( + val count: Int, + val firstSupportedAt: Int, + val lastSupportedAt: Int, +) + +data class ProfileCompactionModelResult( + val response: ProfileCompactionResponse, + val rawResponse: String, + val usage: ProfileTokenUsage, +) + +data class ProfileCompactionPlan( + val reduction: ProfileReduction, + val supportReassignments: Map, + val mergedGroups: Int, + val rewrittenItems: Int, + val deletedItems: Int, + val skippedOperations: List = emptyList(), +) + +data class ProfileCompactionReport( + val userId: Long, + val beforeItems: Int, + val afterItems: Int, + val mergedGroups: Int, + val rewrittenItems: Int, + val deletedItems: Int, + val summaryChanged: Boolean, + val skippedOperations: Int, + val usage: ProfileTokenUsage, + val profile: UserProfileSnapshot, +) diff --git a/src/main/kotlin/profile/ProfileContentRules.kt b/src/main/kotlin/profile/ProfileContentRules.kt new file mode 100644 index 0000000..afa1f2d --- /dev/null +++ b/src/main/kotlin/profile/ProfileContentRules.kt @@ -0,0 +1,27 @@ +package top.jie65535.mirai.profile + +object ProfileContentRules { + fun validate(raw: String, label: String): String { + val content = raw.trim() + require(content.isNotEmpty()) { "$label 缺少 content" } + require(content.length <= MAX_CONTENT_LENGTH) { "$label.content 超过 $MAX_CONTENT_LENGTH 字符" } + require(!overclaimPattern.containsMatchIn(content)) { "$label.content 包含夸张身份或能力判断" } + require(!itemReferencePattern.containsMatchIn(content)) { "$label.content 包含临时画像条目编号" } + return content + } + + fun validateSummary(raw: String, maxLength: Int): String { + val summary = raw.trim() + require(summary.length <= maxLength) { "画像摘要超过 $maxLength 字符" } + require(!itemReferencePattern.containsMatchIn(summary)) { "画像摘要包含临时画像条目编号" } + return summary + } + + fun normalizedKey(value: String): String = value + .lowercase() + .replace(Regex("[\\s\\p{Punct}，。；、！？（）【】‘’“”]+"), "") + + private const val MAX_CONTENT_LENGTH = 120 + private val overclaimPattern = Regex("深厚|扎实|精通|专家|导师|领袖|天才|极强|全栈|核心成员|公认") + private val itemReferencePattern = Regex("(? + connection.prepareStatement( + """ + SELECT bot_id, MIN(time) AS min_time, MAX(time) AS max_time + FROM message_record + WHERE kind = ? AND recalled = 0 AND target_id = ? + GROUP BY bot_id + ORDER BY max_time DESC, bot_id ASC + LIMIT 1 + """.trimIndent() + ).use { statement -> + statement.setInt(1, MessageSourceKind.GROUP.ordinal) + statement.setLong(2, groupId) + statement.executeQuery().use { results -> + if (!results.next()) return@use null + GroupTimeBounds( + botId = results.getLong("bot_id"), + startTime = results.getInt("min_time"), + endTime = results.getInt("max_time").safeNextSecond(), + ) + } + } + } + fun loadNextBatch( userId: Long, startTime: Int, @@ -159,6 +184,40 @@ class ProfileHistoryReader(private val databaseFile: File) { } } + fun loadNextConversationBatch( + botId: Long, + groupId: Long, + startTime: Int, + snapshotEndTime: Int, + messageLimit: Int, + maxMessageChars: Int, + ): ConversationProfileBatch? { + require(startTime <= snapshotEndTime) { "startTime must not be after snapshotEndTime" } + if (startTime == snapshotEndTime) return null + return openReadConnection().use { connection -> + val firstPage = queryOldestConversationMessages( + connection = connection, + botId = botId, + groupId = groupId, + startTime = startTime, + endTime = snapshotEndTime, + limit = messageLimit.coerceAtLeast(1), + ) + if (firstPage.isEmpty()) return@use null + + val endTime = firstPage.last().time.safeNextSecond() + val records = queryOldestConversationMessages( + connection = connection, + botId = botId, + groupId = groupId, + startTime = startTime, + endTime = endTime, + limit = Int.MAX_VALUE, + ) + createConversationBatch(botId, groupId, startTime, endTime, records, maxMessageChars) + } + } + private fun queryTargetMessages( connection: Connection, userId: Long, @@ -218,6 +277,35 @@ class ProfileHistoryReader(private val databaseFile: File) { } } + private fun queryOldestConversationMessages( + connection: Connection, + botId: Long, + groupId: Long, + startTime: Int, + endTime: Int, + limit: Int, + ): List { + return connection.prepareStatement( + """ + SELECT id, bot_id, from_id, target_id, ids, internal_ids, + time, kind, code, recalled + FROM message_record + WHERE bot_id = ? AND target_id = ? AND kind = ? AND recalled = 0 + AND time >= ? AND time < ? + ORDER BY time ASC, id ASC + LIMIT ? + """.trimIndent() + ).use { statement -> + statement.setLong(1, botId) + statement.setLong(2, groupId) + statement.setInt(3, MessageSourceKind.GROUP.ordinal) + statement.setInt(4, startTime) + statement.setInt(5, endTime) + statement.setInt(6, limit) + statement.executeQuery().use(::readRecords) + } + } + private fun createConversationBatch( botId: Long, groupId: Long, diff --git a/src/main/kotlin/profile/ProfileItemReferences.kt b/src/main/kotlin/profile/ProfileItemReferences.kt new file mode 100644 index 0000000..782ade7 --- /dev/null +++ b/src/main/kotlin/profile/ProfileItemReferences.kt @@ -0,0 +1,25 @@ +package top.jie65535.mirai.profile + +object ProfileItemReferences { + data class Entry( + val reference: String, + val item: UserProfileItem, + ) + + fun entries(profile: UserProfileSnapshot): List = profile.items.mapIndexed { index, item -> + Entry(reference = "P${index + 1}", item = item) + } + + fun resolve(profile: UserProfileSnapshot, rawReference: String?): UserProfileItem? { + val index = rawReference + ?.trim() + ?.let(REFERENCE_PATTERN::matchEntire) + ?.groupValues + ?.get(1) + ?.toIntOrNull() + ?: return null + return profile.items.getOrNull(index - 1) + } + + private val REFERENCE_PATTERN = Regex("P([1-9]\\d*)") +} diff --git a/src/main/kotlin/profile/ProfileModelClient.kt b/src/main/kotlin/profile/ProfileModelClient.kt index 613f802..3dc1cda 100644 --- a/src/main/kotlin/profile/ProfileModelClient.kt +++ b/src/main/kotlin/profile/ProfileModelClient.kt @@ -11,6 +11,11 @@ import kotlinx.serialization.json.Json import top.jie65535.mirai.llm.LargeLanguageModels import top.jie65535.mirai.llm.ModelService +internal val profileResponseJson = Json { + ignoreUnknownKeys = true + explicitNulls = false +} + interface ProfileModel { val modelName: String @@ -30,13 +35,19 @@ interface ConversationProfileModel { ): ConversationProfileModelResult } +interface ProfileCompactionModel { + val modelName: String + + suspend fun compact( + profile: UserProfileSnapshot, + supportStats: Map, + ): ProfileCompactionModelResult +} + class ProfileModelClient( private val endpoint: LargeLanguageModels.ProfileEndpoint, -) : ProfileModel, ConversationProfileModel { - private val json = Json { - ignoreUnknownKeys = false - explicitNulls = false - } +) : ProfileModel, ConversationProfileModel, ProfileCompactionModel { + private val json = profileResponseJson override val modelName: String get() = endpoint.model @@ -115,15 +126,54 @@ class ProfileModelClient( ) } + override suspend fun compact( + profile: UserProfileSnapshot, + supportStats: Map, + ): ProfileCompactionModelResult { + val content = StringBuilder() + var lastUsage: Usage? = null + var cacheUsage: ModelService.CacheUsage? = null + endpoint.service.chatCompletions( + ChatCompletionRequest( + model = ModelId(endpoint.model), + temperature = endpoint.temperature, + responseFormat = ChatResponseFormat.JsonObject, + streamOptions = StreamOptions(includeUsage = true), + messages = listOf( + ChatMessage.System(ProfilePromptStore.compactionSystemPrompt), + ChatMessage.User(ProfilePromptStore.buildCompactionUserPrompt(profile, supportStats)), + ), + ) + ) { cacheUsage = it }.collect { chunk -> + chunk.choices.firstOrNull()?.delta?.content?.let(content::append) + chunk.usage?.let { lastUsage = it } + } + + val raw = content.toString().replace(THINK_REGEX, "").trim() + return ProfileCompactionModelResult( + response = json.decodeFromString(extractObject(raw)), + rawResponse = raw, + usage = ProfileTokenUsage( + promptTokens = lastUsage?.promptTokens ?: 0, + completionTokens = lastUsage?.completionTokens ?: 0, + cachedTokens = cacheUsage?.hitTokens ?: 0, + ), + ) + } + private fun parseResponse(raw: String): ProfileModelResponse { return parseObject(raw) } private inline fun parseObject(raw: String): T { + return json.decodeFromString(extractObject(raw)) + } + + private fun extractObject(raw: String): String { val unfenced = raw .removePrefix("```json").removePrefix("```") .removeSuffix("```").trim() - val objectText = if (unfenced.startsWith('{') && unfenced.endsWith('}')) { + return if (unfenced.startsWith('{') && unfenced.endsWith('}')) { unfenced } else { val start = unfenced.indexOf('{') @@ -131,7 +181,6 @@ class ProfileModelClient( if (start < 0 || end <= start) throw SerializationException("模型响应中没有完整 JSON object") unfenced.substring(start, end + 1) } - return json.decodeFromString(objectText) } companion object { diff --git a/src/main/kotlin/profile/ProfilePersistentText.kt b/src/main/kotlin/profile/ProfilePersistentText.kt new file mode 100644 index 0000000..c62b16d --- /dev/null +++ b/src/main/kotlin/profile/ProfilePersistentText.kt @@ -0,0 +1,65 @@ +package top.jie65535.mirai.profile + +object ProfilePersistentText { + fun normalizeSummary(raw: String, batch: ProfileHistoryBatch): String { + var result = raw + batch.aliases[batch.userId]?.let { result = replaceAlias(result, it, "该用户") } + result = replaceAlias(result, "TARGET", "该用户") + return normalize(result) { alias -> + when (alias) { + "TARGET" -> "该用户" + "BOT" -> "机器人" + else -> "其他用户" + } + } + } + + fun normalizeItemContent( + raw: String, + batch: ProfileHistoryBatch, + relatedUserId: Long?, + ): String { + var result = raw + batch.aliases[batch.userId]?.let { result = replaceAlias(result, it, "本人") } + result = replaceAlias(result, "TARGET", "本人") + relatedUserId + ?.let(batch.aliases::get) + ?.let { result = replaceAlias(result, it, "对方") } + return normalize(result) { alias -> if (alias == "BOT") "机器人" else "其他用户" } + } + + fun summaryForDisplay(raw: String): String = normalize(raw) { alias -> + when (alias) { + "TARGET" -> "该用户" + "BOT" -> "机器人" + else -> "其他用户" + } + } + + fun itemForDisplay(raw: String, relationship: Boolean): String = normalize(raw) { alias -> + when (alias) { + "TARGET" -> "本人" + "BOT" -> "机器人" + else -> if (relationship) "对方" else "其他用户" + } + } + + private fun normalize(raw: String, replacement: (String) -> String): String = INTERNAL_ALIAS_PATTERN + .replace(raw.trim()) { match -> replacement(match.value) } + .replace(WHITESPACE_PATTERN, " ") + .replace(CJK_SPACE_PATTERN, "") + + private fun replaceAlias(source: String, alias: String, replacement: String): String { + if (alias.isBlank()) return source + val pattern = Regex("(? file.readText() - else -> defaultPrompt.also { default -> - file.parentFile?.mkdirs() - file.writeText(default) - } - } - } catch (cause: Throwable) { - JChatGPT.logger.warning("载入${label}提示词失败，暂时使用内置提示词", cause) - defaultPrompt - } - } + val systemPrompt: String = DEFAULT_SYSTEM_PROMPT + val conversationSystemPrompt: String = DEFAULT_CONVERSATION_SYSTEM_PROMPT + val compactionSystemPrompt: String = DEFAULT_COMPACTION_SYSTEM_PROMPT fun buildUserPrompt(profile: UserProfileSnapshot, batch: ProfileHistoryBatch): String = buildString { appendLine("## 目标") @@ -59,11 +26,14 @@ object ProfilePromptStore { if (profile.items.isEmpty()) { appendLine("（尚无画像条目）") } else { - profile.items.forEach { item -> - append("[P:").append(item.id).append("] ") + ProfileItemReferences.entries(profile).forEach { (reference, item) -> + append('[').append(reference).append("] ") append(item.category.wireName()).append(" | ") append(item.confidence.wireName()).append(" | ") - append(item.content) + append(ProfilePersistentText.itemForDisplay( + item.content, + relationship = item.category == ProfileCategory.RELATIONSHIP_NOTE, + )) item.relatedUserId?.let { related -> append(" | related=").append(batch.aliases[related] ?: "历史用户") } @@ -72,7 +42,7 @@ object ProfilePromptStore { appendLine() } } - appendLine("当前短摘要: ${profile.summary.ifBlank { "（空）" }}") + appendLine("当前短摘要: ${ProfilePersistentText.summaryForDisplay(profile.summary).ifBlank { "（空）" }}") appendLine() appendLine("## 本批参与者别名") @@ -104,7 +74,8 @@ object ProfilePromptStore { eligibleUserIds: Set, ): String = buildString { appendLine("## 任务") - appendLine("分析这一段已经闭合的群聊，一次性更新所有出现可靠长期信息的候选用户画像。") + appendLine("分析这一段已经闭合的群聊，一次性更新所有出现可靠画像信息的候选用户画像。") + appendLine("候选用户别名: ${eligibleUserIds.mapNotNull(batch.aliases::get).sorted().joinToString(", ")}") appendLine("会话时间范围: [${formatTime(batch.startTime)}, ${formatTime(batch.endTime)})") appendLine("别名只在本批有效，不要输出 QQ 号或数据库消息 ID。") appendLine() @@ -117,11 +88,14 @@ object ProfilePromptStore { if (profile == null || profile.items.isEmpty()) { appendLine("（尚无画像条目）") } else { - profile.items.forEach { item -> - append("[P:").append(item.id).append("] ") + ProfileItemReferences.entries(profile).forEach { (reference, item) -> + append('[').append(reference).append("] ") append(item.category.wireName()).append(" | ") append(item.confidence.wireName()).append(" | ") - append(item.content) + append(ProfilePersistentText.itemForDisplay( + item.content, + relationship = item.category == ProfileCategory.RELATIONSHIP_NOTE, + )) item.relatedUserId?.let { related -> append(" | related=").append(batch.aliases[related] ?: "历史用户") } @@ -130,7 +104,8 @@ object ProfilePromptStore { appendLine() } } - appendLine("当前短摘要: ${profile?.summary?.ifBlank { "（空）" } ?: "（空）"}") + val summary = profile?.summary?.let(ProfilePersistentText::summaryForDisplay).orEmpty() + appendLine("当前短摘要: ${summary.ifBlank { "（空）" }}") } appendLine() @@ -145,6 +120,44 @@ object ProfilePromptStore { } } + fun buildCompactionUserPrompt( + profile: UserProfileSnapshot, + supportStats: Map, + ): String = buildString { + appendLine("## 待整理画像") + appendLine("当前条目数: ${profile.items.size}") + appendLine("当前摘要: ${ProfilePersistentText.summaryForDisplay(profile.summary).ifBlank { "（空）" }}") + appendLine() + + val relatedReferences = profile.items.asSequence() + .mapNotNull(UserProfileItem::relatedUserId) + .distinct() + .sorted() + .mapIndexed { index, userId -> userId to "R${index + 1}" } + .toMap() + ProfileItemReferences.entries(profile).forEach { (reference, item) -> + val supports = supportStats[item.id] + append('[').append(reference).append("] ") + append(item.category.wireName()).append(" | ") + append(item.confidence.wireName()).append(" | ") + append(ProfilePersistentText.itemForDisplay( + item.content, + relationship = item.category == ProfileCategory.RELATIONSHIP_NOTE, + )) + item.relatedUserId?.let { related -> + append(" | related_group=").append(relatedReferences.getValue(related)) + } + append(" | supports=").append(supports?.count ?: 0) + supports?.takeIf { it.count > 0 }?.let { + append(" | support_range=").append(formatTime(it.firstSupportedAt)) + .append(" ~ ").append(formatTime(it.lastSupportedAt)) + } + append(" | item_range=").append(formatTime(item.firstSeenAt)) + .append(" ~ ").append(formatTime(item.lastConfirmedAt)) + appendLine() + } + } + private fun formatTime(epochSecond: Int): String = dateTimeFormatter.format(Instant.ofEpochSecond(epochSecond.toLong())) @@ -154,10 +167,10 @@ object ProfilePromptStore { private const val DEFAULT_SYSTEM_PROMPT = """你是保守、严谨的群聊人物画像归纳器。 你会收到一个目标用户的当前画像，以及一个带完整发言者、时间、回复引用和相邻消息的原始群聊批次。 -你的任务是判断本批信息是否应当 ADD、UPDATE、CONFIRM 或 DELETE 长期画像条目；没有可靠变化时返回空 operations。 +你的任务是判断本批信息是否应当 ADD、UPDATE、CONFIRM 或 DELETE 画像条目；没有可靠变化时返回空 operations。 画像只描述： -- notable_fact：本人明确披露且半年后仍有助于认识此人的事实 +- notable_fact：本人明确披露的稳定事实，或当前仍有认识价值的阶段状态、近期事件 - interest：持续关注或参与的领域 - expertise_signal：反复表现出的具体知识或解决问题能力，不授予专家头衔 - thinking_style：分析、判断和解决问题的方式 @@ -174,18 +187,22 @@ object ProfilePromptStore { 5. 一次技术回答或同一话题中的连续补充只算一个语境。新增 expertise_signal、thinking_style、expression_style、social_mode 或 relationship_note，必须由至少两个独立对话片段中的一致表现支持；单个片段最多用于 CONFIRM 已有条目。本人明确自述的 notable_fact 和 preference 不受此限制。 6. 每个条目只表达一个主题。禁止把不同时间、不同领域的内容拼成一个所谓稳定特点。 7. 禁止使用“精通、专家、导师、领袖、天才、极强、全栈、核心成员”等拔高表述。 -8. ADD 不填写 item_id；UPDATE、CONFIRM、DELETE 必须填写当前画像中的 item_id。 -9. relationship_note 必须填写本批存在的 related_user_alias，只描述互动方式，不推断现实亲疏。 +8. ADD 不填写 item_ref；UPDATE、CONFIRM、DELETE 必须填写当前画像中的 P 编号作为 item_ref。 +9. relationship_note 必须填写本批存在的 related_user_alias；content 只描述互动方式，不重复人物别名，不推断现实亲疏。 10. 新证据与旧画像无关时不要勉强更新。未输出的旧条目由程序自动保留。 11. DELETE 仅用于新证据明确证明旧条目归因错误或已被本人否定，不能因为本批没提到就删除。 -12. summary 是更新后的日常短摘要，必须自然、克制，不写证据编号、时间、QQ 号、进度或内部条目 ID。 +12. summary 是应用 operations 并保留所有未操作旧条目之后，对完整画像的综合人物摘要，不是本批聊天摘要，也不是本批新增条目的复述。 +13. 综合摘要应优先概括最有代表性的身份/能力、兴趣/偏好、思考/表达/社交方式等不同维度；已有多个维度时不得只写最后编辑的一项。轻微新增或单纯确认不改变整体形象时，原样保留当前短摘要。 +14. summary 必须自然、克制，不写证据编号、QQ 号、内部条目 ID、逐条清单或具体关系流水。 +15. evidence_refs 只输出 JSON 整数，例如 [128, 129]；不要输出 "e:128" 这样的字符串，也不要引用输入中不存在的编号。 +16. 对求职、健康、经济状况、近期进度等有时效的信息，content 和 summary 不得悬空使用“本月、最近、目前、当前、正在”；必须依据证据时间改写成“截至 YYYY-MM-DD”或“YYYY-MM-DD 提及”的绝对时间表达。 只输出一个 JSON object，不要 Markdown 代码围栏，不要解释。格式： { "operations": [ { "action": "ADD|UPDATE|CONFIRM|DELETE", - "item_id": "UPDATE/CONFIRM/DELETE 时填写，ADD 为 null", + "item_ref": "UPDATE/CONFIRM/DELETE 时填写 P1 这样的编号，ADD 为 null", "category": "notable_fact|interest|expertise_signal|thinking_style|expression_style|social_mode|preference|relationship_note", "content": "ADD/UPDATE 时填写的单一、克制结论；其余操作可为 null", "confidence": "low|medium|high", @@ -199,10 +216,10 @@ object ProfilePromptStore { private const val DEFAULT_CONVERSATION_SYSTEM_PROMPT = """你是保守、严谨的群聊人物画像归纳器。 -你会收到一段已经闭合的真实群聊、候选用户别名，以及他们各自的当前画像。你的任务是一次性判断这段对话是否足以 ADD、UPDATE、CONFIRM 或 DELETE 各候选用户的长期画像条目。没有可靠变化的用户不要输出，每名用户最多输出 4 个真正有长期价值的操作。 +你会收到一段已经闭合的真实群聊、候选用户别名，以及他们各自的当前画像。你的任务是一次性判断这段对话是否足以 ADD、UPDATE、CONFIRM 或 DELETE 各候选用户的画像条目。没有可靠变化的用户不要输出，只输出稳定事实或当前仍有认识价值的阶段状态、近期事件。 画像只描述： -- notable_fact：本人明确披露且半年后仍有助于认识此人的事实 +- notable_fact：本人明确披露的稳定事实，或当前仍有认识价值的阶段状态、近期事件 - interest：持续关注或参与的领域 - expertise_signal：反复表现出的具体知识或解决问题能力，不授予专家头衔 - thinking_style：分析、判断和解决问题的方式 @@ -219,13 +236,18 @@ object ProfilePromptStore { 5. 一次明确自述可以支持 notable_fact 或 preference。新增 expertise_signal、thinking_style、expression_style、social_mode 或 relationship_note，必须在本会话中有多条分离的本人证据；证据不足时宁可不写。 6. 每个条目只表达一个主题，禁止把不同人的特点或不同领域拼接在一起。 7. 禁止使用“精通、专家、导师、领袖、天才、极强、全栈、核心成员”等拔高表述。 -8. ADD 不填写 item_id；UPDATE、CONFIRM、DELETE 必须填写该用户当前画像中的 item_id，不能引用其他用户的条目。 -9. relationship_note 必须填写本批存在的 related_user_alias，只描述互动方式，不推断现实亲疏。 +8. ADD 不填写 item_ref；UPDATE、CONFIRM、DELETE 必须填写该用户当前画像中的 P 编号作为 item_ref，不能引用其他用户的条目。 +9. relationship_note 必须填写本批存在的 related_user_alias；content 只描述互动方式，不重复人物别名，不推断现实亲疏。 10. 未输出的用户和旧条目由程序自动保留。DELETE 仅用于新证据明确证明旧条目归因错误或已被本人否定。 -11. summary 是该用户更新后的日常短摘要，必须自然、克制，不写证据编号、时间、QQ 号、进度或内部 ID。 -12. 不生成或修改好感度、代号、主观印象和标签；这些属于另一套 Bot 关系状态。 -13. 本批只是一段会话。除非当前画像已有同类条目且本批在确认它，否则不得使用“长期、持续、一贯、总是、通常”等跨时间措辞；只能描述本批确实支持的事实、关注点或表现。 -14. 对尚无同类旧条目的用户，thinking_style、expression_style、social_mode、expertise_signal 和 relationship_note 必须有多个彼此分离的本人证据才可新增，并保持 low 或 medium 可信度；同一问答链中的连续补充不算多次独立表现。 +11. summary 是应用该用户 operations 并保留所有未操作旧条目之后，对其完整画像的综合人物摘要，不是本批聊天摘要，也不是本批新增条目的复述。 +12. 综合摘要应优先概括最有代表性的身份/能力、兴趣/偏好、思考/表达/社交方式等不同维度；已有多个维度时不得只写最后编辑的一项。轻微新增或单纯确认不改变整体形象时，原样保留当前短摘要。 +13. summary 必须自然、克制，不写证据编号、QQ 号、内部 ID、逐条清单或具体关系流水。 +14. 不生成或修改好感度、代号、主观印象和标签；这些属于另一套 Bot 关系状态。 +15. 本批只是一段会话。除非当前画像已有同类条目且本批在确认它，否则不得使用“长期、持续、一贯、总是、通常”等跨时间措辞；只能描述本批确实支持的事实、关注点或表现。 +16. 对尚无同类旧条目的用户，thinking_style、expression_style、social_mode、expertise_signal 和 relationship_note 必须有多个彼此分离的本人证据才可新增，并保持 low 或 medium 可信度；同一问答链中的连续补充不算多次独立表现。 +17. evidence_refs 只输出 JSON 整数，例如 [128, 129]；不要输出 "e:128" 这样的字符串，也不要引用输入中不存在的编号。 +18. 对求职、健康、经济状况、近期进度等有时效的信息，content 和 summary 不得悬空使用“本月、最近、目前、当前、正在”；必须依据证据时间改写成“截至 YYYY-MM-DD”或“YYYY-MM-DD 提及”的绝对时间表达。 +19. users 只能输出“候选用户别名”中明确列出的用户；消息里出现但不在候选名单中的上下文用户不要输出。 只输出一个 JSON object，不要 Markdown 代码围栏，不要解释。格式： { @@ -235,7 +257,7 @@ object ProfilePromptStore { "operations": [ { "action": "ADD|UPDATE|CONFIRM|DELETE", - "item_id": "UPDATE/CONFIRM/DELETE 时填写，ADD 为 null", + "item_ref": "UPDATE/CONFIRM/DELETE 时填写 P1 这样的编号，ADD 为 null", "category": "notable_fact|interest|expertise_signal|thinking_style|expression_style|social_mode|preference|relationship_note", "content": "ADD/UPDATE 时填写的单一、克制结论；其余操作可为 null", "confidence": "low|medium|high", @@ -247,5 +269,31 @@ object ProfilePromptStore { } ] } +""" + + private const val DEFAULT_COMPACTION_SYSTEM_PROMPT = """你是保守的用户画像编辑器。你会收到一份已有画像，条目本身不是新的事实证据；supports 只表示程序保存的历史支持次数。 + +你的任务仅是减少重复和噪声，不得补充输入中不存在的新事实： +1. merges 用于合并语义高度重叠、粒度过细的条目。item_refs 至少两个，必须同类别；relationship_note 还必须具有相同 related_group。content 写合并后的单一概括，不拼接无关主题。 +2. rewrites 用于把一个条目改写得更概括、自然，不改变事实含义、类别、关联对象和置信度。对“正在、本月、最近、目前”等有时效的旧表述，应依据 item_range 改写成带 YYYY-MM-DD 的绝对时间表达。 +3. deletes 仅删除 low/medium 置信、supports 不超过 1，且明显属于过细例子或根本不应成为画像的内容。 +4. 同一 P 编号最多出现在一个操作中。未提及的条目自动保留。拿不准时不要操作。 +5. 禁止输出 TARGET、BOT、U 编号、R 编号、QQ 号、昵称、P 编号或 UUID 到 content/summary。 +6. summary 必须基于所有操作完成后的全部保留条目，综合最有代表性的多个维度；不是本次 merges/rewrites/deletes 的变更摘要，也不得只描述最后编辑的条目。若当前摘要已经综合且整理没有改变整体人物形象，原样保留；若当前摘要明显偏向单条或遗漏主要维度，即使没有条目操作也应重写。 +7. summary 应自然、克制，不写逐条清单或具体关系流水；有时效的信息必须带绝对日期，不得保留悬空相对表达。 + +只输出一个 JSON object，不要 Markdown 代码围栏，不要解释。格式： +{ + "merges": [ + {"item_refs": ["P1", "P2"], "content": "合并后的单一结论"} + ], + "rewrites": [ + {"item_ref": "P3", "content": "更概括但不增加事实的结论"} + ], + "deletes": [ + {"item_ref": "P4", "reason": "one_off|over_specific|transient|not_profile"} + ], + "summary": "整理后的短摘要" +} """ } diff --git a/src/main/kotlin/profile/UserProfileAnalysisService.kt b/src/main/kotlin/profile/UserProfileAnalysisService.kt index 2242ae1..051f364 100644 --- a/src/main/kotlin/profile/UserProfileAnalysisService.kt +++ b/src/main/kotlin/profile/UserProfileAnalysisService.kt @@ -10,10 +10,12 @@ import top.jie65535.mirai.config.PluginConfig import top.jie65535.mirai.data.ChatHistoryStore import top.jie65535.mirai.llm.LargeLanguageModels import java.io.File +import java.security.MessageDigest import java.util.concurrent.ConcurrentHashMap object UserProfileAnalysisService { private val runningUsers = ConcurrentHashMap.newKeySet() + private val runningGroups = ConcurrentHashMap.newKeySet() private val concurrencyLimiter = Semaphore(1) suspend fun analyze( @@ -32,6 +34,7 @@ object UserProfileAnalysisService { processedBatches = 0, processedMessages = 0, appliedOperations = 0, + skippedOperations = 0, usage = ProfileTokenUsage(), profile = UserProfileStore.load(userId), caughtUp = false, @@ -48,6 +51,90 @@ object UserProfileAnalysisService { } } + suspend fun compact(userId: Long): ProfileCompactionReport { + require(userId > 0) { "userId 必须是正数" } + check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" } + check(UserProfileStore.isAvailable) { "用户画像数据库不可用" } + + return concurrencyLimiter.withPermit { + val profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) } + ?: throw IllegalArgumentException("用户 $userId 尚无画像") + if (profile.items.isEmpty()) { + return@withPermit ProfileCompactionReport( + userId = userId, + beforeItems = profile.items.size, + afterItems = profile.items.size, + mergedGroups = 0, + rewrittenItems = 0, + deletedItems = 0, + summaryChanged = false, + skippedOperations = 0, + usage = ProfileTokenUsage(), + profile = profile, + ) + } + + val endpoint = checkNotNull(LargeLanguageModels.profile) { "画像分析模型未配置" } + val model: ProfileCompactionModel = ProfileModelClient(endpoint) + val supportStats = withContext(Dispatchers.IO) { UserProfileStore.loadSupportStats(userId) } + val (result, plan) = compactWithRetry(model, profile, supportStats) + if (plan.reduction.profile.version != profile.version) { + val batch = compactionBatch(profile, result.rawResponse) + withContext(Dispatchers.IO) { + UserProfileStore.commitCompaction(plan, batch, result.usage) + } + } + ProfileCompactionReport( + userId = userId, + beforeItems = profile.items.size, + afterItems = plan.reduction.profile.items.size, + mergedGroups = plan.mergedGroups, + rewrittenItems = plan.rewrittenItems, + deletedItems = plan.deletedItems, + summaryChanged = plan.reduction.profile.summary != profile.summary, + skippedOperations = plan.skippedOperations.size, + usage = result.usage, + profile = plan.reduction.profile, + ) + } + } + + suspend fun analyzeGroup( + groupId: Long, + maxBatches: Int, + onProgress: suspend (GroupProfileAnalysisProgress) -> Unit = {}, + ): GroupProfileAnalysisReport { + require(groupId > 0) { "groupId 必须是正数" } + require(maxBatches > 0) { "maxBatches 必须是正数" } + check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" } + check(UserProfileStore.isAvailable) { "用户画像数据库不可用" } + + if (!runningGroups.add(groupId)) { + return GroupProfileAnalysisReport( + botId = null, + groupId = groupId, + processedBatches = 0, + processedMessages = 0, + analyzedUsers = 0, + appliedOperations = 0, + skippedOperations = 0, + usage = ProfileTokenUsage(), + cursorTime = 0, + snapshotEndTime = 0, + caughtUp = false, + alreadyRunning = true, + ) + } + + try { + return concurrencyLimiter.withPermit { + analyzeGroupExclusive(groupId, maxBatches, onProgress) + } + } finally { + runningGroups.remove(groupId) + } + } + private suspend fun analyzeExclusive( userId: Long, maxBatches: Int, @@ -75,6 +162,7 @@ object UserProfileAnalysisService { var processedBatches = 0 var processedMessages = 0 var appliedOperations = 0 + var skippedOperations = 0 var totalUsage = ProfileTokenUsage() var caughtUp = false @@ -111,6 +199,7 @@ object UserProfileAnalysisService { processedBatches++ processedMessages += batch.messages.size appliedOperations += reduction.operations.size + skippedOperations += reduction.skippedOperations.size totalUsage += result.usage onProgress( ProfileAnalysisProgress( @@ -119,6 +208,7 @@ object UserProfileAnalysisService { endTime = batch.endTime, messageCount = batch.messages.size, operationCount = reduction.operations.size, + skippedOperationCount = reduction.skippedOperations.size, usage = result.usage, ) ) @@ -130,12 +220,117 @@ object UserProfileAnalysisService { processedBatches = processedBatches, processedMessages = processedMessages, appliedOperations = appliedOperations, + skippedOperations = skippedOperations, usage = totalUsage, profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) } ?: profile, caughtUp = caughtUp, ) } + private suspend fun analyzeGroupExclusive( + groupId: Long, + maxBatches: Int, + onProgress: suspend (GroupProfileAnalysisProgress) -> Unit, + ): GroupProfileAnalysisReport { + val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(resolveHistoryFile()) } + val bounds = withContext(Dispatchers.IO) { reader.findGroupTimeBounds(groupId) } + ?: return emptyGroupReport(groupId) + val endpoint = checkNotNull(LargeLanguageModels.profile) { + "画像分析模型未配置，请设置 profileModelApi/profileModelToken，或配置可继承的聊天模型接入点" + } + val model: ConversationProfileModel = ProfileModelClient(endpoint) + var cursor = withContext(Dispatchers.IO) { + UserProfileStore.loadGroupCursor(bounds.botId, groupId) + } ?: GroupProfileCursor( + botId = bounds.botId, + groupId = groupId, + cursorTime = bounds.startTime, + snapshotEndTime = bounds.endTime, + ) + if (cursor.cursorTime >= cursor.snapshotEndTime && bounds.endTime > cursor.snapshotEndTime) { + cursor = cursor.copy(snapshotEndTime = bounds.endTime) + } + + var processedBatches = 0 + var processedMessages = 0 + var analyzedUsers = 0 + var appliedOperations = 0 + var skippedOperations = 0 + var totalUsage = ProfileTokenUsage() + var caughtUp = cursor.cursorTime >= cursor.snapshotEndTime + + while (processedBatches < maxBatches && !caughtUp) { + val batch = withContext(Dispatchers.IO) { + reader.loadNextConversationBatch( + botId = cursor.botId, + groupId = groupId, + startTime = cursor.cursorTime, + snapshotEndTime = cursor.snapshotEndTime, + messageLimit = PluginConfig.profileAutoConversationMessageLimit.coerceAtLeast(1), + maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80), + ) + } + if (batch == null) { + cursor = cursor.copy( + cursorTime = cursor.snapshotEndTime, + updatedAt = System.currentTimeMillis(), + ) + withContext(Dispatchers.IO) { UserProfileStore.saveGroupCursor(cursor) } + caughtUp = true + break + } + + val report = analyzeConversationBatch( + batch = batch, + minAuthoredTextChars = PluginConfig.profileAutoMinAuthoredTextChars, + model = model, + retryMax = PluginConfig.profileRetryMax, + summaryMaxLength = PluginConfig.profileSummaryMaxLength, + onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) }, + ) + cursor = cursor.copy( + cursorTime = batch.endTime, + updatedAt = System.currentTimeMillis(), + ) + withContext(Dispatchers.IO) { UserProfileStore.saveGroupCursor(cursor) } + + val usage = report?.usage ?: ProfileTokenUsage() + processedBatches++ + processedMessages += batch.messages.size + analyzedUsers += report?.analyzedUsers ?: 0 + appliedOperations += report?.appliedOperations ?: 0 + skippedOperations += report?.skippedOperations ?: 0 + totalUsage += usage + caughtUp = cursor.cursorTime >= cursor.snapshotEndTime + onProgress( + GroupProfileAnalysisProgress( + batchIndex = processedBatches, + startTime = batch.startTime, + endTime = batch.endTime, + messageCount = batch.messages.size, + analyzedUsers = report?.analyzedUsers ?: 0, + appliedOperations = report?.appliedOperations ?: 0, + skippedOperations = report?.skippedOperations ?: 0, + usage = usage, + ) + ) + } + + return GroupProfileAnalysisReport( + botId = cursor.botId, + groupId = groupId, + processedBatches = processedBatches, + processedMessages = processedMessages, + analyzedUsers = analyzedUsers, + appliedOperations = appliedOperations, + skippedOperations = skippedOperations, + usage = totalUsage, + cursorTime = cursor.cursorTime, + snapshotEndTime = cursor.snapshotEndTime, + caughtUp = caughtUp, + ) + } + suspend fun analyzeConversation( botId: Long, groupId: Long, @@ -217,6 +412,7 @@ object UserProfileAnalysisService { analyzedUsers = eligibleUserIds.size, processedMessages = batch.messages.size, appliedOperations = reductions.sumOf { it.operations.size }, + skippedOperations = reductions.sumOf { it.skippedOperations.size }, usage = result.usage, ) } @@ -281,6 +477,10 @@ object UserProfileAnalysisService { summaryMaxLength = PluginConfig.profileSummaryMaxLength.coerceAtLeast(100), advanceBackfillCursor = advanceBackfillCursor, ) + logSkippedOperations( + "用户 ${batch.userId} 画像批次 [${batch.startTime}, ${batch.endTime})", + reduction.skippedOperations, + ) return result to reduction } catch (cause: Exception) { if (cause is CancellationException) throw cause @@ -298,6 +498,57 @@ object UserProfileAnalysisService { ) } + private suspend fun compactWithRetry( + model: ProfileCompactionModel, + profile: UserProfileSnapshot, + supportStats: Map, + ): Pair { + val attempts = PluginConfig.profileRetryMax.coerceIn(0, 3) + 1 + var lastFailure: Throwable? = null + repeat(attempts) { attempt -> + try { + val result = model.compact(profile, supportStats) + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = supportStats, + response = result.response, + model = model.modelName, + promptVersion = ProfilePromptStore.COMPACTION_PROMPT_VERSION, + summaryMaxLength = PluginConfig.profileSummaryMaxLength.coerceAtLeast(100), + ) + return result to plan + } catch (cause: Exception) { + if (cause is CancellationException) throw cause + lastFailure = cause + JChatGPT.logger.warning( + "用户 ${profile.userId} 画像压缩第 ${attempt + 1}/$attempts 次失败", + cause, + ) + } + } + throw IllegalStateException( + "用户 ${profile.userId} 画像压缩连续 $attempts 次失败，未提交任何结果", + lastFailure, + ) + } + + private fun compactionBatch(profile: UserProfileSnapshot, rawResponse: String): ProfileHistoryBatch { + val digest = MessageDigest.getInstance("SHA-256") + .digest("${profile.userId}|${profile.version}|$rawResponse".toByteArray(Charsets.UTF_8)) + .joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) } + val startTime = profile.items.minOfOrNull(UserProfileItem::firstSeenAt) ?: profile.cursorTime + val lastConfirmedAt = profile.items.maxOfOrNull(UserProfileItem::lastConfirmedAt) ?: startTime + val endTime = if (lastConfirmedAt == Int.MAX_VALUE) lastConfirmedAt else lastConfirmedAt + 1 + return ProfileHistoryBatch( + userId = profile.userId, + startTime = startTime, + endTime = endTime, + messages = emptyList(), + aliases = mapOf(profile.userId to "TARGET"), + inputHash = "compact-$digest", + ) + } + private fun resolveHistoryFile(): File { val configured = PluginConfig.profileHistoryDatabasePath.trim() return if (configured.isNotEmpty()) { @@ -315,14 +566,37 @@ object UserProfileAnalysisService { processedBatches = 0, processedMessages = 0, appliedOperations = 0, + skippedOperations = 0, usage = ProfileTokenUsage(), profile = null, caughtUp = true, ) + private fun emptyGroupReport(groupId: Long) = GroupProfileAnalysisReport( + botId = null, + groupId = groupId, + processedBatches = 0, + processedMessages = 0, + analyzedUsers = 0, + appliedOperations = 0, + skippedOperations = 0, + usage = ProfileTokenUsage(), + cursorTime = 0, + snapshotEndTime = 0, + caughtUp = true, + ) + private operator fun ProfileTokenUsage.plus(other: ProfileTokenUsage) = ProfileTokenUsage( promptTokens = promptTokens + other.promptTokens, completionTokens = completionTokens + other.completionTokens, cachedTokens = cachedTokens + other.cachedTokens, ) + + private fun logSkippedOperations(context: String, skipped: List) { + if (skipped.isEmpty()) return + JChatGPT.logger.warning( + "$context 跳过 ${skipped.size} 项无效建议：" + skipped.take(8).joinToString("；") + + if (skipped.size > 8) "；其余 ${skipped.size - 8} 项已省略" else "" + ) + } } diff --git a/src/main/kotlin/profile/UserProfileCompactor.kt b/src/main/kotlin/profile/UserProfileCompactor.kt new file mode 100644 index 0000000..4abdb3c --- /dev/null +++ b/src/main/kotlin/profile/UserProfileCompactor.kt @@ -0,0 +1,154 @@ +package top.jie65535.mirai.profile + +object UserProfileCompactor { + fun reduce( + current: UserProfileSnapshot, + supportStats: Map, + response: ProfileCompactionResponse, + model: String, + promptVersion: String, + summaryMaxLength: Int, + ): ProfileCompactionPlan { + val items = current.items.associateBy { it.id }.toMutableMap() + val usedItemIds = mutableSetOf() + val supportReassignments = mutableMapOf() + val applied = mutableListOf() + val skippedOperations = mutableListOf() + var mergedGroups = 0 + var rewrittenItems = 0 + var deletedItems = 0 + + response.merges.forEachIndexed { index, merge -> + try { + require(merge.itemRefs.size >= 2) { "至少需要两个 item_refs" } + require(merge.itemRefs.distinct().size == merge.itemRefs.size) { "包含重复 item_ref" } + val sourceItems = merge.itemRefs.map { ref -> resolve(current, ref, "merges[$index]") } + require(sourceItems.none { it.id in usedItemIds }) { "重复操作了画像条目" } + val category = sourceItems.first().category + val relatedUserId = sourceItems.first().relatedUserId + require(sourceItems.all { it.category == category }) { "只能合并同类别条目" } + require(sourceItems.all { it.relatedUserId == relatedUserId }) { + "只能合并指向同一用户的关系条目" + } + val target = sourceItems.minWith(compareBy({ it.firstSeenAt }, { it.id })) + val content = normalizeAndValidate( + merge.content, + relationship = category == ProfileCategory.RELATIONSHIP_NOTE, + label = "merges[$index]", + ) + val updated = target.copy( + content = content, + confidence = sourceItems.minBy { it.confidence.ordinal }.confidence, + firstSeenAt = sourceItems.minOf { it.firstSeenAt }, + lastConfirmedAt = sourceItems.maxOf { it.lastConfirmedAt }, + ) + usedItemIds += sourceItems.map(UserProfileItem::id) + sourceItems.forEach { source -> items.remove(source.id) } + items[updated.id] = updated + applied += updated.toApplied(ProfileOperationAction.UPDATE) + sourceItems.filter { it.id != updated.id }.forEach { source -> + supportReassignments[source.id] = updated.id + applied += source.toApplied(ProfileOperationAction.DELETE) + } + mergedGroups++ + } catch (cause: IllegalArgumentException) { + skippedOperations += "merges[$index]: ${cause.message}" + } + } + response.rewrites.forEachIndexed { index, rewrite -> + try { + val old = resolve(current, rewrite.itemRef, "rewrites[$index]") + require(old.id !in usedItemIds) { "重复操作了画像条目" } + val content = normalizeAndValidate( + rewrite.content, + relationship = old.category == ProfileCategory.RELATIONSHIP_NOTE, + label = "rewrites[$index]", + ) + if (ProfileContentRules.normalizedKey(content) == ProfileContentRules.normalizedKey(old.content)) { + return@forEachIndexed + } + usedItemIds += old.id + val updated = old.copy( + content = content, + ) + items[old.id] = updated + applied += updated.toApplied(ProfileOperationAction.UPDATE) + rewrittenItems++ + } catch (cause: IllegalArgumentException) { + skippedOperations += "rewrites[$index]: ${cause.message}" + } + } + response.deletes.forEachIndexed { index, delete -> + try { + val old = resolve(current, delete.itemRef, "deletes[$index]") + require(old.id !in usedItemIds) { "重复操作了画像条目" } + require(old.confidence != ProfileConfidence.HIGH) { "不能自动删除 high 置信条目" } + val supports = supportStats[old.id]?.count ?: 0 + require(supports <= MAX_DELETE_SUPPORTS) { + "画像条目已有 $supports 次支持，不能自动删除" + } + usedItemIds += old.id + items.remove(old.id) + applied += old.toApplied(ProfileOperationAction.DELETE) + deletedItems++ + } catch (cause: IllegalArgumentException) { + skippedOperations += "deletes[$index]: ${cause.message}" + } + } + val summary = if (skippedOperations.isNotEmpty()) { + current.summary + } else { + runCatching { + ProfileContentRules.validateSummary( + ProfilePersistentText.summaryForDisplay(response.summary), + summaryMaxLength, + ) + }.getOrElse { cause -> + skippedOperations += "summary: ${cause.message}" + current.summary + }.ifBlank { current.summary } + } + val summaryChanged = summary != current.summary + val profile = if (applied.isEmpty() && !summaryChanged) current else current.copy( + summary = summary, + version = current.version + 1, + reliable = items.isNotEmpty(), + model = model, + promptVersion = promptVersion, + updatedAt = System.currentTimeMillis(), + items = items.values.sortedWith( + compareBy({ it.category.ordinal }, { it.firstSeenAt }, { it.id }) + ), + ) + return ProfileCompactionPlan( + reduction = ProfileReduction(profile, applied), + supportReassignments = supportReassignments, + mergedGroups = mergedGroups, + rewrittenItems = rewrittenItems, + deletedItems = deletedItems, + skippedOperations = skippedOperations, + ) + } + + private fun resolve(profile: UserProfileSnapshot, reference: String, label: String): UserProfileItem = + ProfileItemReferences.resolve(profile, reference) + ?: throw IllegalArgumentException("$label 指向不存在的画像条目 $reference") + + private fun normalizeAndValidate(raw: String, relationship: Boolean, label: String): String = + ProfileContentRules.validate( + ProfilePersistentText.itemForDisplay(raw, relationship), + label, + ) + + private fun UserProfileItem.toApplied(action: ProfileOperationAction) = AppliedProfileOperation( + action = action, + itemId = id, + category = category, + content = content, + confidence = confidence, + relatedUserId = relatedUserId, + evidenceRefs = emptyList(), + ) + + private const val MAX_DELETE_SUPPORTS = 1 +} diff --git a/src/main/kotlin/profile/UserProfileContextRenderer.kt b/src/main/kotlin/profile/UserProfileContextRenderer.kt index c2353de..205ea1f 100644 --- a/src/main/kotlin/profile/UserProfileContextRenderer.kt +++ b/src/main/kotlin/profile/UserProfileContextRenderer.kt @@ -11,7 +11,9 @@ object UserProfileContextRenderer { summaryMaxChars: Int, ): String { val profilesByUserId = profiles - .filter { it.reliable && it.summary.isNotBlank() } + .filter { profile -> + profile.reliable && ProfilePersistentText.summaryForDisplay(profile.summary).isNotBlank() + } .associateBy { it.userId } val userIds = activeUserIds.filter { userId -> userId in profilesByUserId || favorabilityByUserId[userId]?.hasVisibleContext() == true @@ -21,7 +23,7 @@ object UserProfileContextRenderer { val maxChars = summaryMaxChars.coerceAtLeast(50) return buildString { appendLine("## 你对相关群友的认识") - appendLine("好感度、代号和主观印象代表你的关系状态；长期认识来自可修正的历史归纳。仅在当前话题相关时自然运用，不要逐条复述或提及信息来源。") + appendLine("好感度、代号和主观印象代表你的关系状态；画像认识来自可修正的历史归纳。仅在当前话题相关时自然运用，不要逐条复述或提及信息来源。") userIds.forEach { userId -> val profile = profilesByUserId[userId] val favorability = favorabilityByUserId[userId] @@ -34,8 +36,10 @@ object UserProfileContextRenderer { if (info.tags.isNotEmpty()) append(" | 标签：").append(info.tags.joinToString("、")) if (info.impression.isNotBlank()) append(" | 主观印象：").append(info.impression.normalized()) } - profile?.let { - append(" | 长期认识：").append(it.summary.normalized().take(maxChars)) + profile?.summary?.let(ProfilePersistentText::summaryForDisplay) + ?.takeIf(String::isNotBlank)?.let { summary -> + append(" | 画像认识：") + .append(summary.normalized().take(maxChars)) } profile?.items?.asSequence() @@ -46,8 +50,11 @@ object UserProfileContextRenderer { ?.take(2) ?.forEach { item -> val relatedId = checkNotNull(item.relatedUserId) - val relatedName = displayNames[relatedId].orEmpty().ifBlank { relatedId.toString() } - append("；与").append(relatedName).append("：").append(item.content.normalized()) + val relatedName = favorabilityByUserId[relatedId]?.name.orEmpty().ifBlank { + displayNames[relatedId].orEmpty().ifBlank { relatedId.toString() } + } + append("；与").append(relatedName).append("：") + .append(ProfilePersistentText.itemForDisplay(item.content, relationship = true).normalized()) } appendLine() } diff --git a/src/main/kotlin/profile/UserProfileModels.kt b/src/main/kotlin/profile/UserProfileModels.kt index fbf864a..29c5ba2 100644 --- a/src/main/kotlin/profile/UserProfileModels.kt +++ b/src/main/kotlin/profile/UserProfileModels.kt @@ -54,6 +54,7 @@ enum class ProfileOperationAction { enum class ProfileRevisionSource { BACKFILL, CONVERSATION, + COMPACTION, } data class UserProfileItem( @@ -82,14 +83,15 @@ data class UserProfileSnapshot( @Serializable data class ProfileModelOperation( val action: ProfileOperationAction, - @SerialName("item_id") - val itemId: String? = null, + @SerialName("item_ref") + val itemRef: String? = null, val category: ProfileCategory? = null, val content: String? = null, val confidence: ProfileConfidence? = null, @SerialName("related_user_alias") val relatedUserAlias: String? = null, @SerialName("evidence_refs") + @Serializable(with = EvidenceReferenceListSerializer::class) val evidenceRefs: List = emptyList(), ) @@ -202,6 +204,7 @@ data class AppliedProfileOperation( data class ProfileReduction( val profile: UserProfileSnapshot, val operations: List, + val skippedOperations: List = emptyList(), ) data class ProfileAnalysisProgress( @@ -210,6 +213,7 @@ data class ProfileAnalysisProgress( val endTime: Int, val messageCount: Int, val operationCount: Int, + val skippedOperationCount: Int, val usage: ProfileTokenUsage, ) @@ -217,14 +221,50 @@ data class ConversationProfileAnalysisReport( val analyzedUsers: Int, val processedMessages: Int, val appliedOperations: Int, + val skippedOperations: Int, val usage: ProfileTokenUsage, ) +data class GroupProfileCursor( + val botId: Long, + val groupId: Long, + val cursorTime: Int, + val snapshotEndTime: Int, + val updatedAt: Long = 0, +) + +data class GroupProfileAnalysisProgress( + val batchIndex: Int, + val startTime: Int, + val endTime: Int, + val messageCount: Int, + val analyzedUsers: Int, + val appliedOperations: Int, + val skippedOperations: Int, + val usage: ProfileTokenUsage, +) + +data class GroupProfileAnalysisReport( + val botId: Long?, + val groupId: Long, + val processedBatches: Int, + val processedMessages: Int, + val analyzedUsers: Int, + val appliedOperations: Int, + val skippedOperations: Int, + val usage: ProfileTokenUsage, + val cursorTime: Int, + val snapshotEndTime: Int, + val caughtUp: Boolean, + val alreadyRunning: Boolean = false, +) + data class ProfileAnalysisReport( val userId: Long, val processedBatches: Int, val processedMessages: Int, val appliedOperations: Int, + val skippedOperations: Int, val usage: ProfileTokenUsage, val profile: UserProfileSnapshot?, val caughtUp: Boolean, diff --git a/src/main/kotlin/profile/UserProfileReducer.kt b/src/main/kotlin/profile/UserProfileReducer.kt index 463c929..200f1ca 100644 --- a/src/main/kotlin/profile/UserProfileReducer.kt +++ b/src/main/kotlin/profile/UserProfileReducer.kt @@ -3,8 +3,6 @@ package top.jie65535.mirai.profile import java.util.UUID object UserProfileReducer { - private val overclaimPattern = Regex("深厚|扎实|精通|专家|导师|领袖|天才|极强|全栈|核心成员|公认") - fun reduce( current: UserProfileSnapshot, batch: ProfileHistoryBatch, @@ -14,108 +12,34 @@ object UserProfileReducer { summaryMaxLength: Int, advanceBackfillCursor: Boolean = true, ): ProfileReduction { - require(response.summary.length <= summaryMaxLength) { - "画像摘要超过 ${summaryMaxLength} 字符" - } - val items = current.items.associateBy { it.id }.toMutableMap() val applied = mutableListOf() + val skipped = mutableListOf() response.operations.forEachIndexed { index, operation -> - val evidence = operation.evidenceRefs.distinct().map { ref -> - batch.evidenceByRef[ref] - ?: throw IllegalArgumentException("operations[$index] 引用了不存在的证据 e:$ref") - } - require(evidence.isNotEmpty()) { "operations[$index] 缺少证据" } - require(evidence.any { it.record.fromId == batch.userId }) { - "operations[$index] 没有目标用户自己的发言证据" - } - - val evidenceTime = evidence - .asSequence() - .filter { it.record.fromId == batch.userId } - .maxOf { it.record.time } - val firstEvidenceTime = evidence - .asSequence() - .filter { it.record.fromId == batch.userId } - .minOf { it.record.time } - - when (operation.action) { - ProfileOperationAction.ADD -> { - require(operation.itemId.isNullOrBlank()) { - "operations[$index] ADD 不能指定 item_id" - } - val category = requireNotNull(operation.category) { - "operations[$index] ADD 缺少 category" - } - val content = validateContent(index, operation.content) - val confidence = requireNotNull(operation.confidence) { - "operations[$index] ADD 缺少 confidence" - } - val relatedUserId = resolveRelatedUser(index, category, operation.relatedUserAlias, batch) - val duplicate = items.values.any { - it.category == category && normalize(it.content) == normalize(content) - } - if (duplicate) return@forEachIndexed - - val item = UserProfileItem( - id = UUID.randomUUID().toString(), - category = category, - content = content, - confidence = confidence, - relatedUserId = relatedUserId, - firstSeenAt = firstEvidenceTime, - lastConfirmedAt = evidenceTime, - ) - items[item.id] = item - applied += item.toApplied(operation.action, operation.evidenceRefs) - } - - ProfileOperationAction.UPDATE -> { - val old = requireExistingItem(index, operation, items) - val category = operation.category ?: old.category - val content = validateContent(index, operation.content) - val confidence = operation.confidence ?: old.confidence - val relatedUserId = resolveRelatedUser( - index, - category, - operation.relatedUserAlias, - batch, - fallback = old.relatedUserId, - ) - val updated = old.copy( - category = category, - content = content, - confidence = confidence, - relatedUserId = relatedUserId, - lastConfirmedAt = evidenceTime, - ) - items[old.id] = updated - applied += updated.toApplied(operation.action, operation.evidenceRefs) - } - - ProfileOperationAction.CONFIRM -> { - val old = requireExistingItem(index, operation, items) - val updated = old.copy( - confidence = operation.confidence ?: old.confidence, - lastConfirmedAt = evidenceTime, - ) - items[old.id] = updated - applied += updated.toApplied(operation.action, operation.evidenceRefs) - } - - ProfileOperationAction.DELETE -> { - val old = requireExistingItem(index, operation, items) - items.remove(old.id) - applied += old.toApplied(operation.action, operation.evidenceRefs) - } + try { + applyOperation(index, operation, current, batch, items)?.let(applied::add) + } catch (cause: IllegalArgumentException) { + skipped += cause.message ?: "operations[$index] 不符合画像协议" } } - val summary = if (applied.isEmpty()) { + val hasSummaryRelevantChange = applied.any { it.action != ProfileOperationAction.CONFIRM } + val summary = if ( + applied.isEmpty() || skipped.isNotEmpty() || + (!hasSummaryRelevantChange && current.summary.isNotBlank()) + ) { current.summary } else { - response.summary.trim().ifEmpty { current.summary } + try { + ProfileContentRules.validateSummary( + ProfilePersistentText.normalizeSummary(response.summary, batch), + summaryMaxLength, + ).ifEmpty { current.summary } + } catch (cause: IllegalArgumentException) { + skipped += "summary: ${cause.message}" + current.summary + } } val changed = applied.isNotEmpty() || summary != current.summary val profile = current.copy( @@ -130,28 +54,129 @@ object UserProfileReducer { compareBy({ it.category.ordinal }, { it.firstSeenAt }, { it.id }) ), ) - return ProfileReduction(profile, applied) + return ProfileReduction(profile, applied, skipped) } - private fun validateContent(index: Int, raw: String?): String { - val content = raw?.trim().orEmpty() - require(content.isNotEmpty()) { "operations[$index] 缺少 content" } - require(content.length <= 120) { "operations[$index].content 超过 120 字符" } - require(!overclaimPattern.containsMatchIn(content)) { - "operations[$index].content 包含夸张身份或能力判断" + private fun applyOperation( + index: Int, + operation: ProfileModelOperation, + current: UserProfileSnapshot, + batch: ProfileHistoryBatch, + items: MutableMap, + ): AppliedProfileOperation? { + val evidence = operation.evidenceRefs.distinct().map { ref -> + batch.evidenceByRef[ref] + ?: throw IllegalArgumentException("operations[$index] 引用了不存在的证据 e:$ref") } - return content + require(evidence.isNotEmpty()) { "operations[$index] 缺少证据" } + require(evidence.any { it.record.fromId == batch.userId }) { + "operations[$index] 没有目标用户自己的发言证据" + } + + val targetEvidence = evidence.filter { it.record.fromId == batch.userId } + val evidenceTime = targetEvidence.maxOf { it.record.time } + val firstEvidenceTime = targetEvidence.minOf { it.record.time } + + return when (operation.action) { + ProfileOperationAction.ADD -> { + require(operation.itemRef.isNullOrBlank()) { + "operations[$index] ADD 不能指定 item_ref" + } + val category = requireNotNull(operation.category) { + "operations[$index] ADD 缺少 category" + } + val confidence = requireNotNull(operation.confidence) { + "operations[$index] ADD 缺少 confidence" + } + val relatedUserId = resolveRelatedUser(index, category, operation.relatedUserAlias, batch) + val content = validateContent(index, operation.content, batch, relatedUserId) + val duplicate = items.values.any { + it.category == category && + ProfileContentRules.normalizedKey(it.content) == ProfileContentRules.normalizedKey(content) + } + if (duplicate) return null + + val item = UserProfileItem( + id = UUID.randomUUID().toString(), + category = category, + content = content, + confidence = confidence, + relatedUserId = relatedUserId, + firstSeenAt = firstEvidenceTime, + lastConfirmedAt = evidenceTime, + ) + items[item.id] = item + item.toApplied(operation.action, operation.evidenceRefs) + } + + ProfileOperationAction.UPDATE -> { + val old = requireExistingItem(index, operation, current, items) + val category = operation.category ?: old.category + val confidence = operation.confidence ?: old.confidence + val relatedUserId = resolveRelatedUser( + index, + category, + operation.relatedUserAlias, + batch, + fallback = old.relatedUserId, + ) + val content = validateContent(index, operation.content, batch, relatedUserId) + val updated = old.copy( + category = category, + content = content, + confidence = confidence, + relatedUserId = relatedUserId, + lastConfirmedAt = evidenceTime, + ) + items[old.id] = updated + updated.toApplied(operation.action, operation.evidenceRefs) + } + + ProfileOperationAction.CONFIRM -> { + val old = requireExistingItem(index, operation, current, items) + val updated = old.copy( + confidence = operation.confidence ?: old.confidence, + lastConfirmedAt = evidenceTime, + ) + items[old.id] = updated + updated.toApplied(operation.action, operation.evidenceRefs) + } + + ProfileOperationAction.DELETE -> { + val old = requireExistingItem(index, operation, current, items) + items.remove(old.id) + old.toApplied(operation.action, operation.evidenceRefs) + } + } + } + + private fun validateContent( + index: Int, + raw: String?, + batch: ProfileHistoryBatch, + relatedUserId: Long?, + ): String { + return ProfileContentRules.validate( + ProfilePersistentText.normalizeItemContent(raw.orEmpty(), batch, relatedUserId), + "operations[$index]", + ) } private fun requireExistingItem( index: Int, operation: ProfileModelOperation, + referenceProfile: UserProfileSnapshot, items: Map, ): UserProfileItem { - val itemId = operation.itemId?.takeIf { it.isNotBlank() } - ?: throw IllegalArgumentException("operations[$index] ${operation.action} 缺少 item_id") - return items[itemId] - ?: throw IllegalArgumentException("operations[$index] 指向不存在的画像条目 $itemId") + val itemRef = operation.itemRef?.takeIf { it.isNotBlank() } + val referencedItem = ProfileItemReferences.resolve(referenceProfile, itemRef) + ?: throw IllegalArgumentException( + "operations[$index] ${operation.action} 指向不存在的画像条目 ${itemRef ?: "（缺少 item_ref）"}" + ) + return items[referencedItem.id] + ?: throw IllegalArgumentException( + "operations[$index] ${operation.action} 重复操作了画像条目 $itemRef" + ) } private fun resolveRelatedUser( @@ -186,7 +211,4 @@ object UserProfileReducer { evidenceRefs = evidenceRefs.distinct(), ) - private fun normalize(value: String): String = value - .lowercase() - .replace(Regex("[\\s\\p{Punct}，。；、！？（）【】‘’“”]+"), "") } diff --git a/src/main/kotlin/profile/UserProfileStore.kt b/src/main/kotlin/profile/UserProfileStore.kt index 24f94c1..2c00508 100644 --- a/src/main/kotlin/profile/UserProfileStore.kt +++ b/src/main/kotlin/profile/UserProfileStore.kt @@ -8,7 +8,7 @@ import java.sql.DriverManager import java.sql.ResultSet object UserProfileStore { - private const val SCHEMA_VERSION = 2 + private const val SCHEMA_VERSION = 3 private const val BUSY_TIMEOUT_MS = 30_000 private val lifecycleLock = Any() @@ -98,6 +98,38 @@ object UserProfileStore { } } + fun loadSupportStats(userId: Long): Map { + check(initialized) { "用户画像数据库尚未初始化" } + return openReadConnection().use { connection -> + connection.prepareStatement( + """ + SELECT item_id, COUNT(*) AS support_count, + MIN(start_time) AS first_supported_at, + MAX(end_time) AS last_supported_at + FROM profile_support + WHERE user_id = ? + GROUP BY item_id + """.trimIndent() + ).use { statement -> + statement.setLong(1, userId) + statement.executeQuery().use { results -> + buildMap { + while (results.next()) { + put( + results.getString("item_id"), + ProfileItemSupportStats( + count = results.getInt("support_count"), + firstSupportedAt = results.getInt("first_supported_at"), + lastSupportedAt = results.getInt("last_supported_at"), + ) + ) + } + } + } + } + } + } + fun commit( reduction: ProfileReduction, batch: ProfileHistoryBatch, @@ -110,6 +142,17 @@ object UserProfileStore { usage: ProfileTokenUsage, ) = commitAll(reductions, usage, ProfileRevisionSource.CONVERSATION) + fun commitCompaction( + plan: ProfileCompactionPlan, + batch: ProfileHistoryBatch, + usage: ProfileTokenUsage, + ) = commitAll( + entries = listOf(plan.reduction to batch), + usage = usage, + source = ProfileRevisionSource.COMPACTION, + supportReassignments = plan.supportReassignments, + ) + fun isConversationProcessed(inputHash: String): Boolean { check(initialized) { "用户画像数据库尚未初始化" } return openReadConnection().use { connection -> @@ -122,10 +165,61 @@ object UserProfileStore { } } + fun loadGroupCursor(botId: Long, groupId: Long): GroupProfileCursor? { + check(initialized) { "用户画像数据库尚未初始化" } + return openReadConnection().use { connection -> + connection.prepareStatement( + """ + SELECT bot_id, group_id, cursor_time, snapshot_end_time, updated_at + FROM profile_group_cursor + WHERE bot_id = ? AND group_id = ? + """.trimIndent() + ).use { statement -> + statement.setLong(1, botId) + statement.setLong(2, groupId) + statement.executeQuery().use { results -> + if (!results.next()) return@use null + GroupProfileCursor( + botId = results.getLong("bot_id"), + groupId = results.getLong("group_id"), + cursorTime = results.getInt("cursor_time"), + snapshotEndTime = results.getInt("snapshot_end_time"), + updatedAt = results.getLong("updated_at"), + ) + } + } + } + } + + fun saveGroupCursor(cursor: GroupProfileCursor) { + check(initialized) { "用户画像数据库尚未初始化" } + withWriteConnection { connection -> + connection.prepareStatement( + """ + INSERT INTO profile_group_cursor( + bot_id, group_id, cursor_time, snapshot_end_time, updated_at + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(bot_id, group_id) DO UPDATE SET + cursor_time = excluded.cursor_time, + snapshot_end_time = excluded.snapshot_end_time, + updated_at = excluded.updated_at + """.trimIndent() + ).use { statement -> + statement.setLong(1, cursor.botId) + statement.setLong(2, cursor.groupId) + statement.setInt(3, cursor.cursorTime) + statement.setInt(4, cursor.snapshotEndTime) + statement.setLong(5, cursor.updatedAt) + statement.executeUpdate() + } + } + } + private fun commitAll( entries: List>, usage: ProfileTokenUsage, source: ProfileRevisionSource, + supportReassignments: Map = emptyMap(), ) { check(initialized) { "用户画像数据库尚未初始化" } if (entries.isEmpty()) return @@ -142,6 +236,20 @@ object UserProfileStore { source = source, ) } + if (supportReassignments.isNotEmpty()) { + val userId = entries.single().first.profile.userId + connection.prepareStatement( + "UPDATE profile_support SET item_id = ? WHERE user_id = ? AND item_id = ?" + ).use { statement -> + supportReassignments.forEach { (sourceItemId, targetItemId) -> + statement.setString(1, targetItemId) + statement.setLong(2, userId) + statement.setString(3, sourceItemId) + statement.addBatch() + } + statement.executeBatch() + } + } connection.commit() } catch (cause: Throwable) { connection.rollback() @@ -216,30 +324,32 @@ object UserProfileStore { statement.executeBatch() } - connection.prepareStatement( - """ - INSERT INTO profile_support( - item_id, user_id, group_ids, start_time, end_time, - input_hash, action, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """.trimIndent() - ).use { statement -> - reduction.operations.forEach { operation -> - val evidence = operation.evidenceRefs.mapNotNull(batch.evidenceByRef::get) - val startTime = evidence.minOf { it.record.time } - val endTime = evidence.maxOf { it.record.time }.safeNextSecond() - val groupIds = evidence.map { it.record.targetId }.distinct().sorted().joinToString(",") - statement.setString(1, operation.itemId) - statement.setLong(2, profile.userId) - statement.setString(3, groupIds) - statement.setInt(4, startTime) - statement.setInt(5, endTime) - statement.setString(6, batch.inputHash) - statement.setString(7, operation.action.name) - statement.setLong(8, System.currentTimeMillis()) - statement.addBatch() + if (source != ProfileRevisionSource.COMPACTION) { + connection.prepareStatement( + """ + INSERT INTO profile_support( + item_id, user_id, group_ids, start_time, end_time, + input_hash, action, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """.trimIndent() + ).use { statement -> + reduction.operations.forEach { operation -> + val evidence = operation.evidenceRefs.mapNotNull(batch.evidenceByRef::get) + val startTime = evidence.minOf { it.record.time } + val endTime = evidence.maxOf { it.record.time }.safeNextSecond() + val groupIds = evidence.map { it.record.targetId }.distinct().sorted().joinToString(",") + statement.setString(1, operation.itemId) + statement.setLong(2, profile.userId) + statement.setString(3, groupIds) + statement.setInt(4, startTime) + statement.setInt(5, endTime) + statement.setString(6, batch.inputHash) + statement.setString(7, operation.action.name) + statement.setLong(8, System.currentTimeMillis()) + statement.addBatch() + } + statement.executeBatch() } - statement.executeBatch() } connection.prepareStatement( @@ -376,6 +486,18 @@ object UserProfileStore { ) """.trimIndent() ) + statement.executeUpdate( + """ + CREATE TABLE IF NOT EXISTS profile_group_cursor( + bot_id INTEGER NOT NULL, + group_id INTEGER NOT NULL, + cursor_time INTEGER NOT NULL, + snapshot_end_time INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY(bot_id, group_id) + ) + """.trimIndent() + ) } connection.prepareStatement( "INSERT INTO user_profile_meta(key, value) VALUES ('schema_version', ?) " + diff --git a/src/test/kotlin/profile/ConversationProfileReducerTest.kt b/src/test/kotlin/profile/ConversationProfileReducerTest.kt index 6d2dc7b..22b37bf 100644 --- a/src/test/kotlin/profile/ConversationProfileReducerTest.kt +++ b/src/test/kotlin/profile/ConversationProfileReducerTest.kt @@ -4,7 +4,7 @@ import net.mamoe.mirai.message.data.MessageSourceKind import top.jie65535.mirai.data.ChatMessageRecord import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertFailsWith +import kotlin.test.assertFalse import kotlin.test.assertTrue class ConversationProfileReducerTest { @@ -59,36 +59,36 @@ class ConversationProfileReducerTest { } @Test - fun rejectsEvidenceAuthoredByAnotherUser() { - val failure = assertFailsWith { - ConversationProfileReducer.reduce( - profiles = USERS.associateWith(::emptyProfile), - batch = batch(), - eligibleUserIds = USERS, - response = ConversationProfileModelResponse( - users = listOf( - ConversationProfileUserResponse( - userAlias = "U1", - operations = listOf( - ProfileModelOperation( - action = ProfileOperationAction.ADD, - category = ProfileCategory.NOTABLE_FACT, - content = "关注本地大模型", - confidence = ProfileConfidence.MEDIUM, - evidenceRefs = listOf(2), - ) - ), - summary = "关注本地大模型。", - ) + fun skipsEvidenceAuthoredByAnotherUser() { + val reductions = ConversationProfileReducer.reduce( + profiles = USERS.associateWith(::emptyProfile), + batch = batch(), + eligibleUserIds = USERS, + response = ConversationProfileModelResponse( + users = listOf( + ConversationProfileUserResponse( + userAlias = "U1", + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.NOTABLE_FACT, + content = "关注本地大模型", + confidence = ProfileConfidence.MEDIUM, + evidenceRefs = listOf(2), + ) + ), + summary = "关注本地大模型。", ) - ), - model = "test-model", - promptVersion = "test-prompt", - summaryMaxLength = 500, - ) - } + ) + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) - assertTrue(failure.message.orEmpty().contains("目标用户")) + val userReduction = reductions.single { it.profile.userId == USER_A } + assertTrue(userReduction.operations.isEmpty()) + assertTrue(userReduction.skippedOperations.single().contains("目标用户")) } @Test @@ -120,12 +120,12 @@ class ConversationProfileReducerTest { operations = listOf( ProfileModelOperation( action = ProfileOperationAction.CONFIRM, - itemId = oldItem.id, + itemRef = "P1", confidence = ProfileConfidence.MEDIUM, evidenceRefs = listOf(1), ) ), - summary = "关注 Kotlin 开发。", + summary = "本批再次提到 Kotlin。", ) ) ), @@ -138,14 +138,14 @@ class ConversationProfileReducerTest { assertEquals(2, updated.version) assertEquals("existing-item", updated.items.single().id) assertEquals(ProfileConfidence.MEDIUM, updated.items.single().confidence) - assertTrue( - ProfilePromptStore.buildConversationUserPrompt(profiles, batch(), USERS) - .contains("[P:existing-item]") - ) + assertEquals(existing.summary, updated.summary) + val prompt = ProfilePromptStore.buildConversationUserPrompt(profiles, batch(), USERS) + assertTrue(prompt.contains("[P1]")) + assertFalse(prompt.contains(oldItem.id)) } @Test - fun rejectsMoreThanFourOperationsForOneUser() { + fun acceptsMoreThanFourOperationsForOneUser() { val operations = (1..5).map { index -> ProfileModelOperation( action = ProfileOperationAction.ADD, @@ -156,27 +156,126 @@ class ConversationProfileReducerTest { ) } - val failure = assertFailsWith { - ConversationProfileReducer.reduce( - profiles = USERS.associateWith(::emptyProfile), - batch = batch(), - eligibleUserIds = USERS, - response = ConversationProfileModelResponse( - users = listOf( - ConversationProfileUserResponse( - userAlias = "U1", - operations = operations, - summary = "包含过多事实。", - ) + val reductions = ConversationProfileReducer.reduce( + profiles = USERS.associateWith(::emptyProfile), + batch = batch(), + eligibleUserIds = USERS, + response = ConversationProfileModelResponse( + users = listOf( + ConversationProfileUserResponse( + userAlias = "U1", + operations = operations, + summary = "包含多项事实。", ) - ), - model = "test-model", - promptVersion = "test-prompt", - summaryMaxLength = 500, - ) - } + ) + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) - assertTrue(failure.message.orEmpty().contains("超过 4 项")) + assertEquals(5, reductions.single { it.profile.userId == USER_A }.operations.size) + } + + @Test + fun mergesDuplicateUserResultGroupsAndDeduplicatesOperations() { + val operation = ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.NOTABLE_FACT, + content = "日常使用 Kotlin 开发", + confidence = ProfileConfidence.MEDIUM, + evidenceRefs = listOf(1), + ) + + val reductions = ConversationProfileReducer.reduce( + profiles = USERS.associateWith(::emptyProfile), + batch = batch(), + eligibleUserIds = USERS, + response = ConversationProfileModelResponse( + users = listOf( + ConversationProfileUserResponse( + userAlias = "U1", + operations = listOf(operation), + summary = "使用 Kotlin。", + ), + ConversationProfileUserResponse( + userAlias = "U1", + operations = listOf( + operation, + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.PREFERENCE, + content = "偏好 Kotlin", + confidence = ProfileConfidence.LOW, + evidenceRefs = listOf(1), + ), + ), + summary = "使用并偏好 Kotlin。", + ), + ) + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + val reduction = reductions.single { it.profile.userId == USER_A } + assertEquals(2, reduction.operations.size) + assertEquals(2, reduction.profile.items.size) + assertEquals("使用并偏好 Kotlin。", reduction.profile.summary) + } + + @Test + fun ignoresModelResultsForNonCandidateContextUsers() { + val contextUser = 400L + val batch = batch().copy(aliases = batch().aliases + (contextUser to "U4")) + val reductions = ConversationProfileReducer.reduce( + profiles = USERS.associateWith(::emptyProfile), + batch = batch, + eligibleUserIds = USERS, + response = ConversationProfileModelResponse( + users = listOf( + ConversationProfileUserResponse( + userAlias = "U4", + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.INTEREST, + content = "关注 Kotlin", + confidence = ProfileConfidence.LOW, + evidenceRefs = listOf(1), + ) + ), + summary = "关注 Kotlin。", + ), + ConversationProfileUserResponse( + userAlias = "U1", + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.NOTABLE_FACT, + content = "日常使用 Kotlin 开发", + confidence = ProfileConfidence.MEDIUM, + evidenceRefs = listOf(1), + ) + ), + summary = "日常使用 Kotlin 开发。", + ), + ) + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + assertEquals(1, reductions.single { it.profile.userId == USER_A }.operations.size) + assertTrue(reductions.single { it.profile.userId == USER_B }.operations.isEmpty()) + val prompt = ProfilePromptStore.buildConversationUserPrompt( + profiles = USERS.associateWith(::emptyProfile), + batch = batch, + eligibleUserIds = USERS, + ) + assertTrue(prompt.contains("候选用户别名: U1, U2")) } private fun batch() = ConversationProfileBatch( diff --git a/src/test/kotlin/profile/ProfileHistoryReaderTest.kt b/src/test/kotlin/profile/ProfileHistoryReaderTest.kt index 1682d1d..6a012ce 100644 --- a/src/test/kotlin/profile/ProfileHistoryReaderTest.kt +++ b/src/test/kotlin/profile/ProfileHistoryReaderTest.kt @@ -49,6 +49,7 @@ class ProfileHistoryReaderTest { } insert(TARGET, 10, 100, "目标发言一") insert(OTHER, 10, 110, "用于理解语境的回复") + insert(OTHER, 10, 110, "同一秒的补充回复") insert(TARGET, 10, 130, "目标发言二") insert(TARGET, 20, 130, "同一秒的另一群发言") insert(OTHER, 20, 140, "后续上下文") @@ -110,6 +111,35 @@ class ProfileHistoryReaderTest { assertTrue(conversation.authoredTextCharsByUser.getValue(TARGET) > 0) assertTrue(conversation.messages.all { it.record.targetId == 10L }) assertTrue(conversation.messages.all { it.record.time in 90 until 150 }) + + val groupBounds = assertNotNull(reader.findGroupTimeBounds(10)) + assertEquals(1, groupBounds.botId) + assertEquals(100, groupBounds.startTime) + assertEquals(201, groupBounds.endTime) + val firstGroupBatch = assertNotNull( + reader.loadNextConversationBatch( + botId = groupBounds.botId, + groupId = 10, + startTime = groupBounds.startTime, + snapshotEndTime = groupBounds.endTime, + messageLimit = 2, + maxMessageChars = 200, + ) + ) + assertEquals(111, firstGroupBatch.endTime) + assertEquals(listOf(100, 110, 110), firstGroupBatch.messages.map { it.record.time }) + val secondGroupBatch = assertNotNull( + reader.loadNextConversationBatch( + botId = groupBounds.botId, + groupId = 10, + startTime = firstGroupBatch.endTime, + snapshotEndTime = groupBounds.endTime, + messageLimit = 2, + maxMessageChars = 200, + ) + ) + assertEquals(201, secondGroupBatch.endTime) + assertEquals(listOf(130, 200), secondGroupBatch.messages.map { it.record.time }) } finally { directory.toFile().deleteRecursively() } diff --git a/src/test/kotlin/profile/ProfileModelResponseParsingTest.kt b/src/test/kotlin/profile/ProfileModelResponseParsingTest.kt new file mode 100644 index 0000000..6eaf86f --- /dev/null +++ b/src/test/kotlin/profile/ProfileModelResponseParsingTest.kt @@ -0,0 +1,40 @@ +package top.jie65535.mirai.profile + +import kotlinx.serialization.decodeFromString +import kotlin.test.Test +import kotlin.test.assertEquals + +class ProfileModelResponseParsingTest { + @Test + fun ignoresLegacyFieldsAndNormalizesDisplayedEvidenceReferences() { + val response = profileResponseJson.decodeFromString( + """ + { + "operations": [ + { + "action": "ADD", + "item_id": null, + "category": "interest", + "content": "关注 Kotlin", + "confidence": "low", + "evidence_refs": [127, "e:128", "129"] + } + ], + "summary": "关注 Kotlin。", + "unexpected": true + } + """.trimIndent() + ) + + assertEquals(listOf(127, 128, 129), response.operations.single().evidenceRefs) + } + + @Test + fun preservesMalformedEvidenceAsAnInvalidReferenceForOperationValidation() { + val response = profileResponseJson.decodeFromString( + """{"operations":[{"action":"CONFIRM","item_ref":"P1","evidence_refs":["invalid"]}]}""" + ) + + assertEquals(listOf(0), response.operations.single().evidenceRefs) + } +} diff --git a/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt b/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt index 527a034..7f4314c 100644 --- a/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt +++ b/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt @@ -11,6 +11,7 @@ import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertNotNull import kotlin.test.assertNull +import kotlin.test.assertTrue class UserProfileAnalysisServiceTest { @Test @@ -22,13 +23,14 @@ class UserProfileAnalysisServiceTest { assertEquals(1, model.calls) assertEquals(2, report.analyzedUsers) assertEquals(2, report.appliedOperations) + assertEquals(0, report.skippedOperations) assertEquals("日常使用 Kotlin 开发", UserProfileStore.load(USER_A)?.items?.single()?.content) assertEquals("持续关注本地大模型", UserProfileStore.load(USER_B)?.items?.single()?.content) assertEquals(true, UserProfileStore.isConversationProcessed(INPUT_HASH)) } @Test - fun rejectsWronglyAttributedEvidenceWithoutPartialCommit() = withProfileStore { + fun skipsWronglyAttributedEvidenceAndMarksConversationProcessed() = withProfileStore { val model = FakeConversationProfileModel { result( responseFor( @@ -39,14 +41,14 @@ class UserProfileAnalysisServiceTest { ) } - assertFailsWith { - analyze(batch(), model, retryMax = 1) - } + val report = assertNotNull(analyze(batch(), model, retryMax = 1)) - assertEquals(2, model.calls) - assertNull(UserProfileStore.load(USER_A)) - assertNull(UserProfileStore.load(USER_B)) - assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH)) + assertEquals(1, model.calls) + assertEquals(0, report.appliedOperations) + assertEquals(1, report.skippedOperations) + assertNotNull(UserProfileStore.load(USER_A)) + assertNotNull(UserProfileStore.load(USER_B)) + assertTrue(UserProfileStore.isConversationProcessed(INPUT_HASH)) } @Test diff --git a/src/test/kotlin/profile/UserProfileCompactorTest.kt b/src/test/kotlin/profile/UserProfileCompactorTest.kt new file mode 100644 index 0000000..d76bf64 --- /dev/null +++ b/src/test/kotlin/profile/UserProfileCompactorTest.kt @@ -0,0 +1,295 @@ +package top.jie65535.mirai.profile + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class UserProfileCompactorTest { + @Test + fun mergesRewritesAndDeletesWithinConservativeBounds() { + val profile = profile() + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = mapOf("style-item" to supportStats(1)), + response = ProfileCompactionResponse( + merges = listOf( + ProfileCompactionMerge( + itemRefs = listOf("P1", "P2"), + content = "关注并实际体验国内外大语言模型", + ) + ), + rewrites = listOf( + ProfileCompactionRewrite( + itemRef = "P4", + content = "从事使用 C# 的上位机开发", + ) + ), + deletes = listOf( + ProfileCompactionDelete( + itemRef = "P3", + reason = ProfileCompactionDeleteReason.OVER_SPECIFIC, + ) + ), + summary = "该用户从事上位机开发，并关注大语言模型。", + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + assertEquals(2, plan.reduction.profile.items.size) + assertEquals(profile.cursorTime, plan.reduction.profile.cursorTime) + assertEquals(profile.snapshotEndTime, plan.reduction.profile.snapshotEndTime) + assertEquals(profile.version + 1, plan.reduction.profile.version) + assertEquals(1, plan.mergedGroups) + assertEquals(1, plan.rewrittenItems) + assertEquals(1, plan.deletedItems) + assertEquals(mapOf("model-b" to "model-a"), plan.supportReassignments) + assertEquals(4, plan.reduction.operations.size) + val merged = plan.reduction.profile.items.single { it.category == ProfileCategory.INTEREST } + assertEquals("model-a", merged.id) + assertEquals(ProfileConfidence.LOW, merged.confidence) + assertEquals("关注并实际体验国内外大语言模型", merged.content) + assertFalse(plan.reduction.profile.items.any { it.id == "style-item" }) + } + + @Test + fun skipsMergingDifferentCategories() { + val profile = profile() + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = emptyMap(), + response = ProfileCompactionResponse( + merges = listOf( + ProfileCompactionMerge( + itemRefs = listOf("P2", "P3"), + content = "关注模型并使用口语化表达", + ) + ) + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + assertTrue(plan.reduction.operations.isEmpty()) + assertEquals(profile.version, plan.reduction.profile.version) + assertTrue(plan.skippedOperations.single().contains("同类别")) + } + + @Test + fun skipsDeletingSupportedOrHighItemsButAllowsLowSupportMedium() { + val supported = delete(profile(), "P3", mapOf("style-item" to supportStats(2))) + assertTrue(supported.reduction.operations.isEmpty()) + assertTrue(supported.skippedOperations.single().contains("2 次支持")) + + val high = delete(profile(), "P4", mapOf("fact-item" to supportStats(1))) + assertTrue(high.reduction.operations.isEmpty()) + assertTrue(high.skippedOperations.single().contains("high")) + + val medium = delete(profile(), "P2", mapOf("model-b" to supportStats(1))) + assertEquals(1, medium.deletedItems) + assertFalse(medium.reduction.profile.items.any { it.id == "model-b" }) + } + + @Test + fun compactionPromptUsesTemporaryReferencesAndSupportCounts() { + val profile = profile() + val prompt = ProfilePromptStore.buildCompactionUserPrompt( + profile, + mapOf("model-a" to supportStats(3)), + ) + + assertTrue(prompt.contains("[P1]")) + assertTrue(prompt.contains("supports=3")) + assertTrue(prompt.contains("item_range=")) + profile.items.forEach { assertFalse(prompt.contains(it.id)) } + } + + @Test + fun profilePromptsRequireAbsoluteDatesForTimeSensitiveFacts() { + assertTrue(ProfilePromptStore.systemPrompt.contains("截至 YYYY-MM-DD")) + assertTrue(ProfilePromptStore.conversationSystemPrompt.contains("截至 YYYY-MM-DD")) + assertTrue(ProfilePromptStore.compactionSystemPrompt.contains("item_range")) + } + + @Test + fun promptsDefineSummaryAsAWholeProfileSynthesis() { + assertTrue(ProfilePromptStore.systemPrompt.contains("不是本批聊天摘要")) + assertTrue(ProfilePromptStore.conversationSystemPrompt.contains("不是本批聊天摘要")) + assertTrue(ProfilePromptStore.compactionSystemPrompt.contains("全部保留条目")) + assertTrue(ProfilePromptStore.compactionSystemPrompt.contains("不得只描述最后编辑的条目")) + assertTrue(ProfilePromptStore.compactionSystemPrompt.contains("即使没有条目操作也应重写")) + } + + @Test + fun removesRelatedGroupAliasFromRewrittenRelationship() { + val relation = item( + id = "relation-item", + category = ProfileCategory.RELATIONSHIP_NOTE, + content = "经常讨论技术方案", + confidence = ProfileConfidence.LOW, + firstSeenAt = 10, + ).copy(relatedUserId = 200) + val profile = profile().copy(items = listOf(relation)) + + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = emptyMap(), + response = ProfileCompactionResponse( + rewrites = listOf( + ProfileCompactionRewrite("P1", "经常与 R1 讨论技术方案并互相提供建议") + ), + summary = profile.summary, + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + assertEquals("经常与对方讨论技术方案并互相提供建议", plan.reduction.profile.items.single().content) + assertFalse(plan.reduction.profile.items.single().content.contains("R1")) + } + + @Test + fun ignoresNoopRewriteWithoutChangingVersion() { + val profile = profile() + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = emptyMap(), + response = ProfileCompactionResponse( + rewrites = listOf( + ProfileCompactionRewrite("P1", "关注 Qwen、DeepSeek 等模型") + ) + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + assertTrue(plan.reduction.operations.isEmpty()) + assertEquals(0, plan.rewrittenItems) + assertEquals(profile.version, plan.reduction.profile.version) + } + + @Test + fun allowsSummaryOnlyCompactionAcrossTheWholeProfile() { + val profile = profile().copy(summary = "最近一次只讨论了 C#。") + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = emptyMap(), + response = ProfileCompactionResponse( + summary = "该用户从事上位机开发，关注大语言模型，表达直接。", + ), + model = "test-model", + promptVersion = "compact-v3", + summaryMaxLength = 500, + ) + + assertTrue(plan.reduction.operations.isEmpty()) + assertEquals(profile.version + 1, plan.reduction.profile.version) + assertEquals( + "该用户从事上位机开发，关注大语言模型，表达直接。", + plan.reduction.profile.summary, + ) + } + + @Test + fun processesMoreThanTwelveOperationsOfOneType() { + val items = (1..15).map { index -> + item( + id = "item-$index", + category = ProfileCategory.EXPRESSION_STYLE, + content = "旧表达方式 $index", + confidence = ProfileConfidence.LOW, + firstSeenAt = index, + ) + } + val profile = profile().copy(items = items) + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = emptyMap(), + response = ProfileCompactionResponse( + rewrites = items.indices.map { index -> + ProfileCompactionRewrite("P${index + 1}", "概括后的表达方式 ${index + 1}") + }, + summary = profile.summary, + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + assertEquals(15, plan.rewrittenItems) + assertTrue(plan.skippedOperations.isEmpty()) + } + + @Test + fun keepsOldSummaryWhenNewSummaryContainsTemporaryReference() { + val profile = profile() + val plan = UserProfileCompactor.reduce( + current = profile, + supportStats = emptyMap(), + response = ProfileCompactionResponse( + rewrites = listOf(ProfileCompactionRewrite("P1", "关注多个国内外大语言模型")), + summary = "P1 已经概括多个模型兴趣。", + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + assertEquals(profile.summary, plan.reduction.profile.summary) + assertTrue(plan.skippedOperations.single().contains("临时画像条目编号")) + } + + private fun delete( + profile: UserProfileSnapshot, + itemRef: String, + stats: Map, + ) = UserProfileCompactor.reduce( + current = profile, + supportStats = stats, + response = ProfileCompactionResponse( + deletes = listOf( + ProfileCompactionDelete(itemRef, ProfileCompactionDeleteReason.ONE_OFF) + ) + ), + model = "test-model", + promptVersion = "compact-v1", + summaryMaxLength = 500, + ) + + private fun profile() = UserProfileSnapshot( + userId = 100, + summary = "关注模型并从事软件开发。", + version = 8, + cursorTime = 300, + snapshotEndTime = 1_000, + reliable = true, + items = listOf( + item("model-a", ProfileCategory.INTEREST, "关注 Qwen、DeepSeek 等模型", ProfileConfidence.LOW, 10), + item("model-b", ProfileCategory.INTEREST, "关注开源大模型部署和性能", ProfileConfidence.MEDIUM, 20), + item("style-item", ProfileCategory.EXPRESSION_STYLE, "会使用“我去”表示惊讶", ProfileConfidence.LOW, 30), + item("fact-item", ProfileCategory.NOTABLE_FACT, "从事上位机开发，使用 C#", ProfileConfidence.HIGH, 40), + ), + ) + + private fun item( + id: String, + category: ProfileCategory, + content: String, + confidence: ProfileConfidence, + firstSeenAt: Int, + ) = UserProfileItem( + id = id, + category = category, + content = content, + confidence = confidence, + firstSeenAt = firstSeenAt, + lastConfirmedAt = firstSeenAt + 100, + ) + + private fun supportStats(count: Int) = ProfileItemSupportStats(count, 1, 2) +} diff --git a/src/test/kotlin/profile/UserProfileContextRendererTest.kt b/src/test/kotlin/profile/UserProfileContextRendererTest.kt index 90ad31d..1147272 100644 --- a/src/test/kotlin/profile/UserProfileContextRendererTest.kt +++ b/src/test/kotlin/profile/UserProfileContextRendererTest.kt @@ -15,7 +15,7 @@ class UserProfileContextRendererTest { snapshotEndTime = 2, reliable = true, items = listOf( - relationship("visible", 200, "经常互相讨论技术方案"), + relationship("visible", 200, "经常与 U12 讨论技术方案"), relationship("hidden", 300, "曾共同讨论游戏"), ), ) @@ -38,8 +38,9 @@ class UserProfileContextRendererTest { assertContains(rendered, "小明代号(100)") assertContains(rendered, "好感度+12") - assertContains(rendered, "长期认识：长期关注 Kotlin 开发") - assertContains(rendered, "与小王：经常互相讨论技术方案") + assertContains(rendered, "画像认识：长期关注 Kotlin 开发") + assertContains(rendered, "与小王：经常与对方讨论技术方案") + assertFalse(rendered.contains("U12")) assertFalse(rendered.contains("小李")) assertFalse(rendered.contains("曾共同讨论游戏")) } @@ -59,7 +60,7 @@ class UserProfileContextRendererTest { assertContains(rendered, "小明(100)") assertContains(rendered, "好感度-8") assertContains(rendered, "主观印象：偶尔喜欢抬杠") - assertFalse(rendered.contains("长期认识：")) + assertFalse(rendered.contains("画像认识：")) } @Test @@ -88,4 +89,5 @@ class UserProfileContextRendererTest { firstSeenAt = 1, lastConfirmedAt = 1, ) + } diff --git a/src/test/kotlin/profile/UserProfileReducerTest.kt b/src/test/kotlin/profile/UserProfileReducerTest.kt index 1b5e66a..a599ca7 100644 --- a/src/test/kotlin/profile/UserProfileReducerTest.kt +++ b/src/test/kotlin/profile/UserProfileReducerTest.kt @@ -4,7 +4,7 @@ import net.mamoe.mirai.message.data.MessageSourceKind import top.jie65535.mirai.data.ChatMessageRecord import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertFailsWith +import kotlin.test.assertFalse import kotlin.test.assertTrue class UserProfileReducerTest { @@ -43,32 +43,33 @@ class UserProfileReducerTest { } @Test - fun rejectsItemSupportedOnlyByAnotherUser() { + fun skipsItemSupportedOnlyByAnotherUser() { val batch = batchOf(message(ref = 1, fromId = OTHER, text = "我平时会写 Kotlin")) - val failure = assertFailsWith { - UserProfileReducer.reduce( - current = emptyProfile(), - batch = batch, - response = ProfileModelResponse( - operations = listOf( - ProfileModelOperation( - action = ProfileOperationAction.ADD, - category = ProfileCategory.NOTABLE_FACT, - content = "从事 Kotlin 开发", - confidence = ProfileConfidence.HIGH, - evidenceRefs = listOf(1), - ) - ), - summary = "从事 Kotlin 开发。", + val reduction = UserProfileReducer.reduce( + current = emptyProfile(), + batch = batch, + response = ProfileModelResponse( + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.NOTABLE_FACT, + content = "从事 Kotlin 开发", + confidence = ProfileConfidence.HIGH, + evidenceRefs = listOf(1), + ) ), - model = "test-model", - promptVersion = "test-prompt", - summaryMaxLength = 500, - ) - } + summary = "从事 Kotlin 开发。", + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) - assertTrue(failure.message.orEmpty().contains("目标用户")) + assertTrue(reduction.operations.isEmpty()) + assertTrue(reduction.profile.items.isEmpty()) + assertTrue(reduction.skippedOperations.single().contains("目标用户")) + assertEquals(batch.endTime, reduction.profile.cursorTime) } @Test @@ -120,12 +121,189 @@ class UserProfileReducerTest { assertEquals(1, reduction.profile.version) } + @Test + fun resolvesPromptLocalItemReferenceWithoutExposingStoredId() { + val item = existingItem() + val current = emptyProfile().copy(items = listOf(item), reliable = true) + val batch = batchOf(message(ref = 1, fromId = TARGET, text = "我现在仍然关注 Kotlin")) + + val reduction = UserProfileReducer.reduce( + current = current, + batch = batch, + response = ProfileModelResponse( + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.CONFIRM, + itemRef = "P1", + confidence = ProfileConfidence.HIGH, + evidenceRefs = listOf(1), + ) + ), + summary = "TARGET 仍然关注 Kotlin。", + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + assertEquals(item.id, reduction.profile.items.single().id) + assertEquals(ProfileConfidence.HIGH, reduction.profile.items.single().confidence) + assertEquals("该用户仍然关注 Kotlin。", reduction.profile.summary) + val prompt = ProfilePromptStore.buildUserPrompt(current, batch) + assertTrue(prompt.contains("[P1]")) + assertFalse(prompt.contains(item.id)) + } + + @Test + fun keepsComprehensiveSummaryWhenBatchOnlyConfirmsOneItem() { + val interest = existingItem() + val work = UserProfileItem( + id = "work-item", + category = ProfileCategory.NOTABLE_FACT, + content = "从事上位机开发", + confidence = ProfileConfidence.HIGH, + firstSeenAt = 70, + lastConfirmedAt = 70, + ) + val current = emptyProfile().copy( + summary = "从事上位机开发，并持续关注 Kotlin 生态。", + reliable = true, + items = listOf(interest, work), + ) + val batch = batchOf(message(ref = 1, fromId = TARGET, text = "我还在关注 Kotlin")) + + val reduction = UserProfileReducer.reduce( + current = current, + batch = batch, + response = ProfileModelResponse( + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.CONFIRM, + itemRef = "P1", + evidenceRefs = listOf(1), + ) + ), + summary = "仍在关注 Kotlin。", + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + assertEquals(current.summary, reduction.profile.summary) + } + + @Test + fun skipsUnknownPromptLocalItemReference() { + val current = emptyProfile().copy(items = listOf(existingItem()), reliable = true) + val batch = batchOf(message(ref = 1, fromId = TARGET, text = "我仍然关注 Kotlin")) + + val reduction = UserProfileReducer.reduce( + current = current, + batch = batch, + response = ProfileModelResponse( + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.CONFIRM, + itemRef = "P2", + evidenceRefs = listOf(1), + ) + ), + summary = "关注 Kotlin。", + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + assertEquals(current.items, reduction.profile.items) + assertTrue(reduction.operations.isEmpty()) + assertTrue(reduction.skippedOperations.single().contains("P2")) + } + + @Test + fun appliesSafeOperationAndSkipsInvalidEvidenceWithoutRewritingSummary() { + val current = emptyProfile().copy(summary = "原摘要") + val batch = batchOf(message(ref = 1, fromId = TARGET, text = "我平时会写 Kotlin")) + + val reduction = UserProfileReducer.reduce( + current = current, + batch = batch, + response = ProfileModelResponse( + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.NOTABLE_FACT, + content = "日常使用 Kotlin 开发", + confidence = ProfileConfidence.MEDIUM, + evidenceRefs = listOf(1), + ), + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.INTEREST, + content = "关注本地大模型", + confidence = ProfileConfidence.LOW, + evidenceRefs = listOf(323), + ), + ), + summary = "使用 Kotlin 并关注本地大模型。", + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + assertEquals(1, reduction.operations.size) + assertEquals("日常使用 Kotlin 开发", reduction.profile.items.single().content) + assertTrue(reduction.skippedOperations.single().contains("e:323")) + assertEquals("原摘要", reduction.profile.summary) + } + + @Test + fun removesTemporaryAliasesFromPersistedRelationshipText() { + val batch = batchOf(message(ref = 1, fromId = TARGET, text = "先找到下家再离职")) + + val reduction = UserProfileReducer.reduce( + current = emptyProfile(), + batch = batch, + response = ProfileModelResponse( + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.RELATIONSHIP_NOTE, + content = "TARGET 在 U1 提及离职时会建议 U1 先找到下家", + confidence = ProfileConfidence.LOW, + relatedUserAlias = "U1", + evidenceRefs = listOf(1), + ) + ), + summary = "TARGET 会给 U1 提供务实建议。", + ), + model = "test-model", + promptVersion = "test-prompt", + summaryMaxLength = 500, + ) + + assertEquals("本人在对方提及离职时会建议对方先找到下家", reduction.profile.items.single().content) + assertEquals("该用户会给其他用户提供务实建议。", reduction.profile.summary) + assertFalse(reduction.profile.items.single().content.contains("U1")) + } + private fun emptyProfile() = UserProfileSnapshot( userId = TARGET, cursorTime = 100, snapshotEndTime = 1_000, ) + private fun existingItem() = UserProfileItem( + id = "b287070d-b7c0-4d50-a18c-fa8348932048", + category = ProfileCategory.INTEREST, + content = "关注 Kotlin 开发", + confidence = ProfileConfidence.MEDIUM, + firstSeenAt = 80, + lastConfirmedAt = 80, + ) + private fun batchOf(vararg messages: ProfilePromptMessage) = ProfileHistoryBatch( userId = TARGET, startTime = 100, diff --git a/src/test/kotlin/profile/UserProfileStoreTest.kt b/src/test/kotlin/profile/UserProfileStoreTest.kt index 591c3b4..e9782a8 100644 --- a/src/test/kotlin/profile/UserProfileStoreTest.kt +++ b/src/test/kotlin/profile/UserProfileStoreTest.kt @@ -136,6 +136,14 @@ class UserProfileStoreTest { } } assertTrue("source" in columns) + val tables = connection.createStatement().use { statement -> + statement.executeQuery("SELECT name FROM sqlite_master WHERE type = 'table'").use { results -> + buildSet { + while (results.next()) add(results.getString("name")) + } + } + } + assertTrue("profile_group_cursor" in tables) val version = connection.createStatement().use { statement -> statement.executeQuery( "SELECT value FROM user_profile_meta WHERE key = 'schema_version'" @@ -144,7 +152,7 @@ class UserProfileStoreTest { results.getString(1) } } - assertEquals("2", version) + assertEquals("3", version) } } finally { UserProfileStore.close() @@ -152,6 +160,31 @@ class UserProfileStoreTest { } } + @Test + fun persistsGroupAnalysisCursor() { + val directory = Files.createTempDirectory("jchatgpt-profile-group-cursor-test-") + try { + UserProfileStore.init(directory.toFile()) + val cursor = GroupProfileCursor( + botId = 1, + groupId = 300, + cursorTime = 200, + snapshotEndTime = 1_000, + updatedAt = 123, + ) + + UserProfileStore.saveGroupCursor(cursor) + assertEquals(cursor, UserProfileStore.loadGroupCursor(1, 300)) + + val advanced = cursor.copy(cursorTime = 400, updatedAt = 456) + UserProfileStore.saveGroupCursor(advanced) + assertEquals(advanced, UserProfileStore.loadGroupCursor(1, 300)) + } finally { + UserProfileStore.close() + directory.toFile().deleteRecursively() + } + } + @Test fun commitsAllConversationProfilesInOneTransactionAndCountsUsageOnce() { val directory = Files.createTempDirectory("jchatgpt-profile-conversation-commit-test-") @@ -198,6 +231,81 @@ class UserProfileStoreTest { } } + @Test + fun commitsCompactionWithoutNewEvidenceAndReassignsExistingSupport() { + val directory = Files.createTempDirectory("jchatgpt-profile-compaction-commit-test-") + try { + UserProfileStore.init(directory.toFile()) + val batch = batchFor(100, "initial-hash") + val first = itemFor("item-1") + val second = itemFor("item-2").copy(content = "关注 Kotlin 生态") + val initialProfile = UserProfileSnapshot( + userId = 100, + summary = "关注 Kotlin。", + version = 1, + cursorTime = 200, + snapshotEndTime = 1_000, + reliable = true, + model = "test-model", + promptVersion = "test-prompt", + items = listOf(first, second), + ) + UserProfileStore.commit( + reduction = ProfileReduction( + profile = initialProfile, + operations = listOf( + applied(first, ProfileOperationAction.ADD, listOf(1)), + applied(second, ProfileOperationAction.ADD, listOf(1)), + ), + ), + batch = batch, + usage = ProfileTokenUsage(), + ) + + val merged = first.copy(content = "关注 Kotlin 及其生态") + val compactedProfile = initialProfile.copy( + summary = "关注 Kotlin 及其生态。", + version = 2, + items = listOf(merged), + ) + val plan = ProfileCompactionPlan( + reduction = ProfileReduction( + profile = compactedProfile, + operations = listOf( + applied(merged, ProfileOperationAction.UPDATE, emptyList()), + applied(second, ProfileOperationAction.DELETE, emptyList()), + ), + ), + supportReassignments = mapOf(second.id to first.id), + mergedGroups = 1, + rewrittenItems = 0, + deletedItems = 0, + ) + UserProfileStore.commitCompaction( + plan = plan, + batch = ProfileHistoryBatch( + userId = 100, + startTime = 150, + endTime = 151, + messages = emptyList(), + aliases = mapOf(100L to "TARGET"), + inputHash = "compact-hash", + ), + usage = ProfileTokenUsage(50, 10, 0), + ) + + val loaded = assertNotNull(UserProfileStore.load(100)) + assertEquals(200, loaded.cursorTime) + assertEquals(listOf(merged), loaded.items) + assertEquals(2, UserProfileStore.loadSupportStats(100).getValue(first.id).count) + assertTrue(second.id !in UserProfileStore.loadSupportStats(100)) + assertNotNull(UserProfileStore.lastRevisionAt(100, ProfileRevisionSource.COMPACTION)) + } finally { + UserProfileStore.close() + directory.toFile().deleteRecursively() + } + } + private fun batchFor(userId: Long, inputHash: String) = ProfileHistoryBatch( userId = userId, startTime = 100, @@ -231,4 +339,18 @@ class UserProfileStoreTest { firstSeenAt = 150, lastConfirmedAt = 150, ) + + private fun applied( + item: UserProfileItem, + action: ProfileOperationAction, + evidenceRefs: List, + ) = AppliedProfileOperation( + action = action, + itemId = item.id, + category = item.category, + content = item.content, + confidence = item.confidence, + relatedUserId = item.relatedUserId, + evidenceRefs = evidenceRefs, + ) }