profile: parallelize and control analysis jobs

This commit is contained in:
2026-08-03 11:55:21 +08:00
parent 2a5e7fd2f9
commit fa93d48002
16 changed files with 650 additions and 162 deletions
+7 -4
View File
@@ -57,10 +57,11 @@ AI 可以自动调用多种工具来完成复杂任务:
### 渐进式历史画像(实验) ### 渐进式历史画像(实验)
- 日常使用无需画像命令:群聊缓存会话闭合后,一次模型调用会静默归纳其中所有有实质发言的参与者 - 日常使用无需画像命令:群聊缓存会话闭合后,一次模型调用会静默归纳其中所有有实质发言的参与者
- `/jgpt profileAnalyze <userId> [batches]` - 手动推进指定用户画像,默认1批 - `/jgpt profileAnalyze <userIds> [batches]` - 分析用户画像,多个 ID 用逗号分隔
- `/jgpt profileAnalyzeGroup <groupId> [batches]` - 按群历史批量推进参与者画像,默认1批 - `/jgpt profileAnalyzeGroup <groupIds> [batches]` - 分析群画像,多个 ID 用逗号分隔
- `/jgpt profileShow <userId>` - 诊断或验收时查看已经提交的完整画像和覆盖时间 - `/jgpt profileShow <userId>` - 查看用户画像
- `/jgpt profileCompact <userId>` - 独立反思并压缩指定用户的重复、过细或低价值画像条目 - `/jgpt profileCompact [userIds]` - 压缩画像,不传 ID 时处理全部用户
- `/jgpt profileStop` - 当前批次完成后停止画像任务
## 配置文件 ## 配置文件
@@ -239,6 +240,8 @@ searchHistoryMaxRecords: 5000
好友昵称、群名片、群角色、签名、年龄等联系人公开资料只作为人物识别提示,不直接视为画像证据。画像模型 好友昵称、群名片、群角色、签名、年龄等联系人公开资料只作为人物识别提示,不直接视为画像证据。画像模型
仍必须依据对应用户本人在聊天历史中的发言,才能新增或确认长期画像条目。 仍必须依据对应用户本人在聊天历史中的发言,才能新增或确认长期画像条目。
画像发生变更时,日志会输出 `PROFILE_OPERATIONS` 供抽查。
画像结果保存在插件数据目录的 `user-profile.sqlite`,不会覆盖现有好感度或旧印象数据。实验时可把历史 画像结果保存在插件数据目录的 `user-profile.sqlite`,不会覆盖现有好感度或旧印象数据。实验时可把历史
备份配置为只读来源,例如: 备份配置为只读来源,例如:
+56 -12
View File
@@ -1,7 +1,9 @@
package top.jie65535.mirai.command package top.jie65535.mirai.command
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import net.mamoe.mirai.console.command.CommandSender import net.mamoe.mirai.console.command.CommandSender
import net.mamoe.mirai.console.command.CompositeCommand import net.mamoe.mirai.console.command.CompositeCommand
import net.mamoe.mirai.console.permission.PermissionService.Companion.cancel import net.mamoe.mirai.console.permission.PermissionService.Companion.cancel
@@ -49,12 +51,15 @@ object PluginCommands : CompositeCommand(
} }
@SubCommand @SubCommand
suspend fun CommandSender.profileAnalyze(userId: Long, batches: Int = 1) { suspend fun CommandSender.profileAnalyze(userIds: String, batches: Int = 1) {
require(batches > 0) { "batches 必须是正数" } require(batches > 0) { "batches 必须是正数" }
sendMessage("已启动用户 $userId 的画像分析,本次最多推进 $batches 个批次。") val parsedUserIds = parseProfileUserIds(userIds)
val runToken = UserProfileAnalysisService.newRunToken()
sendMessage("已启动 ${parsedUserIds.size} 个用户的画像分析,每人最多推进 $batches 个批次。")
parsedUserIds.forEach { userId ->
JChatGPT.launch { JChatGPT.launch {
try { try {
val report = UserProfileAnalysisService.analyze(userId, batches) { progress -> val report = UserProfileAnalysisService.analyze(userId, batches, runToken) { progress ->
JChatGPT.logger.info( JChatGPT.logger.info(
"PROFILE_BATCH user=$userId batch=${progress.batchIndex}/$batches " + "PROFILE_BATCH user=$userId batch=${progress.batchIndex}/$batches " +
"range=${progress.startTime}-${progress.endTime} " + "range=${progress.startTime}-${progress.endTime} " +
@@ -77,14 +82,18 @@ object PluginCommands : CompositeCommand(
} }
} }
} }
}
@SubCommand @SubCommand
suspend fun CommandSender.profileAnalyzeGroup(groupId: Long, batches: Int = 1) { suspend fun CommandSender.profileAnalyzeGroup(groupIds: String, batches: Int = 1) {
require(batches > 0) { "batches 必须是正数" } require(batches > 0) { "batches 必须是正数" }
sendMessage("已启动群 $groupId 的批量画像分析,本次最多推进 $batches 个批次。") val parsedGroupIds = parseProfileGroupIds(groupIds)
val runToken = UserProfileAnalysisService.newRunToken()
sendMessage("已启动 ${parsedGroupIds.size} 个群的画像分析,每群最多推进 $batches 个批次。")
parsedGroupIds.forEach { groupId ->
JChatGPT.launch { JChatGPT.launch {
try { try {
val report = UserProfileAnalysisService.analyzeGroup(groupId, batches) { progress -> val report = UserProfileAnalysisService.analyzeGroup(groupId, batches, runToken) { progress ->
JChatGPT.logger.info( JChatGPT.logger.info(
"PROFILE_GROUP_BATCH group=$groupId batch=${progress.batchIndex}/$batches " + "PROFILE_GROUP_BATCH group=$groupId batch=${progress.batchIndex}/$batches " +
"range=${progress.startTime}-${progress.endTime} " + "range=${progress.startTime}-${progress.endTime} " +
@@ -107,6 +116,7 @@ object PluginCommands : CompositeCommand(
} }
} }
} }
}
@SubCommand @SubCommand
suspend fun CommandSender.profileShow(userId: Long) { suspend fun CommandSender.profileShow(userId: Long) {
@@ -119,12 +129,29 @@ object PluginCommands : CompositeCommand(
} }
@SubCommand @SubCommand
suspend fun CommandSender.profileCompact(userId: Long) { suspend fun CommandSender.profileCompact(userIds: String = "") {
sendMessage("已启动用户 $userId 的画像压缩反思。") check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" }
check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
val parsedUserIds = if (userIds.isBlank()) {
withContext(Dispatchers.IO) { UserProfileStore.listUserIds() }
} else {
parseProfileUserIds(userIds)
}
if (parsedUserIds.isEmpty()) {
sendMessage("当前没有可压缩的用户画像。")
return
}
val runToken = UserProfileAnalysisService.newRunToken()
sendMessage("已启动 ${parsedUserIds.size} 个用户的画像压缩反思。")
parsedUserIds.forEach { userId ->
JChatGPT.launch { JChatGPT.launch {
try { try {
val report = UserProfileAnalysisService.compact(userId) val report = UserProfileAnalysisService.compact(userId, runToken)
sendMessage(formatProfileCompactionReport(report)) when {
report.alreadyRunning -> sendMessage("用户 $userId 已有画像压缩任务在运行。")
report.stopped -> sendMessage("用户 $userId 的画像压缩已按请求停止,未开始新的压缩轮次。")
else -> sendMessage(formatProfileCompactionReport(report))
}
} catch (cause: CancellationException) { } catch (cause: CancellationException) {
throw cause throw cause
} catch (cause: Exception) { } catch (cause: Exception) {
@@ -133,6 +160,17 @@ object PluginCommands : CompositeCommand(
} }
} }
} }
}
@SubCommand
suspend fun CommandSender.profileStop() {
val report = UserProfileAnalysisService.stopAll()
sendMessage(
"已发出画像任务停止请求;当前检测到 ${report.totalTasks} 个任务" +
"(用户分析 ${report.userTasks},群分析 ${report.groupTasks},压缩 ${report.compactionTasks})。" +
"当前轮次会正常完成,之后不再开始新一轮。"
)
}
@SubCommand @SubCommand
suspend fun CommandSender.skills() { suspend fun CommandSender.skills() {
@@ -325,7 +363,7 @@ object PluginCommands : CompositeCommand(
"${formatNumber(report.usage.completionTokens)},缓存命中 " + "${formatNumber(report.usage.completionTokens)},缓存命中 " +
formatNumber(report.usage.cachedTokens) formatNumber(report.usage.cachedTokens)
) )
appendLine("状态:${if (report.caughtUp) "已追平当前快照" else "可继续推进"}") appendLine("状态:${profileAnalysisStatus(report.caughtUp, report.stopped)}")
append(formatProfile(checkNotNull(report.profile))) append(formatProfile(checkNotNull(report.profile)))
}.trim() }.trim()
@@ -341,9 +379,15 @@ object PluginCommands : CompositeCommand(
formatNumber(report.usage.cachedTokens) formatNumber(report.usage.cachedTokens)
) )
appendLine("群历史覆盖至 ${formatProfileTime(report.cursorTime)}") appendLine("群历史覆盖至 ${formatProfileTime(report.cursorTime)}")
append("状态:${if (report.caughtUp) "已追平当前快照" else "可继续推进"}") append("状态:${profileAnalysisStatus(report.caughtUp, report.stopped)}")
}.trim() }.trim()
private fun profileAnalysisStatus(caughtUp: Boolean, stopped: Boolean): String = when {
caughtUp -> "已追平当前快照"
stopped -> "已按请求停止,可继续推进"
else -> "可继续推进"
}
private fun formatProfileCompactionReport(report: ProfileCompactionReport): String = buildString { private fun formatProfileCompactionReport(report: ProfileCompactionReport): String = buildString {
appendLine( appendLine(
"画像压缩完成:${report.beforeItems} -> ${report.afterItems} 条," + "画像压缩完成:${report.beforeItems} -> ${report.afterItems} 条," +
@@ -0,0 +1,15 @@
package top.jie65535.mirai.command
internal fun parseProfileGroupIds(raw: String): List<Long> = parseProfileIds(raw, "群号")
internal fun parseProfileUserIds(raw: String): List<Long> = parseProfileIds(raw, "用户号")
private fun parseProfileIds(raw: String, label: String): List<Long> {
val ids = raw.split(',', '', ';', '')
.map(String::trim)
.filter(String::isNotEmpty)
.map { value -> value.toLongOrNull()?.takeIf { it > 0 } ?: error("无效$label: $value") }
.distinct()
require(ids.isNotEmpty()) { "至少需要一个$label" }
return ids
}
@@ -0,0 +1,28 @@
package top.jie65535.mirai.profile
import java.util.concurrent.atomic.AtomicLong
class ProfileAnalysisRunToken internal constructor(
internal val generation: Long,
)
data class ProfileAnalysisStopReport(
val userTasks: Int,
val groupTasks: Int,
val compactionTasks: Int,
) {
val totalTasks: Int
get() = userTasks + groupTasks + compactionTasks
}
internal class ProfileAnalysisRunGate {
private val generation = AtomicLong()
fun newToken(): ProfileAnalysisRunToken = ProfileAnalysisRunToken(generation.get())
fun stopCurrentRuns() {
generation.incrementAndGet()
}
fun canContinue(token: ProfileAnalysisRunToken): Boolean = token.generation == generation.get()
}
@@ -79,4 +79,6 @@ data class ProfileCompactionReport(
val skippedOperations: Int, val skippedOperations: Int,
val usage: ProfileTokenUsage, val usage: ProfileTokenUsage,
val profile: UserProfileSnapshot, val profile: UserProfileSnapshot,
val alreadyRunning: Boolean = false,
val stopped: Boolean = false,
) )
@@ -0,0 +1,31 @@
package top.jie65535.mirai.profile
import top.jie65535.mirai.JChatGPT
internal object ProfileOperationLogger {
fun log(context: String, reductions: Collection<ProfileReduction>) {
format(context, reductions)?.let(JChatGPT.logger::info)
}
internal fun format(context: String, reductions: Collection<ProfileReduction>): String? {
val operations = reductions.flatMap { reduction ->
reduction.operations.map { operation -> reduction.profile.userId to operation }
}
if (operations.isEmpty()) return null
return buildString {
append("PROFILE_OPERATIONS ").append(context)
.append(" operations=").appendLine(operations.size)
operations.forEach { (userId, operation) ->
append("- user=").append(userId)
.append(" action=").append(operation.action)
.append(" category=").append(operation.category.name.lowercase())
.append(" confidence=").append(operation.confidence.name.lowercase())
operation.relatedUserId?.let { append(" related=").append(it) }
append(" content=").appendLine(operation.content.normalized())
}
}.trimEnd()
}
private fun String.normalized(): String = trim().replace(Regex("\\s+"), " ")
}
@@ -0,0 +1,47 @@
package top.jie65535.mirai.profile
import kotlinx.coroutines.sync.Mutex
import java.util.concurrent.ConcurrentHashMap
internal class ProfileUserLockManager {
private val entries = ConcurrentHashMap<Long, LockEntry>()
suspend fun <T> withUserLocks(userIds: Collection<Long>, block: suspend () -> T): T {
val reserved = userIds.asSequence()
.distinct()
.sorted()
.map { userId -> userId to reserve(userId) }
.toList()
val acquired = mutableListOf<LockEntry>()
return try {
reserved.forEach { (_, entry) ->
entry.mutex.lock()
acquired += entry
}
block()
} finally {
acquired.asReversed().forEach { entry -> entry.mutex.unlock() }
reserved.forEach { (userId, entry) -> release(userId, entry) }
}
}
internal val activeLockCount: Int
get() = entries.size
private fun reserve(userId: Long): LockEntry = entries.compute(userId) { _, current ->
(current ?: LockEntry()).also { it.references++ }
} ?: error("无法创建用户画像锁: $userId")
private fun release(userId: Long, expected: LockEntry) {
entries.computeIfPresent(userId) { _, current ->
check(current === expected) { "用户画像锁状态不一致: $userId" }
current.references--
current.takeIf { it.references > 0 }
}
}
private class LockEntry(
val mutex: Mutex = Mutex(),
var references: Int = 0,
)
}
@@ -2,8 +2,6 @@ package top.jie65535.mirai.profile
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import top.jie65535.mirai.JChatGPT import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig import top.jie65535.mirai.config.PluginConfig
@@ -16,11 +14,26 @@ import java.util.concurrent.ConcurrentHashMap
object UserProfileAnalysisService { object UserProfileAnalysisService {
private val runningUsers = ConcurrentHashMap.newKeySet<Long>() private val runningUsers = ConcurrentHashMap.newKeySet<Long>()
private val runningGroups = ConcurrentHashMap.newKeySet<Long>() private val runningGroups = ConcurrentHashMap.newKeySet<Long>()
private val concurrencyLimiter = Semaphore(1) private val runningCompactions = ConcurrentHashMap.newKeySet<Long>()
private val userLocks = ProfileUserLockManager()
private val runGate = ProfileAnalysisRunGate()
fun newRunToken(): ProfileAnalysisRunToken = runGate.newToken()
fun stopAll(): ProfileAnalysisStopReport {
val report = ProfileAnalysisStopReport(
userTasks = runningUsers.size,
groupTasks = runningGroups.size,
compactionTasks = runningCompactions.size,
)
runGate.stopCurrentRuns()
return report
}
suspend fun analyze( suspend fun analyze(
userId: Long, userId: Long,
maxBatches: Int, maxBatches: Int,
runToken: ProfileAnalysisRunToken = newRunToken(),
onProgress: suspend (ProfileAnalysisProgress) -> Unit = {}, onProgress: suspend (ProfileAnalysisProgress) -> Unit = {},
): ProfileAnalysisReport { ): ProfileAnalysisReport {
require(userId > 0) { "userId 必须是正数" } require(userId > 0) { "userId 必须是正数" }
@@ -43,35 +56,37 @@ object UserProfileAnalysisService {
} }
try { try {
return concurrencyLimiter.withPermit { return userLocks.withUserLocks(listOf(userId)) {
analyzeExclusive(userId, maxBatches, onProgress) analyzeExclusive(userId, maxBatches, runToken, onProgress)
} }
} finally { } finally {
runningUsers.remove(userId) runningUsers.remove(userId)
} }
} }
suspend fun compact(userId: Long): ProfileCompactionReport { suspend fun compact(
userId: Long,
runToken: ProfileAnalysisRunToken = newRunToken(),
): ProfileCompactionReport {
require(userId > 0) { "userId 必须是正数" } require(userId > 0) { "userId 必须是正数" }
check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" } check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" }
check(UserProfileStore.isAvailable) { "用户画像数据库不可用" } check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
return concurrencyLimiter.withPermit { if (!runningCompactions.add(userId)) {
val profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) } val profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) }
?: throw IllegalArgumentException("用户 $userId 尚无画像") ?: throw IllegalArgumentException("用户 $userId 尚无画像")
return unchangedCompactionReport(profile, alreadyRunning = true)
}
try {
return userLocks.withUserLocks(listOf(userId)) locked@{
val profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) }
?: throw IllegalArgumentException("用户 $userId 尚无画像")
if (!runGate.canContinue(runToken)) {
return@locked unchangedCompactionReport(profile, stopped = true)
}
if (profile.items.isEmpty()) { if (profile.items.isEmpty()) {
return@withPermit ProfileCompactionReport( return@locked unchangedCompactionReport(profile)
userId = userId,
beforeItems = profile.items.size,
afterItems = profile.items.size,
mergedGroups = 0,
rewrittenItems = 0,
deletedItems = 0,
summaryChanged = false,
skippedOperations = 0,
usage = ProfileTokenUsage(),
profile = profile,
)
} }
val endpoint = checkNotNull(LargeLanguageModels.profile) { "画像分析模型未配置" } val endpoint = checkNotNull(LargeLanguageModels.profile) { "画像分析模型未配置" }
@@ -83,6 +98,10 @@ object UserProfileAnalysisService {
withContext(Dispatchers.IO) { withContext(Dispatchers.IO) {
UserProfileStore.commitCompaction(plan, batch, result.usage) UserProfileStore.commitCompaction(plan, batch, result.usage)
} }
ProfileOperationLogger.log(
context = "source=COMPACTION user=$userId",
reductions = listOf(plan.reduction),
)
} }
ProfileCompactionReport( ProfileCompactionReport(
userId = userId, userId = userId,
@@ -97,11 +116,15 @@ object UserProfileAnalysisService {
profile = plan.reduction.profile, profile = plan.reduction.profile,
) )
} }
} finally {
runningCompactions.remove(userId)
}
} }
suspend fun analyzeGroup( suspend fun analyzeGroup(
groupId: Long, groupId: Long,
maxBatches: Int, maxBatches: Int,
runToken: ProfileAnalysisRunToken = newRunToken(),
onProgress: suspend (GroupProfileAnalysisProgress) -> Unit = {}, onProgress: suspend (GroupProfileAnalysisProgress) -> Unit = {},
): GroupProfileAnalysisReport { ): GroupProfileAnalysisReport {
require(groupId > 0) { "groupId 必须是正数" } require(groupId > 0) { "groupId 必须是正数" }
@@ -127,9 +150,7 @@ object UserProfileAnalysisService {
} }
try { try {
return concurrencyLimiter.withPermit { return analyzeGroupExclusive(groupId, maxBatches, runToken, onProgress)
analyzeGroupExclusive(groupId, maxBatches, onProgress)
}
} finally { } finally {
runningGroups.remove(groupId) runningGroups.remove(groupId)
} }
@@ -138,6 +159,7 @@ object UserProfileAnalysisService {
private suspend fun analyzeExclusive( private suspend fun analyzeExclusive(
userId: Long, userId: Long,
maxBatches: Int, maxBatches: Int,
runToken: ProfileAnalysisRunToken,
onProgress: suspend (ProfileAnalysisProgress) -> Unit, onProgress: suspend (ProfileAnalysisProgress) -> Unit,
): ProfileAnalysisReport { ): ProfileAnalysisReport {
val endpoint = checkNotNull(LargeLanguageModels.profile) { val endpoint = checkNotNull(LargeLanguageModels.profile) {
@@ -166,7 +188,7 @@ object UserProfileAnalysisService {
var totalUsage = ProfileTokenUsage() var totalUsage = ProfileTokenUsage()
var caughtUp = false var caughtUp = false
while (processedBatches < maxBatches) { while (processedBatches < maxBatches && runGate.canContinue(runToken)) {
val batch = withContext(Dispatchers.IO) { val batch = withContext(Dispatchers.IO) {
reader.loadNextBatch( reader.loadNextBatch(
userId = userId, userId = userId,
@@ -195,6 +217,10 @@ object UserProfileAnalysisService {
source = ProfileRevisionSource.BACKFILL, source = ProfileRevisionSource.BACKFILL,
) )
} }
ProfileOperationLogger.log(
context = "source=BACKFILL batch=[${batch.startTime},${batch.endTime})",
reductions = listOf(reduction),
)
profile = reduction.profile profile = reduction.profile
processedBatches++ processedBatches++
processedMessages += batch.messages.size processedMessages += batch.messages.size
@@ -215,6 +241,7 @@ object UserProfileAnalysisService {
} }
if (!caughtUp && profile.cursorTime >= profile.snapshotEndTime) caughtUp = true if (!caughtUp && profile.cursorTime >= profile.snapshotEndTime) caughtUp = true
val stopped = processedBatches < maxBatches && !caughtUp && !runGate.canContinue(runToken)
return ProfileAnalysisReport( return ProfileAnalysisReport(
userId = userId, userId = userId,
processedBatches = processedBatches, processedBatches = processedBatches,
@@ -224,12 +251,14 @@ object UserProfileAnalysisService {
usage = totalUsage, usage = totalUsage,
profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) } ?: profile, profile = withContext(Dispatchers.IO) { UserProfileStore.load(userId) } ?: profile,
caughtUp = caughtUp, caughtUp = caughtUp,
stopped = stopped,
) )
} }
private suspend fun analyzeGroupExclusive( private suspend fun analyzeGroupExclusive(
groupId: Long, groupId: Long,
maxBatches: Int, maxBatches: Int,
runToken: ProfileAnalysisRunToken,
onProgress: suspend (GroupProfileAnalysisProgress) -> Unit, onProgress: suspend (GroupProfileAnalysisProgress) -> Unit,
): GroupProfileAnalysisReport { ): GroupProfileAnalysisReport {
val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(resolveHistoryFile()) } val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(resolveHistoryFile()) }
@@ -259,7 +288,7 @@ object UserProfileAnalysisService {
var totalUsage = ProfileTokenUsage() var totalUsage = ProfileTokenUsage()
var caughtUp = cursor.cursorTime >= cursor.snapshotEndTime var caughtUp = cursor.cursorTime >= cursor.snapshotEndTime
while (processedBatches < maxBatches && !caughtUp) { while (processedBatches < maxBatches && !caughtUp && runGate.canContinue(runToken)) {
val batch = withContext(Dispatchers.IO) { val batch = withContext(Dispatchers.IO) {
reader.loadNextConversationBatch( reader.loadNextConversationBatch(
botId = cursor.botId, botId = cursor.botId,
@@ -287,6 +316,7 @@ object UserProfileAnalysisService {
retryMax = PluginConfig.profileRetryMax, retryMax = PluginConfig.profileRetryMax,
summaryMaxLength = PluginConfig.profileSummaryMaxLength, summaryMaxLength = PluginConfig.profileSummaryMaxLength,
onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) }, onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) },
onCommittedOperations = ProfileOperationLogger::log,
) )
cursor = cursor.copy( cursor = cursor.copy(
cursorTime = batch.endTime, cursorTime = batch.endTime,
@@ -328,6 +358,7 @@ object UserProfileAnalysisService {
cursorTime = cursor.cursorTime, cursorTime = cursor.cursorTime,
snapshotEndTime = cursor.snapshotEndTime, snapshotEndTime = cursor.snapshotEndTime,
caughtUp = caughtUp, caughtUp = caughtUp,
stopped = processedBatches < maxBatches && !caughtUp && !runGate.canContinue(runToken),
) )
} }
@@ -342,7 +373,6 @@ object UserProfileAnalysisService {
check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" } check(PluginConfig.profileEnabled) { "历史用户画像分析未启用" }
check(UserProfileStore.isAvailable) { "用户画像数据库不可用" } check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
return concurrencyLimiter.withPermit {
val endpoint = checkNotNull(LargeLanguageModels.profile) { "画像分析模型未配置" } val endpoint = checkNotNull(LargeLanguageModels.profile) { "画像分析模型未配置" }
val model: ConversationProfileModel = ProfileModelClient(endpoint) val model: ConversationProfileModel = ProfileModelClient(endpoint)
val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(resolveLiveHistoryFile()) } val reader = withContext(Dispatchers.IO) { ProfileHistoryReader(resolveLiveHistoryFile()) }
@@ -355,17 +385,17 @@ object UserProfileAnalysisService {
messageLimit = PluginConfig.profileAutoConversationMessageLimit.coerceIn(20, 500), messageLimit = PluginConfig.profileAutoConversationMessageLimit.coerceIn(20, 500),
maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80), maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80),
) )
} ?: return@withPermit null } ?: return null
analyzeConversationBatch( return analyzeConversationBatch(
batch = batch, batch = batch,
minAuthoredTextChars = minAuthoredTextChars, minAuthoredTextChars = minAuthoredTextChars,
model = model, model = model,
retryMax = PluginConfig.profileRetryMax, retryMax = PluginConfig.profileRetryMax,
summaryMaxLength = PluginConfig.profileSummaryMaxLength, summaryMaxLength = PluginConfig.profileSummaryMaxLength,
onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) }, onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) },
onCommittedOperations = ProfileOperationLogger::log,
) )
} }
}
internal suspend fun analyzeConversationBatch( internal suspend fun analyzeConversationBatch(
batch: ConversationProfileBatch, batch: ConversationProfileBatch,
@@ -374,14 +404,16 @@ object UserProfileAnalysisService {
retryMax: Int, retryMax: Int,
summaryMaxLength: Int, summaryMaxLength: Int,
onRetryFailure: (String, Throwable) -> Unit = { _, _ -> }, onRetryFailure: (String, Throwable) -> Unit = { _, _ -> },
onCommittedOperations: (String, Collection<ProfileReduction>) -> Unit = { _, _ -> },
): ConversationProfileAnalysisReport? { ): ConversationProfileAnalysisReport? {
check(UserProfileStore.isAvailable) { "用户画像数据库不可用" } check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
val eligibleUserIds = batch.authoredTextCharsByUser val eligibleUserIds = batch.authoredTextCharsByUser
.filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) } .filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) }
.keys .keys
if (eligibleUserIds.isEmpty()) return null if (eligibleUserIds.isEmpty()) return null
return userLocks.withUserLocks(eligibleUserIds) locked@{
if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) { if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) {
return null return@locked null
} }
val profiles = withContext(Dispatchers.IO) { val profiles = withContext(Dispatchers.IO) {
eligibleUserIds.associateWith { userId -> eligibleUserIds.associateWith { userId ->
@@ -408,7 +440,12 @@ object UserProfileAnalysisService {
usage = result.usage, usage = result.usage,
) )
} }
return ConversationProfileAnalysisReport( onCommittedOperations(
"source=CONVERSATION bot=${batch.botId} group=${batch.groupId} " +
"batch=[${batch.startTime},${batch.endTime})",
reductions,
)
ConversationProfileAnalysisReport(
analyzedUsers = eligibleUserIds.size, analyzedUsers = eligibleUserIds.size,
processedMessages = batch.messages.size, processedMessages = batch.messages.size,
appliedOperations = reductions.sumOf { it.operations.size }, appliedOperations = reductions.sumOf { it.operations.size },
@@ -416,6 +453,7 @@ object UserProfileAnalysisService {
usage = result.usage, usage = result.usage,
) )
} }
}
private suspend fun analyzeConversationWithRetry( private suspend fun analyzeConversationWithRetry(
model: ConversationProfileModel, model: ConversationProfileModel,
@@ -586,6 +624,25 @@ object UserProfileAnalysisService {
caughtUp = true, caughtUp = true,
) )
private fun unchangedCompactionReport(
profile: UserProfileSnapshot,
alreadyRunning: Boolean = false,
stopped: Boolean = false,
) = ProfileCompactionReport(
userId = profile.userId,
beforeItems = profile.items.size,
afterItems = profile.items.size,
mergedGroups = 0,
rewrittenItems = 0,
deletedItems = 0,
summaryChanged = false,
skippedOperations = 0,
usage = ProfileTokenUsage(),
profile = profile,
alreadyRunning = alreadyRunning,
stopped = stopped,
)
private operator fun ProfileTokenUsage.plus(other: ProfileTokenUsage) = ProfileTokenUsage( private operator fun ProfileTokenUsage.plus(other: ProfileTokenUsage) = ProfileTokenUsage(
promptTokens = promptTokens + other.promptTokens, promptTokens = promptTokens + other.promptTokens,
completionTokens = completionTokens + other.completionTokens, completionTokens = completionTokens + other.completionTokens,
@@ -261,6 +261,7 @@ data class GroupProfileAnalysisReport(
val snapshotEndTime: Int, val snapshotEndTime: Int,
val caughtUp: Boolean, val caughtUp: Boolean,
val alreadyRunning: Boolean = false, val alreadyRunning: Boolean = false,
val stopped: Boolean = false,
) )
data class ProfileAnalysisReport( data class ProfileAnalysisReport(
@@ -273,4 +274,5 @@ data class ProfileAnalysisReport(
val profile: UserProfileSnapshot?, val profile: UserProfileSnapshot?,
val caughtUp: Boolean, val caughtUp: Boolean,
val alreadyRunning: Boolean = false, val alreadyRunning: Boolean = false,
val stopped: Boolean = false,
) )
@@ -98,6 +98,19 @@ object UserProfileStore {
} }
} }
fun listUserIds(): List<Long> {
check(initialized) { "用户画像数据库尚未初始化" }
return openReadConnection().use { connection ->
connection.prepareStatement("SELECT user_id FROM user_profile ORDER BY user_id").use { statement ->
statement.executeQuery().use { results ->
buildList {
while (results.next()) add(results.getLong("user_id"))
}
}
}
}
}
fun loadSupportStats(userId: Long): Map<String, ProfileItemSupportStats> { fun loadSupportStats(userId: Long): Map<String, ProfileItemSupportStats> {
check(initialized) { "用户画像数据库尚未初始化" } check(initialized) { "用户画像数据库尚未初始化" }
return openReadConnection().use { connection -> return openReadConnection().use { connection ->
@@ -0,0 +1,30 @@
package top.jie65535.mirai.command
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
class ProfileCommandArgumentsTest {
@Test
fun parsesAndDeduplicatesMultipleGroupIds() {
assertEquals(
listOf(111L, 222L, 333L),
parseProfileGroupIds("111, 222333;111"),
)
}
@Test
fun keepsSingleGroupCommandCompatible() {
assertEquals(listOf(818800431L), parseProfileGroupIds("818800431"))
}
@Test
fun parsesMultipleUserIds() {
assertEquals(listOf(100L, 200L), parseProfileUserIds("100,200,100"))
}
@Test
fun rejectsInvalidGroupId() {
assertFailsWith<IllegalStateException> { parseProfileGroupIds("111,abc") }
}
}
@@ -0,0 +1,18 @@
package top.jie65535.mirai.profile
import kotlin.test.Test
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class ProfileAnalysisRunGateTest {
@Test
fun stopInvalidatesExistingTokensButNotFutureRuns() {
val gate = ProfileAnalysisRunGate()
val existing = gate.newToken()
gate.stopCurrentRuns()
assertFalse(gate.canContinue(existing))
assertTrue(gate.canContinue(gate.newToken()))
}
}
@@ -0,0 +1,59 @@
package top.jie65535.mirai.profile
import kotlin.test.Test
import kotlin.test.assertContains
import kotlin.test.assertFalse
import kotlin.test.assertNull
class ProfileOperationLoggerTest {
@Test
fun formatsOnlyAppliedOperations() {
val reduction = ProfileReduction(
profile = UserProfileSnapshot(
userId = 100,
summary = "不应打印的摘要",
cursorTime = 0,
snapshotEndTime = 0,
),
operations = listOf(
AppliedProfileOperation(
action = ProfileOperationAction.ADD,
itemId = "item-1",
category = ProfileCategory.INTEREST,
content = "关注 Kotlin\n开发",
confidence = ProfileConfidence.MEDIUM,
relatedUserId = null,
evidenceRefs = listOf(1),
),
AppliedProfileOperation(
action = ProfileOperationAction.DELETE,
itemId = "item-2",
category = ProfileCategory.NOTABLE_FACT,
content = "已经过期的信息",
confidence = ProfileConfidence.LOW,
relatedUserId = null,
evidenceRefs = listOf(2),
),
),
)
val output = ProfileOperationLogger.format("source=CONVERSATION group=10", listOf(reduction))
assertContains(checkNotNull(output), "operations=2")
assertContains(output, "user=100 action=ADD category=interest confidence=medium content=关注 Kotlin 开发")
assertContains(output, "action=DELETE")
assertFalse(output.contains("不应打印的摘要"))
assertFalse(output.contains("item-1"))
assertFalse(output.contains("evidence"))
}
@Test
fun omitsBatchWhenThereAreNoOperations() {
val reduction = ProfileReduction(
profile = UserProfileSnapshot(userId = 100, cursorTime = 0, snapshotEndTime = 0),
operations = emptyList(),
)
assertNull(ProfileOperationLogger.format("source=BACKFILL", listOf(reduction)))
}
}
@@ -0,0 +1,43 @@
package top.jie65535.mirai.profile
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.async
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import kotlin.test.Test
import kotlin.test.assertEquals
class ProfileUserLockManagerTest {
@Test
fun cancellationWhileWaitingReleasesAlreadyAcquiredUserLocks() = runBlocking {
val locks = ProfileUserLockManager()
val secondUserLocked = CompletableDeferred<Unit>()
val releaseSecondUser = CompletableDeferred<Unit>()
coroutineScope {
val holder = async {
locks.withUserLocks(listOf(2)) {
secondUserLocked.complete(Unit)
releaseSecondUser.await()
}
}
secondUserLocked.await()
val waiter = async {
locks.withUserLocks(listOf(1, 2)) { error("已取消任务不应进入临界区") }
}
delay(50)
waiter.cancelAndJoin()
withTimeout(1_000) {
locks.withUserLocks(listOf(1)) { }
}
releaseSecondUser.complete(Unit)
holder.await()
}
assertEquals(0, locks.activeLockCount)
}
}
@@ -1,10 +1,16 @@
package top.jie65535.mirai.profile package top.jie65535.mirai.profile
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.async
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.withTimeoutOrNull
import net.mamoe.mirai.message.data.MessageSourceKind import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatMessageRecord import top.jie65535.mirai.data.ChatMessageRecord
import java.io.IOException import java.io.IOException
import java.nio.file.Files import java.nio.file.Files
import java.util.concurrent.atomic.AtomicInteger
import kotlin.test.Test import kotlin.test.Test
import kotlin.test.assertEquals import kotlin.test.assertEquals
import kotlin.test.assertFailsWith import kotlin.test.assertFailsWith
@@ -88,6 +94,69 @@ class UserProfileAnalysisServiceTest {
assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH)) assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH))
} }
@Test
fun analyzesDisjointGroupsConcurrently() = withProfileStore {
val entered = AtomicInteger()
val bothEntered = CompletableDeferred<Unit>()
val release = CompletableDeferred<Unit>()
fun concurrentModel(content: String) = InspectingConversationProfileModel {
if (entered.incrementAndGet() == 2) bothEntered.complete(Unit)
release.await()
result(responseFor("U1", content, evidenceRef = 1))
}
coroutineScope {
val first = async {
analyze(singleUserBatch(USER_A, 10, "parallel-a"), concurrentModel("用户 A 的信息"))
}
val second = async {
analyze(singleUserBatch(USER_B, 20, "parallel-b"), concurrentModel("用户 B 的信息"))
}
withTimeout(1_000) { bothEntered.await() }
release.complete(Unit)
assertNotNull(first.await())
assertNotNull(second.await())
}
}
@Test
fun sharedUserBatchWaitsAndLoadsLatestCommittedProfile() = withProfileStore {
val firstEntered = CompletableDeferred<Unit>()
val releaseFirst = CompletableDeferred<Unit>()
val secondStarted = CompletableDeferred<Unit>()
val secondEntered = CompletableDeferred<Unit>()
val firstModel = InspectingConversationProfileModel { profiles ->
assertEquals(0, profiles.getValue(USER_A).version)
firstEntered.complete(Unit)
releaseFirst.await()
result(responseFor("U1", "第一群归纳的信息", evidenceRef = 1))
}
val secondModel = InspectingConversationProfileModel { profiles ->
assertEquals(1, profiles.getValue(USER_A).version)
secondEntered.complete(Unit)
result(responseFor("U1", "第二群归纳的信息", evidenceRef = 1))
}
coroutineScope {
val first = async { analyze(singleUserBatch(USER_A, 10, "shared-a"), firstModel) }
firstEntered.await()
val second = async {
secondStarted.complete(Unit)
analyze(singleUserBatch(USER_A, 20, "shared-b"), secondModel)
}
secondStarted.await()
assertNull(withTimeoutOrNull(100) { secondEntered.await() })
releaseFirst.complete(Unit)
assertNotNull(first.await())
withTimeout(1_000) { secondEntered.await() }
assertNotNull(second.await())
}
val profile = assertNotNull(UserProfileStore.load(USER_A))
assertEquals(2, profile.version)
assertEquals(2, profile.items.size)
}
private suspend fun analyze( private suspend fun analyze(
batch: ConversationProfileBatch, batch: ConversationProfileBatch,
model: ConversationProfileModel, model: ConversationProfileModel,
@@ -145,11 +214,25 @@ class UserProfileAnalysisServiceTest {
inputHash = INPUT_HASH, inputHash = INPUT_HASH,
) )
private fun message(ref: Int, fromId: Long, text: String) = ProfilePromptMessage( private fun singleUserBatch(
userId: Long,
groupId: Long,
inputHash: String,
) = ConversationProfileBatch(
botId = BOT,
groupId = groupId,
startTime = 100,
endTime = 200,
messages = listOf(message(1, userId, "用于并发画像测试的信息", groupId)),
aliases = mapOf(BOT to "BOT", userId to "U1"),
inputHash = inputHash,
)
private fun message(ref: Int, fromId: Long, text: String, groupId: Long = GROUP) = ProfilePromptMessage(
record = ChatMessageRecord( record = ChatMessageRecord(
botId = BOT, botId = BOT,
fromId = fromId, fromId = fromId,
targetId = GROUP, targetId = groupId,
ids = null, ids = null,
internalIds = null, internalIds = null,
time = 120 + ref, time = 120 + ref,
@@ -190,6 +273,18 @@ class UserProfileAnalysisServiceTest {
} }
} }
private class InspectingConversationProfileModel(
private val behavior: suspend (Map<Long, UserProfileSnapshot>) -> ConversationProfileModelResult,
) : ConversationProfileModel {
override val modelName: String = "inspecting-profile-model"
override suspend fun analyzeConversation(
profiles: Map<Long, UserProfileSnapshot>,
batch: ConversationProfileBatch,
eligibleUserIds: Set<Long>,
): ConversationProfileModelResult = behavior(profiles)
}
companion object { companion object {
private const val BOT = 1L private const val BOT = 1L
private const val GROUP = 10L private const val GROUP = 10L
@@ -210,6 +210,7 @@ class UserProfileStoreTest {
assertNotNull(UserProfileStore.load(100)) assertNotNull(UserProfileStore.load(100))
assertNotNull(UserProfileStore.load(200)) assertNotNull(UserProfileStore.load(200))
assertEquals(listOf(100L, 200L), UserProfileStore.listUserIds())
assertTrue(UserProfileStore.isConversationProcessed("shared-conversation-hash")) assertTrue(UserProfileStore.isConversationProcessed("shared-conversation-hash"))
val database = directory.resolve("user-profile.sqlite") val database = directory.resolve("user-profile.sqlite")
DriverManager.getConnection("jdbc:sqlite:${database.toAbsolutePath()}").use { connection -> DriverManager.getConnection("jdbc:sqlite:${database.toAbsolutePath()}").use { connection ->