From aa67305d802599dc2353b977871db0b54baa1266 Mon Sep 17 00:00:00 2001 From: jie65535 Date: Sun, 2 Aug 2026 22:18:17 +0800 Subject: [PATCH] test: cover automatic profile analysis Exercise multi-user commits, attribution failures, model retries, deduplication, and eligibility gates with a deterministic fake model. Move the real-model experiment to an opt-in live test task. --- build.gradle.kts | 17 +- .../profile/UserProfileAnalysisService.kt | 89 +++++--- .../ConversationProfileLiveExperiment.kt | 6 +- .../profile/UserProfileAnalysisServiceTest.kt | 198 ++++++++++++++++++ 4 files changed, 276 insertions(+), 34 deletions(-) create mode 100644 src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 00365aa..a3b4f04 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,5 @@ +import org.gradle.api.tasks.testing.Test + plugins { val kotlinVersion = "2.0.20" kotlin("jvm") version kotlinVersion @@ -47,5 +49,18 @@ dependencies { } tasks.test { - useJUnitPlatform() + useJUnitPlatform { + excludeTags("live") + } +} + +tasks.register("liveProfileTest") { + group = "verification" + description = "Runs the opt-in profile experiment against a configured live model." + testClassesDirs = tasks.test.get().testClassesDirs + classpath = tasks.test.get().classpath + useJUnitPlatform { + includeTags("live") + } + shouldRunAfter(tasks.test) } diff --git a/src/main/kotlin/profile/UserProfileAnalysisService.kt b/src/main/kotlin/profile/UserProfileAnalysisService.kt index d35b08c..2242ae1 100644 --- a/src/main/kotlin/profile/UserProfileAnalysisService.kt +++ b/src/main/kotlin/profile/UserProfileAnalysisService.kt @@ -161,42 +161,64 @@ object UserProfileAnalysisService { maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80), ) } ?: return@withPermit null - val eligibleUserIds = batch.authoredTextCharsByUser - .filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) } - .keys - if (eligibleUserIds.isEmpty()) return@withPermit null - if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) { - return@withPermit null - } - val profiles = withContext(Dispatchers.IO) { - eligibleUserIds.associateWith { userId -> - UserProfileStore.load(userId) ?: UserProfileSnapshot( - userId = userId, - cursorTime = 0, - snapshotEndTime = 0, - ) - } - } - - val (result, reductions) = analyzeConversationWithRetry( - model = model, - profiles = profiles, + analyzeConversationBatch( batch = batch, - eligibleUserIds = eligibleUserIds, + minAuthoredTextChars = minAuthoredTextChars, + model = model, + retryMax = PluginConfig.profileRetryMax, + summaryMaxLength = PluginConfig.profileSummaryMaxLength, + onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) }, ) - withContext(Dispatchers.IO) { - UserProfileStore.commitConversation( - reductions = reductions.map { reduction -> reduction to batch.forUser(reduction.profile.userId) }, - usage = result.usage, + } + } + + internal suspend fun analyzeConversationBatch( + batch: ConversationProfileBatch, + minAuthoredTextChars: Int, + model: ConversationProfileModel, + retryMax: Int, + summaryMaxLength: Int, + onRetryFailure: (String, Throwable) -> Unit = { _, _ -> }, + ): ConversationProfileAnalysisReport? { + check(UserProfileStore.isAvailable) { "用户画像数据库不可用" } + val eligibleUserIds = batch.authoredTextCharsByUser + .filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) } + .keys + if (eligibleUserIds.isEmpty()) return null + if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) { + return null + } + val profiles = withContext(Dispatchers.IO) { + eligibleUserIds.associateWith { userId -> + UserProfileStore.load(userId) ?: UserProfileSnapshot( + userId = userId, + cursorTime = 0, + snapshotEndTime = 0, ) } - ConversationProfileAnalysisReport( - analyzedUsers = eligibleUserIds.size, - processedMessages = batch.messages.size, - appliedOperations = reductions.sumOf { it.operations.size }, + } + + val (result, reductions) = analyzeConversationWithRetry( + model = model, + profiles = profiles, + batch = batch, + eligibleUserIds = eligibleUserIds, + retryMax = retryMax, + summaryMaxLength = summaryMaxLength, + onRetryFailure = onRetryFailure, + ) + withContext(Dispatchers.IO) { + UserProfileStore.commitConversation( + reductions = reductions.map { reduction -> reduction to batch.forUser(reduction.profile.userId) }, usage = result.usage, ) } + return ConversationProfileAnalysisReport( + analyzedUsers = eligibleUserIds.size, + processedMessages = batch.messages.size, + appliedOperations = reductions.sumOf { it.operations.size }, + usage = result.usage, + ) } private suspend fun analyzeConversationWithRetry( @@ -204,8 +226,11 @@ object UserProfileAnalysisService { profiles: Map, batch: ConversationProfileBatch, eligibleUserIds: Set, + retryMax: Int, + summaryMaxLength: Int, + onRetryFailure: (String, Throwable) -> Unit, ): Pair> { - val attempts = PluginConfig.profileRetryMax.coerceIn(0, 3) + 1 + val attempts = retryMax.coerceIn(0, 3) + 1 var lastFailure: Throwable? = null repeat(attempts) { attempt -> try { @@ -217,13 +242,13 @@ object UserProfileAnalysisService { response = result.response, model = model.modelName, promptVersion = ProfilePromptStore.PROMPT_VERSION, - summaryMaxLength = PluginConfig.profileSummaryMaxLength.coerceAtLeast(100), + summaryMaxLength = summaryMaxLength.coerceAtLeast(100), ) return result to reductions } catch (cause: Exception) { if (cause is CancellationException) throw cause lastFailure = cause - JChatGPT.logger.warning( + onRetryFailure( "群 ${batch.groupId} 会话画像 [${batch.startTime}, ${batch.endTime}) " + "第 ${attempt + 1}/$attempts 次分析失败", cause, diff --git a/src/test/kotlin/profile/ConversationProfileLiveExperiment.kt b/src/test/kotlin/profile/ConversationProfileLiveExperiment.kt index 1a96a8a..2682e2f 100644 --- a/src/test/kotlin/profile/ConversationProfileLiveExperiment.kt +++ b/src/test/kotlin/profile/ConversationProfileLiveExperiment.kt @@ -3,6 +3,7 @@ package top.jie65535.mirai.profile import kotlinx.coroutines.runBlocking import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Tag import top.jie65535.mirai.llm.LargeLanguageModels import top.jie65535.mirai.llm.ModelService import java.io.File @@ -11,8 +12,11 @@ import kotlin.time.Duration.Companion.seconds class ConversationProfileLiveExperiment { @Test + @Tag("live") fun analyzeConfiguredHistoricalConversation() = runBlocking { - val apiKey = System.getenv("PROFILE_EXPERIMENT_API_KEY") ?: return@runBlocking + val apiKey = checkNotNull(System.getenv("PROFILE_EXPERIMENT_API_KEY")) { + "PROFILE_EXPERIMENT_API_KEY is required for the live profile experiment" + } val source = File(checkNotNull(System.getenv("PROFILE_EXPERIMENT_SOURCE"))) val botId = checkNotNull(System.getenv("PROFILE_EXPERIMENT_BOT_ID")).toLong() val groupId = checkNotNull(System.getenv("PROFILE_EXPERIMENT_GROUP_ID")).toLong() diff --git a/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt b/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt new file mode 100644 index 0000000..527a034 --- /dev/null +++ b/src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt @@ -0,0 +1,198 @@ +package top.jie65535.mirai.profile + +import kotlinx.coroutines.runBlocking +import net.mamoe.mirai.message.data.MessageSourceKind +import top.jie65535.mirai.data.ChatMessageRecord +import java.io.IOException +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull + +class UserProfileAnalysisServiceTest { + @Test + fun analyzesAllEligibleUsersWithOneModelCall() = withProfileStore { + val model = FakeConversationProfileModel { successfulResult() } + + val report = assertNotNull(analyze(batch(), model)) + + assertEquals(1, model.calls) + assertEquals(2, report.analyzedUsers) + assertEquals(2, report.appliedOperations) + assertEquals("日常使用 Kotlin 开发", UserProfileStore.load(USER_A)?.items?.single()?.content) + assertEquals("持续关注本地大模型", UserProfileStore.load(USER_B)?.items?.single()?.content) + assertEquals(true, UserProfileStore.isConversationProcessed(INPUT_HASH)) + } + + @Test + fun rejectsWronglyAttributedEvidenceWithoutPartialCommit() = withProfileStore { + val model = FakeConversationProfileModel { + result( + responseFor( + alias = "U1", + content = "持续关注本地大模型", + evidenceRef = 2, + ) + ) + } + + assertFailsWith { + analyze(batch(), model, retryMax = 1) + } + + assertEquals(2, model.calls) + assertNull(UserProfileStore.load(USER_A)) + assertNull(UserProfileStore.load(USER_B)) + assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH)) + } + + @Test + fun modelFailureDoesNotCommitOrMarkConversationProcessed() = withProfileStore { + val model = FakeConversationProfileModel { throw IOException("model unavailable") } + + assertFailsWith { + analyze(batch(), model, retryMax = 1) + } + + assertEquals(2, model.calls) + assertNull(UserProfileStore.load(USER_A)) + assertNull(UserProfileStore.load(USER_B)) + assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH)) + } + + @Test + fun skipsConversationAlreadyProcessedByAnEarlierCall() = withProfileStore { + val model = FakeConversationProfileModel { successfulResult() } + + assertNotNull(analyze(batch(), model)) + assertNull(analyze(batch(), model)) + + assertEquals(1, model.calls) + } + + @Test + fun skipsModelWhenNoParticipantMeetsTheTextThreshold() = withProfileStore { + val model = FakeConversationProfileModel { + throw AssertionError("model must not be called") + } + + val report = analyze(batch(), model, minAuthoredTextChars = 1_000) + + assertNull(report) + assertEquals(0, model.calls) + assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH)) + } + + private suspend fun analyze( + batch: ConversationProfileBatch, + model: ConversationProfileModel, + minAuthoredTextChars: Int = 1, + retryMax: Int = 0, + ): ConversationProfileAnalysisReport? = UserProfileAnalysisService.analyzeConversationBatch( + batch = batch, + minAuthoredTextChars = minAuthoredTextChars, + model = model, + retryMax = retryMax, + summaryMaxLength = 500, + ) + + private fun successfulResult() = result( + responseFor("U1", "日常使用 Kotlin 开发", evidenceRef = 1), + responseFor("U2", "持续关注本地大模型", evidenceRef = 2), + ) + + private fun result( + vararg users: ConversationProfileUserResponse, + ) = ConversationProfileModelResult( + response = ConversationProfileModelResponse(users.toList()), + rawResponse = "test-response", + usage = ProfileTokenUsage(promptTokens = 100, completionTokens = 20, cachedTokens = 40), + ) + + private fun responseFor( + alias: String, + content: String, + evidenceRef: Int, + ) = ConversationProfileUserResponse( + userAlias = alias, + operations = listOf( + ProfileModelOperation( + action = ProfileOperationAction.ADD, + category = ProfileCategory.NOTABLE_FACT, + content = content, + confidence = ProfileConfidence.MEDIUM, + evidenceRefs = listOf(evidenceRef), + ) + ), + summary = "$content。", + ) + + private fun batch() = ConversationProfileBatch( + botId = BOT, + groupId = GROUP, + startTime = 100, + endTime = 200, + messages = listOf( + message(1, USER_A, "我日常使用 Kotlin 开发"), + message(2, USER_B, "我持续关注本地大模型"), + ), + aliases = mapOf(BOT to "BOT", USER_A to "U1", USER_B to "U2"), + inputHash = INPUT_HASH, + ) + + private fun message(ref: Int, fromId: Long, text: String) = ProfilePromptMessage( + record = ChatMessageRecord( + botId = BOT, + fromId = fromId, + targetId = GROUP, + ids = null, + internalIds = null, + time = 120 + ref, + kind = MessageSourceKind.GROUP, + code = """[{"type":"PlainText","content":"$text"}]""", + ), + text = text, + evidenceRef = ref, + episodeIndex = 1, + ) + + private fun withProfileStore(block: suspend () -> Unit) = runBlocking { + val directory = Files.createTempDirectory("jchatgpt-profile-service-test-") + try { + UserProfileStore.init(directory.toFile()) + block() + } finally { + UserProfileStore.close() + directory.toFile().deleteRecursively() + } + } + + private class FakeConversationProfileModel( + private val behavior: suspend () -> ConversationProfileModelResult, + ) : ConversationProfileModel { + var calls: Int = 0 + private set + + override val modelName: String = "fake-profile-model" + + override suspend fun analyzeConversation( + profiles: Map, + batch: ConversationProfileBatch, + eligibleUserIds: Set, + ): ConversationProfileModelResult { + calls++ + return behavior() + } + } + + companion object { + private const val BOT = 1L + private const val GROUP = 10L + private const val USER_A = 100L + private const val USER_B = 200L + private const val INPUT_HASH = "service-conversation-hash" + } +}