test: cover automatic profile analysis

Exercise multi-user commits, attribution failures, model retries, deduplication, and eligibility gates with a deterministic fake model. Move the real-model experiment to an opt-in live test task.

build.gradle.kts

@@ -1,3 +1,5 @@
+import org.gradle.api.tasks.testing.Test
+
 plugins {
     val kotlinVersion = "2.0.20"
     kotlin("jvm") version kotlinVersion
@@ -47,5 +49,18 @@ dependencies {
 }
 
 tasks.test {
-    useJUnitPlatform()
+    useJUnitPlatform {
+        excludeTags("live")
+    }
+}
+
+tasks.register<Test>("liveProfileTest") {
+    group = "verification"
+    description = "Runs the opt-in profile experiment against a configured live model."
+    testClassesDirs = tasks.test.get().testClassesDirs
+    classpath = tasks.test.get().classpath
+    useJUnitPlatform {
+        includeTags("live")
+    }
+    shouldRunAfter(tasks.test)
 }

src/main/kotlin/profile/UserProfileAnalysisService.kt

@@ -161,42 +161,64 @@ object UserProfileAnalysisService {
                     maxMessageChars = PluginConfig.profileMaxMessageChars.coerceAtLeast(80),
                 )
             } ?: return@withPermit null
-            val eligibleUserIds = batch.authoredTextCharsByUser
+            analyzeConversationBatch(
-                .filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) }
-                .keys
-            if (eligibleUserIds.isEmpty()) return@withPermit null
-            if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) {
-                return@withPermit null
-            }
-            val profiles = withContext(Dispatchers.IO) {
-                eligibleUserIds.associateWith { userId ->
-                    UserProfileStore.load(userId) ?: UserProfileSnapshot(
-                        userId = userId,
-                        cursorTime = 0,
-                        snapshotEndTime = 0,
-                    )
-                }
-            }
-
-            val (result, reductions) = analyzeConversationWithRetry(
-                model = model,
-                profiles = profiles,
                 batch = batch,
-                eligibleUserIds = eligibleUserIds,
+                minAuthoredTextChars = minAuthoredTextChars,
+                model = model,
+                retryMax = PluginConfig.profileRetryMax,
+                summaryMaxLength = PluginConfig.profileSummaryMaxLength,
+                onRetryFailure = { message, cause -> JChatGPT.logger.warning(message, cause) },
             )
-            withContext(Dispatchers.IO) {
+        }
-                UserProfileStore.commitConversation(
+    }
-                    reductions = reductions.map { reduction -> reduction to batch.forUser(reduction.profile.userId) },
+
-                    usage = result.usage,
+    internal suspend fun analyzeConversationBatch(
+        batch: ConversationProfileBatch,
+        minAuthoredTextChars: Int,
+        model: ConversationProfileModel,
+        retryMax: Int,
+        summaryMaxLength: Int,
+        onRetryFailure: (String, Throwable) -> Unit = { _, _ -> },
+    ): ConversationProfileAnalysisReport? {
+        check(UserProfileStore.isAvailable) { "用户画像数据库不可用" }
+        val eligibleUserIds = batch.authoredTextCharsByUser
+            .filterValues { it >= minAuthoredTextChars.coerceAtLeast(1) }
+            .keys
+        if (eligibleUserIds.isEmpty()) return null
+        if (withContext(Dispatchers.IO) { UserProfileStore.isConversationProcessed(batch.inputHash) }) {
+            return null
+        }
+        val profiles = withContext(Dispatchers.IO) {
+            eligibleUserIds.associateWith { userId ->
+                UserProfileStore.load(userId) ?: UserProfileSnapshot(
+                    userId = userId,
+                    cursorTime = 0,
+                    snapshotEndTime = 0,
                 )
             }
-            ConversationProfileAnalysisReport(
+        }
-                analyzedUsers = eligibleUserIds.size,
+
-                processedMessages = batch.messages.size,
+        val (result, reductions) = analyzeConversationWithRetry(
-                appliedOperations = reductions.sumOf { it.operations.size },
+            model = model,
+            profiles = profiles,
+            batch = batch,
+            eligibleUserIds = eligibleUserIds,
+            retryMax = retryMax,
+            summaryMaxLength = summaryMaxLength,
+            onRetryFailure = onRetryFailure,
+        )
+        withContext(Dispatchers.IO) {
+            UserProfileStore.commitConversation(
+                reductions = reductions.map { reduction -> reduction to batch.forUser(reduction.profile.userId) },
                 usage = result.usage,
             )
         }
+        return ConversationProfileAnalysisReport(
+            analyzedUsers = eligibleUserIds.size,
+            processedMessages = batch.messages.size,
+            appliedOperations = reductions.sumOf { it.operations.size },
+            usage = result.usage,
+        )
     }
 
     private suspend fun analyzeConversationWithRetry(
@@ -204,8 +226,11 @@ object UserProfileAnalysisService {
         profiles: Map<Long, UserProfileSnapshot>,
         batch: ConversationProfileBatch,
         eligibleUserIds: Set<Long>,
+        retryMax: Int,
+        summaryMaxLength: Int,
+        onRetryFailure: (String, Throwable) -> Unit,
     ): Pair<ConversationProfileModelResult, List<ProfileReduction>> {
-        val attempts = PluginConfig.profileRetryMax.coerceIn(0, 3) + 1
+        val attempts = retryMax.coerceIn(0, 3) + 1
         var lastFailure: Throwable? = null
         repeat(attempts) { attempt ->
             try {
@@ -217,13 +242,13 @@ object UserProfileAnalysisService {
                     response = result.response,
                     model = model.modelName,
                     promptVersion = ProfilePromptStore.PROMPT_VERSION,
-                    summaryMaxLength = PluginConfig.profileSummaryMaxLength.coerceAtLeast(100),
+                    summaryMaxLength = summaryMaxLength.coerceAtLeast(100),
                 )
                 return result to reductions
             } catch (cause: Exception) {
                 if (cause is CancellationException) throw cause
                 lastFailure = cause
-                JChatGPT.logger.warning(
+                onRetryFailure(
                     "群 ${batch.groupId} 会话画像 [${batch.startTime}, ${batch.endTime}) " +
                             "第 ${attempt + 1}/$attempts 次分析失败",
                     cause,

src/test/kotlin/profile/ConversationProfileLiveExperiment.kt

@@ -3,6 +3,7 @@ package top.jie65535.mirai.profile
 import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
+import org.junit.jupiter.api.Tag
 import top.jie65535.mirai.llm.LargeLanguageModels
 import top.jie65535.mirai.llm.ModelService
 import java.io.File
@@ -11,8 +12,11 @@ import kotlin.time.Duration.Companion.seconds
 
 class ConversationProfileLiveExperiment {
     @Test
+    @Tag("live")
     fun analyzeConfiguredHistoricalConversation() = runBlocking {
-        val apiKey = System.getenv("PROFILE_EXPERIMENT_API_KEY") ?: return@runBlocking
+        val apiKey = checkNotNull(System.getenv("PROFILE_EXPERIMENT_API_KEY")) {
+            "PROFILE_EXPERIMENT_API_KEY is required for the live profile experiment"
+        }
         val source = File(checkNotNull(System.getenv("PROFILE_EXPERIMENT_SOURCE")))
         val botId = checkNotNull(System.getenv("PROFILE_EXPERIMENT_BOT_ID")).toLong()
         val groupId = checkNotNull(System.getenv("PROFILE_EXPERIMENT_GROUP_ID")).toLong()

src/test/kotlin/profile/UserProfileAnalysisServiceTest.kt

@@ -0,0 +1,198 @@
+package top.jie65535.mirai.profile
+
+import kotlinx.coroutines.runBlocking
+import net.mamoe.mirai.message.data.MessageSourceKind
+import top.jie65535.mirai.data.ChatMessageRecord
+import java.io.IOException
+import java.nio.file.Files
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertFalse
+import kotlin.test.assertNotNull
+import kotlin.test.assertNull
+
+class UserProfileAnalysisServiceTest {
+    @Test
+    fun analyzesAllEligibleUsersWithOneModelCall() = withProfileStore {
+        val model = FakeConversationProfileModel { successfulResult() }
+
+        val report = assertNotNull(analyze(batch(), model))
+
+        assertEquals(1, model.calls)
+        assertEquals(2, report.analyzedUsers)
+        assertEquals(2, report.appliedOperations)
+        assertEquals("日常使用 Kotlin 开发", UserProfileStore.load(USER_A)?.items?.single()?.content)
+        assertEquals("持续关注本地大模型", UserProfileStore.load(USER_B)?.items?.single()?.content)
+        assertEquals(true, UserProfileStore.isConversationProcessed(INPUT_HASH))
+    }
+
+    @Test
+    fun rejectsWronglyAttributedEvidenceWithoutPartialCommit() = withProfileStore {
+        val model = FakeConversationProfileModel {
+            result(
+                responseFor(
+                    alias = "U1",
+                    content = "持续关注本地大模型",
+                    evidenceRef = 2,
+                )
+            )
+        }
+
+        assertFailsWith<IllegalStateException> {
+            analyze(batch(), model, retryMax = 1)
+        }
+
+        assertEquals(2, model.calls)
+        assertNull(UserProfileStore.load(USER_A))
+        assertNull(UserProfileStore.load(USER_B))
+        assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH))
+    }
+
+    @Test
+    fun modelFailureDoesNotCommitOrMarkConversationProcessed() = withProfileStore {
+        val model = FakeConversationProfileModel { throw IOException("model unavailable") }
+
+        assertFailsWith<IllegalStateException> {
+            analyze(batch(), model, retryMax = 1)
+        }
+
+        assertEquals(2, model.calls)
+        assertNull(UserProfileStore.load(USER_A))
+        assertNull(UserProfileStore.load(USER_B))
+        assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH))
+    }
+
+    @Test
+    fun skipsConversationAlreadyProcessedByAnEarlierCall() = withProfileStore {
+        val model = FakeConversationProfileModel { successfulResult() }
+
+        assertNotNull(analyze(batch(), model))
+        assertNull(analyze(batch(), model))
+
+        assertEquals(1, model.calls)
+    }
+
+    @Test
+    fun skipsModelWhenNoParticipantMeetsTheTextThreshold() = withProfileStore {
+        val model = FakeConversationProfileModel {
+            throw AssertionError("model must not be called")
+        }
+
+        val report = analyze(batch(), model, minAuthoredTextChars = 1_000)
+
+        assertNull(report)
+        assertEquals(0, model.calls)
+        assertFalse(UserProfileStore.isConversationProcessed(INPUT_HASH))
+    }
+
+    private suspend fun analyze(
+        batch: ConversationProfileBatch,
+        model: ConversationProfileModel,
+        minAuthoredTextChars: Int = 1,
+        retryMax: Int = 0,
+    ): ConversationProfileAnalysisReport? = UserProfileAnalysisService.analyzeConversationBatch(
+        batch = batch,
+        minAuthoredTextChars = minAuthoredTextChars,
+        model = model,
+        retryMax = retryMax,
+        summaryMaxLength = 500,
+    )
+
+    private fun successfulResult() = result(
+        responseFor("U1", "日常使用 Kotlin 开发", evidenceRef = 1),
+        responseFor("U2", "持续关注本地大模型", evidenceRef = 2),
+    )
+
+    private fun result(
+        vararg users: ConversationProfileUserResponse,
+    ) = ConversationProfileModelResult(
+        response = ConversationProfileModelResponse(users.toList()),
+        rawResponse = "test-response",
+        usage = ProfileTokenUsage(promptTokens = 100, completionTokens = 20, cachedTokens = 40),
+    )
+
+    private fun responseFor(
+        alias: String,
+        content: String,
+        evidenceRef: Int,
+    ) = ConversationProfileUserResponse(
+        userAlias = alias,
+        operations = listOf(
+            ProfileModelOperation(
+                action = ProfileOperationAction.ADD,
+                category = ProfileCategory.NOTABLE_FACT,
+                content = content,
+                confidence = ProfileConfidence.MEDIUM,
+                evidenceRefs = listOf(evidenceRef),
+            )
+        ),
+        summary = "$content。",
+    )
+
+    private fun batch() = ConversationProfileBatch(
+        botId = BOT,
+        groupId = GROUP,
+        startTime = 100,
+        endTime = 200,
+        messages = listOf(
+            message(1, USER_A, "我日常使用 Kotlin 开发"),
+            message(2, USER_B, "我持续关注本地大模型"),
+        ),
+        aliases = mapOf(BOT to "BOT", USER_A to "U1", USER_B to "U2"),
+        inputHash = INPUT_HASH,
+    )
+
+    private fun message(ref: Int, fromId: Long, text: String) = ProfilePromptMessage(
+        record = ChatMessageRecord(
+            botId = BOT,
+            fromId = fromId,
+            targetId = GROUP,
+            ids = null,
+            internalIds = null,
+            time = 120 + ref,
+            kind = MessageSourceKind.GROUP,
+            code = """[{"type":"PlainText","content":"$text"}]""",
+        ),
+        text = text,
+        evidenceRef = ref,
+        episodeIndex = 1,
+    )
+
+    private fun withProfileStore(block: suspend () -> Unit) = runBlocking {
+        val directory = Files.createTempDirectory("jchatgpt-profile-service-test-")
+        try {
+            UserProfileStore.init(directory.toFile())
+            block()
+        } finally {
+            UserProfileStore.close()
+            directory.toFile().deleteRecursively()
+        }
+    }
+
+    private class FakeConversationProfileModel(
+        private val behavior: suspend () -> ConversationProfileModelResult,
+    ) : ConversationProfileModel {
+        var calls: Int = 0
+            private set
+
+        override val modelName: String = "fake-profile-model"
+
+        override suspend fun analyzeConversation(
+            profiles: Map<Long, UserProfileSnapshot>,
+            batch: ConversationProfileBatch,
+            eligibleUserIds: Set<Long>,
+        ): ConversationProfileModelResult {
+            calls++
+            return behavior()
+        }
+    }
+
+    companion object {
+        private const val BOT = 1L
+        private const val GROUP = 10L
+        private const val USER_A = 100L
+        private const val USER_B = 200L
+        private const val INPUT_HASH = "service-conversation-hash"
+    }
+}