jie65535/JChatGPT
1
0
Fork 0
mirror of https://github.com/jie65535/JChatGPT.git synced 2026-09-15 02:56:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

profile: preserve valid text and evidence ranges

Browse Source
This commit is contained in:
jie65535
2026-08-05 00:05:01 +08:00
parent c853e9afcc
commit 0dd221a80f
5 changed files with 182 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/kotlin/profile/ProfileContentRules.kt
+5
View File
@@ -7,6 +7,7 @@ object ProfileContentRules {
require(content.length <= MAX_CONTENT_LENGTH) { "$label.content 超过 $MAX_CONTENT_LENGTH 字符" }
require(!overclaimPattern.containsMatchIn(content)) { "$label.content 包含夸张身份或能力判断" }
require(!itemReferencePattern.containsMatchIn(content)) { "$label.content 包含临时画像条目编号" }
require(!batchScopedPattern.containsMatchIn(content)) { "$label.content 包含批次化处理措辞" }
return content
}
@@ -14,6 +15,7 @@ object ProfileContentRules {
val summary = raw.trim()
require(summary.length <= maxLength) { "画像摘要超过 $maxLength 字符" }
require(!itemReferencePattern.containsMatchIn(summary)) { "画像摘要包含临时画像条目编号" }
require(!batchScopedPattern.containsMatchIn(summary)) { "画像摘要包含批次化处理措辞" }
return summary
}
@@ -21,7 +23,10 @@ object ProfileContentRules {
.lowercase()
.replace(Regex("[\\s\\p{Punct}，。；、！？（）【】‘’“”]+"), "")
fun containsBatchScopedText(value: String): Boolean = batchScopedPattern.containsMatchIn(value)
private const val MAX_CONTENT_LENGTH = 120
private val overclaimPattern = Regex("深厚|扎实|精通|专家|导师|领袖|天才|极强|全栈|核心成员|公认")
private val itemReferencePattern = Regex("(?<![A-Za-z0-9_])P[1-9]\\d*(?![A-Za-z0-9_])")
private val batchScopedPattern = Regex("本批|本轮(?:分析|整理|对话|更新)|此次对话|本次对话|这段对话")
}
src/main/kotlin/profile/ProfileMessageRenderer.kt
+49 -6
View File
@@ -28,10 +28,10 @@ object ProfileMessageRenderer {
renderChain(record.toMessageChain(), aliases)
}.getOrElse {
"[消息内容解析失败]"
}.replace(Regex("[\\r\\n]+"), " ").trim()
}.replace(Regex("[\\r\\n]+"), " ").trim().replaceUnpairedSurrogates()
if (content.length <= maxChars) return content.ifEmpty { "[无文本消息]" }
return content.take(maxChars).trimEnd() + "...[截断]"
return content.takeUtf16Safely(maxChars).trimEnd() + "...[截断]"
}
fun referencedUserIds(record: ChatMessageRecord): Set<Long> = runCatching {
@@ -95,7 +95,7 @@ object ProfileMessageRenderer {
?.let { renderJsonMessages(it, aliases) }
.orEmpty()
.replace(Regex("[\\r\\n]+"), " ")
.take(160)
.takeUtf16Safely(160)
return "[引用 $author: $original]"
}
@@ -107,7 +107,7 @@ object ProfileMessageRenderer {
val sender = node.string("senderName") ?: "未知用户"
val chain = node["messageChain"] as? JsonArray
append(' ').append(sender).append(": ")
append(chain?.let { renderJsonMessages(it, aliases) }.orEmpty().take(200))
append(chain?.let { renderJsonMessages(it, aliases) }.orEmpty().takeUtf16Safely(200))
}
if (nodes.size > 20) append(" ...[转发内容截断]")
}
@@ -161,17 +161,60 @@ object ProfileMessageRenderer {
val author = aliases[message.source.fromId] ?: "其他用户"
val quoted = renderChain(message.source.originalMessage, aliases)
.replace(Regex("[\\r\\n]+"), " ")
.take(160)
.takeUtf16Safely(160)
"[引用 $author: $quoted]"
}
is ForwardMessage -> buildString {
append("[转发消息]")
message.nodeList.take(20).forEach { node ->
append(" ").append(node.senderName).append(": ")
append(renderChain(node.messageChain, aliases).replace(Regex("[\\r\\n]+"), " ").take(200))
append(
renderChain(node.messageChain, aliases)
.replace(Regex("[\\r\\n]+"), " ")
.takeUtf16Safely(200)
)
}
if (message.nodeList.size > 20) append(" ...[转发内容截断]")
}
else -> message.content
}
private fun String.takeUtf16Safely(maxLength: Int): String {
require(maxLength >= 0) { "maxLength must not be negative" }
if (length <= maxLength) return this
val endIndex = if (maxLength > 0 &&
Character.isHighSurrogate(this[maxLength - 1]) &&
Character.isLowSurrogate(this[maxLength])
) {
maxLength - 1
} else {
maxLength
}
return substring(0, endIndex)
}
private fun String.replaceUnpairedSurrogates(): String {
var output: StringBuilder? = null
var index = 0
while (index < length) {
val current = this[index]
when {
Character.isHighSurrogate(current) &&
index + 1 < length && Character.isLowSurrogate(this[index + 1]) -> {
output?.append(current)?.append(this[index + 1])
index += 2
}
Character.isSurrogate(current) -> {
if (output == null) output = StringBuilder(length).append(this, 0, index)
output.append('\uFFFD')
index++
}
else -> {
output?.append(current)
index++
}
}
}
return output?.toString() ?: this
}
}
src/main/kotlin/profile/UserProfileReducer.kt
+4 -2
View File
@@ -126,7 +126,8 @@ object UserProfileReducer {
content = content,
confidence = confidence,
relatedUserId = relatedUserId,
lastConfirmedAt = evidenceTime,
firstSeenAt = minOf(old.firstSeenAt, firstEvidenceTime),
lastConfirmedAt = maxOf(old.lastConfirmedAt, evidenceTime),
)
items[old.id] = updated
updated.toApplied(operation.action, operation.evidenceRefs)
@@ -136,7 +137,8 @@ object UserProfileReducer {
val old = requireExistingItem(index, operation, current, items)
val updated = old.copy(
confidence = operation.confidence ?: old.confidence,
lastConfirmedAt = evidenceTime,
firstSeenAt = minOf(old.firstSeenAt, firstEvidenceTime),
lastConfirmedAt = maxOf(old.lastConfirmedAt, evidenceTime),
)
items[old.id] = updated
updated.toApplied(operation.action, operation.evidenceRefs)
src/test/kotlin/profile/ProfileMessageRendererTest.kt
+61
View File
@@ -33,6 +33,67 @@ class ProfileMessageRendererTest {
assertEquals("12345678...[截断]", ProfileMessageRenderer.render(record, emptyMap(), 8))
}
@Test
fun doesNotSplitSurrogatePairsAtTruncationBoundaries() {
val emoji = "\uD83D\uDE00"
val finalText = "a".repeat(7) + emoji + "tail"
assertEquals(
"a".repeat(7) + "...[截断]",
ProfileMessageRenderer.render(
record("""[{"type":"PlainText","content":"$finalText"}]"""),
emptyMap(),
8,
),
)
val quotedText = "q".repeat(159) + emoji + "tail"
assertEquals(
"[引用 U1: ${"q".repeat(159)}]",
ProfileMessageRenderer.render(
record(
"""
[{"type":"QuoteReply","source":{"fromId":200,"originalMessage":[
{"type":"PlainText","content":"$quotedText"}
]}}]
""".trimIndent()
),
mapOf(200L to "U1"),
1_000,
),
)
val forwardedText = "f".repeat(199) + emoji + "tail"
assertEquals(
"[转发消息] sender: ${"f".repeat(199)}",
ProfileMessageRenderer.render(
record(
"""
[{"type":"ForwardMessage","nodeList":[{
"senderName":"sender",
"messageChain":[{"type":"PlainText","content":"$forwardedText"}]
}]}]
""".trimIndent()
),
emptyMap(),
1_000,
),
)
}
@Test
fun replacesUnpairedSurrogatesBeforeBuildingModelPrompts() {
val loneHighSurrogate = "\uD83D"
assertEquals(
"prefix\uFFFDsuffix",
ProfileMessageRenderer.render(
record("""[{"type":"PlainText","content":"prefix${loneHighSurrogate}suffix"}]"""),
emptyMap(),
1_000,
),
)
}
private fun record(code: String) = ChatMessageRecord(
botId = 1,
fromId = 100,
src/test/kotlin/profile/UserProfileReducerTest.kt
+63
View File
@@ -291,6 +291,69 @@ class UserProfileReducerTest {
assertEquals("原摘要", reduction.profile.summary)
}
@Test
fun rejectsBatchScopedPersistentText() {
val current = emptyProfile().copy(summary = "原摘要")
val batch = batchOf(message(ref = 1, fromId = TARGET, text = "喵喵喵"))
val reduction = UserProfileReducer.reduce(
current = current,
batch = batch,
response = ProfileModelResponse(
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.ADD,
category = ProfileCategory.EXPRESSION_STYLE,
content = "本批多次使用拟猫化措辞",
confidence = ProfileConfidence.LOW,
evidenceRefs = listOf(1),
)
),
summary = "本批主要使用拟猫化措辞。",
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
assertTrue(reduction.operations.isEmpty())
assertTrue(reduction.skippedOperations.single().contains("批次化"))
assertEquals("原摘要", reduction.profile.summary)
}
@Test
fun keepsEvidenceRangeMonotonicWhenOlderBatchUpdatesExistingItem() {
val old = existingItem().copy(
firstSeenAt = 150,
lastConfirmedAt = 180,
)
val current = emptyProfile().copy(items = listOf(old), reliable = true)
val batch = batchOf(message(ref = 1, fromId = TARGET, text = "我更早也关注 Kotlin"))
val reduction = UserProfileReducer.reduce(
current = current,
batch = batch,
response = ProfileModelResponse(
operations = listOf(
ProfileModelOperation(
action = ProfileOperationAction.UPDATE,
itemRef = "P1",
content = "关注 Kotlin 开发",
evidenceRefs = listOf(1),
)
),
summary = "关注 Kotlin 开发。",
),
model = "test-model",
promptVersion = "test-prompt",
summaryMaxLength = 500,
)
val updated = reduction.profile.items.single()
assertEquals(121, updated.firstSeenAt)
assertEquals(180, updated.lastConfirmedAt)
}
@Test
fun removesTemporaryAliasesFromPersistedRelationshipText() {
val batch = batchOf(message(ref = 1, fromId = TARGET, text = "先找到下家再离职"))