diff --git a/README.md b/README.md index 8e98fdf..94b9dba 100644 --- a/README.md +++ b/README.md @@ -238,10 +238,11 @@ searchHistoryMaxRecords: 5000 聊天记录与联系人快照保存在插件数据目录的 `chat-history.sqlite` 中，并使用 SQLite WAL 模式支持记录与查询并行进行。 数据库由插件在首次启动时自动创建和维护，无需安装额外的聊天记录插件。 -聊天记录搜索使用 SQLite FTS5 中文三元索引。升级已有数据库时只会新增独立的派生搜索文本表和索引表， +聊天记录搜索使用 SQLite FTS5 中文二元 + 三元索引。升级已有数据库时只会新增独立的派生搜索文本表和索引表， 不会改列、重建或删除 `message_record` 原始记录；旧记录的搜索文本会在插件启动后分批后台回填。回填期间新消息仍会正常保存， 搜索结果会明确标注索引尚未完成。搜索工具默认只查询当前群聊或当前私聊，明确指定 `from` 和 `to` 后可以跨越任意历史时间范围， -并通过 `nextCursor` 分页。升级后新收到的消息还会记录发送者在该会话中使用过的名称，名称搜索不依赖对方仍是当前群成员； +并通过 `nextCursor` 分页。普通文字和可解析的 `@` 目标会合并检索，不要求模型预先判断查询类型； +升级后新收到的消息还会记录发送者在该会话中使用过的名称，名称搜索不依赖对方仍是当前群成员； `getChatHistoryContext` 可按 `messageId` 获取命中消息前后的对话上下文。 联系人刷新在后台通过 Overflow 的公开 `RemoteBot.executeAction` 调用 OneBot @@ -506,7 +507,7 @@ fallbackCooldownMinutes: 5 14. **GetChatHistoryContext** - 根据历史消息 ID 获取目标消息前后的对话上下文 15. **QueryUserProfile** - 按当前发送者、QQ 号、群名片、昵称或好友备注读取长期画像，并针对该联系人按需读取公开资料卡 -`SearchChatHistory` 的常用参数只有消息内容、发送者、时间范围和翻页游标；结果默认按最近消息返回， +`SearchChatHistory` 的常用参数只有消息内容、发送者、时间范围和翻页游标；普通文字和可解析的 `@` 目标会合并检索，结果默认按最近消息返回， 单页大小和匹配策略由插件控制。`senderId` 与 `senderName` 可以同时提供，此时 QQ 号作为精确筛选条件；有 QQ 号时名称不参与筛选。 ### GitHub 查询工具 diff --git a/src/main/kotlin/data/ChatHistorySearch.kt b/src/main/kotlin/data/ChatHistorySearch.kt index de6dd64..5e38ffb 100644 --- a/src/main/kotlin/data/ChatHistorySearch.kt +++ b/src/main/kotlin/data/ChatHistorySearch.kt @@ -42,6 +42,7 @@ data class ChatHistoryCursor( data class ChatHistorySearchRequest( val subject: ChatHistorySubject, val query: String? = null, + val atTargetIds: Set = emptySet(), val matchMode: ChatHistoryMatchMode = ChatHistoryMatchMode.ALL, val fromId: Long? = null, val start: Int? = null, diff --git a/src/main/kotlin/data/ChatHistorySearchText.kt b/src/main/kotlin/data/ChatHistorySearchText.kt index 892a112..ed08e3d 100644 --- a/src/main/kotlin/data/ChatHistorySearchText.kt +++ b/src/main/kotlin/data/ChatHistorySearchText.kt @@ -18,28 +18,49 @@ object ChatHistorySearchText { private val json = Json { ignoreUnknownKeys = true } private val whitespace = Regex("\\s+") - fun extract(code: String): String { - val rendered = runCatching { renderJsonCode(code) } + fun extract(code: String, atNames: Map = emptyMap()): String { + val rendered = runCatching { renderJsonCode(code, atNames) } .recoverCatching { decodeMessageCode(code).content } .getOrDefault("") return rendered.normalize().takeUtf16Safely(MAX_INDEXED_CHARS) } - private fun renderJsonCode(code: String): String { - val messages = json.parseToJsonElement(code) as? JsonArray - ?: throw IllegalArgumentException("消息记录不是 JSON array") - return renderMessages(messages) + fun extractAtTargets(code: String): Set = runCatching { + val messages = json.parseToJsonElement(code) as? JsonArray ?: return@runCatching emptySet() + buildSet { collectAtTargets(messages, this) } + }.getOrDefault(emptySet()) + + fun bigrams(text: String): String { + val codePoints = text.lowercase().codePoints().toArray() + if (codePoints.size < 2) return "" + return buildString(codePoints.size * 3) { + var tokenCount = 0 + for (index in 0 until codePoints.lastIndex) { + if (Character.isWhitespace(codePoints[index]) || Character.isWhitespace(codePoints[index + 1])) { + continue + } + if (tokenCount++ > 0) append(' ') + appendCodePoint(codePoints[index]) + appendCodePoint(codePoints[index + 1]) + } + } } - private fun renderMessages(messages: JsonArray): String = messages.joinToString("") { element -> + private fun renderJsonCode(code: String, atNames: Map): String { + val messages = json.parseToJsonElement(code) as? JsonArray + ?: throw IllegalArgumentException("消息记录不是 JSON array") + return renderMessages(messages, atNames) + } + + private fun renderMessages(messages: JsonArray, atNames: Map): String = messages.joinToString("") { element -> val message = element as? JsonObject ?: return@joinToString "" when (val type = message.string("type")) { "PlainText" -> message.string("content").orEmpty() - "At" -> message.long("target")?.let { "@$it" }.orEmpty() + "At" -> message.long("target")?.let { target -> "@${atNames[target] ?: target}" }.orEmpty() "AtAll" -> "@全体成员" "Image", "FlashImage" -> if (message.boolean("isEmoji") == true) "[表情包]" else "[图片]" - "QuoteReply" -> renderQuote(message) - "ForwardMessage" -> renderForward(message) + "QuoteReply" -> renderQuote(message, atNames) + "ForwardMessage" -> renderForward(message, atNames) "MessageOrigin", "MessageSource", "ShowImageFlag" -> "" "Face", "MarketFace", "VipFace" -> "[表情]" "Audio" -> "[语音]" @@ -51,18 +72,18 @@ object ChatHistorySearchText { } } - private fun renderQuote(message: JsonObject): String { + private fun renderQuote(message: JsonObject, atNames: Map): String { val source = message["source"] as? JsonObject ?: return "[引用消息]" val author = source.long("fromId")?.toString() ?: "其他用户" val original = (source["originalMessage"] as? JsonArray) - ?.let(::renderMessages) + ?.let { renderMessages(it, atNames) } .orEmpty() .normalize() .takeUtf16Safely(MAX_QUOTED_CHARS) return "[引用 $author: $original]" } - private fun renderForward(message: JsonObject): String = buildString { + private fun renderForward(message: JsonObject, atNames: Map): String = buildString { append("[转发消息]") val nodes = message["nodeList"] as? JsonArray ?: return@buildString nodes.take(MAX_FORWARD_NODES).forEach { element -> @@ -71,7 +92,7 @@ object ChatHistorySearchText { val chain = node["messageChain"] as? JsonArray append(' ').append(sender).append(": ") append( - chain?.let(::renderMessages) + chain?.let { renderMessages(it, atNames) } .orEmpty() .normalize() .takeUtf16Safely(MAX_FORWARD_NODE_CHARS) @@ -80,6 +101,24 @@ object ChatHistorySearchText { if (nodes.size > MAX_FORWARD_NODES) append(" ...[转发内容截断]") } + private fun collectAtTargets(messages: JsonArray, targets: MutableSet) { + messages.forEach { element -> + val message = element as? JsonObject ?: return@forEach + when (message.string("type")) { + "At" -> message.long("target")?.let(targets::add) + "QuoteReply" -> { + val original = (message["source"] as? JsonObject) + ?.get("originalMessage") as? JsonArray + original?.let { collectAtTargets(it, targets) } + } + "ForwardMessage" -> (message["nodeList"] as? JsonArray).orEmpty().forEach { node -> + ((node as? JsonObject)?.get("messageChain") as? JsonArray) + ?.let { collectAtTargets(it, targets) } + } + } + } + } + private fun String.normalize(): String = whitespace.replace(this, " ") .trim() .replaceUnpairedSurrogates() diff --git a/src/main/kotlin/data/ChatHistoryStore.kt b/src/main/kotlin/data/ChatHistoryStore.kt index 1b2e5cc..18888d0 100644 --- a/src/main/kotlin/data/ChatHistoryStore.kt +++ b/src/main/kotlin/data/ChatHistoryStore.kt @@ -24,13 +24,16 @@ import java.sql.ResultSet * 写连接由 [writeLock] 串行保护；SQLite 使用 WAL + NORMAL synchronous，允许并发读取。 */ object ChatHistoryStore { - private const val SCHEMA_VERSION = 3 + private const val SCHEMA_VERSION = 4 private const val BUSY_TIMEOUT_MS = 30_000 private const val SEARCH_FTS_TABLE = "message_record_fts" + private const val SEARCH_BIGRAM_FTS_TABLE = "message_record_bigram_fts" private const val SEARCH_TEXT_TABLE = "message_record_search" private const val SENDER_ALIAS_TABLE = "chat_history_sender_alias" private const val SEARCH_BACKFILL_CURSOR_KEY = "search_backfill_cursor" private const val SEARCH_INDEX_COMPLETE_KEY = "search_index_complete" + private const val SEARCH_BIGRAM_VERSION_KEY = "search_bigram_version" + private const val SEARCH_BIGRAM_VERSION = "1" private const val DEFAULT_BACKFILL_BATCH_SIZE = 250 private const val MAX_PAGE_SIZE = 200 private const val MAX_CONTEXT_MESSAGES = 100 @@ -43,6 +46,8 @@ object ChatHistoryStore { @Volatile private var searchIndexAvailable = false @Volatile + private var bigramIndexAvailable = false + @Volatile private var searchIndexReady = false private lateinit var databaseFile: File private var writeConnection: Connection? = null @@ -79,6 +84,7 @@ object ChatHistoryStore { writeConnection = connection initialized = true searchIndexAvailable = hasSearchIndex(connection) + bigramIndexAvailable = hasBigramSearchIndex(connection) searchIndexReady = searchIndexAvailable && readMeta(connection, SEARCH_INDEX_COMPLETE_KEY) == "1" } catch (cause: Throwable) { connection.close() @@ -102,6 +108,7 @@ object ChatHistoryStore { writeConnection = null initialized = false searchIndexAvailable = false + bigramIndexAvailable = false searchIndexReady = false warningLogger = null } @@ -299,17 +306,17 @@ object ChatHistoryStore { } val query = request.query?.trim().orEmpty() - val textPredicate = buildTextPredicate(query, request.matchMode, parameters) + val textPredicate = buildTextPredicate(query, request.matchMode, request.atTargetIds, parameters) if ((textPredicate.usesFts || textPredicate.usesSearchText) && !isSearchIndexAvailable) { error("聊天记录全文索引尚未就绪") } + if (textPredicate.usesBigram && !bigramIndexAvailable) { + error("聊天记录二元索引尚未就绪") + } val joins = buildString { if (textPredicate.usesSearchText) { append("JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id ") } - if (textPredicate.usesFts) { - append("JOIN $SEARCH_FTS_TABLE ON $SEARCH_FTS_TABLE.rowid = mr.id") - } } textPredicate.sql?.let(conditions::add) @@ -499,16 +506,24 @@ object ChatHistoryStore { var cursor = withWriteConnectionResult { connection -> readMeta(connection, SEARCH_BACKFILL_CURSOR_KEY)?.toLongOrNull() ?: 0L } + val hasBigramIndex = bigramIndexAvailable while (shouldContinue()) { val batch = openReadConnection().use { connection -> + val missingBigramExpression = if (hasBigramIndex) { + "NOT EXISTS (SELECT 1 FROM $SEARCH_BIGRAM_FTS_TABLE bigram WHERE bigram.rowid = mr.id)" + } else { + "0" + } + val missingBigramCondition = if (hasBigramIndex) "OR $missingBigramExpression" else "" connection.prepareStatement( """ - SELECT mr.id, mr.code + SELECT mr.id, mr.code, + search.message_id IS NULL AS missing_search, + $missingBigramExpression AS missing_bigram FROM message_record mr + LEFT JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id WHERE mr.id > ? - AND NOT EXISTS ( - SELECT 1 FROM $SEARCH_TEXT_TABLE search WHERE search.message_id = mr.id - ) + AND (search.message_id IS NULL $missingBigramCondition) ORDER BY mr.id LIMIT ? """.trimIndent() @@ -518,7 +533,14 @@ object ChatHistoryStore { statement.executeQuery().use { results -> buildList { while (results.next()) { - add(results.getLong("id") to results.getString("code")) + add( + SearchBackfillRow( + id = results.getLong("id"), + code = results.getString("code"), + missingSearch = results.getBoolean("missing_search"), + missingBigram = results.getBoolean("missing_bigram"), + ) + ) } } } @@ -541,20 +563,39 @@ object ChatHistoryStore { connection.prepareStatement( "INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)" ).use { insertFts -> - batch.forEach { (id, code) -> - val text = ChatHistorySearchText.extract(code) - insertText.setLong(1, id) - insertText.setString(2, text) - insertText.addBatch() - insertFts.setLong(1, id) - insertFts.setString(2, text) - insertFts.addBatch() + val insertBigram = if (hasBigramIndex) { + connection.prepareStatement( + "INSERT INTO $SEARCH_BIGRAM_FTS_TABLE(rowid, bigram_text) VALUES (?, ?)" + ) + } else { + null + } + try { + batch.forEach { row -> + val text = ChatHistorySearchText.extract(row.code) + if (row.missingSearch) { + insertText.setLong(1, row.id) + insertText.setString(2, text) + insertText.addBatch() + insertFts.setLong(1, row.id) + insertFts.setString(2, text) + insertFts.addBatch() + } + if (row.missingBigram) { + insertBigram?.setLong(1, row.id) + insertBigram?.setString(2, ChatHistorySearchText.bigrams(text)) + insertBigram?.addBatch() + } + } + insertText.executeBatch() + insertFts.executeBatch() + insertBigram?.executeBatch() + } finally { + insertBigram?.close() } - insertText.executeBatch() - insertFts.executeBatch() } } - cursor = batch.last().first + cursor = batch.last().id writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, cursor.toString()) writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0") connection.commit() @@ -571,6 +612,13 @@ object ChatHistoryStore { return processed } + private data class SearchBackfillRow( + val id: Long, + val code: String, + val missingSearch: Boolean, + val missingBigram: Boolean, + ) + internal fun insertRecord(record: ChatMessageRecord, senderName: String = "") { var recordId = 0L var indexFailure: Throwable? = null @@ -639,6 +687,15 @@ object ChatHistoryStore { statement.setString(2, searchText) statement.executeUpdate() } + if (bigramIndexAvailable) { + connection.prepareStatement( + "INSERT INTO $SEARCH_BIGRAM_FTS_TABLE(rowid, bigram_text) VALUES (?, ?)" + ).use { statement -> + statement.setLong(1, recordId) + statement.setString(2, ChatHistorySearchText.bigrams(searchText)) + statement.executeUpdate() + } + } connection.commit() } catch (cause: Throwable) { connection.rollback() @@ -810,6 +867,20 @@ object ChatHistoryStore { } }.onFailure { warn("初始化 SQLite FTS5 聊天记录索引失败，消息记录仍会继续保存", it) } + runCatching { + connection.createStatement().use { statement -> + statement.executeUpdate( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS $SEARCH_BIGRAM_FTS_TABLE USING fts5( + bigram_text, + content='', + tokenize='unicode61' + ) + """.trimIndent() + ) + } + }.onFailure { warn("初始化 SQLite FTS5 聊天记录二元索引失败，将回退到短词扫描", it) } + if (hasSearchIndex(connection)) { val hasMessages = connection.createStatement().use { statement -> statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next) @@ -825,6 +896,27 @@ object ChatHistoryStore { statement.executeUpdate() } } + + if (hasBigramSearchIndex(connection) && + readMeta(connection, SEARCH_BIGRAM_VERSION_KEY) != SEARCH_BIGRAM_VERSION + ) { + val hasMessages = connection.createStatement().use { statement -> + statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next) + } + val oldAutoCommitForMigration = connection.autoCommit + connection.autoCommit = false + try { + writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, "0") + writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, if (hasMessages) "0" else "1") + writeMeta(connection, SEARCH_BIGRAM_VERSION_KEY, SEARCH_BIGRAM_VERSION) + connection.commit() + } catch (cause: Throwable) { + connection.rollback() + throw cause + } finally { + connection.autoCommit = oldAutoCommitForMigration + } + } } private fun subjectConditions( @@ -852,9 +944,40 @@ object ChatHistoryStore { val sql: String?, val usesFts: Boolean, val usesSearchText: Boolean, + val usesBigram: Boolean = false, ) private fun buildTextPredicate( + query: String, + mode: ChatHistoryMatchMode, + atTargetIds: Set, + parameters: MutableList, + ): TextPredicate { + if (query.isBlank() && atTargetIds.isEmpty()) return TextPredicate(null, false, false) + val contentPredicate = if (query.isBlank()) { + TextPredicate(null, false, false) + } else { + buildContentTextPredicate(query, mode, parameters) + } + if (atTargetIds.isEmpty()) return contentPredicate + + val mentionPredicate = atTargetIds.sorted().joinToString(" OR ", prefix = "(", postfix = ")") { + parameters += quoteFtsTerm("@$it") + ftsRowPredicate(SEARCH_FTS_TABLE) + } + val sql = when { + contentPredicate.sql == null -> mentionPredicate + else -> "(${contentPredicate.sql} OR $mentionPredicate)" + } + return TextPredicate( + sql = sql, + usesFts = true, + usesSearchText = contentPredicate.usesSearchText, + usesBigram = contentPredicate.usesBigram, + ) + } + + private fun buildContentTextPredicate( query: String, mode: ChatHistoryMatchMode, parameters: MutableList, @@ -862,49 +985,78 @@ object ChatHistoryStore { if (query.isBlank()) return TextPredicate(null, false, false) val normalized = query.lowercase() val terms = normalized.split(Regex("\\s+")).filter(String::isNotBlank) - val hasShortTerm = terms.any { it.codePointCount(0, it.length) < 3 } - if (mode == ChatHistoryMatchMode.ANY && hasShortTerm) { - val predicates = terms.map { - parameters += it - "instr(lower(search.search_text), ?) > 0" + if (mode == ChatHistoryMatchMode.PHRASE) { + val codePointCount = normalized.codePointCount(0, normalized.length) + return when { + codePointCount >= 3 -> { + parameters += quoteFtsTerm(normalized) + TextPredicate(ftsRowPredicate(SEARCH_FTS_TABLE), usesFts = true, usesSearchText = false) + } + codePointCount == 2 && bigramIndexAvailable -> { + parameters += quoteFtsTerm(normalized) + TextPredicate( + ftsRowPredicate(SEARCH_BIGRAM_FTS_TABLE), + usesFts = false, + usesSearchText = false, + usesBigram = true, + ) + } + else -> { + parameters += normalized + TextPredicate("instr(lower(search.search_text), ?) > 0", false, true) + } } - return TextPredicate(predicates.joinToString(" OR ", prefix = "(", postfix = ")"), false, true) } - val ftsTerms = if (mode == ChatHistoryMatchMode.PHRASE) { - listOf(normalized) - } else { - terms + + val longTerms = terms.filter { it.codePointCount(0, it.length) >= 3 } + val bigramTerms = terms.filter { it.codePointCount(0, it.length) == 2 } + val shortTerms = terms.filter { it.codePointCount(0, it.length) < 2 } + val predicates = mutableListOf() + var usesFts = false + var usesBigram = false + var usesSearchText = false + + if (longTerms.isNotEmpty()) { + parameters += longTerms.joinToString( + if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND ", + transform = ::quoteFtsTerm, + ) + predicates += ftsRowPredicate(SEARCH_FTS_TABLE) + usesFts = true } - val ftsOnly = ftsTerms.filter { it.codePointCount(0, it.length) >= 3 } - if (ftsOnly.isEmpty()) { - val predicates = terms.map { - parameters += it - "instr(lower(search.search_text), ?) > 0" + if (bigramTerms.isNotEmpty()) { + if (bigramIndexAvailable) { + parameters += bigramTerms.joinToString( + if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND ", + transform = ::quoteFtsTerm, + ) + predicates += ftsRowPredicate(SEARCH_BIGRAM_FTS_TABLE) + usesBigram = true + } else { + bigramTerms.forEach { + parameters += it + predicates += "instr(lower(search.search_text), ?) > 0" + } + usesSearchText = true } - val joiner = if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND " - return TextPredicate(predicates.joinToString(joiner, prefix = "(", postfix = ")"), false, true) } - val ftsQuery = when (mode) { - ChatHistoryMatchMode.PHRASE -> quoteFtsTerm(normalized) - ChatHistoryMatchMode.ALL -> ftsOnly.joinToString(" AND ", transform = ::quoteFtsTerm) - ChatHistoryMatchMode.ANY -> ftsOnly.joinToString(" OR ", transform = ::quoteFtsTerm) - } - parameters += ftsQuery - val ftsPredicate = "$SEARCH_FTS_TABLE MATCH ?" - if (mode != ChatHistoryMatchMode.ALL || !hasShortTerm) { - return TextPredicate(ftsPredicate, true, false) - } - val shortPredicates = terms.filter { it.codePointCount(0, it.length) < 3 }.map { + shortTerms.forEach { parameters += it - "instr(lower(search.search_text), ?) > 0" + predicates += "instr(lower(search.search_text), ?) > 0" + usesSearchText = true } + val joiner = if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND " return TextPredicate( - (listOf(ftsPredicate) + shortPredicates).joinToString(" AND ", prefix = "(", postfix = ")"), - true, - true, + predicates.joinToString(joiner, prefix = "(", postfix = ")"), + usesFts, + usesSearchText, + usesBigram, ) } + private fun ftsRowPredicate(table: String): String = + "mr.id IN (SELECT rowid FROM $table WHERE $table MATCH ?)" + private fun quoteFtsTerm(term: String): String = "\"${term.replace("\"", "\"\"")}\"" @@ -948,6 +1100,17 @@ object ChatHistoryStore { } } + private fun hasBigramSearchIndex(connection: Connection): Boolean { + connection.prepareStatement( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1" + ).use { statement -> + statement.setString(1, SEARCH_BIGRAM_FTS_TABLE) + statement.executeQuery().use { results -> + return results.next() + } + } + } + private fun readMeta(connection: Connection, key: String): String? { connection.prepareStatement( "SELECT value FROM chat_history_meta WHERE key = ?" diff --git a/src/main/kotlin/tools/ChatHistoryToolFormatter.kt b/src/main/kotlin/tools/ChatHistoryToolFormatter.kt index 30ea1d9..d9e1749 100644 --- a/src/main/kotlin/tools/ChatHistoryToolFormatter.kt +++ b/src/main/kotlin/tools/ChatHistoryToolFormatter.kt @@ -26,6 +26,16 @@ internal object ChatHistoryToolFormatter { val snapshotNames = runCatching { ContactSnapshotStore.loadDisplayNames(event.bot.id, group?.id, userIds) }.getOrDefault(emptyMap()) + val atNames = if (group != null) { + val targetIds = records.asSequence() + .flatMap { ChatHistorySearchText.extractAtTargets(it.code).asSequence() } + .toSet() + runCatching { + ContactSnapshotStore.loadDisplayNames(event.bot.id, group.id, targetIds) + }.getOrDefault(emptyMap()) + } else { + emptyMap() + } records.forEach { record -> val marker = if (record.id == targetId) ">>> " else "" @@ -40,7 +50,7 @@ internal object ChatHistoryToolFormatter { val time = timeFormatter.format( Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault()) ) - val content = buildSnippet(ChatHistorySearchText.extract(record.code), query) + val content = buildSnippet(ChatHistorySearchText.extract(record.code, atNames), query) output.append(marker) .append("[messageId=").append(record.id).append("] ") .append(time).append(' ') diff --git a/src/main/kotlin/tools/GetChatHistoryContext.kt b/src/main/kotlin/tools/GetChatHistoryContext.kt index 1225ad7..8919be1 100644 --- a/src/main/kotlin/tools/GetChatHistoryContext.kt +++ b/src/main/kotlin/tools/GetChatHistoryContext.kt @@ -18,22 +18,21 @@ import top.jie65535.mirai.data.ChatHistorySubject class GetChatHistoryContext : BaseAgent( tool = Tool.function( name = "getChatHistoryContext", - description = "根据 searchChatHistory 返回的 messageId，读取该消息在当前群聊或私聊中的前后文。" + - "结果按时间顺序排列，并用 >>> 标记目标消息。", + description = "读取某条聊天记录前后的消息。", parameters = Parameters.buildJsonObject { put("type", "object") putJsonObject("properties") { putJsonObject("messageId") { put("type", "integer") - put("description", "searchChatHistory 返回的稳定消息 ID") + put("description", "搜索结果中的 messageId") } putJsonObject("before") { put("type", "integer") - put("description", "目标消息之前的消息数，默认8，最大15") + put("description", "前文条数，默认8，最大15") } putJsonObject("after") { put("type", "integer") - put("description", "目标消息之后的消息数，默认8，最大15") + put("description", "后文条数，默认8，最大15") } } putJsonArray("required") { add(JsonPrimitive("messageId")) } diff --git a/src/main/kotlin/tools/SearchChatHistory.kt b/src/main/kotlin/tools/SearchChatHistory.kt index 8c1886e..695938a 100644 --- a/src/main/kotlin/tools/SearchChatHistory.kt +++ b/src/main/kotlin/tools/SearchChatHistory.kt @@ -33,35 +33,33 @@ import java.util.Base64 class SearchChatHistory : BaseAgent( tool = Tool.function( name = "searchChatHistory", - description = "在当前群聊或私聊的 SQLite 历史中搜索消息，可按内容、发送者和时间筛选。" + - "默认返回最近匹配并在结果过多时提供 cursor；需要查看命中消息前后讨论时调用 getChatHistoryContext。" + - "未指定 from 时默认搜索配置的近期天数，明确指定时间时可以搜索任意历史跨度。", + description = "搜索当前聊天的历史消息，支持文本、发送者、时间和分页；普通文本与可解析的 @ 目标会一起匹配。", parameters = Parameters.buildJsonObject { put("type", "object") putJsonObject("properties") { putJsonObject("query") { put("type", "string") - put("description", "要搜索的消息文本；省略时按其他条件浏览时间线") + put("description", "消息文本；也会匹配名称对应的 @ 目标") } putJsonObject("senderId") { put("type", "integer") - put("description", "发送者 QQ 号；仅在用户明确给出或上下文可靠提供 QQ 号时使用，不要根据昵称猜测") + put("description", "发送者 QQ 号；用户明确给出时使用") } putJsonObject("senderName") { put("type", "string") - put("description", "用户用名称指代发送者时优先原样传入群名片、昵称或好友备注；可与 senderId 同时提供，没有 senderId 时解析发送者，有重名时返回候选 QQ 号") + put("description", "发送者名称；用户按昵称或群名片称呼时使用") } putJsonObject("from") { put("type", "string") - put("description", "起始时间，格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd") + put("description", "起始时间：yyyy-MM-dd HH:mm 或 yyyy-MM-dd") } putJsonObject("to") { put("type", "string") - put("description", "结束时间，格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd；仅日期表示当天结束") + put("description", "结束时间：yyyy-MM-dd HH:mm 或 yyyy-MM-dd") } putJsonObject("cursor") { put("type", "string") - put("description", "上一页返回的 nextCursor；继续同一搜索时原样传回") + put("description", "上一页的 nextCursor") } } }, @@ -119,9 +117,11 @@ class SearchChatHistory : BaseAgent( return "聊天记录全文索引当前不可用，无法执行关键词搜索" } + val atTargetIds = query?.let { resolveMentionTargets(it, event) }.orEmpty() val request = ChatHistorySearchRequest( subject = ChatHistorySubject.from(event.subject), query = query, + atTargetIds = atTargetIds, matchMode = matchMode, fromId = resolvedSenderId, start = start.toEpochSecond().toInt(), @@ -182,6 +182,20 @@ class SearchChatHistory : BaseAgent( } private fun resolveSender(name: String, event: MessageEvent): SenderResolution { + val sorted = findNameMatches(name, event) + val first = sorted.firstOrNull() ?: return SenderResolution.NotFound + val second = sorted.getOrNull(1) + return if (second == null || first.matchRank < second.matchRank) { + SenderResolution.Found(first.userId) + } else { + SenderResolution.Ambiguous(sorted) + } + } + + private fun resolveMentionTargets(name: String, event: MessageEvent): Set = + findNameMatches(name, event).mapTo(linkedSetOf(), ContactNameMatch::userId) + + private fun findNameMatches(name: String, event: MessageEvent): List { val groupId = (event as? GroupMessageEvent)?.group?.id val matches = runCatching { ContactSnapshotStore.findUsersByName(event.bot.id, groupId, name, limit = 6) @@ -202,14 +216,8 @@ class SearchChatHistory : BaseAgent( } } } - val sorted = matches.sortedWith(compareBy({ it.matchRank }, { it.userId })) - val first = sorted.firstOrNull() ?: return SenderResolution.NotFound - val second = sorted.getOrNull(1) - return if (second == null || first.matchRank < second.matchRank) { - SenderResolution.Found(first.userId) - } else { - SenderResolution.Ambiguous(sorted) - } + return matches.sortedWith(compareBy({ it.matchRank }, { it.userId })) + .take(MAX_NAME_MATCHES) } private sealed interface SenderResolution { @@ -221,6 +229,7 @@ class SearchChatHistory : BaseAgent( companion object { private const val DEFAULT_PAGE_SIZE = 20 private const val DIAGNOSTIC_PAGE_SIZE = 6 + private const val MAX_NAME_MATCHES = 12 private const val MAX_PAGE_SIZE = 200 private const val OUTPUT_RESERVE_CHARS = 1_000 private const val APPROXIMATE_RECORD_CHARS = 450 diff --git a/src/test/kotlin/data/ChatHistorySearchTextTest.kt b/src/test/kotlin/data/ChatHistorySearchTextTest.kt new file mode 100644 index 0000000..e6c50be --- /dev/null +++ b/src/test/kotlin/data/ChatHistorySearchTextTest.kt @@ -0,0 +1,17 @@ +package top.jie65535.mirai.data + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals + +class ChatHistorySearchTextTest { + @Test + fun rendersAtTargetNamesAndBuildsBigrams() { + val code = """ + [{"type":"At","target":2180487691}, {"type":"PlainText","content":" 筱玥"}] + """.trimIndent() + + assertEquals(setOf(2180487691L), ChatHistorySearchText.extractAtTargets(code)) + assertEquals("@筱玥 筱玥", ChatHistorySearchText.extract(code, mapOf(2180487691L to "筱玥"))) + assertEquals("筱玥 筱玥", ChatHistorySearchText.bigrams("筱玥 筱玥")) + } +} diff --git a/src/test/kotlin/data/ChatHistoryStoreTest.kt b/src/test/kotlin/data/ChatHistoryStoreTest.kt index 23dd081..544da15 100644 --- a/src/test/kotlin/data/ChatHistoryStoreTest.kt +++ b/src/test/kotlin/data/ChatHistoryStoreTest.kt @@ -62,15 +62,16 @@ class ChatHistoryStoreTest { "SELECT value FROM chat_history_meta WHERE key = 'schema_version'" ).use { results -> assertTrue(results.next()) - assertEquals("3", results.getString(1)) + assertEquals("4", results.getString(1)) } statement.executeQuery( "SELECT COUNT(*) FROM sqlite_master " + "WHERE type = 'table' AND name IN " + - "('message_record_search', 'message_record_fts', 'chat_history_sender_alias')" + "('message_record_search', 'message_record_fts', 'message_record_bigram_fts', " + + "'chat_history_sender_alias')" ).use { results -> assertTrue(results.next()) - assertEquals(3, results.getInt(1)) + assertEquals(4, results.getInt(1)) } } } @@ -137,6 +138,33 @@ class ChatHistoryStoreTest { assertEquals(2L, shortQuery.totalMatches) assertEquals(listOf(first, third), shortQuery.records.map { it.id }) + val mention = insert( + directory, + time = 103, + fromId = 11, + text = "", + code = buildJsonArray { + add(buildJsonObject { + put("type", "At") + put("target", 2180487691L) + }) + }.toString(), + ) + val literal = insert(directory, 104, 11, "筱玥检查一下") + val mentionQuery = ChatHistoryStore.search( + ChatHistorySearchRequest( + subject = subject, + query = "筱玥", + atTargetIds = setOf(2180487691L), + fromId = 11, + start = 0, + end = 200, + limit = 10, + ) + ) + assertEquals(2L, mentionQuery.totalMatches) + assertEquals(listOf(literal, mention), mentionQuery.records.map { it.id }) + val aliases = ChatHistoryStore.findSenderAliases(subject, "旧名") assertEquals(listOf(ChatHistorySenderAliasMatch(11, "旧名张三", 1)), aliases) } finally { @@ -245,12 +273,81 @@ class ChatHistoryStoreTest { } } + @Test + fun migratesCompletedTrigramIndexAndBackfillsBigramIndex() { + val directory = Files.createTempDirectory("jchatgpt-history-bigram-migration-test-") + val database = directory.resolve("chat-history.sqlite") + try { + ChatHistoryStore.init(directory.toFile()) + val messageId = insert(directory, 100, 11, "部署完成") + ChatHistoryStore.close() + + DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection -> + connection.createStatement().use { statement -> + statement.executeUpdate("DROP TABLE message_record_bigram_fts") + statement.executeUpdate( + "DELETE FROM chat_history_meta WHERE key = 'search_bigram_version'" + ) + statement.executeUpdate( + "UPDATE chat_history_meta SET value = '3' WHERE key = 'schema_version'" + ) + statement.executeUpdate( + "UPDATE chat_history_meta SET value = '1' WHERE key = 'search_index_complete'" + ) + statement.executeUpdate( + "UPDATE chat_history_meta SET value = '$messageId' WHERE key = 'search_backfill_cursor'" + ) + } + } + + ChatHistoryStore.init(directory.toFile()) + assertFalse(ChatHistoryStore.isSearchIndexReady) + assertEquals(1, ChatHistoryStore.backfillSearchIndex(batchSize = 1)) + assertTrue(ChatHistoryStore.isSearchIndexReady) + + val search = ChatHistoryStore.search( + ChatHistorySearchRequest( + subject = ChatHistorySubject(1, MessageSourceKind.GROUP, 1000), + query = "部署", + start = 0, + end = 200, + ) + ) + assertEquals(listOf(messageId), search.records.map { it.id }) + + DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection -> + connection.createStatement().use { statement -> + statement.executeQuery( + "SELECT search_text FROM message_record_search WHERE message_id = $messageId" + ).use { results -> + assertTrue(results.next()) + assertEquals("部署完成", results.getString(1)) + } + statement.executeQuery( + "SELECT rowid FROM message_record_bigram_fts WHERE rowid = $messageId" + ).use { results -> + assertTrue(results.next()) + } + } + } + } finally { + ChatHistoryStore.close() + directory.toFile().deleteRecursively() + } + } + private fun insert( directory: java.nio.file.Path, time: Int, fromId: Long, text: String, senderName: String = "", + code: String = buildJsonArray { + add(buildJsonObject { + put("type", "PlainText") + put("content", text) + }) + }.toString(), ): Long { val record = ChatMessageRecord( botId = 1, @@ -260,12 +357,7 @@ class ChatHistoryStoreTest { internalIds = "$time", time = time, kind = MessageSourceKind.GROUP, - code = buildJsonArray { - add(buildJsonObject { - put("type", "PlainText") - put("content", text) - }) - }.toString(), + code = code, ) ChatHistoryStore.insertRecord(record, senderName) DriverManager.getConnection("jdbc:sqlite:${directory.resolve("chat-history.sqlite").absolutePathString()}").use {