mirror of
https://github.com/jie65535/JChatGPT.git
synced 2026-09-15 02:56:10 +08:00
history: index short terms and resolve mentions
This commit is contained in:
@@ -238,10 +238,11 @@ searchHistoryMaxRecords: 5000
|
|||||||
聊天记录与联系人快照保存在插件数据目录的 `chat-history.sqlite` 中,并使用 SQLite WAL 模式支持记录与查询并行进行。
|
聊天记录与联系人快照保存在插件数据目录的 `chat-history.sqlite` 中,并使用 SQLite WAL 模式支持记录与查询并行进行。
|
||||||
数据库由插件在首次启动时自动创建和维护,无需安装额外的聊天记录插件。
|
数据库由插件在首次启动时自动创建和维护,无需安装额外的聊天记录插件。
|
||||||
|
|
||||||
聊天记录搜索使用 SQLite FTS5 中文三元索引。升级已有数据库时只会新增独立的派生搜索文本表和索引表,
|
聊天记录搜索使用 SQLite FTS5 中文二元 + 三元索引。升级已有数据库时只会新增独立的派生搜索文本表和索引表,
|
||||||
不会改列、重建或删除 `message_record` 原始记录;旧记录的搜索文本会在插件启动后分批后台回填。回填期间新消息仍会正常保存,
|
不会改列、重建或删除 `message_record` 原始记录;旧记录的搜索文本会在插件启动后分批后台回填。回填期间新消息仍会正常保存,
|
||||||
搜索结果会明确标注索引尚未完成。搜索工具默认只查询当前群聊或当前私聊,明确指定 `from` 和 `to` 后可以跨越任意历史时间范围,
|
搜索结果会明确标注索引尚未完成。搜索工具默认只查询当前群聊或当前私聊,明确指定 `from` 和 `to` 后可以跨越任意历史时间范围,
|
||||||
并通过 `nextCursor` 分页。升级后新收到的消息还会记录发送者在该会话中使用过的名称,名称搜索不依赖对方仍是当前群成员;
|
并通过 `nextCursor` 分页。普通文字和可解析的 `@` 目标会合并检索,不要求模型预先判断查询类型;
|
||||||
|
升级后新收到的消息还会记录发送者在该会话中使用过的名称,名称搜索不依赖对方仍是当前群成员;
|
||||||
`getChatHistoryContext` 可按 `messageId` 获取命中消息前后的对话上下文。
|
`getChatHistoryContext` 可按 `messageId` 获取命中消息前后的对话上下文。
|
||||||
|
|
||||||
联系人刷新在后台通过 Overflow 的公开 `RemoteBot.executeAction` 调用 OneBot
|
联系人刷新在后台通过 Overflow 的公开 `RemoteBot.executeAction` 调用 OneBot
|
||||||
@@ -506,7 +507,7 @@ fallbackCooldownMinutes: 5
|
|||||||
14. **GetChatHistoryContext** - 根据历史消息 ID 获取目标消息前后的对话上下文
|
14. **GetChatHistoryContext** - 根据历史消息 ID 获取目标消息前后的对话上下文
|
||||||
15. **QueryUserProfile** - 按当前发送者、QQ 号、群名片、昵称或好友备注读取长期画像,并针对该联系人按需读取公开资料卡
|
15. **QueryUserProfile** - 按当前发送者、QQ 号、群名片、昵称或好友备注读取长期画像,并针对该联系人按需读取公开资料卡
|
||||||
|
|
||||||
`SearchChatHistory` 的常用参数只有消息内容、发送者、时间范围和翻页游标;结果默认按最近消息返回,
|
`SearchChatHistory` 的常用参数只有消息内容、发送者、时间范围和翻页游标;普通文字和可解析的 `@` 目标会合并检索,结果默认按最近消息返回,
|
||||||
单页大小和匹配策略由插件控制。`senderId` 与 `senderName` 可以同时提供,此时 QQ 号作为精确筛选条件;有 QQ 号时名称不参与筛选。
|
单页大小和匹配策略由插件控制。`senderId` 与 `senderName` 可以同时提供,此时 QQ 号作为精确筛选条件;有 QQ 号时名称不参与筛选。
|
||||||
|
|
||||||
### GitHub 查询工具
|
### GitHub 查询工具
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ data class ChatHistoryCursor(
|
|||||||
data class ChatHistorySearchRequest(
|
data class ChatHistorySearchRequest(
|
||||||
val subject: ChatHistorySubject,
|
val subject: ChatHistorySubject,
|
||||||
val query: String? = null,
|
val query: String? = null,
|
||||||
|
val atTargetIds: Set<Long> = emptySet(),
|
||||||
val matchMode: ChatHistoryMatchMode = ChatHistoryMatchMode.ALL,
|
val matchMode: ChatHistoryMatchMode = ChatHistoryMatchMode.ALL,
|
||||||
val fromId: Long? = null,
|
val fromId: Long? = null,
|
||||||
val start: Int? = null,
|
val start: Int? = null,
|
||||||
|
|||||||
@@ -18,28 +18,49 @@ object ChatHistorySearchText {
|
|||||||
private val json = Json { ignoreUnknownKeys = true }
|
private val json = Json { ignoreUnknownKeys = true }
|
||||||
private val whitespace = Regex("\\s+")
|
private val whitespace = Regex("\\s+")
|
||||||
|
|
||||||
fun extract(code: String): String {
|
fun extract(code: String, atNames: Map<Long, String> = emptyMap()): String {
|
||||||
val rendered = runCatching { renderJsonCode(code) }
|
val rendered = runCatching { renderJsonCode(code, atNames) }
|
||||||
.recoverCatching { decodeMessageCode(code).content }
|
.recoverCatching { decodeMessageCode(code).content }
|
||||||
.getOrDefault("")
|
.getOrDefault("")
|
||||||
return rendered.normalize().takeUtf16Safely(MAX_INDEXED_CHARS)
|
return rendered.normalize().takeUtf16Safely(MAX_INDEXED_CHARS)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun renderJsonCode(code: String): String {
|
fun extractAtTargets(code: String): Set<Long> = runCatching {
|
||||||
val messages = json.parseToJsonElement(code) as? JsonArray
|
val messages = json.parseToJsonElement(code) as? JsonArray ?: return@runCatching emptySet()
|
||||||
?: throw IllegalArgumentException("消息记录不是 JSON array")
|
buildSet { collectAtTargets(messages, this) }
|
||||||
return renderMessages(messages)
|
}.getOrDefault(emptySet())
|
||||||
|
|
||||||
|
fun bigrams(text: String): String {
|
||||||
|
val codePoints = text.lowercase().codePoints().toArray()
|
||||||
|
if (codePoints.size < 2) return ""
|
||||||
|
return buildString(codePoints.size * 3) {
|
||||||
|
var tokenCount = 0
|
||||||
|
for (index in 0 until codePoints.lastIndex) {
|
||||||
|
if (Character.isWhitespace(codePoints[index]) || Character.isWhitespace(codePoints[index + 1])) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (tokenCount++ > 0) append(' ')
|
||||||
|
appendCodePoint(codePoints[index])
|
||||||
|
appendCodePoint(codePoints[index + 1])
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun renderMessages(messages: JsonArray): String = messages.joinToString("") { element ->
|
private fun renderJsonCode(code: String, atNames: Map<Long, String>): String {
|
||||||
|
val messages = json.parseToJsonElement(code) as? JsonArray
|
||||||
|
?: throw IllegalArgumentException("消息记录不是 JSON array")
|
||||||
|
return renderMessages(messages, atNames)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun renderMessages(messages: JsonArray, atNames: Map<Long, String>): String = messages.joinToString("") { element ->
|
||||||
val message = element as? JsonObject ?: return@joinToString ""
|
val message = element as? JsonObject ?: return@joinToString ""
|
||||||
when (val type = message.string("type")) {
|
when (val type = message.string("type")) {
|
||||||
"PlainText" -> message.string("content").orEmpty()
|
"PlainText" -> message.string("content").orEmpty()
|
||||||
"At" -> message.long("target")?.let { "@$it" }.orEmpty()
|
"At" -> message.long("target")?.let { target -> "@${atNames[target] ?: target}" }.orEmpty()
|
||||||
"AtAll" -> "@全体成员"
|
"AtAll" -> "@全体成员"
|
||||||
"Image", "FlashImage" -> if (message.boolean("isEmoji") == true) "[表情包]" else "[图片]"
|
"Image", "FlashImage" -> if (message.boolean("isEmoji") == true) "[表情包]" else "[图片]"
|
||||||
"QuoteReply" -> renderQuote(message)
|
"QuoteReply" -> renderQuote(message, atNames)
|
||||||
"ForwardMessage" -> renderForward(message)
|
"ForwardMessage" -> renderForward(message, atNames)
|
||||||
"MessageOrigin", "MessageSource", "ShowImageFlag" -> ""
|
"MessageOrigin", "MessageSource", "ShowImageFlag" -> ""
|
||||||
"Face", "MarketFace", "VipFace" -> "[表情]"
|
"Face", "MarketFace", "VipFace" -> "[表情]"
|
||||||
"Audio" -> "[语音]"
|
"Audio" -> "[语音]"
|
||||||
@@ -51,18 +72,18 @@ object ChatHistorySearchText {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun renderQuote(message: JsonObject): String {
|
private fun renderQuote(message: JsonObject, atNames: Map<Long, String>): String {
|
||||||
val source = message["source"] as? JsonObject ?: return "[引用消息]"
|
val source = message["source"] as? JsonObject ?: return "[引用消息]"
|
||||||
val author = source.long("fromId")?.toString() ?: "其他用户"
|
val author = source.long("fromId")?.toString() ?: "其他用户"
|
||||||
val original = (source["originalMessage"] as? JsonArray)
|
val original = (source["originalMessage"] as? JsonArray)
|
||||||
?.let(::renderMessages)
|
?.let { renderMessages(it, atNames) }
|
||||||
.orEmpty()
|
.orEmpty()
|
||||||
.normalize()
|
.normalize()
|
||||||
.takeUtf16Safely(MAX_QUOTED_CHARS)
|
.takeUtf16Safely(MAX_QUOTED_CHARS)
|
||||||
return "[引用 $author: $original]"
|
return "[引用 $author: $original]"
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun renderForward(message: JsonObject): String = buildString {
|
private fun renderForward(message: JsonObject, atNames: Map<Long, String>): String = buildString {
|
||||||
append("[转发消息]")
|
append("[转发消息]")
|
||||||
val nodes = message["nodeList"] as? JsonArray ?: return@buildString
|
val nodes = message["nodeList"] as? JsonArray ?: return@buildString
|
||||||
nodes.take(MAX_FORWARD_NODES).forEach { element ->
|
nodes.take(MAX_FORWARD_NODES).forEach { element ->
|
||||||
@@ -71,7 +92,7 @@ object ChatHistorySearchText {
|
|||||||
val chain = node["messageChain"] as? JsonArray
|
val chain = node["messageChain"] as? JsonArray
|
||||||
append(' ').append(sender).append(": ")
|
append(' ').append(sender).append(": ")
|
||||||
append(
|
append(
|
||||||
chain?.let(::renderMessages)
|
chain?.let { renderMessages(it, atNames) }
|
||||||
.orEmpty()
|
.orEmpty()
|
||||||
.normalize()
|
.normalize()
|
||||||
.takeUtf16Safely(MAX_FORWARD_NODE_CHARS)
|
.takeUtf16Safely(MAX_FORWARD_NODE_CHARS)
|
||||||
@@ -80,6 +101,24 @@ object ChatHistorySearchText {
|
|||||||
if (nodes.size > MAX_FORWARD_NODES) append(" ...[转发内容截断]")
|
if (nodes.size > MAX_FORWARD_NODES) append(" ...[转发内容截断]")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun collectAtTargets(messages: JsonArray, targets: MutableSet<Long>) {
|
||||||
|
messages.forEach { element ->
|
||||||
|
val message = element as? JsonObject ?: return@forEach
|
||||||
|
when (message.string("type")) {
|
||||||
|
"At" -> message.long("target")?.let(targets::add)
|
||||||
|
"QuoteReply" -> {
|
||||||
|
val original = (message["source"] as? JsonObject)
|
||||||
|
?.get("originalMessage") as? JsonArray
|
||||||
|
original?.let { collectAtTargets(it, targets) }
|
||||||
|
}
|
||||||
|
"ForwardMessage" -> (message["nodeList"] as? JsonArray).orEmpty().forEach { node ->
|
||||||
|
((node as? JsonObject)?.get("messageChain") as? JsonArray)
|
||||||
|
?.let { collectAtTargets(it, targets) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun String.normalize(): String = whitespace.replace(this, " ")
|
private fun String.normalize(): String = whitespace.replace(this, " ")
|
||||||
.trim()
|
.trim()
|
||||||
.replaceUnpairedSurrogates()
|
.replaceUnpairedSurrogates()
|
||||||
|
|||||||
@@ -24,13 +24,16 @@ import java.sql.ResultSet
|
|||||||
* 写连接由 [writeLock] 串行保护;SQLite 使用 WAL + NORMAL synchronous,允许并发读取。
|
* 写连接由 [writeLock] 串行保护;SQLite 使用 WAL + NORMAL synchronous,允许并发读取。
|
||||||
*/
|
*/
|
||||||
object ChatHistoryStore {
|
object ChatHistoryStore {
|
||||||
private const val SCHEMA_VERSION = 3
|
private const val SCHEMA_VERSION = 4
|
||||||
private const val BUSY_TIMEOUT_MS = 30_000
|
private const val BUSY_TIMEOUT_MS = 30_000
|
||||||
private const val SEARCH_FTS_TABLE = "message_record_fts"
|
private const val SEARCH_FTS_TABLE = "message_record_fts"
|
||||||
|
private const val SEARCH_BIGRAM_FTS_TABLE = "message_record_bigram_fts"
|
||||||
private const val SEARCH_TEXT_TABLE = "message_record_search"
|
private const val SEARCH_TEXT_TABLE = "message_record_search"
|
||||||
private const val SENDER_ALIAS_TABLE = "chat_history_sender_alias"
|
private const val SENDER_ALIAS_TABLE = "chat_history_sender_alias"
|
||||||
private const val SEARCH_BACKFILL_CURSOR_KEY = "search_backfill_cursor"
|
private const val SEARCH_BACKFILL_CURSOR_KEY = "search_backfill_cursor"
|
||||||
private const val SEARCH_INDEX_COMPLETE_KEY = "search_index_complete"
|
private const val SEARCH_INDEX_COMPLETE_KEY = "search_index_complete"
|
||||||
|
private const val SEARCH_BIGRAM_VERSION_KEY = "search_bigram_version"
|
||||||
|
private const val SEARCH_BIGRAM_VERSION = "1"
|
||||||
private const val DEFAULT_BACKFILL_BATCH_SIZE = 250
|
private const val DEFAULT_BACKFILL_BATCH_SIZE = 250
|
||||||
private const val MAX_PAGE_SIZE = 200
|
private const val MAX_PAGE_SIZE = 200
|
||||||
private const val MAX_CONTEXT_MESSAGES = 100
|
private const val MAX_CONTEXT_MESSAGES = 100
|
||||||
@@ -43,6 +46,8 @@ object ChatHistoryStore {
|
|||||||
@Volatile
|
@Volatile
|
||||||
private var searchIndexAvailable = false
|
private var searchIndexAvailable = false
|
||||||
@Volatile
|
@Volatile
|
||||||
|
private var bigramIndexAvailable = false
|
||||||
|
@Volatile
|
||||||
private var searchIndexReady = false
|
private var searchIndexReady = false
|
||||||
private lateinit var databaseFile: File
|
private lateinit var databaseFile: File
|
||||||
private var writeConnection: Connection? = null
|
private var writeConnection: Connection? = null
|
||||||
@@ -79,6 +84,7 @@ object ChatHistoryStore {
|
|||||||
writeConnection = connection
|
writeConnection = connection
|
||||||
initialized = true
|
initialized = true
|
||||||
searchIndexAvailable = hasSearchIndex(connection)
|
searchIndexAvailable = hasSearchIndex(connection)
|
||||||
|
bigramIndexAvailable = hasBigramSearchIndex(connection)
|
||||||
searchIndexReady = searchIndexAvailable && readMeta(connection, SEARCH_INDEX_COMPLETE_KEY) == "1"
|
searchIndexReady = searchIndexAvailable && readMeta(connection, SEARCH_INDEX_COMPLETE_KEY) == "1"
|
||||||
} catch (cause: Throwable) {
|
} catch (cause: Throwable) {
|
||||||
connection.close()
|
connection.close()
|
||||||
@@ -102,6 +108,7 @@ object ChatHistoryStore {
|
|||||||
writeConnection = null
|
writeConnection = null
|
||||||
initialized = false
|
initialized = false
|
||||||
searchIndexAvailable = false
|
searchIndexAvailable = false
|
||||||
|
bigramIndexAvailable = false
|
||||||
searchIndexReady = false
|
searchIndexReady = false
|
||||||
warningLogger = null
|
warningLogger = null
|
||||||
}
|
}
|
||||||
@@ -299,17 +306,17 @@ object ChatHistoryStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
val query = request.query?.trim().orEmpty()
|
val query = request.query?.trim().orEmpty()
|
||||||
val textPredicate = buildTextPredicate(query, request.matchMode, parameters)
|
val textPredicate = buildTextPredicate(query, request.matchMode, request.atTargetIds, parameters)
|
||||||
if ((textPredicate.usesFts || textPredicate.usesSearchText) && !isSearchIndexAvailable) {
|
if ((textPredicate.usesFts || textPredicate.usesSearchText) && !isSearchIndexAvailable) {
|
||||||
error("聊天记录全文索引尚未就绪")
|
error("聊天记录全文索引尚未就绪")
|
||||||
}
|
}
|
||||||
|
if (textPredicate.usesBigram && !bigramIndexAvailable) {
|
||||||
|
error("聊天记录二元索引尚未就绪")
|
||||||
|
}
|
||||||
val joins = buildString {
|
val joins = buildString {
|
||||||
if (textPredicate.usesSearchText) {
|
if (textPredicate.usesSearchText) {
|
||||||
append("JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id ")
|
append("JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id ")
|
||||||
}
|
}
|
||||||
if (textPredicate.usesFts) {
|
|
||||||
append("JOIN $SEARCH_FTS_TABLE ON $SEARCH_FTS_TABLE.rowid = mr.id")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
textPredicate.sql?.let(conditions::add)
|
textPredicate.sql?.let(conditions::add)
|
||||||
|
|
||||||
@@ -499,16 +506,24 @@ object ChatHistoryStore {
|
|||||||
var cursor = withWriteConnectionResult { connection ->
|
var cursor = withWriteConnectionResult { connection ->
|
||||||
readMeta(connection, SEARCH_BACKFILL_CURSOR_KEY)?.toLongOrNull() ?: 0L
|
readMeta(connection, SEARCH_BACKFILL_CURSOR_KEY)?.toLongOrNull() ?: 0L
|
||||||
}
|
}
|
||||||
|
val hasBigramIndex = bigramIndexAvailable
|
||||||
while (shouldContinue()) {
|
while (shouldContinue()) {
|
||||||
val batch = openReadConnection().use { connection ->
|
val batch = openReadConnection().use { connection ->
|
||||||
|
val missingBigramExpression = if (hasBigramIndex) {
|
||||||
|
"NOT EXISTS (SELECT 1 FROM $SEARCH_BIGRAM_FTS_TABLE bigram WHERE bigram.rowid = mr.id)"
|
||||||
|
} else {
|
||||||
|
"0"
|
||||||
|
}
|
||||||
|
val missingBigramCondition = if (hasBigramIndex) "OR $missingBigramExpression" else ""
|
||||||
connection.prepareStatement(
|
connection.prepareStatement(
|
||||||
"""
|
"""
|
||||||
SELECT mr.id, mr.code
|
SELECT mr.id, mr.code,
|
||||||
|
search.message_id IS NULL AS missing_search,
|
||||||
|
$missingBigramExpression AS missing_bigram
|
||||||
FROM message_record mr
|
FROM message_record mr
|
||||||
|
LEFT JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id
|
||||||
WHERE mr.id > ?
|
WHERE mr.id > ?
|
||||||
AND NOT EXISTS (
|
AND (search.message_id IS NULL $missingBigramCondition)
|
||||||
SELECT 1 FROM $SEARCH_TEXT_TABLE search WHERE search.message_id = mr.id
|
|
||||||
)
|
|
||||||
ORDER BY mr.id
|
ORDER BY mr.id
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
""".trimIndent()
|
""".trimIndent()
|
||||||
@@ -518,7 +533,14 @@ object ChatHistoryStore {
|
|||||||
statement.executeQuery().use { results ->
|
statement.executeQuery().use { results ->
|
||||||
buildList {
|
buildList {
|
||||||
while (results.next()) {
|
while (results.next()) {
|
||||||
add(results.getLong("id") to results.getString("code"))
|
add(
|
||||||
|
SearchBackfillRow(
|
||||||
|
id = results.getLong("id"),
|
||||||
|
code = results.getString("code"),
|
||||||
|
missingSearch = results.getBoolean("missing_search"),
|
||||||
|
missingBigram = results.getBoolean("missing_bigram"),
|
||||||
|
)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -541,20 +563,39 @@ object ChatHistoryStore {
|
|||||||
connection.prepareStatement(
|
connection.prepareStatement(
|
||||||
"INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)"
|
"INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)"
|
||||||
).use { insertFts ->
|
).use { insertFts ->
|
||||||
batch.forEach { (id, code) ->
|
val insertBigram = if (hasBigramIndex) {
|
||||||
val text = ChatHistorySearchText.extract(code)
|
connection.prepareStatement(
|
||||||
insertText.setLong(1, id)
|
"INSERT INTO $SEARCH_BIGRAM_FTS_TABLE(rowid, bigram_text) VALUES (?, ?)"
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
batch.forEach { row ->
|
||||||
|
val text = ChatHistorySearchText.extract(row.code)
|
||||||
|
if (row.missingSearch) {
|
||||||
|
insertText.setLong(1, row.id)
|
||||||
insertText.setString(2, text)
|
insertText.setString(2, text)
|
||||||
insertText.addBatch()
|
insertText.addBatch()
|
||||||
insertFts.setLong(1, id)
|
insertFts.setLong(1, row.id)
|
||||||
insertFts.setString(2, text)
|
insertFts.setString(2, text)
|
||||||
insertFts.addBatch()
|
insertFts.addBatch()
|
||||||
}
|
}
|
||||||
|
if (row.missingBigram) {
|
||||||
|
insertBigram?.setLong(1, row.id)
|
||||||
|
insertBigram?.setString(2, ChatHistorySearchText.bigrams(text))
|
||||||
|
insertBigram?.addBatch()
|
||||||
|
}
|
||||||
|
}
|
||||||
insertText.executeBatch()
|
insertText.executeBatch()
|
||||||
insertFts.executeBatch()
|
insertFts.executeBatch()
|
||||||
|
insertBigram?.executeBatch()
|
||||||
|
} finally {
|
||||||
|
insertBigram?.close()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cursor = batch.last().first
|
}
|
||||||
|
cursor = batch.last().id
|
||||||
writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, cursor.toString())
|
writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, cursor.toString())
|
||||||
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0")
|
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0")
|
||||||
connection.commit()
|
connection.commit()
|
||||||
@@ -571,6 +612,13 @@ object ChatHistoryStore {
|
|||||||
return processed
|
return processed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private data class SearchBackfillRow(
|
||||||
|
val id: Long,
|
||||||
|
val code: String,
|
||||||
|
val missingSearch: Boolean,
|
||||||
|
val missingBigram: Boolean,
|
||||||
|
)
|
||||||
|
|
||||||
internal fun insertRecord(record: ChatMessageRecord, senderName: String = "") {
|
internal fun insertRecord(record: ChatMessageRecord, senderName: String = "") {
|
||||||
var recordId = 0L
|
var recordId = 0L
|
||||||
var indexFailure: Throwable? = null
|
var indexFailure: Throwable? = null
|
||||||
@@ -639,6 +687,15 @@ object ChatHistoryStore {
|
|||||||
statement.setString(2, searchText)
|
statement.setString(2, searchText)
|
||||||
statement.executeUpdate()
|
statement.executeUpdate()
|
||||||
}
|
}
|
||||||
|
if (bigramIndexAvailable) {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO $SEARCH_BIGRAM_FTS_TABLE(rowid, bigram_text) VALUES (?, ?)"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setLong(1, recordId)
|
||||||
|
statement.setString(2, ChatHistorySearchText.bigrams(searchText))
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
}
|
||||||
connection.commit()
|
connection.commit()
|
||||||
} catch (cause: Throwable) {
|
} catch (cause: Throwable) {
|
||||||
connection.rollback()
|
connection.rollback()
|
||||||
@@ -810,6 +867,20 @@ object ChatHistoryStore {
|
|||||||
}
|
}
|
||||||
}.onFailure { warn("初始化 SQLite FTS5 聊天记录索引失败,消息记录仍会继续保存", it) }
|
}.onFailure { warn("初始化 SQLite FTS5 聊天记录索引失败,消息记录仍会继续保存", it) }
|
||||||
|
|
||||||
|
runCatching {
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS $SEARCH_BIGRAM_FTS_TABLE USING fts5(
|
||||||
|
bigram_text,
|
||||||
|
content='',
|
||||||
|
tokenize='unicode61'
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}.onFailure { warn("初始化 SQLite FTS5 聊天记录二元索引失败,将回退到短词扫描", it) }
|
||||||
|
|
||||||
if (hasSearchIndex(connection)) {
|
if (hasSearchIndex(connection)) {
|
||||||
val hasMessages = connection.createStatement().use { statement ->
|
val hasMessages = connection.createStatement().use { statement ->
|
||||||
statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next)
|
statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next)
|
||||||
@@ -825,6 +896,27 @@ object ChatHistoryStore {
|
|||||||
statement.executeUpdate()
|
statement.executeUpdate()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hasBigramSearchIndex(connection) &&
|
||||||
|
readMeta(connection, SEARCH_BIGRAM_VERSION_KEY) != SEARCH_BIGRAM_VERSION
|
||||||
|
) {
|
||||||
|
val hasMessages = connection.createStatement().use { statement ->
|
||||||
|
statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next)
|
||||||
|
}
|
||||||
|
val oldAutoCommitForMigration = connection.autoCommit
|
||||||
|
connection.autoCommit = false
|
||||||
|
try {
|
||||||
|
writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, "0")
|
||||||
|
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, if (hasMessages) "0" else "1")
|
||||||
|
writeMeta(connection, SEARCH_BIGRAM_VERSION_KEY, SEARCH_BIGRAM_VERSION)
|
||||||
|
connection.commit()
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
connection.rollback()
|
||||||
|
throw cause
|
||||||
|
} finally {
|
||||||
|
connection.autoCommit = oldAutoCommitForMigration
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun subjectConditions(
|
private fun subjectConditions(
|
||||||
@@ -852,9 +944,40 @@ object ChatHistoryStore {
|
|||||||
val sql: String?,
|
val sql: String?,
|
||||||
val usesFts: Boolean,
|
val usesFts: Boolean,
|
||||||
val usesSearchText: Boolean,
|
val usesSearchText: Boolean,
|
||||||
|
val usesBigram: Boolean = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
private fun buildTextPredicate(
|
private fun buildTextPredicate(
|
||||||
|
query: String,
|
||||||
|
mode: ChatHistoryMatchMode,
|
||||||
|
atTargetIds: Set<Long>,
|
||||||
|
parameters: MutableList<Any>,
|
||||||
|
): TextPredicate {
|
||||||
|
if (query.isBlank() && atTargetIds.isEmpty()) return TextPredicate(null, false, false)
|
||||||
|
val contentPredicate = if (query.isBlank()) {
|
||||||
|
TextPredicate(null, false, false)
|
||||||
|
} else {
|
||||||
|
buildContentTextPredicate(query, mode, parameters)
|
||||||
|
}
|
||||||
|
if (atTargetIds.isEmpty()) return contentPredicate
|
||||||
|
|
||||||
|
val mentionPredicate = atTargetIds.sorted().joinToString(" OR ", prefix = "(", postfix = ")") {
|
||||||
|
parameters += quoteFtsTerm("@$it")
|
||||||
|
ftsRowPredicate(SEARCH_FTS_TABLE)
|
||||||
|
}
|
||||||
|
val sql = when {
|
||||||
|
contentPredicate.sql == null -> mentionPredicate
|
||||||
|
else -> "(${contentPredicate.sql} OR $mentionPredicate)"
|
||||||
|
}
|
||||||
|
return TextPredicate(
|
||||||
|
sql = sql,
|
||||||
|
usesFts = true,
|
||||||
|
usesSearchText = contentPredicate.usesSearchText,
|
||||||
|
usesBigram = contentPredicate.usesBigram,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun buildContentTextPredicate(
|
||||||
query: String,
|
query: String,
|
||||||
mode: ChatHistoryMatchMode,
|
mode: ChatHistoryMatchMode,
|
||||||
parameters: MutableList<Any>,
|
parameters: MutableList<Any>,
|
||||||
@@ -862,48 +985,77 @@ object ChatHistoryStore {
|
|||||||
if (query.isBlank()) return TextPredicate(null, false, false)
|
if (query.isBlank()) return TextPredicate(null, false, false)
|
||||||
val normalized = query.lowercase()
|
val normalized = query.lowercase()
|
||||||
val terms = normalized.split(Regex("\\s+")).filter(String::isNotBlank)
|
val terms = normalized.split(Regex("\\s+")).filter(String::isNotBlank)
|
||||||
val hasShortTerm = terms.any { it.codePointCount(0, it.length) < 3 }
|
if (mode == ChatHistoryMatchMode.PHRASE) {
|
||||||
if (mode == ChatHistoryMatchMode.ANY && hasShortTerm) {
|
val codePointCount = normalized.codePointCount(0, normalized.length)
|
||||||
val predicates = terms.map {
|
return when {
|
||||||
parameters += it
|
codePointCount >= 3 -> {
|
||||||
"instr(lower(search.search_text), ?) > 0"
|
parameters += quoteFtsTerm(normalized)
|
||||||
|
TextPredicate(ftsRowPredicate(SEARCH_FTS_TABLE), usesFts = true, usesSearchText = false)
|
||||||
}
|
}
|
||||||
return TextPredicate(predicates.joinToString(" OR ", prefix = "(", postfix = ")"), false, true)
|
codePointCount == 2 && bigramIndexAvailable -> {
|
||||||
}
|
parameters += quoteFtsTerm(normalized)
|
||||||
val ftsTerms = if (mode == ChatHistoryMatchMode.PHRASE) {
|
TextPredicate(
|
||||||
listOf(normalized)
|
ftsRowPredicate(SEARCH_BIGRAM_FTS_TABLE),
|
||||||
} else {
|
usesFts = false,
|
||||||
terms
|
usesSearchText = false,
|
||||||
}
|
usesBigram = true,
|
||||||
val ftsOnly = ftsTerms.filter { it.codePointCount(0, it.length) >= 3 }
|
|
||||||
if (ftsOnly.isEmpty()) {
|
|
||||||
val predicates = terms.map {
|
|
||||||
parameters += it
|
|
||||||
"instr(lower(search.search_text), ?) > 0"
|
|
||||||
}
|
|
||||||
val joiner = if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND "
|
|
||||||
return TextPredicate(predicates.joinToString(joiner, prefix = "(", postfix = ")"), false, true)
|
|
||||||
}
|
|
||||||
val ftsQuery = when (mode) {
|
|
||||||
ChatHistoryMatchMode.PHRASE -> quoteFtsTerm(normalized)
|
|
||||||
ChatHistoryMatchMode.ALL -> ftsOnly.joinToString(" AND ", transform = ::quoteFtsTerm)
|
|
||||||
ChatHistoryMatchMode.ANY -> ftsOnly.joinToString(" OR ", transform = ::quoteFtsTerm)
|
|
||||||
}
|
|
||||||
parameters += ftsQuery
|
|
||||||
val ftsPredicate = "$SEARCH_FTS_TABLE MATCH ?"
|
|
||||||
if (mode != ChatHistoryMatchMode.ALL || !hasShortTerm) {
|
|
||||||
return TextPredicate(ftsPredicate, true, false)
|
|
||||||
}
|
|
||||||
val shortPredicates = terms.filter { it.codePointCount(0, it.length) < 3 }.map {
|
|
||||||
parameters += it
|
|
||||||
"instr(lower(search.search_text), ?) > 0"
|
|
||||||
}
|
|
||||||
return TextPredicate(
|
|
||||||
(listOf(ftsPredicate) + shortPredicates).joinToString(" AND ", prefix = "(", postfix = ")"),
|
|
||||||
true,
|
|
||||||
true,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
else -> {
|
||||||
|
parameters += normalized
|
||||||
|
TextPredicate("instr(lower(search.search_text), ?) > 0", false, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val longTerms = terms.filter { it.codePointCount(0, it.length) >= 3 }
|
||||||
|
val bigramTerms = terms.filter { it.codePointCount(0, it.length) == 2 }
|
||||||
|
val shortTerms = terms.filter { it.codePointCount(0, it.length) < 2 }
|
||||||
|
val predicates = mutableListOf<String>()
|
||||||
|
var usesFts = false
|
||||||
|
var usesBigram = false
|
||||||
|
var usesSearchText = false
|
||||||
|
|
||||||
|
if (longTerms.isNotEmpty()) {
|
||||||
|
parameters += longTerms.joinToString(
|
||||||
|
if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND ",
|
||||||
|
transform = ::quoteFtsTerm,
|
||||||
|
)
|
||||||
|
predicates += ftsRowPredicate(SEARCH_FTS_TABLE)
|
||||||
|
usesFts = true
|
||||||
|
}
|
||||||
|
if (bigramTerms.isNotEmpty()) {
|
||||||
|
if (bigramIndexAvailable) {
|
||||||
|
parameters += bigramTerms.joinToString(
|
||||||
|
if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND ",
|
||||||
|
transform = ::quoteFtsTerm,
|
||||||
|
)
|
||||||
|
predicates += ftsRowPredicate(SEARCH_BIGRAM_FTS_TABLE)
|
||||||
|
usesBigram = true
|
||||||
|
} else {
|
||||||
|
bigramTerms.forEach {
|
||||||
|
parameters += it
|
||||||
|
predicates += "instr(lower(search.search_text), ?) > 0"
|
||||||
|
}
|
||||||
|
usesSearchText = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
shortTerms.forEach {
|
||||||
|
parameters += it
|
||||||
|
predicates += "instr(lower(search.search_text), ?) > 0"
|
||||||
|
usesSearchText = true
|
||||||
|
}
|
||||||
|
val joiner = if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND "
|
||||||
|
return TextPredicate(
|
||||||
|
predicates.joinToString(joiner, prefix = "(", postfix = ")"),
|
||||||
|
usesFts,
|
||||||
|
usesSearchText,
|
||||||
|
usesBigram,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun ftsRowPredicate(table: String): String =
|
||||||
|
"mr.id IN (SELECT rowid FROM $table WHERE $table MATCH ?)"
|
||||||
|
|
||||||
private fun quoteFtsTerm(term: String): String =
|
private fun quoteFtsTerm(term: String): String =
|
||||||
"\"${term.replace("\"", "\"\"")}\""
|
"\"${term.replace("\"", "\"\"")}\""
|
||||||
@@ -948,6 +1100,17 @@ object ChatHistoryStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun hasBigramSearchIndex(connection: Connection): Boolean {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setString(1, SEARCH_BIGRAM_FTS_TABLE)
|
||||||
|
statement.executeQuery().use { results ->
|
||||||
|
return results.next()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun readMeta(connection: Connection, key: String): String? {
|
private fun readMeta(connection: Connection, key: String): String? {
|
||||||
connection.prepareStatement(
|
connection.prepareStatement(
|
||||||
"SELECT value FROM chat_history_meta WHERE key = ?"
|
"SELECT value FROM chat_history_meta WHERE key = ?"
|
||||||
|
|||||||
@@ -26,6 +26,16 @@ internal object ChatHistoryToolFormatter {
|
|||||||
val snapshotNames = runCatching {
|
val snapshotNames = runCatching {
|
||||||
ContactSnapshotStore.loadDisplayNames(event.bot.id, group?.id, userIds)
|
ContactSnapshotStore.loadDisplayNames(event.bot.id, group?.id, userIds)
|
||||||
}.getOrDefault(emptyMap())
|
}.getOrDefault(emptyMap())
|
||||||
|
val atNames = if (group != null) {
|
||||||
|
val targetIds = records.asSequence()
|
||||||
|
.flatMap { ChatHistorySearchText.extractAtTargets(it.code).asSequence() }
|
||||||
|
.toSet()
|
||||||
|
runCatching {
|
||||||
|
ContactSnapshotStore.loadDisplayNames(event.bot.id, group.id, targetIds)
|
||||||
|
}.getOrDefault(emptyMap())
|
||||||
|
} else {
|
||||||
|
emptyMap()
|
||||||
|
}
|
||||||
|
|
||||||
records.forEach { record ->
|
records.forEach { record ->
|
||||||
val marker = if (record.id == targetId) ">>> " else ""
|
val marker = if (record.id == targetId) ">>> " else ""
|
||||||
@@ -40,7 +50,7 @@ internal object ChatHistoryToolFormatter {
|
|||||||
val time = timeFormatter.format(
|
val time = timeFormatter.format(
|
||||||
Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault())
|
Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault())
|
||||||
)
|
)
|
||||||
val content = buildSnippet(ChatHistorySearchText.extract(record.code), query)
|
val content = buildSnippet(ChatHistorySearchText.extract(record.code, atNames), query)
|
||||||
output.append(marker)
|
output.append(marker)
|
||||||
.append("[messageId=").append(record.id).append("] ")
|
.append("[messageId=").append(record.id).append("] ")
|
||||||
.append(time).append(' ')
|
.append(time).append(' ')
|
||||||
|
|||||||
@@ -18,22 +18,21 @@ import top.jie65535.mirai.data.ChatHistorySubject
|
|||||||
class GetChatHistoryContext : BaseAgent(
|
class GetChatHistoryContext : BaseAgent(
|
||||||
tool = Tool.function(
|
tool = Tool.function(
|
||||||
name = "getChatHistoryContext",
|
name = "getChatHistoryContext",
|
||||||
description = "根据 searchChatHistory 返回的 messageId,读取该消息在当前群聊或私聊中的前后文。" +
|
description = "读取某条聊天记录前后的消息。",
|
||||||
"结果按时间顺序排列,并用 >>> 标记目标消息。",
|
|
||||||
parameters = Parameters.buildJsonObject {
|
parameters = Parameters.buildJsonObject {
|
||||||
put("type", "object")
|
put("type", "object")
|
||||||
putJsonObject("properties") {
|
putJsonObject("properties") {
|
||||||
putJsonObject("messageId") {
|
putJsonObject("messageId") {
|
||||||
put("type", "integer")
|
put("type", "integer")
|
||||||
put("description", "searchChatHistory 返回的稳定消息 ID")
|
put("description", "搜索结果中的 messageId")
|
||||||
}
|
}
|
||||||
putJsonObject("before") {
|
putJsonObject("before") {
|
||||||
put("type", "integer")
|
put("type", "integer")
|
||||||
put("description", "目标消息之前的消息数,默认8,最大15")
|
put("description", "前文条数,默认8,最大15")
|
||||||
}
|
}
|
||||||
putJsonObject("after") {
|
putJsonObject("after") {
|
||||||
put("type", "integer")
|
put("type", "integer")
|
||||||
put("description", "目标消息之后的消息数,默认8,最大15")
|
put("description", "后文条数,默认8,最大15")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
putJsonArray("required") { add(JsonPrimitive("messageId")) }
|
putJsonArray("required") { add(JsonPrimitive("messageId")) }
|
||||||
|
|||||||
@@ -33,35 +33,33 @@ import java.util.Base64
|
|||||||
class SearchChatHistory : BaseAgent(
|
class SearchChatHistory : BaseAgent(
|
||||||
tool = Tool.function(
|
tool = Tool.function(
|
||||||
name = "searchChatHistory",
|
name = "searchChatHistory",
|
||||||
description = "在当前群聊或私聊的 SQLite 历史中搜索消息,可按内容、发送者和时间筛选。" +
|
description = "搜索当前聊天的历史消息,支持文本、发送者、时间和分页;普通文本与可解析的 @ 目标会一起匹配。",
|
||||||
"默认返回最近匹配并在结果过多时提供 cursor;需要查看命中消息前后讨论时调用 getChatHistoryContext。" +
|
|
||||||
"未指定 from 时默认搜索配置的近期天数,明确指定时间时可以搜索任意历史跨度。",
|
|
||||||
parameters = Parameters.buildJsonObject {
|
parameters = Parameters.buildJsonObject {
|
||||||
put("type", "object")
|
put("type", "object")
|
||||||
putJsonObject("properties") {
|
putJsonObject("properties") {
|
||||||
putJsonObject("query") {
|
putJsonObject("query") {
|
||||||
put("type", "string")
|
put("type", "string")
|
||||||
put("description", "要搜索的消息文本;省略时按其他条件浏览时间线")
|
put("description", "消息文本;也会匹配名称对应的 @ 目标")
|
||||||
}
|
}
|
||||||
putJsonObject("senderId") {
|
putJsonObject("senderId") {
|
||||||
put("type", "integer")
|
put("type", "integer")
|
||||||
put("description", "发送者 QQ 号;仅在用户明确给出或上下文可靠提供 QQ 号时使用,不要根据昵称猜测")
|
put("description", "发送者 QQ 号;用户明确给出时使用")
|
||||||
}
|
}
|
||||||
putJsonObject("senderName") {
|
putJsonObject("senderName") {
|
||||||
put("type", "string")
|
put("type", "string")
|
||||||
put("description", "用户用名称指代发送者时优先原样传入群名片、昵称或好友备注;可与 senderId 同时提供,没有 senderId 时解析发送者,有重名时返回候选 QQ 号")
|
put("description", "发送者名称;用户按昵称或群名片称呼时使用")
|
||||||
}
|
}
|
||||||
putJsonObject("from") {
|
putJsonObject("from") {
|
||||||
put("type", "string")
|
put("type", "string")
|
||||||
put("description", "起始时间,格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd")
|
put("description", "起始时间:yyyy-MM-dd HH:mm 或 yyyy-MM-dd")
|
||||||
}
|
}
|
||||||
putJsonObject("to") {
|
putJsonObject("to") {
|
||||||
put("type", "string")
|
put("type", "string")
|
||||||
put("description", "结束时间,格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd;仅日期表示当天结束")
|
put("description", "结束时间:yyyy-MM-dd HH:mm 或 yyyy-MM-dd")
|
||||||
}
|
}
|
||||||
putJsonObject("cursor") {
|
putJsonObject("cursor") {
|
||||||
put("type", "string")
|
put("type", "string")
|
||||||
put("description", "上一页返回的 nextCursor;继续同一搜索时原样传回")
|
put("description", "上一页的 nextCursor")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -119,9 +117,11 @@ class SearchChatHistory : BaseAgent(
|
|||||||
return "聊天记录全文索引当前不可用,无法执行关键词搜索"
|
return "聊天记录全文索引当前不可用,无法执行关键词搜索"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
val atTargetIds = query?.let { resolveMentionTargets(it, event) }.orEmpty()
|
||||||
val request = ChatHistorySearchRequest(
|
val request = ChatHistorySearchRequest(
|
||||||
subject = ChatHistorySubject.from(event.subject),
|
subject = ChatHistorySubject.from(event.subject),
|
||||||
query = query,
|
query = query,
|
||||||
|
atTargetIds = atTargetIds,
|
||||||
matchMode = matchMode,
|
matchMode = matchMode,
|
||||||
fromId = resolvedSenderId,
|
fromId = resolvedSenderId,
|
||||||
start = start.toEpochSecond().toInt(),
|
start = start.toEpochSecond().toInt(),
|
||||||
@@ -182,6 +182,20 @@ class SearchChatHistory : BaseAgent(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun resolveSender(name: String, event: MessageEvent): SenderResolution {
|
private fun resolveSender(name: String, event: MessageEvent): SenderResolution {
|
||||||
|
val sorted = findNameMatches(name, event)
|
||||||
|
val first = sorted.firstOrNull() ?: return SenderResolution.NotFound
|
||||||
|
val second = sorted.getOrNull(1)
|
||||||
|
return if (second == null || first.matchRank < second.matchRank) {
|
||||||
|
SenderResolution.Found(first.userId)
|
||||||
|
} else {
|
||||||
|
SenderResolution.Ambiguous(sorted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun resolveMentionTargets(name: String, event: MessageEvent): Set<Long> =
|
||||||
|
findNameMatches(name, event).mapTo(linkedSetOf(), ContactNameMatch::userId)
|
||||||
|
|
||||||
|
private fun findNameMatches(name: String, event: MessageEvent): List<ContactNameMatch> {
|
||||||
val groupId = (event as? GroupMessageEvent)?.group?.id
|
val groupId = (event as? GroupMessageEvent)?.group?.id
|
||||||
val matches = runCatching {
|
val matches = runCatching {
|
||||||
ContactSnapshotStore.findUsersByName(event.bot.id, groupId, name, limit = 6)
|
ContactSnapshotStore.findUsersByName(event.bot.id, groupId, name, limit = 6)
|
||||||
@@ -202,14 +216,8 @@ class SearchChatHistory : BaseAgent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
val sorted = matches.sortedWith(compareBy({ it.matchRank }, { it.userId }))
|
return matches.sortedWith(compareBy({ it.matchRank }, { it.userId }))
|
||||||
val first = sorted.firstOrNull() ?: return SenderResolution.NotFound
|
.take(MAX_NAME_MATCHES)
|
||||||
val second = sorted.getOrNull(1)
|
|
||||||
return if (second == null || first.matchRank < second.matchRank) {
|
|
||||||
SenderResolution.Found(first.userId)
|
|
||||||
} else {
|
|
||||||
SenderResolution.Ambiguous(sorted)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed interface SenderResolution {
|
private sealed interface SenderResolution {
|
||||||
@@ -221,6 +229,7 @@ class SearchChatHistory : BaseAgent(
|
|||||||
companion object {
|
companion object {
|
||||||
private const val DEFAULT_PAGE_SIZE = 20
|
private const val DEFAULT_PAGE_SIZE = 20
|
||||||
private const val DIAGNOSTIC_PAGE_SIZE = 6
|
private const val DIAGNOSTIC_PAGE_SIZE = 6
|
||||||
|
private const val MAX_NAME_MATCHES = 12
|
||||||
private const val MAX_PAGE_SIZE = 200
|
private const val MAX_PAGE_SIZE = 200
|
||||||
private const val OUTPUT_RESERVE_CHARS = 1_000
|
private const val OUTPUT_RESERVE_CHARS = 1_000
|
||||||
private const val APPROXIMATE_RECORD_CHARS = 450
|
private const val APPROXIMATE_RECORD_CHARS = 450
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package top.jie65535.mirai.data
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
import kotlin.test.assertEquals
|
||||||
|
|
||||||
|
class ChatHistorySearchTextTest {
|
||||||
|
@Test
|
||||||
|
fun rendersAtTargetNamesAndBuildsBigrams() {
|
||||||
|
val code = """
|
||||||
|
[{"type":"At","target":2180487691}, {"type":"PlainText","content":" 筱玥"}]
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
assertEquals(setOf(2180487691L), ChatHistorySearchText.extractAtTargets(code))
|
||||||
|
assertEquals("@筱玥 筱玥", ChatHistorySearchText.extract(code, mapOf(2180487691L to "筱玥")))
|
||||||
|
assertEquals("筱玥 筱玥", ChatHistorySearchText.bigrams("筱玥 筱玥"))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -62,15 +62,16 @@ class ChatHistoryStoreTest {
|
|||||||
"SELECT value FROM chat_history_meta WHERE key = 'schema_version'"
|
"SELECT value FROM chat_history_meta WHERE key = 'schema_version'"
|
||||||
).use { results ->
|
).use { results ->
|
||||||
assertTrue(results.next())
|
assertTrue(results.next())
|
||||||
assertEquals("3", results.getString(1))
|
assertEquals("4", results.getString(1))
|
||||||
}
|
}
|
||||||
statement.executeQuery(
|
statement.executeQuery(
|
||||||
"SELECT COUNT(*) FROM sqlite_master " +
|
"SELECT COUNT(*) FROM sqlite_master " +
|
||||||
"WHERE type = 'table' AND name IN " +
|
"WHERE type = 'table' AND name IN " +
|
||||||
"('message_record_search', 'message_record_fts', 'chat_history_sender_alias')"
|
"('message_record_search', 'message_record_fts', 'message_record_bigram_fts', " +
|
||||||
|
"'chat_history_sender_alias')"
|
||||||
).use { results ->
|
).use { results ->
|
||||||
assertTrue(results.next())
|
assertTrue(results.next())
|
||||||
assertEquals(3, results.getInt(1))
|
assertEquals(4, results.getInt(1))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -137,6 +138,33 @@ class ChatHistoryStoreTest {
|
|||||||
assertEquals(2L, shortQuery.totalMatches)
|
assertEquals(2L, shortQuery.totalMatches)
|
||||||
assertEquals(listOf(first, third), shortQuery.records.map { it.id })
|
assertEquals(listOf(first, third), shortQuery.records.map { it.id })
|
||||||
|
|
||||||
|
val mention = insert(
|
||||||
|
directory,
|
||||||
|
time = 103,
|
||||||
|
fromId = 11,
|
||||||
|
text = "",
|
||||||
|
code = buildJsonArray {
|
||||||
|
add(buildJsonObject {
|
||||||
|
put("type", "At")
|
||||||
|
put("target", 2180487691L)
|
||||||
|
})
|
||||||
|
}.toString(),
|
||||||
|
)
|
||||||
|
val literal = insert(directory, 104, 11, "筱玥检查一下")
|
||||||
|
val mentionQuery = ChatHistoryStore.search(
|
||||||
|
ChatHistorySearchRequest(
|
||||||
|
subject = subject,
|
||||||
|
query = "筱玥",
|
||||||
|
atTargetIds = setOf(2180487691L),
|
||||||
|
fromId = 11,
|
||||||
|
start = 0,
|
||||||
|
end = 200,
|
||||||
|
limit = 10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertEquals(2L, mentionQuery.totalMatches)
|
||||||
|
assertEquals(listOf(literal, mention), mentionQuery.records.map { it.id })
|
||||||
|
|
||||||
val aliases = ChatHistoryStore.findSenderAliases(subject, "旧名")
|
val aliases = ChatHistoryStore.findSenderAliases(subject, "旧名")
|
||||||
assertEquals(listOf(ChatHistorySenderAliasMatch(11, "旧名张三", 1)), aliases)
|
assertEquals(listOf(ChatHistorySenderAliasMatch(11, "旧名张三", 1)), aliases)
|
||||||
} finally {
|
} finally {
|
||||||
@@ -245,12 +273,81 @@ class ChatHistoryStoreTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun migratesCompletedTrigramIndexAndBackfillsBigramIndex() {
|
||||||
|
val directory = Files.createTempDirectory("jchatgpt-history-bigram-migration-test-")
|
||||||
|
val database = directory.resolve("chat-history.sqlite")
|
||||||
|
try {
|
||||||
|
ChatHistoryStore.init(directory.toFile())
|
||||||
|
val messageId = insert(directory, 100, 11, "部署完成")
|
||||||
|
ChatHistoryStore.close()
|
||||||
|
|
||||||
|
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeUpdate("DROP TABLE message_record_bigram_fts")
|
||||||
|
statement.executeUpdate(
|
||||||
|
"DELETE FROM chat_history_meta WHERE key = 'search_bigram_version'"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"UPDATE chat_history_meta SET value = '3' WHERE key = 'schema_version'"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"UPDATE chat_history_meta SET value = '1' WHERE key = 'search_index_complete'"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"UPDATE chat_history_meta SET value = '$messageId' WHERE key = 'search_backfill_cursor'"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ChatHistoryStore.init(directory.toFile())
|
||||||
|
assertFalse(ChatHistoryStore.isSearchIndexReady)
|
||||||
|
assertEquals(1, ChatHistoryStore.backfillSearchIndex(batchSize = 1))
|
||||||
|
assertTrue(ChatHistoryStore.isSearchIndexReady)
|
||||||
|
|
||||||
|
val search = ChatHistoryStore.search(
|
||||||
|
ChatHistorySearchRequest(
|
||||||
|
subject = ChatHistorySubject(1, MessageSourceKind.GROUP, 1000),
|
||||||
|
query = "部署",
|
||||||
|
start = 0,
|
||||||
|
end = 200,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertEquals(listOf(messageId), search.records.map { it.id })
|
||||||
|
|
||||||
|
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeQuery(
|
||||||
|
"SELECT search_text FROM message_record_search WHERE message_id = $messageId"
|
||||||
|
).use { results ->
|
||||||
|
assertTrue(results.next())
|
||||||
|
assertEquals("部署完成", results.getString(1))
|
||||||
|
}
|
||||||
|
statement.executeQuery(
|
||||||
|
"SELECT rowid FROM message_record_bigram_fts WHERE rowid = $messageId"
|
||||||
|
).use { results ->
|
||||||
|
assertTrue(results.next())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
ChatHistoryStore.close()
|
||||||
|
directory.toFile().deleteRecursively()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun insert(
|
private fun insert(
|
||||||
directory: java.nio.file.Path,
|
directory: java.nio.file.Path,
|
||||||
time: Int,
|
time: Int,
|
||||||
fromId: Long,
|
fromId: Long,
|
||||||
text: String,
|
text: String,
|
||||||
senderName: String = "",
|
senderName: String = "",
|
||||||
|
code: String = buildJsonArray {
|
||||||
|
add(buildJsonObject {
|
||||||
|
put("type", "PlainText")
|
||||||
|
put("content", text)
|
||||||
|
})
|
||||||
|
}.toString(),
|
||||||
): Long {
|
): Long {
|
||||||
val record = ChatMessageRecord(
|
val record = ChatMessageRecord(
|
||||||
botId = 1,
|
botId = 1,
|
||||||
@@ -260,12 +357,7 @@ class ChatHistoryStoreTest {
|
|||||||
internalIds = "$time",
|
internalIds = "$time",
|
||||||
time = time,
|
time = time,
|
||||||
kind = MessageSourceKind.GROUP,
|
kind = MessageSourceKind.GROUP,
|
||||||
code = buildJsonArray {
|
code = code,
|
||||||
add(buildJsonObject {
|
|
||||||
put("type", "PlainText")
|
|
||||||
put("content", text)
|
|
||||||
})
|
|
||||||
}.toString(),
|
|
||||||
)
|
)
|
||||||
ChatHistoryStore.insertRecord(record, senderName)
|
ChatHistoryStore.insertRecord(record, senderName)
|
||||||
DriverManager.getConnection("jdbc:sqlite:${directory.resolve("chat-history.sqlite").absolutePathString()}").use {
|
DriverManager.getConnection("jdbc:sqlite:${directory.resolve("chat-history.sqlite").absolutePathString()}").use {
|
||||||
|
|||||||
Reference in New Issue
Block a user