history: add indexed search and context retrieval

This commit is contained in:
2026-08-06 21:26:37 +08:00
parent 0303bf0ac8
commit c8cdea6fab
12 changed files with 1514 additions and 222 deletions
+29
View File
@@ -1,5 +1,10 @@
package top.jie65535.mirai
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import net.mamoe.mirai.console.command.CommandManager.INSTANCE.register
import net.mamoe.mirai.console.command.CommandSender.Companion.toCommandSender
import net.mamoe.mirai.console.permission.PermissionId
@@ -48,6 +53,7 @@ object JChatGPT : KotlinPlugin(
}
) {
internal var includeHistory: Boolean = false
private var historyIndexJob: Job? = null
val chatPermission = PermissionId("JChatGPT", "Chat")
@@ -69,6 +75,27 @@ object JChatGPT : KotlinPlugin(
logger.error("初始化 SQLite 聊天记录失败,历史上下文与搜索将暂时禁用", cause)
false
}
if (includeHistory && ChatHistoryStore.isSearchIndexAvailable && !ChatHistoryStore.isSearchIndexReady) {
historyIndexJob = launch(Dispatchers.IO) {
try {
val processed = ChatHistoryStore.backfillSearchIndex(
shouldContinue = { isActive },
onProgress = { count ->
if (count % 10_000 == 0) {
logger.info("聊天记录全文索引回填进度: 已处理 $count")
}
},
)
if (isActive && processed > 0) {
logger.info("聊天记录全文索引回填完成: 共处理 $processed")
}
} catch (cause: CancellationException) {
throw cause
} catch (cause: Throwable) {
logger.warning("聊天记录全文索引回填失败,后续启动将继续尝试", cause)
}
}
}
if (includeHistory) {
runCatching { ContactSnapshotStore.init(dataFolder) }
.onFailure { logger.error("初始化联系人快照数据库失败,联系人画像辅助将暂时禁用", it) }
@@ -103,6 +130,8 @@ object JChatGPT : KotlinPlugin(
}
override fun onDisable() {
historyIndexJob?.cancel()
historyIndexJob = null
ConversationEngine.clear()
ConversationContext.clearAll()
ProfileAutoMaintenance.clear()
+2 -2
View File
@@ -292,9 +292,9 @@ object PluginConfig : AutoSavePluginConfig("Config") {
@ValueDescription("单个工具调用返回内容的最大字符数,超过将被截断并标注")
val maxToolOutputLength: Int by value(15000)
@ValueDescription("聊天记录搜索最大天数")
@ValueDescription("未指定起始时间时聊天记录搜索默认回溯天数;明确指定时间时不限制历史跨度")
val searchHistoryMaxDays: Int by value(30)
@ValueDescription("聊天记录搜索最大查询条数,防止内存溢出")
@ValueDescription("聊天记录搜索单页消息数上限;工具硬上限为200,保留此配置键以兼容旧配置")
val searchHistoryMaxRecords: Int by value(5000)
}
@@ -29,6 +29,7 @@ import top.jie65535.mirai.tools.AdjustUserFavorabilityAgent
import top.jie65535.mirai.tools.BaseAgent
import top.jie65535.mirai.tools.DeleteSkill
import top.jie65535.mirai.tools.GroupManageAgent
import top.jie65535.mirai.tools.GetChatHistoryContext
import top.jie65535.mirai.tools.GithubAgent
import top.jie65535.mirai.tools.ImageAgent
import top.jie65535.mirai.tools.LoadSkill
@@ -70,6 +71,7 @@ internal object ConversationEngine {
SaveSkill(),
DeleteSkill(),
SearchChatHistory(),
GetChatHistoryContext(),
QueryUserProfileAgent(),
WebSearch(),
GithubAgent(),
+69
View File
@@ -0,0 +1,69 @@
package top.jie65535.mirai.data
import net.mamoe.mirai.contact.Contact
import net.mamoe.mirai.contact.Friend
import net.mamoe.mirai.contact.Group
import net.mamoe.mirai.contact.Member
import net.mamoe.mirai.contact.Stranger
import net.mamoe.mirai.message.data.MessageSourceKind
data class ChatHistorySubject(
val botId: Long,
val kind: MessageSourceKind,
val subjectId: Long,
) {
companion object {
fun from(contact: Contact): ChatHistorySubject = when (contact) {
is Group -> ChatHistorySubject(contact.bot.id, MessageSourceKind.GROUP, contact.id)
is Member -> ChatHistorySubject(contact.bot.id, MessageSourceKind.GROUP, contact.group.id)
is Friend -> ChatHistorySubject(contact.bot.id, MessageSourceKind.FRIEND, contact.id)
is Stranger -> ChatHistorySubject(contact.bot.id, MessageSourceKind.STRANGER, contact.id)
else -> error("不支持查询的联系人 $contact")
}
}
}
enum class ChatHistoryMatchMode {
ALL,
ANY,
PHRASE,
}
enum class ChatHistorySortOrder {
NEWEST,
OLDEST,
}
data class ChatHistoryCursor(
val time: Int,
val id: Long,
)
data class ChatHistorySearchRequest(
val subject: ChatHistorySubject,
val query: String? = null,
val matchMode: ChatHistoryMatchMode = ChatHistoryMatchMode.ALL,
val fromId: Long? = null,
val start: Int? = null,
val end: Int? = null,
val sortOrder: ChatHistorySortOrder = ChatHistorySortOrder.NEWEST,
val limit: Int = 20,
val cursor: ChatHistoryCursor? = null,
)
data class ChatHistorySearchPage(
val records: List<ChatMessageRecord>,
val totalMatches: Long?,
val nextCursor: ChatHistoryCursor?,
)
data class ChatHistoryContext(
val targetId: Long,
val records: List<ChatMessageRecord>,
)
data class ChatHistorySenderAliasMatch(
val userId: Long,
val displayName: String,
val matchRank: Int,
)
@@ -0,0 +1,133 @@
package top.jie65535.mirai.data
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.booleanOrNull
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.longOrNull
import net.mamoe.mirai.message.data.content
object ChatHistorySearchText {
private const val MAX_INDEXED_CHARS = 16_384
private const val MAX_QUOTED_CHARS = 320
private const val MAX_FORWARD_NODES = 30
private const val MAX_FORWARD_NODE_CHARS = 500
private val json = Json { ignoreUnknownKeys = true }
private val whitespace = Regex("\\s+")
fun extract(code: String): String {
val rendered = runCatching { renderJsonCode(code) }
.recoverCatching { decodeMessageCode(code).content }
.getOrDefault("")
return rendered.normalize().takeUtf16Safely(MAX_INDEXED_CHARS)
}
private fun renderJsonCode(code: String): String {
val messages = json.parseToJsonElement(code) as? JsonArray
?: throw IllegalArgumentException("消息记录不是 JSON array")
return renderMessages(messages)
}
private fun renderMessages(messages: JsonArray): String = messages.joinToString("") { element ->
val message = element as? JsonObject ?: return@joinToString ""
when (val type = message.string("type")) {
"PlainText" -> message.string("content").orEmpty()
"At" -> message.long("target")?.let { "@$it" }.orEmpty()
"AtAll" -> "@全体成员"
"Image", "FlashImage" -> if (message.boolean("isEmoji") == true) "[表情包]" else "[图片]"
"QuoteReply" -> renderQuote(message)
"ForwardMessage" -> renderForward(message)
"MessageOrigin", "MessageSource", "ShowImageFlag" -> ""
"Face", "MarketFace", "VipFace" -> "[表情]"
"Audio" -> "[语音]"
"FileMessage" -> "[文件${message.string("name")?.let { ": $it" }.orEmpty()}]"
"LightApp", "SimpleServiceMessage", "MusicShare" -> "[卡片消息]"
"PokeMessage" -> "[戳一戳]"
null -> ""
else -> message.string("content") ?: "[$type]"
}
}
private fun renderQuote(message: JsonObject): String {
val source = message["source"] as? JsonObject ?: return "[引用消息]"
val author = source.long("fromId")?.toString() ?: "其他用户"
val original = (source["originalMessage"] as? JsonArray)
?.let(::renderMessages)
.orEmpty()
.normalize()
.takeUtf16Safely(MAX_QUOTED_CHARS)
return "[引用 $author: $original]"
}
private fun renderForward(message: JsonObject): String = buildString {
append("[转发消息]")
val nodes = message["nodeList"] as? JsonArray ?: return@buildString
nodes.take(MAX_FORWARD_NODES).forEach { element ->
val node = element as? JsonObject ?: return@forEach
val sender = node.string("senderName") ?: "未知用户"
val chain = node["messageChain"] as? JsonArray
append(' ').append(sender).append(": ")
append(
chain?.let(::renderMessages)
.orEmpty()
.normalize()
.takeUtf16Safely(MAX_FORWARD_NODE_CHARS)
)
}
if (nodes.size > MAX_FORWARD_NODES) append(" ...[转发内容截断]")
}
private fun String.normalize(): String = whitespace.replace(this, " ")
.trim()
.replaceUnpairedSurrogates()
private fun String.takeUtf16Safely(maxLength: Int): String {
if (length <= maxLength) return this
val endIndex = if (maxLength > 0 &&
Character.isHighSurrogate(this[maxLength - 1]) &&
Character.isLowSurrogate(this[maxLength])
) {
maxLength - 1
} else {
maxLength
}
return substring(0, endIndex)
}
private fun String.replaceUnpairedSurrogates(): String {
var output: StringBuilder? = null
var index = 0
while (index < length) {
val current = this[index]
when {
Character.isHighSurrogate(current) &&
index + 1 < length && Character.isLowSurrogate(this[index + 1]) -> {
output?.append(current)?.append(this[index + 1])
index += 2
}
Character.isSurrogate(current) -> {
if (output == null) output = StringBuilder(length).append(this, 0, index)
output.append('\uFFFD')
index++
}
else -> {
output?.append(current)
index++
}
}
}
return output?.toString() ?: this
}
private fun JsonObject.string(key: String): String? =
(get(key) as? JsonPrimitive)?.contentOrNull
private fun JsonObject.long(key: String): Long? =
(get(key) as? JsonPrimitive)?.longOrNull
private fun JsonObject.boolean(key: String): Boolean? =
(get(key) as? JsonPrimitive)?.booleanOrNull
}
+688 -58
View File
@@ -24,14 +24,26 @@ import java.sql.ResultSet
* 写连接由 [writeLock] 串行保护;SQLite 使用 WAL + NORMAL synchronous,允许并发读取。
*/
object ChatHistoryStore {
private const val SCHEMA_VERSION = 2
private const val SCHEMA_VERSION = 3
private const val BUSY_TIMEOUT_MS = 30_000
private const val SEARCH_FTS_TABLE = "message_record_fts"
private const val SEARCH_TEXT_TABLE = "message_record_search"
private const val SENDER_ALIAS_TABLE = "chat_history_sender_alias"
private const val SEARCH_BACKFILL_CURSOR_KEY = "search_backfill_cursor"
private const val SEARCH_INDEX_COMPLETE_KEY = "search_index_complete"
private const val DEFAULT_BACKFILL_BATCH_SIZE = 250
private const val MAX_PAGE_SIZE = 200
private const val MAX_CONTEXT_MESSAGES = 100
private val lifecycleLock = Any()
private val writeLock = Any()
@Volatile
private var initialized = false
@Volatile
private var searchIndexAvailable = false
@Volatile
private var searchIndexReady = false
private lateinit var databaseFile: File
private var writeConnection: Connection? = null
private var warningLogger: ((String, Throwable?) -> Unit)? = null
@@ -39,6 +51,12 @@ object ChatHistoryStore {
val isAvailable: Boolean
get() = initialized
val isSearchIndexAvailable: Boolean
get() = initialized && searchIndexAvailable
val isSearchIndexReady: Boolean
get() = isSearchIndexAvailable && searchIndexReady
val databaseFileOrNull: File?
get() = if (initialized) databaseFile else null
@@ -60,6 +78,8 @@ object ChatHistoryStore {
createSchema(connection)
writeConnection = connection
initialized = true
searchIndexAvailable = hasSearchIndex(connection)
searchIndexReady = searchIndexAvailable && readMeta(connection, SEARCH_INDEX_COMPLETE_KEY) == "1"
} catch (cause: Throwable) {
connection.close()
throw cause
@@ -81,6 +101,8 @@ object ChatHistoryStore {
}
writeConnection = null
initialized = false
searchIndexAvailable = false
searchIndexReady = false
warningLogger = null
}
}
@@ -91,7 +113,7 @@ object ChatHistoryStore {
val message = event.message.asSequence()
.filterNot { it is MessageSource }
.toMessageChain()
insert(ChatMessageRecord.fromSuccess(event.message.source, message))
insertRecord(ChatMessageRecord.fromSuccess(event.message.source, message), event.senderName)
}
fun record(event: MessagePostSendEvent<*>) {
@@ -100,7 +122,7 @@ object ChatHistoryStore {
val message = event.message.asSequence()
.filterNot { it is MessageSource }
.toMessageChain()
insert(ChatMessageRecord.fromSuccess(source, message))
insertRecord(ChatMessageRecord.fromSuccess(source, message), event.bot.nick)
}
fun markRecalled(event: MessageRecallEvent) {
@@ -254,31 +276,410 @@ object ChatHistoryStore {
}
}
private fun insert(record: ChatMessageRecord) {
withWriteConnection { connection ->
fun search(request: ChatHistorySearchRequest): ChatHistorySearchPage {
check(initialized) { "聊天记录数据库尚未初始化" }
require(request.limit > 0) { "limit must be positive" }
require(request.start == null || request.end == null || request.start <= request.end) {
"start must not be after end"
}
val parameters = mutableListOf<Any>()
val conditions = subjectConditions(request.subject, parameters).toMutableList()
request.start?.let {
conditions += "mr.time >= ?"
parameters += it
}
request.end?.let {
conditions += "mr.time <= ?"
parameters += it
}
request.fromId?.let {
conditions += "mr.from_id = ?"
parameters += it
}
val query = request.query?.trim().orEmpty()
val textPredicate = buildTextPredicate(query, request.matchMode, parameters)
if ((textPredicate.usesFts || textPredicate.usesSearchText) && !isSearchIndexAvailable) {
error("聊天记录全文索引尚未就绪")
}
val joins = buildString {
if (textPredicate.usesSearchText) {
append("JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id ")
}
if (textPredicate.usesFts) {
append("JOIN $SEARCH_FTS_TABLE ON $SEARCH_FTS_TABLE.rowid = mr.id")
}
}
textPredicate.sql?.let(conditions::add)
request.cursor?.let { cursor ->
val operator = if (request.sortOrder == ChatHistorySortOrder.NEWEST) "<" else ">"
conditions += "(mr.time $operator ? OR (mr.time = ? AND mr.id $operator ?))"
parameters += cursor.time
parameters += cursor.time
parameters += cursor.id
}
val where = conditions.joinToString(" AND ")
val order = if (request.sortOrder == ChatHistorySortOrder.NEWEST) {
"mr.time DESC, mr.id DESC"
} else {
"mr.time ASC, mr.id ASC"
}
val records = openReadConnection().use { connection ->
val sql = """
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
mr.time, mr.kind, mr.code, mr.recalled
FROM message_record mr
$joins
WHERE $where
ORDER BY $order
LIMIT ?
""".trimIndent()
val queryParameters = parameters + (request.limit.coerceAtMost(MAX_PAGE_SIZE) + 1)
loadRecords(connection, sql, queryParameters)
}
val hasMore = records.size > request.limit.coerceAtMost(MAX_PAGE_SIZE)
val pageRecords = records.take(request.limit.coerceAtMost(MAX_PAGE_SIZE))
val last = pageRecords.lastOrNull()
val nextCursor = if (hasMore && last != null) {
ChatHistoryCursor(last.time, last.id)
} else {
null
}
val total = if (request.cursor == null) {
openReadConnection().use { connection ->
val sql = "SELECT COUNT(*) FROM message_record mr $joins WHERE $where"
connection.prepareStatement(sql).use { statement ->
bindParameters(statement, parameters)
statement.executeQuery().use { results ->
check(results.next())
results.getLong(1)
}
}
}
} else {
null
}
return ChatHistorySearchPage(pageRecords, total, nextCursor)
}
fun findAround(
subject: ChatHistorySubject,
messageId: Long,
before: Int = 8,
after: Int = 8,
): ChatHistoryContext? {
check(initialized) { "聊天记录数据库尚未初始化" }
require(before >= 0 && after >= 0) { "before and after must not be negative" }
require(before <= MAX_CONTEXT_MESSAGES && after <= MAX_CONTEXT_MESSAGES) {
"context window is too large"
}
return openReadConnection().use { connection ->
val targetParameters = mutableListOf<Any>()
val targetConditions = subjectConditions(subject, targetParameters).toMutableList()
targetConditions += "mr.id = ?"
targetParameters += messageId
val target = loadRecords(
connection,
"""
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
mr.time, mr.kind, mr.code, mr.recalled
FROM message_record mr
WHERE ${targetConditions.joinToString(" AND ")}
""".trimIndent(),
targetParameters,
).firstOrNull() ?: return@use null
val beforeRecords = if (before == 0) {
emptyList()
} else {
val parameters = mutableListOf<Any>()
val conditions = subjectConditions(subject, parameters).toMutableList()
conditions += "(mr.time < ? OR (mr.time = ? AND mr.id < ?))"
parameters += target.time
parameters += target.time
parameters += target.id
conditions += "mr.id <> ?"
parameters += target.id
loadRecords(
connection,
"""
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
mr.time, mr.kind, mr.code, mr.recalled
FROM message_record mr
WHERE ${conditions.joinToString(" AND ")}
ORDER BY mr.time DESC, mr.id DESC
LIMIT ?
""".trimIndent(),
parameters + before,
).asReversed()
}
val afterRecords = if (after == 0) {
emptyList()
} else {
val parameters = mutableListOf<Any>()
val conditions = subjectConditions(subject, parameters).toMutableList()
conditions += "(mr.time > ? OR (mr.time = ? AND mr.id > ?))"
parameters += target.time
parameters += target.time
parameters += target.id
conditions += "mr.id <> ?"
parameters += target.id
loadRecords(
connection,
"""
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
mr.time, mr.kind, mr.code, mr.recalled
FROM message_record mr
WHERE ${conditions.joinToString(" AND ")}
ORDER BY mr.time ASC, mr.id ASC
LIMIT ?
""".trimIndent(),
parameters + after,
)
}
ChatHistoryContext(messageId, beforeRecords + target + afterRecords)
}
}
fun findSenderAliases(
subject: ChatHistorySubject,
query: String,
limit: Int = 10,
): List<ChatHistorySenderAliasMatch> {
check(initialized) { "聊天记录数据库尚未初始化" }
if (query.isBlank() || limit <= 0) return emptyList()
val normalized = query.trim()
return openReadConnection().use { connection ->
connection.prepareStatement(
"""
INSERT INTO message_record(
bot_id, from_id, target_id, ids, internal_ids,
time, kind, code, recalled
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
SELECT user_id, name
FROM $SENDER_ALIAS_TABLE
WHERE bot_id = ? AND kind = ? AND subject_id = ?
AND lower(name) LIKE ? ESCAPE '\'
ORDER BY last_seen DESC
LIMIT ?
""".trimIndent()
).use { statement ->
statement.setLong(1, record.botId)
statement.setLong(2, record.fromId)
statement.setLong(3, record.targetId)
statement.setString(4, record.ids)
statement.setString(5, record.internalIds)
statement.setInt(6, record.time)
statement.setInt(7, record.kind.ordinal)
statement.setString(8, record.code)
statement.setInt(9, record.recalled)
statement.executeUpdate()
statement.setLong(1, subject.botId)
statement.setInt(2, subject.kind.ordinal)
statement.setLong(3, subject.subjectId)
statement.setString(4, "%${normalized.lowercase().escapeLike()}%")
statement.setInt(5, limit.coerceAtMost(50))
statement.executeQuery().use { results ->
buildList {
while (results.next()) {
val name = results.getString("name")
add(
ChatHistorySenderAliasMatch(
userId = results.getLong("user_id"),
displayName = name,
matchRank = name.aliasMatchRank(normalized),
)
)
}
}.sortedWith(compareBy({ it.matchRank }, { it.userId }))
.distinctBy(ChatHistorySenderAliasMatch::userId)
.take(limit)
}
}
}
}
fun backfillSearchIndex(
batchSize: Int = DEFAULT_BACKFILL_BATCH_SIZE,
shouldContinue: () -> Boolean = { true },
onProgress: (processed: Int) -> Unit = {},
): Int {
if (!isSearchIndexAvailable || batchSize <= 0) return 0
var processed = 0
var cursor = withWriteConnectionResult { connection ->
readMeta(connection, SEARCH_BACKFILL_CURSOR_KEY)?.toLongOrNull() ?: 0L
}
while (shouldContinue()) {
val batch = openReadConnection().use { connection ->
connection.prepareStatement(
"""
SELECT mr.id, mr.code
FROM message_record mr
WHERE mr.id > ?
AND NOT EXISTS (
SELECT 1 FROM $SEARCH_TEXT_TABLE search WHERE search.message_id = mr.id
)
ORDER BY mr.id
LIMIT ?
""".trimIndent()
).use { statement ->
statement.setLong(1, cursor)
statement.setInt(2, batchSize)
statement.executeQuery().use { results ->
buildList {
while (results.next()) {
add(results.getLong("id") to results.getString("code"))
}
}
}
}
}
if (batch.isEmpty()) {
withWriteConnection { connection ->
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "1")
}
searchIndexReady = true
return processed
}
withWriteConnection { connection ->
val oldAutoCommit = connection.autoCommit
connection.autoCommit = false
try {
connection.prepareStatement(
"INSERT INTO $SEARCH_TEXT_TABLE(message_id, search_text) VALUES (?, ?)"
).use { insertText ->
connection.prepareStatement(
"INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)"
).use { insertFts ->
batch.forEach { (id, code) ->
val text = ChatHistorySearchText.extract(code)
insertText.setLong(1, id)
insertText.setString(2, text)
insertText.addBatch()
insertFts.setLong(1, id)
insertFts.setString(2, text)
insertFts.addBatch()
}
insertText.executeBatch()
insertFts.executeBatch()
}
}
cursor = batch.last().first
writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, cursor.toString())
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0")
connection.commit()
} catch (cause: Throwable) {
connection.rollback()
throw cause
} finally {
connection.autoCommit = oldAutoCommit
}
}
processed += batch.size
onProgress(processed)
}
return processed
}
internal fun insertRecord(record: ChatMessageRecord, senderName: String = "") {
var recordId = 0L
var indexFailure: Throwable? = null
withWriteConnection { connection ->
recordId = insertPrimaryRecord(connection, record)
if (senderName.isNotBlank()) {
runCatching { upsertSenderAlias(connection, record, senderName) }
.onFailure { warn("保存聊天记录发送者别名失败: user=${record.fromId}", it) }
}
if (isSearchIndexAvailable) {
runCatching { indexRecordOnConnection(connection, recordId, record.code) }
.onFailure { indexFailure = it }
}
}
indexFailure?.let { cause ->
searchIndexReady = false
markSearchIndexIncomplete()
warn("消息已保存,但写入聊天记录全文索引失败: id=$recordId", cause)
}
}
private fun insertPrimaryRecord(connection: Connection, record: ChatMessageRecord): Long {
connection.prepareStatement(
"""
INSERT INTO message_record(
bot_id, from_id, target_id, ids, internal_ids,
time, kind, code, recalled
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent()
).use { statement ->
statement.setLong(1, record.botId)
statement.setLong(2, record.fromId)
statement.setLong(3, record.targetId)
statement.setString(4, record.ids)
statement.setString(5, record.internalIds)
statement.setInt(6, record.time)
statement.setInt(7, record.kind.ordinal)
statement.setString(8, record.code)
statement.setInt(9, record.recalled)
statement.executeUpdate()
}
connection.createStatement().use { statement ->
statement.executeQuery("SELECT last_insert_rowid()").use { results ->
check(results.next())
return results.getLong(1)
}
}
}
private fun indexRecordOnConnection(connection: Connection, recordId: Long, code: String) {
val searchText = ChatHistorySearchText.extract(code)
val oldAutoCommit = connection.autoCommit
connection.autoCommit = false
try {
connection.prepareStatement(
"INSERT INTO $SEARCH_TEXT_TABLE(message_id, search_text) VALUES (?, ?)"
).use { statement ->
statement.setLong(1, recordId)
statement.setString(2, searchText)
statement.executeUpdate()
}
connection.prepareStatement(
"INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)"
).use { statement ->
statement.setLong(1, recordId)
statement.setString(2, searchText)
statement.executeUpdate()
}
connection.commit()
} catch (cause: Throwable) {
connection.rollback()
throw cause
} finally {
connection.autoCommit = oldAutoCommit
}
}
private fun upsertSenderAlias(connection: Connection, record: ChatMessageRecord, senderName: String) {
val subjectId = if (record.kind == MessageSourceKind.GROUP) {
record.targetId
} else if (record.fromId == record.botId) {
record.targetId
} else {
record.fromId
}
connection.prepareStatement(
"""
INSERT INTO $SENDER_ALIAS_TABLE(
bot_id, kind, subject_id, user_id, name, last_seen
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(bot_id, kind, subject_id, user_id, name) DO UPDATE SET
last_seen = MAX(last_seen, excluded.last_seen)
""".trimIndent()
).use { statement ->
statement.setLong(1, record.botId)
statement.setInt(2, record.kind.ordinal)
statement.setLong(3, subjectId)
statement.setLong(4, record.fromId)
statement.setString(5, senderName.trim())
statement.setInt(6, record.time)
statement.executeUpdate()
}
}
private fun withWriteConnection(block: (Connection) -> Unit) {
withWriteConnectionResult { connection -> block(connection) }
}
private fun <T> withWriteConnectionResult(block: (Connection) -> T): T {
synchronized(writeLock) {
check(initialized) { "聊天记录数据库尚未初始化" }
val connection = writeConnection?.takeUnless(Connection::isClosed)
@@ -286,7 +687,7 @@ object ChatHistoryStore {
configureWriteConnection(it)
writeConnection = it
}
block(connection)
return block(connection)
}
}
@@ -313,53 +714,282 @@ object ChatHistoryStore {
}
private fun createSchema(connection: Connection) {
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS message_record(
id INTEGER PRIMARY KEY AUTOINCREMENT,
bot_id INTEGER NOT NULL,
from_id INTEGER NOT NULL,
target_id INTEGER NOT NULL,
ids TEXT,
internal_ids TEXT,
time INTEGER NOT NULL,
kind INTEGER NOT NULL,
code TEXT NOT NULL,
recalled INTEGER NOT NULL DEFAULT 0
val oldAutoCommit = connection.autoCommit
connection.autoCommit = false
try {
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS message_record(
id INTEGER PRIMARY KEY AUTOINCREMENT,
bot_id INTEGER NOT NULL,
from_id INTEGER NOT NULL,
target_id INTEGER NOT NULL,
ids TEXT,
internal_ids TEXT,
time INTEGER NOT NULL,
kind INTEGER NOT NULL,
code TEXT NOT NULL,
recalled INTEGER NOT NULL DEFAULT 0
)
""".trimIndent()
)
""".trimIndent()
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_message_subject_time " +
"ON message_record(bot_id, kind, target_id, time DESC, id DESC)"
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_message_sender_subject_time " +
"ON message_record(bot_id, kind, target_id, from_id, time DESC, id DESC)"
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_message_recall_identity " +
"ON message_record(bot_id, kind, target_id, from_id, time, ids)"
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS chat_history_meta(
key TEXT PRIMARY KEY,
value TEXT NOT NULL
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_message_subject_time " +
"ON message_record(bot_id, kind, target_id, time DESC, id DESC)"
)
""".trimIndent()
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_message_sender_subject_time " +
"ON message_record(bot_id, kind, target_id, from_id, time DESC, id DESC)"
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_message_recall_identity " +
"ON message_record(bot_id, kind, target_id, from_id, time, ids)"
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS chat_history_meta(
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)
""".trimIndent()
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS $SEARCH_TEXT_TABLE(
message_id INTEGER PRIMARY KEY,
search_text TEXT NOT NULL
)
""".trimIndent()
)
statement.executeUpdate(
"""
CREATE TABLE IF NOT EXISTS $SENDER_ALIAS_TABLE(
bot_id INTEGER NOT NULL,
kind INTEGER NOT NULL,
subject_id INTEGER NOT NULL,
user_id INTEGER NOT NULL,
name TEXT NOT NULL,
last_seen INTEGER NOT NULL,
PRIMARY KEY(bot_id, kind, subject_id, user_id, name)
)
""".trimIndent()
)
statement.executeUpdate(
"CREATE INDEX IF NOT EXISTS idx_chat_history_sender_alias_scope " +
"ON $SENDER_ALIAS_TABLE(bot_id, kind, subject_id, last_seen DESC)"
)
}
connection.prepareStatement(
"INSERT INTO chat_history_meta(key, value) VALUES ('schema_version', ?) " +
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
).use { statement ->
statement.setString(1, SCHEMA_VERSION.toString())
statement.executeUpdate()
}
connection.commit()
} catch (cause: Throwable) {
connection.rollback()
throw cause
} finally {
connection.autoCommit = oldAutoCommit
}
runCatching {
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE VIRTUAL TABLE IF NOT EXISTS $SEARCH_FTS_TABLE USING fts5(
search_text,
content='$SEARCH_TEXT_TABLE',
content_rowid='message_id',
tokenize='trigram'
)
""".trimIndent()
)
}
}.onFailure { warn("初始化 SQLite FTS5 聊天记录索引失败,消息记录仍会继续保存", it) }
if (hasSearchIndex(connection)) {
val hasMessages = connection.createStatement().use { statement ->
statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next)
}
connection.prepareStatement(
"INSERT INTO chat_history_meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING"
).use { statement ->
statement.setString(1, SEARCH_BACKFILL_CURSOR_KEY)
statement.setString(2, "0")
statement.executeUpdate()
statement.setString(1, SEARCH_INDEX_COMPLETE_KEY)
statement.setString(2, if (hasMessages) "0" else "1")
statement.executeUpdate()
}
}
}
private fun subjectConditions(
subject: ChatHistorySubject,
parameters: MutableList<Any>,
): List<String> {
val conditions = mutableListOf(
"mr.bot_id = ?",
"mr.kind = ?",
)
parameters += subject.botId
parameters += subject.kind.ordinal
if (subject.kind == MessageSourceKind.GROUP) {
conditions += "mr.target_id = ?"
parameters += subject.subjectId
} else {
conditions += "(mr.from_id = ? OR mr.target_id = ?)"
parameters += subject.subjectId
parameters += subject.subjectId
}
return conditions
}
private data class TextPredicate(
val sql: String?,
val usesFts: Boolean,
val usesSearchText: Boolean,
)
private fun buildTextPredicate(
query: String,
mode: ChatHistoryMatchMode,
parameters: MutableList<Any>,
): TextPredicate {
if (query.isBlank()) return TextPredicate(null, false, false)
val normalized = query.lowercase()
val terms = normalized.split(Regex("\\s+")).filter(String::isNotBlank)
val hasShortTerm = terms.any { it.codePointCount(0, it.length) < 3 }
if (mode == ChatHistoryMatchMode.ANY && hasShortTerm) {
val predicates = terms.map {
parameters += it
"instr(lower(search.search_text), ?) > 0"
}
return TextPredicate(predicates.joinToString(" OR ", prefix = "(", postfix = ")"), false, true)
}
val ftsTerms = if (mode == ChatHistoryMatchMode.PHRASE) {
listOf(normalized)
} else {
terms
}
val ftsOnly = ftsTerms.filter { it.codePointCount(0, it.length) >= 3 }
if (ftsOnly.isEmpty()) {
val predicates = terms.map {
parameters += it
"instr(lower(search.search_text), ?) > 0"
}
val joiner = if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND "
return TextPredicate(predicates.joinToString(joiner, prefix = "(", postfix = ")"), false, true)
}
val ftsQuery = when (mode) {
ChatHistoryMatchMode.PHRASE -> quoteFtsTerm(normalized)
ChatHistoryMatchMode.ALL -> ftsOnly.joinToString(" AND ", transform = ::quoteFtsTerm)
ChatHistoryMatchMode.ANY -> ftsOnly.joinToString(" OR ", transform = ::quoteFtsTerm)
}
parameters += ftsQuery
val ftsPredicate = "$SEARCH_FTS_TABLE MATCH ?"
if (mode != ChatHistoryMatchMode.ALL || !hasShortTerm) {
return TextPredicate(ftsPredicate, true, false)
}
val shortPredicates = terms.filter { it.codePointCount(0, it.length) < 3 }.map {
parameters += it
"instr(lower(search.search_text), ?) > 0"
}
return TextPredicate(
(listOf(ftsPredicate) + shortPredicates).joinToString(" AND ", prefix = "(", postfix = ")"),
true,
true,
)
}
private fun quoteFtsTerm(term: String): String =
"\"${term.replace("\"", "\"\"")}\""
private fun loadRecords(
connection: Connection,
sql: String,
parameters: List<Any>,
): List<ChatMessageRecord> {
return connection.prepareStatement(sql).use { statement ->
bindParameters(statement, parameters)
statement.executeQuery().use { results ->
buildList {
while (results.next()) add(results.toRecord())
}
}
}
}
private fun bindParameters(
statement: java.sql.PreparedStatement,
parameters: List<Any>,
) {
parameters.forEachIndexed { index, value ->
when (value) {
is Int -> statement.setInt(index + 1, value)
is Long -> statement.setLong(index + 1, value)
is String -> statement.setString(index + 1, value)
else -> error("不支持的查询参数类型 ${value::class}")
}
}
}
private fun hasSearchIndex(connection: Connection): Boolean {
connection.prepareStatement(
"INSERT INTO chat_history_meta(key, value) VALUES ('schema_version', ?) " +
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1"
).use { statement ->
statement.setString(1, SCHEMA_VERSION.toString())
statement.setString(1, SEARCH_FTS_TABLE)
statement.executeQuery().use { results ->
return results.next()
}
}
}
private fun readMeta(connection: Connection, key: String): String? {
connection.prepareStatement(
"SELECT value FROM chat_history_meta WHERE key = ?"
).use { statement ->
statement.setString(1, key)
statement.executeQuery().use { results ->
return if (results.next()) results.getString(1) else null
}
}
}
private fun writeMeta(connection: Connection, key: String, value: String) {
connection.prepareStatement(
"""
INSERT INTO chat_history_meta(key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
""".trimIndent()
).use { statement ->
statement.setString(1, key)
statement.setString(2, value)
statement.executeUpdate()
}
}
private fun markSearchIndexIncomplete() {
runCatching {
withWriteConnection { connection ->
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0")
}
}.onFailure { warn("标记聊天记录全文索引未完成失败", it) }
}
private fun String.escapeLike(): String =
replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
private fun String.aliasMatchRank(query: String): Int = when {
equals(query, ignoreCase = true) -> 0
startsWith(query, ignoreCase = true) -> 1
contains(query, ignoreCase = true) -> 2
else -> Int.MAX_VALUE
}
private fun ResultSet.toRecord(): ChatMessageRecord {
val kindOrdinal = getInt("kind")
val kind = MessageSourceKind.values().getOrNull(kindOrdinal)
+13 -9
View File
@@ -37,15 +37,7 @@ data class ChatMessageRecord(
}
fun toMessageChain(): MessageChain {
return try {
MessageChain.deserializeFromJsonString(code)
} catch (cause: SerializationException) {
try {
MiraiCode.deserializeMiraiCode(code)
} catch (_: Throwable) {
throw cause
}
}
return decodeMessageCode(code)
}
companion object {
@@ -69,3 +61,15 @@ data class ChatMessageRecord(
}
}
}
internal fun decodeMessageCode(code: String): MessageChain {
return try {
MessageChain.deserializeFromJsonString(code)
} catch (cause: SerializationException) {
try {
MiraiCode.deserializeMiraiCode(code)
} catch (_: Throwable) {
throw cause
}
}
}
@@ -0,0 +1,66 @@
package top.jie65535.mirai.tools
import net.mamoe.mirai.contact.nameCardOrNick
import net.mamoe.mirai.event.events.GroupMessageEvent
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.data.ChatHistorySearchText
import top.jie65535.mirai.data.ChatMessageRecord
import top.jie65535.mirai.data.ContactSnapshotStore
import java.time.Instant
import java.time.ZoneId
import java.time.format.DateTimeFormatter
internal object ChatHistoryToolFormatter {
private const val SNIPPET_LENGTH = 360
private val timeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
fun appendRecords(
output: StringBuilder,
records: List<ChatMessageRecord>,
event: MessageEvent,
query: String? = null,
targetId: Long? = null,
) {
val group = (event as? GroupMessageEvent)?.group
val userIds = records.map(ChatMessageRecord::fromId).distinct()
val snapshotNames = runCatching {
ContactSnapshotStore.loadDisplayNames(event.bot.id, group?.id, userIds)
}.getOrDefault(emptyMap())
records.forEach { record ->
val marker = if (record.id == targetId) ">>> " else ""
val sender = when {
record.fromId == event.bot.id -> ""
group != null -> group[record.fromId]?.nameCardOrNick
?: snapshotNames[record.fromId]
?: "未知群员"
record.fromId == event.sender.id -> event.senderName
else -> snapshotNames[record.fromId] ?: "用户"
}
val time = timeFormatter.format(
Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault())
)
val content = buildSnippet(ChatHistorySearchText.extract(record.code), query)
output.append(marker)
.append("[messageId=").append(record.id).append("] ")
.append(time).append(' ')
.append(sender).append('(').append(record.fromId).append("): ")
.appendLine(content.ifEmpty { "[无文本消息]" })
}
}
private fun buildSnippet(content: String, query: String?): String {
if (content.length <= SNIPPET_LENGTH) return content
val terms = query.orEmpty().split(Regex("\\s+")).filter(String::isNotBlank)
val matchIndex = terms.mapNotNull { term ->
content.indexOf(term, ignoreCase = true).takeIf { it >= 0 }
}.minOrNull() ?: 0
val start = (matchIndex - SNIPPET_LENGTH / 3).coerceAtLeast(0)
val end = (start + SNIPPET_LENGTH).coerceAtMost(content.length)
return buildString {
if (start > 0) append("...")
append(content, start, end)
if (end < content.length) append("...")
}
}
}
@@ -0,0 +1,83 @@
package top.jie65535.mirai.tools
import com.aallam.openai.api.chat.Tool
import com.aallam.openai.api.core.Parameters
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.intOrNull
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.longOrNull
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.MessageEvent
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.data.ChatHistoryStore
import top.jie65535.mirai.data.ChatHistorySubject
class GetChatHistoryContext : BaseAgent(
tool = Tool.function(
name = "getChatHistoryContext",
description = "根据 searchChatHistory 返回的 messageId,读取该消息在当前群聊或私聊中的前后文。" +
"结果按时间顺序排列,并用 >>> 标记目标消息。",
parameters = Parameters.buildJsonObject {
put("type", "object")
putJsonObject("properties") {
putJsonObject("messageId") {
put("type", "integer")
put("description", "searchChatHistory 返回的稳定消息 ID")
}
putJsonObject("before") {
put("type", "integer")
put("description", "目标消息之前的消息数,默认8,最大15")
}
putJsonObject("after") {
put("type", "integer")
put("description", "目标消息之后的消息数,默认8,最大15")
}
}
putJsonArray("required") { add(JsonPrimitive("messageId")) }
},
)
) {
override val isEnabled: Boolean
get() = JChatGPT.includeHistory
override val loadingMessage: String
get() = "读取聊天记录上下文中..."
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
val parameters = requireNotNull(args)
val messageId = parameters["messageId"]?.jsonPrimitive?.longOrNull
?: return "缺少有效的 messageId"
val before = parameters["before"]?.jsonPrimitive?.intOrNull?.coerceIn(0, MAX_CONTEXT) ?: DEFAULT_CONTEXT
val after = parameters["after"]?.jsonPrimitive?.intOrNull?.coerceIn(0, MAX_CONTEXT) ?: DEFAULT_CONTEXT
val context = try {
ChatHistoryStore.findAround(
subject = ChatHistorySubject.from(event.subject),
messageId = messageId,
before = before,
after = after,
)
} catch (cause: Throwable) {
JChatGPT.logger.warning("读取聊天记录上下文失败: messageId=$messageId", cause)
return "读取聊天记录上下文失败: ${cause.message}"
} ?: return "当前会话中不存在 messageId=$messageId 的消息"
return buildString {
appendLine("目标消息及上下文(共 ${context.records.size} 条):")
appendLine()
ChatHistoryToolFormatter.appendRecords(
output = this,
records = context.records,
event = event,
targetId = context.targetId,
)
}.trimEnd()
}
companion object {
private const val DEFAULT_CONTEXT = 8
private const val MAX_CONTEXT = 15
}
}
+199 -148
View File
@@ -2,57 +2,69 @@ package top.jie65535.mirai.tools
import com.aallam.openai.api.chat.Tool
import com.aallam.openai.api.core.Parameters
import kotlinx.serialization.json.*
import net.mamoe.mirai.contact.Group
import net.mamoe.mirai.contact.nameCardOrNick
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.longOrNull
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonObject
import net.mamoe.mirai.event.events.GroupMessageEvent
import net.mamoe.mirai.event.events.MessageEvent
import net.mamoe.mirai.message.data.Image
import net.mamoe.mirai.message.data.Image.Key.queryUrl
import net.mamoe.mirai.message.data.SingleMessage
import net.mamoe.mirai.message.data.content
import net.mamoe.mirai.contact.nameCardOrNick
import top.jie65535.mirai.JChatGPT
import top.jie65535.mirai.config.PluginConfig
import top.jie65535.mirai.data.ChatHistoryCursor
import top.jie65535.mirai.data.ChatHistoryMatchMode
import top.jie65535.mirai.data.ChatHistorySearchRequest
import top.jie65535.mirai.data.ChatHistorySortOrder
import top.jie65535.mirai.data.ChatHistoryStore
import top.jie65535.mirai.data.ChatMessageRecord
import java.time.Instant
import top.jie65535.mirai.data.ChatHistorySubject
import top.jie65535.mirai.data.ContactNameMatch
import top.jie65535.mirai.data.ContactSnapshotStore
import java.nio.charset.StandardCharsets
import java.time.LocalDate
import java.time.LocalDateTime
import java.time.OffsetDateTime
import java.time.ZoneId
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeParseException
import java.util.Base64
class SearchChatHistory : BaseAgent(
tool = Tool.function(
name = "searchChatHistory",
description = "搜索群聊消息历史,可按关键词、发送者时间范围筛选。用于回溯之前的讨论、查找某人说过的话、统计话题等" +
"不指定时间范围时默认搜索最近30天。指定时间时范围不能超过30天,如需更长跨度可分多次查询" +
"可以通过多轮搜索来实现找到某条消息的上下文",
description = "在当前群聊或私聊的 SQLite 历史中搜索消息,可按内容、发送者时间筛选" +
"默认返回最近匹配并在结果过多时提供 cursor;需要查看命中消息前后讨论时调用 getChatHistoryContext" +
"未指定 from 时默认搜索配置的近期天数,明确指定时间时可以搜索任意历史跨度",
parameters = Parameters.buildJsonObject {
put("type", "object")
putJsonObject("properties") {
putJsonObject("keyword") {
putJsonObject("query") {
put("type", "string")
put("description", "消息内容关键词,人名请用sender")
put("description", "要搜索的消息文本;省略时按其他条件浏览时间线")
}
putJsonObject("sender") {
put("type", "string")
put("description", "发送者名称或QQ号,查找某人的发言")
}
putJsonObject("startTime") {
put("type", "string")
put("description", "起始时间,格式:yyyy-MM-dd HH:mm,不填则默认为7天前")
}
putJsonObject("endTime") {
put("type", "string")
put("description", "结束时间,格式同上,不填则默认到当前时间")
}
putJsonObject("limit") {
putJsonObject("senderId") {
put("type", "integer")
put("description", "返回消息数量上限,默认20,最大200")
put("description", "发送者 QQ 号;仅在用户明确给出或上下文可靠提供 QQ 号时使用,不要根据昵称猜测")
}
putJsonObject("senderName") {
put("type", "string")
put("description", "用户用名称指代发送者时优先原样传入群名片、昵称或好友备注;可与 senderId 同时提供,没有 senderId 时解析发送者,有重名时返回候选 QQ 号")
}
putJsonObject("from") {
put("type", "string")
put("description", "起始时间,格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd")
}
putJsonObject("to") {
put("type", "string")
put("description", "结束时间,格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd;仅日期表示当天结束")
}
putJsonObject("cursor") {
put("type", "string")
put("description", "上一页返回的 nextCursor;继续同一搜索时原样传回")
}
}
}
},
)
) {
override val isEnabled: Boolean
@@ -62,151 +74,190 @@ class SearchChatHistory : BaseAgent(
get() = "搜索聊天记录中..."
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
requireNotNull(args)
val keyword = args["keyword"]?.jsonPrimitive?.contentOrNull
val sender = args["sender"]?.jsonPrimitive?.contentOrNull
val maxDays = PluginConfig.searchHistoryMaxDays
val parameters = args ?: JsonObject(emptyMap())
val query = parameters.string("query")?.trim()?.takeIf(String::isNotEmpty)
val matchMode = ChatHistoryMatchMode.ALL
val sortOrder = ChatHistorySortOrder.NEWEST
val now = OffsetDateTime.now()
val startTime = args["startTime"]?.jsonPrimitive?.contentOrNull?.let {
parseTime(it) ?: return "startTime 格式错误,请使用 yyyy-MM-dd HH:mm"
} ?: now.minusDays(maxDays.toLong())
val endTime = args["endTime"]?.jsonPrimitive?.contentOrNull?.let {
parseTime(it) ?: return "endTime 格式错误,请使用 yyyy-MM-dd HH:mm"
val start = parameters.string("from")?.let {
parseTime(it, endOfDay = false) ?: return "from 格式错误,请使用 yyyy-MM-dd HH:mm 或 yyyy-MM-dd"
} ?: now.minusDays(PluginConfig.searchHistoryMaxDays.coerceAtLeast(1).toLong())
val end = parameters.string("to")?.let {
parseTime(it, endOfDay = true) ?: return "to 格式错误,请使用 yyyy-MM-dd HH:mm 或 yyyy-MM-dd"
} ?: now
if (start > end) return "起始时间必须早于或等于结束时间"
if (startTime >= endTime) {
return "起始时间必须早于结束时间"
}
if (java.time.Duration.between(startTime, endTime).toDays() > maxDays) {
return "搜索时间范围不能超过 ${maxDays}天,请缩小范围后重试"
}
val senderQq = resolveSenderQq(sender, event)
val startEpoch = startTime.toEpochSecond().toInt()
val endEpoch = endTime.toEpochSecond().toInt()
val maxRecords = PluginConfig.searchHistoryMaxRecords
val records = try {
ChatHistoryStore.query(
contact = event.subject,
start = startEpoch,
end = endEpoch,
limit = maxRecords,
fromId = senderQq,
).sortedWith(compareBy<ChatMessageRecord> { it.time }.thenBy { it.id })
} catch (e: Throwable) {
JChatGPT.logger.warning("查询消息历史失败", e)
return "查询消息历史失败: ${e.message}"
}
var filtered = records
// 消息内容在数据库中是序列化存储的,关键词只能在内存中过滤
if (keyword != null) {
filtered = filtered.filter {
it.toMessageChain().content.contains(keyword, ignoreCase = true)
val senderId = parameters["senderId"]?.jsonPrimitive?.longOrNull
val senderName = parameters.string("senderName")?.trim()?.takeIf(String::isNotEmpty)
val resolvedSenderId = senderId ?: senderName?.let { name ->
when (val resolution = resolveSender(name, event)) {
is SenderResolution.Found -> resolution.userId
is SenderResolution.Ambiguous -> return buildString {
appendLine("找到多个名称匹配的发送者,请改用 senderId:")
resolution.candidates.forEach { candidate ->
appendLine("- ${candidate.displayName} (${candidate.userId})")
}
}.trimEnd()
SenderResolution.NotFound -> return "没有找到名称匹配的发送者:$name"
}
}
if (filtered.isEmpty()) {
return "未找到匹配的聊天记录"
val outputSafePageSize = ((PluginConfig.maxToolOutputLength.coerceAtLeast(1) - OUTPUT_RESERVE_CHARS)
.coerceAtLeast(APPROXIMATE_RECORD_CHARS) / APPROXIMATE_RECORD_CHARS)
.coerceIn(1, MAX_PAGE_SIZE)
val configuredPageSize = minOf(
PluginConfig.searchHistoryMaxRecords.coerceIn(1, MAX_PAGE_SIZE),
outputSafePageSize,
)
val pageSize = DEFAULT_PAGE_SIZE.coerceAtMost(configuredPageSize)
val cursor = parameters.string("cursor")?.let {
decodeCursor(it, sortOrder) ?: return "cursor 无效,请重新开始搜索"
}
if (query != null && query.codePointCount(0, query.length) >= 3 &&
!ChatHistoryStore.isSearchIndexAvailable
) {
return "聊天记录全文索引当前不可用,无法执行关键词搜索"
}
val limit = args["limit"]?.jsonPrimitive?.intOrNull?.coerceIn(1, 200) ?: 20
val total = filtered.size
val result = filtered.takeLast(limit)
val request = ChatHistorySearchRequest(
subject = ChatHistorySubject.from(event.subject),
query = query,
matchMode = matchMode,
fromId = resolvedSenderId,
start = start.toEpochSecond().toInt(),
end = end.toEpochSecond().toInt(),
sortOrder = sortOrder,
limit = pageSize,
cursor = cursor,
)
val page = try {
ChatHistoryStore.search(request)
} catch (cause: Throwable) {
JChatGPT.logger.warning("查询消息历史失败", cause)
return "查询消息历史失败: ${cause.message}"
}
if (page.records.isEmpty()) {
if (query != null && resolvedSenderId != null && cursor == null) {
val withoutSender = runCatching {
ChatHistoryStore.search(
request.copy(
fromId = null,
limit = DIAGNOSTIC_PAGE_SIZE,
cursor = ChatHistoryCursor(request.end ?: Int.MAX_VALUE, Long.MAX_VALUE),
)
)
}.getOrNull()
if (withoutSender?.records?.isNotEmpty() == true) {
return buildString {
append("未找到发送者 $resolvedSenderId 的匹配记录。")
appendLine("去掉发送者筛选后找到了匹配消息,可能是 QQ 号不正确;示例:")
ChatHistoryToolFormatter.appendRecords(this, withoutSender.records, event, query)
}.trimEnd()
}
}
return if (!ChatHistoryStore.isSearchIndexReady && query != null) {
"未找到匹配的聊天记录。历史全文索引仍在后台构建,当前结果可能不完整。"
} else if (resolvedSenderId != null) {
"未找到发送者 $resolvedSenderId 在当前聊天和时间范围内的匹配记录"
} else {
"未找到匹配的聊天记录"
}
}
return buildString {
appendLine("找到 $total 条匹配记录,显示最近 ${result.size} 条:")
page.totalMatches?.let { total -> appendLine("找到 $total 条匹配记录,本页 ${page.records.size} 条:") }
?: appendLine("继续搜索,本页 ${page.records.size} 条:")
page.nextCursor?.let { next ->
appendLine("hasMore: true")
appendLine("nextCursor: ${encodeCursor(next, sortOrder)}")
} ?: appendLine("hasMore: false")
appendLine()
appendHistory(this, result, event)
}
ChatHistoryToolFormatter.appendRecords(this, page.records, event, query)
if (!ChatHistoryStore.isSearchIndexReady && query != null) {
appendLine()
append("[索引状态] 历史全文索引仍在后台构建,当前结果可能不完整。")
}
}.trimEnd()
}
/**
* 将 sender 解析为 QQ 号,优先尝试纯数字,再尝试群成员名称匹配
*/
private fun resolveSenderQq(sender: String?, event: MessageEvent): Long? {
if (sender == null) return null
sender.toLongOrNull()?.let { return it }
private fun resolveSender(name: String, event: MessageEvent): SenderResolution {
val groupId = (event as? GroupMessageEvent)?.group?.id
val matches = runCatching {
ContactSnapshotStore.findUsersByName(event.bot.id, groupId, name, limit = 6)
}.getOrDefault(emptyList()).toMutableList()
runCatching {
ChatHistoryStore.findSenderAliases(ChatHistorySubject.from(event.subject), name, limit = 6)
}.getOrDefault(emptyList()).forEach { alias ->
if (matches.none { it.userId == alias.userId }) {
matches += ContactNameMatch(alias.userId, alias.displayName, alias.matchRank)
}
}
if (event is GroupMessageEvent) {
return event.group.members.firstOrNull {
it.nameCardOrNick.contains(sender, ignoreCase = true)
}?.id
}
return null
}
private suspend fun appendHistory(
sb: StringBuilder,
records: List<ChatMessageRecord>,
event: MessageEvent
) {
val timeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
var lastFromId = 0L
for (record in records) {
val showSender = lastFromId != record.fromId
if (showSender) {
sb.appendLine()
if (event is GroupMessageEvent) {
if (event.bot.id == record.fromId) {
sb.append("**你** ").append(event.bot.nameCardOrNick)
} else {
sb.append(getNameCard(event.group, record.fromId))
event.group.members.asSequence()
.filter { it.nameCardOrNick.contains(name, ignoreCase = true) }
.forEach { member ->
if (matches.none { it.userId == member.id }) {
matches += ContactNameMatch(member.id, member.nameCardOrNick, 10)
}
}
sb.append(" ")
.append(timeFormatter.format(
Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault())
))
.append("")
}
for (msg in record.toMessageChain()) {
sb.append(singleMessageToText(msg, event.subject.id))
}
sb.appendLine()
lastFromId = record.fromId
}
val sorted = matches.sortedWith(compareBy({ it.matchRank }, { it.userId }))
val first = sorted.firstOrNull() ?: return SenderResolution.NotFound
val second = sorted.getOrNull(1)
return if (second == null || first.matchRank < second.matchRank) {
SenderResolution.Found(first.userId)
} else {
SenderResolution.Ambiguous(sorted)
}
}
private suspend fun singleMessageToText(msg: SingleMessage, subjectId: Long): String {
return when (msg) {
is Image -> {
try {
val imageUrl = msg.queryUrl()
val index = JChatGPT.registerImage(subjectId, msg.imageId, imageUrl)
?: return msg.content
"[${if (msg.isEmoji) "表情包" else "图片"}$index]"
} catch (_: Throwable) {
msg.content
}
}
else -> msg.content
}
}
private fun getNameCard(group: Group, qq: Long): String {
val member = group[qq]
return member?.nameCardOrNick ?: "未知群员($qq)"
private sealed interface SenderResolution {
data class Found(val userId: Long) : SenderResolution
data class Ambiguous(val candidates: List<ContactNameMatch>) : SenderResolution
data object NotFound : SenderResolution
}
companion object {
private val timeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")
private const val DEFAULT_PAGE_SIZE = 20
private const val DIAGNOSTIC_PAGE_SIZE = 6
private const val MAX_PAGE_SIZE = 200
private const val OUTPUT_RESERVE_CHARS = 1_000
private const val APPROXIMATE_RECORD_CHARS = 450
private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")
private val dateFormatter = DateTimeFormatter.ISO_LOCAL_DATE
fun parseTime(text: String): OffsetDateTime? {
fun parseTime(text: String, endOfDay: Boolean): OffsetDateTime? {
val zone = ZoneId.systemDefault()
return try {
LocalDateTime.parse(text, timeFormatter)
.atZone(ZoneId.systemDefault())
.toOffsetDateTime()
LocalDateTime.parse(text, dateTimeFormatter).atZone(zone).toOffsetDateTime()
} catch (_: DateTimeParseException) {
null
try {
val date = LocalDate.parse(text, dateFormatter)
val localDateTime = if (endOfDay) date.plusDays(1).atStartOfDay().minusNanos(1) else date.atStartOfDay()
localDateTime.atZone(zone).toOffsetDateTime()
} catch (_: DateTimeParseException) {
null
}
}
}
private fun encodeCursor(cursor: ChatHistoryCursor, sortOrder: ChatHistorySortOrder): String {
val raw = "${sortOrder.name}:${cursor.time}:${cursor.id}"
return Base64.getUrlEncoder().withoutPadding()
.encodeToString(raw.toByteArray(StandardCharsets.UTF_8))
}
private fun decodeCursor(text: String, sortOrder: ChatHistorySortOrder): ChatHistoryCursor? {
return runCatching {
val raw = String(Base64.getUrlDecoder().decode(text), StandardCharsets.UTF_8)
val parts = raw.split(':')
if (parts.size != 3 || parts[0] != sortOrder.name) return null
ChatHistoryCursor(parts[1].toInt(), parts[2].toLong())
}.getOrNull()
}
private fun JsonObject.string(key: String): String? =
get(key)?.jsonPrimitive?.contentOrNull
}
}
+216 -1
View File
@@ -5,7 +5,14 @@ import java.sql.DriverManager
import kotlin.io.path.absolutePathString
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
import net.mamoe.mirai.message.data.MessageSourceKind
import top.jie65535.mirai.data.ChatHistoryMatchMode.PHRASE
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
class ChatHistoryStoreTest {
@Test
@@ -55,7 +62,15 @@ class ChatHistoryStoreTest {
"SELECT value FROM chat_history_meta WHERE key = 'schema_version'"
).use { results ->
assertTrue(results.next())
assertEquals("2", results.getString(1))
assertEquals("3", results.getString(1))
}
statement.executeQuery(
"SELECT COUNT(*) FROM sqlite_master " +
"WHERE type = 'table' AND name IN " +
"('message_record_search', 'message_record_fts', 'chat_history_sender_alias')"
).use { results ->
assertTrue(results.next())
assertEquals(3, results.getInt(1))
}
}
}
@@ -64,4 +79,204 @@ class ChatHistoryStoreTest {
directory.toFile().deleteRecursively()
}
}
@Test
fun searchesChineseTextInDatabaseAndSupportsCursorAndContext() {
val directory = Files.createTempDirectory("jchatgpt-history-search-test-")
try {
ChatHistoryStore.init(directory.toFile())
val first = insert(directory, 100, 11, "部署失败,需要回滚", senderName = "旧名张三")
val second = insert(directory, 101, 12, "部署成功,服务已恢复")
val third = insert(directory, 102, 11, "确认部署失败的原因")
val subject = ChatHistorySubject(1, MessageSourceKind.GROUP, 1000)
val firstPage = ChatHistoryStore.search(
ChatHistorySearchRequest(
subject = subject,
query = "部署失败",
matchMode = PHRASE,
start = 0,
end = 200,
limit = 1,
)
)
assertEquals(2L, firstPage.totalMatches)
assertEquals(third, firstPage.records.single().id)
assertNotNull(firstPage.nextCursor)
val secondPage = ChatHistoryStore.search(
ChatHistorySearchRequest(
subject = subject,
query = "部署失败",
matchMode = PHRASE,
start = 0,
end = 200,
limit = 1,
cursor = firstPage.nextCursor,
)
)
assertEquals(null, secondPage.totalMatches)
assertEquals(first, secondPage.records.single().id)
assertFalse(secondPage.nextCursor != null)
val context = ChatHistoryStore.findAround(subject, second, before = 1, after = 1)
assertNotNull(context)
assertEquals(listOf(first, second, third), context.records.map { it.id })
val shortQuery = ChatHistoryStore.search(
ChatHistorySearchRequest(
subject = subject,
query = "部署",
fromId = 11,
start = 0,
end = 200,
sortOrder = ChatHistorySortOrder.OLDEST,
limit = 10,
)
)
assertEquals(2L, shortQuery.totalMatches)
assertEquals(listOf(first, third), shortQuery.records.map { it.id })
val aliases = ChatHistoryStore.findSenderAliases(subject, "旧名")
assertEquals(listOf(ChatHistorySenderAliasMatch(11, "旧名张三", 1)), aliases)
} finally {
ChatHistoryStore.close()
directory.toFile().deleteRecursively()
}
}
@Test
fun migratesExistingSchemaAndBackfillsSearchTextWithoutReplacingMessageTable() {
val directory = Files.createTempDirectory("jchatgpt-history-migration-test-")
val database = directory.resolve("chat-history.sqlite")
try {
Class.forName("org.sqlite.JDBC")
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
connection.createStatement().use { statement ->
statement.executeUpdate(
"""
CREATE TABLE message_record(
id INTEGER PRIMARY KEY AUTOINCREMENT,
bot_id INTEGER NOT NULL,
from_id INTEGER NOT NULL,
target_id INTEGER NOT NULL,
ids TEXT,
internal_ids TEXT,
time INTEGER NOT NULL,
kind INTEGER NOT NULL,
code TEXT NOT NULL,
recalled INTEGER NOT NULL DEFAULT 0
)
""".trimIndent()
)
statement.executeUpdate(
"CREATE TABLE chat_history_meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
)
statement.executeUpdate(
"INSERT INTO chat_history_meta(key, value) VALUES ('schema_version', '2')"
)
statement.executeUpdate(
"""
INSERT INTO message_record(
bot_id, from_id, target_id, ids, internal_ids, time, kind, code, recalled
) VALUES (1, 11, 1000, '1', '1', 100, ${MessageSourceKind.GROUP.ordinal},
'[{"type":"PlainText","content":"旧库部署失败"}]', 0)
""".trimIndent()
)
statement.executeUpdate(
"""
INSERT INTO message_record(
bot_id, from_id, target_id, ids, internal_ids, time, kind, code, recalled
) VALUES (1, 12, 1000, '2', '2', 101, ${MessageSourceKind.GROUP.ordinal},
'[{"type":"PlainText","content":"旧库部署失败后已恢复"}]', 0)
""".trimIndent()
)
}
}
ChatHistoryStore.init(directory.toFile())
assertTrue(ChatHistoryStore.isSearchIndexAvailable)
assertFalse(ChatHistoryStore.isSearchIndexReady)
var continueChecks = 0
assertEquals(
1,
ChatHistoryStore.backfillSearchIndex(
batchSize = 1,
shouldContinue = { continueChecks++ == 0 },
)
)
assertFalse(ChatHistoryStore.isSearchIndexReady)
ChatHistoryStore.close()
ChatHistoryStore.init(directory.toFile())
assertFalse(ChatHistoryStore.isSearchIndexReady)
assertEquals(1, ChatHistoryStore.backfillSearchIndex(batchSize = 1))
assertTrue(ChatHistoryStore.isSearchIndexReady)
val migratedSearch = ChatHistoryStore.search(
ChatHistorySearchRequest(
subject = ChatHistorySubject(1, MessageSourceKind.GROUP, 1000),
query = "旧库部署失败",
matchMode = PHRASE,
start = 0,
end = 200,
)
)
assertEquals(2L, migratedSearch.totalMatches)
assertEquals(listOf(2L, 1L), migratedSearch.records.map { it.id })
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
connection.createStatement().use { statement ->
statement.executeQuery("SELECT COUNT(*) FROM message_record").use { results ->
assertTrue(results.next())
assertEquals(2, results.getInt(1))
}
statement.executeQuery(
"SELECT search_text FROM message_record_search WHERE message_id = 1"
).use { results ->
assertTrue(results.next())
assertEquals("旧库部署失败", results.getString(1))
}
}
}
} finally {
ChatHistoryStore.close()
directory.toFile().deleteRecursively()
}
}
private fun insert(
directory: java.nio.file.Path,
time: Int,
fromId: Long,
text: String,
senderName: String = "",
): Long {
val record = ChatMessageRecord(
botId = 1,
fromId = fromId,
targetId = 1000,
ids = "$time",
internalIds = "$time",
time = time,
kind = MessageSourceKind.GROUP,
code = buildJsonArray {
add(buildJsonObject {
put("type", "PlainText")
put("content", text)
})
}.toString(),
)
ChatHistoryStore.insertRecord(record, senderName)
DriverManager.getConnection("jdbc:sqlite:${directory.resolve("chat-history.sqlite").absolutePathString()}").use {
it.prepareStatement("SELECT id FROM message_record WHERE time = ? AND from_id = ?").use { query ->
query.setInt(1, time)
query.setLong(2, fromId)
query.executeQuery().use { results ->
assertTrue(results.next())
return results.getLong(1)
}
}
}
}
}