mirror of
https://github.com/jie65535/JChatGPT.git
synced 2026-09-15 02:56:10 +08:00
history: add indexed search and context retrieval
This commit is contained in:
@@ -229,15 +229,21 @@ memoryEnabled: true
|
|||||||
skillsEnabled: true
|
skillsEnabled: true
|
||||||
# 是否启用好感度系统
|
# 是否启用好感度系统
|
||||||
enableFavorabilitySystem: true
|
enableFavorabilitySystem: true
|
||||||
# 聊天记录搜索最大天数
|
# 未指定起始时间时聊天记录搜索默认回溯天数;明确指定时间时不限制历史跨度
|
||||||
searchHistoryMaxDays: 30
|
searchHistoryMaxDays: 30
|
||||||
# 聊天记录搜索最大查询条数,防止内存溢出
|
# 聊天记录搜索单页消息数上限,工具硬上限为200(兼容旧配置键)
|
||||||
searchHistoryMaxRecords: 5000
|
searchHistoryMaxRecords: 5000
|
||||||
```
|
```
|
||||||
|
|
||||||
聊天记录与联系人快照保存在插件数据目录的 `chat-history.sqlite` 中,并使用 SQLite WAL 模式支持记录与查询并行进行。
|
聊天记录与联系人快照保存在插件数据目录的 `chat-history.sqlite` 中,并使用 SQLite WAL 模式支持记录与查询并行进行。
|
||||||
数据库由插件在首次启动时自动创建和维护,无需安装额外的聊天记录插件。
|
数据库由插件在首次启动时自动创建和维护,无需安装额外的聊天记录插件。
|
||||||
|
|
||||||
|
聊天记录搜索使用 SQLite FTS5 中文三元索引。升级已有数据库时只会新增独立的派生搜索文本表和索引表,
|
||||||
|
不会改列、重建或删除 `message_record` 原始记录;旧记录的搜索文本会在插件启动后分批后台回填。回填期间新消息仍会正常保存,
|
||||||
|
搜索结果会明确标注索引尚未完成。搜索工具默认只查询当前群聊或当前私聊,明确指定 `from` 和 `to` 后可以跨越任意历史时间范围,
|
||||||
|
并通过 `nextCursor` 分页。升级后新收到的消息还会记录发送者在该会话中使用过的名称,名称搜索不依赖对方仍是当前群成员;
|
||||||
|
`getChatHistoryContext` 可按 `messageId` 获取命中消息前后的对话上下文。
|
||||||
|
|
||||||
联系人刷新在后台通过 Overflow 的公开 `RemoteBot.executeAction` 调用 OneBot
|
联系人刷新在后台通过 Overflow 的公开 `RemoteBot.executeAction` 调用 OneBot
|
||||||
`get_friend_list`、`get_group_list` 和 `get_group_member_list`,不使用反射,也不依赖 Overflow 的 internal
|
`get_friend_list`、`get_group_list` 和 `get_group_member_list`,不使用反射,也不依赖 Overflow 的 internal
|
||||||
实现类。默认启动延迟 30 秒后刷新、此后每 24 小时刷新一次;单个群失败时保留上一版成员快照,完整刷新成功后
|
实现类。默认启动延迟 30 秒后刷新、此后每 24 小时刷新一次;单个群失败时保留上一版成员快照,完整刷新成功后
|
||||||
@@ -496,8 +502,12 @@ fallbackCooldownMinutes: 5
|
|||||||
10. **SendVoiceMessage** - 发送语音消息
|
10. **SendVoiceMessage** - 发送语音消息
|
||||||
11. **ImageAgent** - 图像生成与编辑(文生图、单图编辑、多图融合)
|
11. **ImageAgent** - 图像生成与编辑(文生图、单图编辑、多图融合)
|
||||||
12. **WeatherService** - 天气查询
|
12. **WeatherService** - 天气查询
|
||||||
13. **SearchChatHistory** - 按关键词、发送者、时间范围搜索插件内置 SQLite 聊天历史
|
13. **SearchChatHistory** - 在当前群聊或私聊中使用 SQLite 全文索引搜索聊天历史,支持发送者、时间范围和游标翻页
|
||||||
14. **QueryUserProfile** - 按当前发送者、QQ 号、群名片、昵称或好友备注读取长期画像,并针对该联系人按需读取公开资料卡
|
14. **GetChatHistoryContext** - 根据历史消息 ID 获取目标消息前后的对话上下文
|
||||||
|
15. **QueryUserProfile** - 按当前发送者、QQ 号、群名片、昵称或好友备注读取长期画像,并针对该联系人按需读取公开资料卡
|
||||||
|
|
||||||
|
`SearchChatHistory` 的常用参数只有消息内容、发送者、时间范围和翻页游标;结果默认按最近消息返回,
|
||||||
|
单页大小和匹配策略由插件控制。`senderId` 与 `senderName` 可以同时提供,此时 QQ 号作为精确筛选条件;有 QQ 号时名称不参与筛选。
|
||||||
|
|
||||||
### GitHub 查询工具
|
### GitHub 查询工具
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
package top.jie65535.mirai
|
package top.jie65535.mirai
|
||||||
|
|
||||||
|
import kotlinx.coroutines.CancellationException
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.Job
|
||||||
|
import kotlinx.coroutines.isActive
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
import net.mamoe.mirai.console.command.CommandManager.INSTANCE.register
|
import net.mamoe.mirai.console.command.CommandManager.INSTANCE.register
|
||||||
import net.mamoe.mirai.console.command.CommandSender.Companion.toCommandSender
|
import net.mamoe.mirai.console.command.CommandSender.Companion.toCommandSender
|
||||||
import net.mamoe.mirai.console.permission.PermissionId
|
import net.mamoe.mirai.console.permission.PermissionId
|
||||||
@@ -48,6 +53,7 @@ object JChatGPT : KotlinPlugin(
|
|||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
internal var includeHistory: Boolean = false
|
internal var includeHistory: Boolean = false
|
||||||
|
private var historyIndexJob: Job? = null
|
||||||
|
|
||||||
val chatPermission = PermissionId("JChatGPT", "Chat")
|
val chatPermission = PermissionId("JChatGPT", "Chat")
|
||||||
|
|
||||||
@@ -69,6 +75,27 @@ object JChatGPT : KotlinPlugin(
|
|||||||
logger.error("初始化 SQLite 聊天记录失败,历史上下文与搜索将暂时禁用", cause)
|
logger.error("初始化 SQLite 聊天记录失败,历史上下文与搜索将暂时禁用", cause)
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
if (includeHistory && ChatHistoryStore.isSearchIndexAvailable && !ChatHistoryStore.isSearchIndexReady) {
|
||||||
|
historyIndexJob = launch(Dispatchers.IO) {
|
||||||
|
try {
|
||||||
|
val processed = ChatHistoryStore.backfillSearchIndex(
|
||||||
|
shouldContinue = { isActive },
|
||||||
|
onProgress = { count ->
|
||||||
|
if (count % 10_000 == 0) {
|
||||||
|
logger.info("聊天记录全文索引回填进度: 已处理 $count 条")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if (isActive && processed > 0) {
|
||||||
|
logger.info("聊天记录全文索引回填完成: 共处理 $processed 条")
|
||||||
|
}
|
||||||
|
} catch (cause: CancellationException) {
|
||||||
|
throw cause
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
logger.warning("聊天记录全文索引回填失败,后续启动将继续尝试", cause)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if (includeHistory) {
|
if (includeHistory) {
|
||||||
runCatching { ContactSnapshotStore.init(dataFolder) }
|
runCatching { ContactSnapshotStore.init(dataFolder) }
|
||||||
.onFailure { logger.error("初始化联系人快照数据库失败,联系人画像辅助将暂时禁用", it) }
|
.onFailure { logger.error("初始化联系人快照数据库失败,联系人画像辅助将暂时禁用", it) }
|
||||||
@@ -103,6 +130,8 @@ object JChatGPT : KotlinPlugin(
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun onDisable() {
|
override fun onDisable() {
|
||||||
|
historyIndexJob?.cancel()
|
||||||
|
historyIndexJob = null
|
||||||
ConversationEngine.clear()
|
ConversationEngine.clear()
|
||||||
ConversationContext.clearAll()
|
ConversationContext.clearAll()
|
||||||
ProfileAutoMaintenance.clear()
|
ProfileAutoMaintenance.clear()
|
||||||
|
|||||||
@@ -292,9 +292,9 @@ object PluginConfig : AutoSavePluginConfig("Config") {
|
|||||||
@ValueDescription("单个工具调用返回内容的最大字符数,超过将被截断并标注")
|
@ValueDescription("单个工具调用返回内容的最大字符数,超过将被截断并标注")
|
||||||
val maxToolOutputLength: Int by value(15000)
|
val maxToolOutputLength: Int by value(15000)
|
||||||
|
|
||||||
@ValueDescription("聊天记录搜索最大天数")
|
@ValueDescription("未指定起始时间时聊天记录搜索默认回溯天数;明确指定时间时不限制历史跨度")
|
||||||
val searchHistoryMaxDays: Int by value(30)
|
val searchHistoryMaxDays: Int by value(30)
|
||||||
|
|
||||||
@ValueDescription("聊天记录搜索最大查询条数,防止内存溢出")
|
@ValueDescription("聊天记录搜索单页消息数上限;工具硬上限为200,保留此配置键以兼容旧配置")
|
||||||
val searchHistoryMaxRecords: Int by value(5000)
|
val searchHistoryMaxRecords: Int by value(5000)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import top.jie65535.mirai.tools.AdjustUserFavorabilityAgent
|
|||||||
import top.jie65535.mirai.tools.BaseAgent
|
import top.jie65535.mirai.tools.BaseAgent
|
||||||
import top.jie65535.mirai.tools.DeleteSkill
|
import top.jie65535.mirai.tools.DeleteSkill
|
||||||
import top.jie65535.mirai.tools.GroupManageAgent
|
import top.jie65535.mirai.tools.GroupManageAgent
|
||||||
|
import top.jie65535.mirai.tools.GetChatHistoryContext
|
||||||
import top.jie65535.mirai.tools.GithubAgent
|
import top.jie65535.mirai.tools.GithubAgent
|
||||||
import top.jie65535.mirai.tools.ImageAgent
|
import top.jie65535.mirai.tools.ImageAgent
|
||||||
import top.jie65535.mirai.tools.LoadSkill
|
import top.jie65535.mirai.tools.LoadSkill
|
||||||
@@ -70,6 +71,7 @@ internal object ConversationEngine {
|
|||||||
SaveSkill(),
|
SaveSkill(),
|
||||||
DeleteSkill(),
|
DeleteSkill(),
|
||||||
SearchChatHistory(),
|
SearchChatHistory(),
|
||||||
|
GetChatHistoryContext(),
|
||||||
QueryUserProfileAgent(),
|
QueryUserProfileAgent(),
|
||||||
WebSearch(),
|
WebSearch(),
|
||||||
GithubAgent(),
|
GithubAgent(),
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package top.jie65535.mirai.data
|
||||||
|
|
||||||
|
import net.mamoe.mirai.contact.Contact
|
||||||
|
import net.mamoe.mirai.contact.Friend
|
||||||
|
import net.mamoe.mirai.contact.Group
|
||||||
|
import net.mamoe.mirai.contact.Member
|
||||||
|
import net.mamoe.mirai.contact.Stranger
|
||||||
|
import net.mamoe.mirai.message.data.MessageSourceKind
|
||||||
|
|
||||||
|
data class ChatHistorySubject(
|
||||||
|
val botId: Long,
|
||||||
|
val kind: MessageSourceKind,
|
||||||
|
val subjectId: Long,
|
||||||
|
) {
|
||||||
|
companion object {
|
||||||
|
fun from(contact: Contact): ChatHistorySubject = when (contact) {
|
||||||
|
is Group -> ChatHistorySubject(contact.bot.id, MessageSourceKind.GROUP, contact.id)
|
||||||
|
is Member -> ChatHistorySubject(contact.bot.id, MessageSourceKind.GROUP, contact.group.id)
|
||||||
|
is Friend -> ChatHistorySubject(contact.bot.id, MessageSourceKind.FRIEND, contact.id)
|
||||||
|
is Stranger -> ChatHistorySubject(contact.bot.id, MessageSourceKind.STRANGER, contact.id)
|
||||||
|
else -> error("不支持查询的联系人 $contact")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum class ChatHistoryMatchMode {
|
||||||
|
ALL,
|
||||||
|
ANY,
|
||||||
|
PHRASE,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum class ChatHistorySortOrder {
|
||||||
|
NEWEST,
|
||||||
|
OLDEST,
|
||||||
|
}
|
||||||
|
|
||||||
|
data class ChatHistoryCursor(
|
||||||
|
val time: Int,
|
||||||
|
val id: Long,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class ChatHistorySearchRequest(
|
||||||
|
val subject: ChatHistorySubject,
|
||||||
|
val query: String? = null,
|
||||||
|
val matchMode: ChatHistoryMatchMode = ChatHistoryMatchMode.ALL,
|
||||||
|
val fromId: Long? = null,
|
||||||
|
val start: Int? = null,
|
||||||
|
val end: Int? = null,
|
||||||
|
val sortOrder: ChatHistorySortOrder = ChatHistorySortOrder.NEWEST,
|
||||||
|
val limit: Int = 20,
|
||||||
|
val cursor: ChatHistoryCursor? = null,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class ChatHistorySearchPage(
|
||||||
|
val records: List<ChatMessageRecord>,
|
||||||
|
val totalMatches: Long?,
|
||||||
|
val nextCursor: ChatHistoryCursor?,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class ChatHistoryContext(
|
||||||
|
val targetId: Long,
|
||||||
|
val records: List<ChatMessageRecord>,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class ChatHistorySenderAliasMatch(
|
||||||
|
val userId: Long,
|
||||||
|
val displayName: String,
|
||||||
|
val matchRank: Int,
|
||||||
|
)
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package top.jie65535.mirai.data
|
||||||
|
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
import kotlinx.serialization.json.JsonArray
|
||||||
|
import kotlinx.serialization.json.JsonObject
|
||||||
|
import kotlinx.serialization.json.JsonPrimitive
|
||||||
|
import kotlinx.serialization.json.booleanOrNull
|
||||||
|
import kotlinx.serialization.json.contentOrNull
|
||||||
|
import kotlinx.serialization.json.longOrNull
|
||||||
|
import net.mamoe.mirai.message.data.content
|
||||||
|
|
||||||
|
object ChatHistorySearchText {
|
||||||
|
private const val MAX_INDEXED_CHARS = 16_384
|
||||||
|
private const val MAX_QUOTED_CHARS = 320
|
||||||
|
private const val MAX_FORWARD_NODES = 30
|
||||||
|
private const val MAX_FORWARD_NODE_CHARS = 500
|
||||||
|
|
||||||
|
private val json = Json { ignoreUnknownKeys = true }
|
||||||
|
private val whitespace = Regex("\\s+")
|
||||||
|
|
||||||
|
fun extract(code: String): String {
|
||||||
|
val rendered = runCatching { renderJsonCode(code) }
|
||||||
|
.recoverCatching { decodeMessageCode(code).content }
|
||||||
|
.getOrDefault("")
|
||||||
|
return rendered.normalize().takeUtf16Safely(MAX_INDEXED_CHARS)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun renderJsonCode(code: String): String {
|
||||||
|
val messages = json.parseToJsonElement(code) as? JsonArray
|
||||||
|
?: throw IllegalArgumentException("消息记录不是 JSON array")
|
||||||
|
return renderMessages(messages)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun renderMessages(messages: JsonArray): String = messages.joinToString("") { element ->
|
||||||
|
val message = element as? JsonObject ?: return@joinToString ""
|
||||||
|
when (val type = message.string("type")) {
|
||||||
|
"PlainText" -> message.string("content").orEmpty()
|
||||||
|
"At" -> message.long("target")?.let { "@$it" }.orEmpty()
|
||||||
|
"AtAll" -> "@全体成员"
|
||||||
|
"Image", "FlashImage" -> if (message.boolean("isEmoji") == true) "[表情包]" else "[图片]"
|
||||||
|
"QuoteReply" -> renderQuote(message)
|
||||||
|
"ForwardMessage" -> renderForward(message)
|
||||||
|
"MessageOrigin", "MessageSource", "ShowImageFlag" -> ""
|
||||||
|
"Face", "MarketFace", "VipFace" -> "[表情]"
|
||||||
|
"Audio" -> "[语音]"
|
||||||
|
"FileMessage" -> "[文件${message.string("name")?.let { ": $it" }.orEmpty()}]"
|
||||||
|
"LightApp", "SimpleServiceMessage", "MusicShare" -> "[卡片消息]"
|
||||||
|
"PokeMessage" -> "[戳一戳]"
|
||||||
|
null -> ""
|
||||||
|
else -> message.string("content") ?: "[$type]"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun renderQuote(message: JsonObject): String {
|
||||||
|
val source = message["source"] as? JsonObject ?: return "[引用消息]"
|
||||||
|
val author = source.long("fromId")?.toString() ?: "其他用户"
|
||||||
|
val original = (source["originalMessage"] as? JsonArray)
|
||||||
|
?.let(::renderMessages)
|
||||||
|
.orEmpty()
|
||||||
|
.normalize()
|
||||||
|
.takeUtf16Safely(MAX_QUOTED_CHARS)
|
||||||
|
return "[引用 $author: $original]"
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun renderForward(message: JsonObject): String = buildString {
|
||||||
|
append("[转发消息]")
|
||||||
|
val nodes = message["nodeList"] as? JsonArray ?: return@buildString
|
||||||
|
nodes.take(MAX_FORWARD_NODES).forEach { element ->
|
||||||
|
val node = element as? JsonObject ?: return@forEach
|
||||||
|
val sender = node.string("senderName") ?: "未知用户"
|
||||||
|
val chain = node["messageChain"] as? JsonArray
|
||||||
|
append(' ').append(sender).append(": ")
|
||||||
|
append(
|
||||||
|
chain?.let(::renderMessages)
|
||||||
|
.orEmpty()
|
||||||
|
.normalize()
|
||||||
|
.takeUtf16Safely(MAX_FORWARD_NODE_CHARS)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if (nodes.size > MAX_FORWARD_NODES) append(" ...[转发内容截断]")
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.normalize(): String = whitespace.replace(this, " ")
|
||||||
|
.trim()
|
||||||
|
.replaceUnpairedSurrogates()
|
||||||
|
|
||||||
|
private fun String.takeUtf16Safely(maxLength: Int): String {
|
||||||
|
if (length <= maxLength) return this
|
||||||
|
val endIndex = if (maxLength > 0 &&
|
||||||
|
Character.isHighSurrogate(this[maxLength - 1]) &&
|
||||||
|
Character.isLowSurrogate(this[maxLength])
|
||||||
|
) {
|
||||||
|
maxLength - 1
|
||||||
|
} else {
|
||||||
|
maxLength
|
||||||
|
}
|
||||||
|
return substring(0, endIndex)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.replaceUnpairedSurrogates(): String {
|
||||||
|
var output: StringBuilder? = null
|
||||||
|
var index = 0
|
||||||
|
while (index < length) {
|
||||||
|
val current = this[index]
|
||||||
|
when {
|
||||||
|
Character.isHighSurrogate(current) &&
|
||||||
|
index + 1 < length && Character.isLowSurrogate(this[index + 1]) -> {
|
||||||
|
output?.append(current)?.append(this[index + 1])
|
||||||
|
index += 2
|
||||||
|
}
|
||||||
|
Character.isSurrogate(current) -> {
|
||||||
|
if (output == null) output = StringBuilder(length).append(this, 0, index)
|
||||||
|
output.append('\uFFFD')
|
||||||
|
index++
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
output?.append(current)
|
||||||
|
index++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return output?.toString() ?: this
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun JsonObject.string(key: String): String? =
|
||||||
|
(get(key) as? JsonPrimitive)?.contentOrNull
|
||||||
|
|
||||||
|
private fun JsonObject.long(key: String): Long? =
|
||||||
|
(get(key) as? JsonPrimitive)?.longOrNull
|
||||||
|
|
||||||
|
private fun JsonObject.boolean(key: String): Boolean? =
|
||||||
|
(get(key) as? JsonPrimitive)?.booleanOrNull
|
||||||
|
}
|
||||||
@@ -24,14 +24,26 @@ import java.sql.ResultSet
|
|||||||
* 写连接由 [writeLock] 串行保护;SQLite 使用 WAL + NORMAL synchronous,允许并发读取。
|
* 写连接由 [writeLock] 串行保护;SQLite 使用 WAL + NORMAL synchronous,允许并发读取。
|
||||||
*/
|
*/
|
||||||
object ChatHistoryStore {
|
object ChatHistoryStore {
|
||||||
private const val SCHEMA_VERSION = 2
|
private const val SCHEMA_VERSION = 3
|
||||||
private const val BUSY_TIMEOUT_MS = 30_000
|
private const val BUSY_TIMEOUT_MS = 30_000
|
||||||
|
private const val SEARCH_FTS_TABLE = "message_record_fts"
|
||||||
|
private const val SEARCH_TEXT_TABLE = "message_record_search"
|
||||||
|
private const val SENDER_ALIAS_TABLE = "chat_history_sender_alias"
|
||||||
|
private const val SEARCH_BACKFILL_CURSOR_KEY = "search_backfill_cursor"
|
||||||
|
private const val SEARCH_INDEX_COMPLETE_KEY = "search_index_complete"
|
||||||
|
private const val DEFAULT_BACKFILL_BATCH_SIZE = 250
|
||||||
|
private const val MAX_PAGE_SIZE = 200
|
||||||
|
private const val MAX_CONTEXT_MESSAGES = 100
|
||||||
|
|
||||||
private val lifecycleLock = Any()
|
private val lifecycleLock = Any()
|
||||||
private val writeLock = Any()
|
private val writeLock = Any()
|
||||||
|
|
||||||
@Volatile
|
@Volatile
|
||||||
private var initialized = false
|
private var initialized = false
|
||||||
|
@Volatile
|
||||||
|
private var searchIndexAvailable = false
|
||||||
|
@Volatile
|
||||||
|
private var searchIndexReady = false
|
||||||
private lateinit var databaseFile: File
|
private lateinit var databaseFile: File
|
||||||
private var writeConnection: Connection? = null
|
private var writeConnection: Connection? = null
|
||||||
private var warningLogger: ((String, Throwable?) -> Unit)? = null
|
private var warningLogger: ((String, Throwable?) -> Unit)? = null
|
||||||
@@ -39,6 +51,12 @@ object ChatHistoryStore {
|
|||||||
val isAvailable: Boolean
|
val isAvailable: Boolean
|
||||||
get() = initialized
|
get() = initialized
|
||||||
|
|
||||||
|
val isSearchIndexAvailable: Boolean
|
||||||
|
get() = initialized && searchIndexAvailable
|
||||||
|
|
||||||
|
val isSearchIndexReady: Boolean
|
||||||
|
get() = isSearchIndexAvailable && searchIndexReady
|
||||||
|
|
||||||
val databaseFileOrNull: File?
|
val databaseFileOrNull: File?
|
||||||
get() = if (initialized) databaseFile else null
|
get() = if (initialized) databaseFile else null
|
||||||
|
|
||||||
@@ -60,6 +78,8 @@ object ChatHistoryStore {
|
|||||||
createSchema(connection)
|
createSchema(connection)
|
||||||
writeConnection = connection
|
writeConnection = connection
|
||||||
initialized = true
|
initialized = true
|
||||||
|
searchIndexAvailable = hasSearchIndex(connection)
|
||||||
|
searchIndexReady = searchIndexAvailable && readMeta(connection, SEARCH_INDEX_COMPLETE_KEY) == "1"
|
||||||
} catch (cause: Throwable) {
|
} catch (cause: Throwable) {
|
||||||
connection.close()
|
connection.close()
|
||||||
throw cause
|
throw cause
|
||||||
@@ -81,6 +101,8 @@ object ChatHistoryStore {
|
|||||||
}
|
}
|
||||||
writeConnection = null
|
writeConnection = null
|
||||||
initialized = false
|
initialized = false
|
||||||
|
searchIndexAvailable = false
|
||||||
|
searchIndexReady = false
|
||||||
warningLogger = null
|
warningLogger = null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -91,7 +113,7 @@ object ChatHistoryStore {
|
|||||||
val message = event.message.asSequence()
|
val message = event.message.asSequence()
|
||||||
.filterNot { it is MessageSource }
|
.filterNot { it is MessageSource }
|
||||||
.toMessageChain()
|
.toMessageChain()
|
||||||
insert(ChatMessageRecord.fromSuccess(event.message.source, message))
|
insertRecord(ChatMessageRecord.fromSuccess(event.message.source, message), event.senderName)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun record(event: MessagePostSendEvent<*>) {
|
fun record(event: MessagePostSendEvent<*>) {
|
||||||
@@ -100,7 +122,7 @@ object ChatHistoryStore {
|
|||||||
val message = event.message.asSequence()
|
val message = event.message.asSequence()
|
||||||
.filterNot { it is MessageSource }
|
.filterNot { it is MessageSource }
|
||||||
.toMessageChain()
|
.toMessageChain()
|
||||||
insert(ChatMessageRecord.fromSuccess(source, message))
|
insertRecord(ChatMessageRecord.fromSuccess(source, message), event.bot.nick)
|
||||||
}
|
}
|
||||||
|
|
||||||
fun markRecalled(event: MessageRecallEvent) {
|
fun markRecalled(event: MessageRecallEvent) {
|
||||||
@@ -254,31 +276,410 @@ object ChatHistoryStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun insert(record: ChatMessageRecord) {
|
fun search(request: ChatHistorySearchRequest): ChatHistorySearchPage {
|
||||||
withWriteConnection { connection ->
|
check(initialized) { "聊天记录数据库尚未初始化" }
|
||||||
|
require(request.limit > 0) { "limit must be positive" }
|
||||||
|
require(request.start == null || request.end == null || request.start <= request.end) {
|
||||||
|
"start must not be after end"
|
||||||
|
}
|
||||||
|
|
||||||
|
val parameters = mutableListOf<Any>()
|
||||||
|
val conditions = subjectConditions(request.subject, parameters).toMutableList()
|
||||||
|
request.start?.let {
|
||||||
|
conditions += "mr.time >= ?"
|
||||||
|
parameters += it
|
||||||
|
}
|
||||||
|
request.end?.let {
|
||||||
|
conditions += "mr.time <= ?"
|
||||||
|
parameters += it
|
||||||
|
}
|
||||||
|
request.fromId?.let {
|
||||||
|
conditions += "mr.from_id = ?"
|
||||||
|
parameters += it
|
||||||
|
}
|
||||||
|
|
||||||
|
val query = request.query?.trim().orEmpty()
|
||||||
|
val textPredicate = buildTextPredicate(query, request.matchMode, parameters)
|
||||||
|
if ((textPredicate.usesFts || textPredicate.usesSearchText) && !isSearchIndexAvailable) {
|
||||||
|
error("聊天记录全文索引尚未就绪")
|
||||||
|
}
|
||||||
|
val joins = buildString {
|
||||||
|
if (textPredicate.usesSearchText) {
|
||||||
|
append("JOIN $SEARCH_TEXT_TABLE search ON search.message_id = mr.id ")
|
||||||
|
}
|
||||||
|
if (textPredicate.usesFts) {
|
||||||
|
append("JOIN $SEARCH_FTS_TABLE ON $SEARCH_FTS_TABLE.rowid = mr.id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
textPredicate.sql?.let(conditions::add)
|
||||||
|
|
||||||
|
request.cursor?.let { cursor ->
|
||||||
|
val operator = if (request.sortOrder == ChatHistorySortOrder.NEWEST) "<" else ">"
|
||||||
|
conditions += "(mr.time $operator ? OR (mr.time = ? AND mr.id $operator ?))"
|
||||||
|
parameters += cursor.time
|
||||||
|
parameters += cursor.time
|
||||||
|
parameters += cursor.id
|
||||||
|
}
|
||||||
|
|
||||||
|
val where = conditions.joinToString(" AND ")
|
||||||
|
val order = if (request.sortOrder == ChatHistorySortOrder.NEWEST) {
|
||||||
|
"mr.time DESC, mr.id DESC"
|
||||||
|
} else {
|
||||||
|
"mr.time ASC, mr.id ASC"
|
||||||
|
}
|
||||||
|
val records = openReadConnection().use { connection ->
|
||||||
|
val sql = """
|
||||||
|
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
|
||||||
|
mr.time, mr.kind, mr.code, mr.recalled
|
||||||
|
FROM message_record mr
|
||||||
|
$joins
|
||||||
|
WHERE $where
|
||||||
|
ORDER BY $order
|
||||||
|
LIMIT ?
|
||||||
|
""".trimIndent()
|
||||||
|
val queryParameters = parameters + (request.limit.coerceAtMost(MAX_PAGE_SIZE) + 1)
|
||||||
|
loadRecords(connection, sql, queryParameters)
|
||||||
|
}
|
||||||
|
val hasMore = records.size > request.limit.coerceAtMost(MAX_PAGE_SIZE)
|
||||||
|
val pageRecords = records.take(request.limit.coerceAtMost(MAX_PAGE_SIZE))
|
||||||
|
val last = pageRecords.lastOrNull()
|
||||||
|
val nextCursor = if (hasMore && last != null) {
|
||||||
|
ChatHistoryCursor(last.time, last.id)
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
val total = if (request.cursor == null) {
|
||||||
|
openReadConnection().use { connection ->
|
||||||
|
val sql = "SELECT COUNT(*) FROM message_record mr $joins WHERE $where"
|
||||||
|
connection.prepareStatement(sql).use { statement ->
|
||||||
|
bindParameters(statement, parameters)
|
||||||
|
statement.executeQuery().use { results ->
|
||||||
|
check(results.next())
|
||||||
|
results.getLong(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
return ChatHistorySearchPage(pageRecords, total, nextCursor)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun findAround(
|
||||||
|
subject: ChatHistorySubject,
|
||||||
|
messageId: Long,
|
||||||
|
before: Int = 8,
|
||||||
|
after: Int = 8,
|
||||||
|
): ChatHistoryContext? {
|
||||||
|
check(initialized) { "聊天记录数据库尚未初始化" }
|
||||||
|
require(before >= 0 && after >= 0) { "before and after must not be negative" }
|
||||||
|
require(before <= MAX_CONTEXT_MESSAGES && after <= MAX_CONTEXT_MESSAGES) {
|
||||||
|
"context window is too large"
|
||||||
|
}
|
||||||
|
|
||||||
|
return openReadConnection().use { connection ->
|
||||||
|
val targetParameters = mutableListOf<Any>()
|
||||||
|
val targetConditions = subjectConditions(subject, targetParameters).toMutableList()
|
||||||
|
targetConditions += "mr.id = ?"
|
||||||
|
targetParameters += messageId
|
||||||
|
val target = loadRecords(
|
||||||
|
connection,
|
||||||
|
"""
|
||||||
|
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
|
||||||
|
mr.time, mr.kind, mr.code, mr.recalled
|
||||||
|
FROM message_record mr
|
||||||
|
WHERE ${targetConditions.joinToString(" AND ")}
|
||||||
|
""".trimIndent(),
|
||||||
|
targetParameters,
|
||||||
|
).firstOrNull() ?: return@use null
|
||||||
|
|
||||||
|
val beforeRecords = if (before == 0) {
|
||||||
|
emptyList()
|
||||||
|
} else {
|
||||||
|
val parameters = mutableListOf<Any>()
|
||||||
|
val conditions = subjectConditions(subject, parameters).toMutableList()
|
||||||
|
conditions += "(mr.time < ? OR (mr.time = ? AND mr.id < ?))"
|
||||||
|
parameters += target.time
|
||||||
|
parameters += target.time
|
||||||
|
parameters += target.id
|
||||||
|
conditions += "mr.id <> ?"
|
||||||
|
parameters += target.id
|
||||||
|
loadRecords(
|
||||||
|
connection,
|
||||||
|
"""
|
||||||
|
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
|
||||||
|
mr.time, mr.kind, mr.code, mr.recalled
|
||||||
|
FROM message_record mr
|
||||||
|
WHERE ${conditions.joinToString(" AND ")}
|
||||||
|
ORDER BY mr.time DESC, mr.id DESC
|
||||||
|
LIMIT ?
|
||||||
|
""".trimIndent(),
|
||||||
|
parameters + before,
|
||||||
|
).asReversed()
|
||||||
|
}
|
||||||
|
val afterRecords = if (after == 0) {
|
||||||
|
emptyList()
|
||||||
|
} else {
|
||||||
|
val parameters = mutableListOf<Any>()
|
||||||
|
val conditions = subjectConditions(subject, parameters).toMutableList()
|
||||||
|
conditions += "(mr.time > ? OR (mr.time = ? AND mr.id > ?))"
|
||||||
|
parameters += target.time
|
||||||
|
parameters += target.time
|
||||||
|
parameters += target.id
|
||||||
|
conditions += "mr.id <> ?"
|
||||||
|
parameters += target.id
|
||||||
|
loadRecords(
|
||||||
|
connection,
|
||||||
|
"""
|
||||||
|
SELECT mr.id, mr.bot_id, mr.from_id, mr.target_id, mr.ids, mr.internal_ids,
|
||||||
|
mr.time, mr.kind, mr.code, mr.recalled
|
||||||
|
FROM message_record mr
|
||||||
|
WHERE ${conditions.joinToString(" AND ")}
|
||||||
|
ORDER BY mr.time ASC, mr.id ASC
|
||||||
|
LIMIT ?
|
||||||
|
""".trimIndent(),
|
||||||
|
parameters + after,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
ChatHistoryContext(messageId, beforeRecords + target + afterRecords)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun findSenderAliases(
|
||||||
|
subject: ChatHistorySubject,
|
||||||
|
query: String,
|
||||||
|
limit: Int = 10,
|
||||||
|
): List<ChatHistorySenderAliasMatch> {
|
||||||
|
check(initialized) { "聊天记录数据库尚未初始化" }
|
||||||
|
if (query.isBlank() || limit <= 0) return emptyList()
|
||||||
|
val normalized = query.trim()
|
||||||
|
return openReadConnection().use { connection ->
|
||||||
connection.prepareStatement(
|
connection.prepareStatement(
|
||||||
"""
|
"""
|
||||||
INSERT INTO message_record(
|
SELECT user_id, name
|
||||||
bot_id, from_id, target_id, ids, internal_ids,
|
FROM $SENDER_ALIAS_TABLE
|
||||||
time, kind, code, recalled
|
WHERE bot_id = ? AND kind = ? AND subject_id = ?
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
AND lower(name) LIKE ? ESCAPE '\'
|
||||||
|
ORDER BY last_seen DESC
|
||||||
|
LIMIT ?
|
||||||
""".trimIndent()
|
""".trimIndent()
|
||||||
).use { statement ->
|
).use { statement ->
|
||||||
statement.setLong(1, record.botId)
|
statement.setLong(1, subject.botId)
|
||||||
statement.setLong(2, record.fromId)
|
statement.setInt(2, subject.kind.ordinal)
|
||||||
statement.setLong(3, record.targetId)
|
statement.setLong(3, subject.subjectId)
|
||||||
statement.setString(4, record.ids)
|
statement.setString(4, "%${normalized.lowercase().escapeLike()}%")
|
||||||
statement.setString(5, record.internalIds)
|
statement.setInt(5, limit.coerceAtMost(50))
|
||||||
statement.setInt(6, record.time)
|
statement.executeQuery().use { results ->
|
||||||
statement.setInt(7, record.kind.ordinal)
|
buildList {
|
||||||
statement.setString(8, record.code)
|
while (results.next()) {
|
||||||
statement.setInt(9, record.recalled)
|
val name = results.getString("name")
|
||||||
statement.executeUpdate()
|
add(
|
||||||
|
ChatHistorySenderAliasMatch(
|
||||||
|
userId = results.getLong("user_id"),
|
||||||
|
displayName = name,
|
||||||
|
matchRank = name.aliasMatchRank(normalized),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}.sortedWith(compareBy({ it.matchRank }, { it.userId }))
|
||||||
|
.distinctBy(ChatHistorySenderAliasMatch::userId)
|
||||||
|
.take(limit)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun backfillSearchIndex(
|
||||||
|
batchSize: Int = DEFAULT_BACKFILL_BATCH_SIZE,
|
||||||
|
shouldContinue: () -> Boolean = { true },
|
||||||
|
onProgress: (processed: Int) -> Unit = {},
|
||||||
|
): Int {
|
||||||
|
if (!isSearchIndexAvailable || batchSize <= 0) return 0
|
||||||
|
var processed = 0
|
||||||
|
var cursor = withWriteConnectionResult { connection ->
|
||||||
|
readMeta(connection, SEARCH_BACKFILL_CURSOR_KEY)?.toLongOrNull() ?: 0L
|
||||||
|
}
|
||||||
|
while (shouldContinue()) {
|
||||||
|
val batch = openReadConnection().use { connection ->
|
||||||
|
connection.prepareStatement(
|
||||||
|
"""
|
||||||
|
SELECT mr.id, mr.code
|
||||||
|
FROM message_record mr
|
||||||
|
WHERE mr.id > ?
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM $SEARCH_TEXT_TABLE search WHERE search.message_id = mr.id
|
||||||
|
)
|
||||||
|
ORDER BY mr.id
|
||||||
|
LIMIT ?
|
||||||
|
""".trimIndent()
|
||||||
|
).use { statement ->
|
||||||
|
statement.setLong(1, cursor)
|
||||||
|
statement.setInt(2, batchSize)
|
||||||
|
statement.executeQuery().use { results ->
|
||||||
|
buildList {
|
||||||
|
while (results.next()) {
|
||||||
|
add(results.getLong("id") to results.getString("code"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (batch.isEmpty()) {
|
||||||
|
withWriteConnection { connection ->
|
||||||
|
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "1")
|
||||||
|
}
|
||||||
|
searchIndexReady = true
|
||||||
|
return processed
|
||||||
|
}
|
||||||
|
withWriteConnection { connection ->
|
||||||
|
val oldAutoCommit = connection.autoCommit
|
||||||
|
connection.autoCommit = false
|
||||||
|
try {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO $SEARCH_TEXT_TABLE(message_id, search_text) VALUES (?, ?)"
|
||||||
|
).use { insertText ->
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)"
|
||||||
|
).use { insertFts ->
|
||||||
|
batch.forEach { (id, code) ->
|
||||||
|
val text = ChatHistorySearchText.extract(code)
|
||||||
|
insertText.setLong(1, id)
|
||||||
|
insertText.setString(2, text)
|
||||||
|
insertText.addBatch()
|
||||||
|
insertFts.setLong(1, id)
|
||||||
|
insertFts.setString(2, text)
|
||||||
|
insertFts.addBatch()
|
||||||
|
}
|
||||||
|
insertText.executeBatch()
|
||||||
|
insertFts.executeBatch()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cursor = batch.last().first
|
||||||
|
writeMeta(connection, SEARCH_BACKFILL_CURSOR_KEY, cursor.toString())
|
||||||
|
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0")
|
||||||
|
connection.commit()
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
connection.rollback()
|
||||||
|
throw cause
|
||||||
|
} finally {
|
||||||
|
connection.autoCommit = oldAutoCommit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
processed += batch.size
|
||||||
|
onProgress(processed)
|
||||||
|
}
|
||||||
|
return processed
|
||||||
|
}
|
||||||
|
|
||||||
|
internal fun insertRecord(record: ChatMessageRecord, senderName: String = "") {
|
||||||
|
var recordId = 0L
|
||||||
|
var indexFailure: Throwable? = null
|
||||||
|
withWriteConnection { connection ->
|
||||||
|
recordId = insertPrimaryRecord(connection, record)
|
||||||
|
if (senderName.isNotBlank()) {
|
||||||
|
runCatching { upsertSenderAlias(connection, record, senderName) }
|
||||||
|
.onFailure { warn("保存聊天记录发送者别名失败: user=${record.fromId}", it) }
|
||||||
|
}
|
||||||
|
if (isSearchIndexAvailable) {
|
||||||
|
runCatching { indexRecordOnConnection(connection, recordId, record.code) }
|
||||||
|
.onFailure { indexFailure = it }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
indexFailure?.let { cause ->
|
||||||
|
searchIndexReady = false
|
||||||
|
markSearchIndexIncomplete()
|
||||||
|
warn("消息已保存,但写入聊天记录全文索引失败: id=$recordId", cause)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun insertPrimaryRecord(connection: Connection, record: ChatMessageRecord): Long {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"""
|
||||||
|
INSERT INTO message_record(
|
||||||
|
bot_id, from_id, target_id, ids, internal_ids,
|
||||||
|
time, kind, code, recalled
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""".trimIndent()
|
||||||
|
).use { statement ->
|
||||||
|
statement.setLong(1, record.botId)
|
||||||
|
statement.setLong(2, record.fromId)
|
||||||
|
statement.setLong(3, record.targetId)
|
||||||
|
statement.setString(4, record.ids)
|
||||||
|
statement.setString(5, record.internalIds)
|
||||||
|
statement.setInt(6, record.time)
|
||||||
|
statement.setInt(7, record.kind.ordinal)
|
||||||
|
statement.setString(8, record.code)
|
||||||
|
statement.setInt(9, record.recalled)
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeQuery("SELECT last_insert_rowid()").use { results ->
|
||||||
|
check(results.next())
|
||||||
|
return results.getLong(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun indexRecordOnConnection(connection: Connection, recordId: Long, code: String) {
|
||||||
|
val searchText = ChatHistorySearchText.extract(code)
|
||||||
|
val oldAutoCommit = connection.autoCommit
|
||||||
|
connection.autoCommit = false
|
||||||
|
try {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO $SEARCH_TEXT_TABLE(message_id, search_text) VALUES (?, ?)"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setLong(1, recordId)
|
||||||
|
statement.setString(2, searchText)
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO $SEARCH_FTS_TABLE(rowid, search_text) VALUES (?, ?)"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setLong(1, recordId)
|
||||||
|
statement.setString(2, searchText)
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
connection.commit()
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
connection.rollback()
|
||||||
|
throw cause
|
||||||
|
} finally {
|
||||||
|
connection.autoCommit = oldAutoCommit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun upsertSenderAlias(connection: Connection, record: ChatMessageRecord, senderName: String) {
|
||||||
|
val subjectId = if (record.kind == MessageSourceKind.GROUP) {
|
||||||
|
record.targetId
|
||||||
|
} else if (record.fromId == record.botId) {
|
||||||
|
record.targetId
|
||||||
|
} else {
|
||||||
|
record.fromId
|
||||||
|
}
|
||||||
|
connection.prepareStatement(
|
||||||
|
"""
|
||||||
|
INSERT INTO $SENDER_ALIAS_TABLE(
|
||||||
|
bot_id, kind, subject_id, user_id, name, last_seen
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(bot_id, kind, subject_id, user_id, name) DO UPDATE SET
|
||||||
|
last_seen = MAX(last_seen, excluded.last_seen)
|
||||||
|
""".trimIndent()
|
||||||
|
).use { statement ->
|
||||||
|
statement.setLong(1, record.botId)
|
||||||
|
statement.setInt(2, record.kind.ordinal)
|
||||||
|
statement.setLong(3, subjectId)
|
||||||
|
statement.setLong(4, record.fromId)
|
||||||
|
statement.setString(5, senderName.trim())
|
||||||
|
statement.setInt(6, record.time)
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun withWriteConnection(block: (Connection) -> Unit) {
|
private fun withWriteConnection(block: (Connection) -> Unit) {
|
||||||
|
withWriteConnectionResult { connection -> block(connection) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun <T> withWriteConnectionResult(block: (Connection) -> T): T {
|
||||||
synchronized(writeLock) {
|
synchronized(writeLock) {
|
||||||
check(initialized) { "聊天记录数据库尚未初始化" }
|
check(initialized) { "聊天记录数据库尚未初始化" }
|
||||||
val connection = writeConnection?.takeUnless(Connection::isClosed)
|
val connection = writeConnection?.takeUnless(Connection::isClosed)
|
||||||
@@ -286,7 +687,7 @@ object ChatHistoryStore {
|
|||||||
configureWriteConnection(it)
|
configureWriteConnection(it)
|
||||||
writeConnection = it
|
writeConnection = it
|
||||||
}
|
}
|
||||||
block(connection)
|
return block(connection)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -313,53 +714,282 @@ object ChatHistoryStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun createSchema(connection: Connection) {
|
private fun createSchema(connection: Connection) {
|
||||||
connection.createStatement().use { statement ->
|
val oldAutoCommit = connection.autoCommit
|
||||||
statement.executeUpdate(
|
connection.autoCommit = false
|
||||||
"""
|
try {
|
||||||
CREATE TABLE IF NOT EXISTS message_record(
|
connection.createStatement().use { statement ->
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
statement.executeUpdate(
|
||||||
bot_id INTEGER NOT NULL,
|
"""
|
||||||
from_id INTEGER NOT NULL,
|
CREATE TABLE IF NOT EXISTS message_record(
|
||||||
target_id INTEGER NOT NULL,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
ids TEXT,
|
bot_id INTEGER NOT NULL,
|
||||||
internal_ids TEXT,
|
from_id INTEGER NOT NULL,
|
||||||
time INTEGER NOT NULL,
|
target_id INTEGER NOT NULL,
|
||||||
kind INTEGER NOT NULL,
|
ids TEXT,
|
||||||
code TEXT NOT NULL,
|
internal_ids TEXT,
|
||||||
recalled INTEGER NOT NULL DEFAULT 0
|
time INTEGER NOT NULL,
|
||||||
|
kind INTEGER NOT NULL,
|
||||||
|
code TEXT NOT NULL,
|
||||||
|
recalled INTEGER NOT NULL DEFAULT 0
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
)
|
)
|
||||||
""".trimIndent()
|
statement.executeUpdate(
|
||||||
)
|
"CREATE INDEX IF NOT EXISTS idx_message_subject_time " +
|
||||||
statement.executeUpdate(
|
"ON message_record(bot_id, kind, target_id, time DESC, id DESC)"
|
||||||
"CREATE INDEX IF NOT EXISTS idx_message_subject_time " +
|
|
||||||
"ON message_record(bot_id, kind, target_id, time DESC, id DESC)"
|
|
||||||
)
|
|
||||||
statement.executeUpdate(
|
|
||||||
"CREATE INDEX IF NOT EXISTS idx_message_sender_subject_time " +
|
|
||||||
"ON message_record(bot_id, kind, target_id, from_id, time DESC, id DESC)"
|
|
||||||
)
|
|
||||||
statement.executeUpdate(
|
|
||||||
"CREATE INDEX IF NOT EXISTS idx_message_recall_identity " +
|
|
||||||
"ON message_record(bot_id, kind, target_id, from_id, time, ids)"
|
|
||||||
)
|
|
||||||
statement.executeUpdate(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS chat_history_meta(
|
|
||||||
key TEXT PRIMARY KEY,
|
|
||||||
value TEXT NOT NULL
|
|
||||||
)
|
)
|
||||||
""".trimIndent()
|
statement.executeUpdate(
|
||||||
)
|
"CREATE INDEX IF NOT EXISTS idx_message_sender_subject_time " +
|
||||||
|
"ON message_record(bot_id, kind, target_id, from_id, time DESC, id DESC)"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_message_recall_identity " +
|
||||||
|
"ON message_record(bot_id, kind, target_id, from_id, time, ids)"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS chat_history_meta(
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS $SEARCH_TEXT_TABLE(
|
||||||
|
message_id INTEGER PRIMARY KEY,
|
||||||
|
search_text TEXT NOT NULL
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS $SENDER_ALIAS_TABLE(
|
||||||
|
bot_id INTEGER NOT NULL,
|
||||||
|
kind INTEGER NOT NULL,
|
||||||
|
subject_id INTEGER NOT NULL,
|
||||||
|
user_id INTEGER NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
last_seen INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY(bot_id, kind, subject_id, user_id, name)
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_chat_history_sender_alias_scope " +
|
||||||
|
"ON $SENDER_ALIAS_TABLE(bot_id, kind, subject_id, last_seen DESC)"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO chat_history_meta(key, value) VALUES ('schema_version', ?) " +
|
||||||
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setString(1, SCHEMA_VERSION.toString())
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
connection.commit()
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
connection.rollback()
|
||||||
|
throw cause
|
||||||
|
} finally {
|
||||||
|
connection.autoCommit = oldAutoCommit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
runCatching {
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS $SEARCH_FTS_TABLE USING fts5(
|
||||||
|
search_text,
|
||||||
|
content='$SEARCH_TEXT_TABLE',
|
||||||
|
content_rowid='message_id',
|
||||||
|
tokenize='trigram'
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}.onFailure { warn("初始化 SQLite FTS5 聊天记录索引失败,消息记录仍会继续保存", it) }
|
||||||
|
|
||||||
|
if (hasSearchIndex(connection)) {
|
||||||
|
val hasMessages = connection.createStatement().use { statement ->
|
||||||
|
statement.executeQuery("SELECT 1 FROM message_record LIMIT 1").use(ResultSet::next)
|
||||||
|
}
|
||||||
|
connection.prepareStatement(
|
||||||
|
"INSERT INTO chat_history_meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setString(1, SEARCH_BACKFILL_CURSOR_KEY)
|
||||||
|
statement.setString(2, "0")
|
||||||
|
statement.executeUpdate()
|
||||||
|
statement.setString(1, SEARCH_INDEX_COMPLETE_KEY)
|
||||||
|
statement.setString(2, if (hasMessages) "0" else "1")
|
||||||
|
statement.executeUpdate()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun subjectConditions(
|
||||||
|
subject: ChatHistorySubject,
|
||||||
|
parameters: MutableList<Any>,
|
||||||
|
): List<String> {
|
||||||
|
val conditions = mutableListOf(
|
||||||
|
"mr.bot_id = ?",
|
||||||
|
"mr.kind = ?",
|
||||||
|
)
|
||||||
|
parameters += subject.botId
|
||||||
|
parameters += subject.kind.ordinal
|
||||||
|
if (subject.kind == MessageSourceKind.GROUP) {
|
||||||
|
conditions += "mr.target_id = ?"
|
||||||
|
parameters += subject.subjectId
|
||||||
|
} else {
|
||||||
|
conditions += "(mr.from_id = ? OR mr.target_id = ?)"
|
||||||
|
parameters += subject.subjectId
|
||||||
|
parameters += subject.subjectId
|
||||||
|
}
|
||||||
|
return conditions
|
||||||
|
}
|
||||||
|
|
||||||
|
private data class TextPredicate(
|
||||||
|
val sql: String?,
|
||||||
|
val usesFts: Boolean,
|
||||||
|
val usesSearchText: Boolean,
|
||||||
|
)
|
||||||
|
|
||||||
|
private fun buildTextPredicate(
|
||||||
|
query: String,
|
||||||
|
mode: ChatHistoryMatchMode,
|
||||||
|
parameters: MutableList<Any>,
|
||||||
|
): TextPredicate {
|
||||||
|
if (query.isBlank()) return TextPredicate(null, false, false)
|
||||||
|
val normalized = query.lowercase()
|
||||||
|
val terms = normalized.split(Regex("\\s+")).filter(String::isNotBlank)
|
||||||
|
val hasShortTerm = terms.any { it.codePointCount(0, it.length) < 3 }
|
||||||
|
if (mode == ChatHistoryMatchMode.ANY && hasShortTerm) {
|
||||||
|
val predicates = terms.map {
|
||||||
|
parameters += it
|
||||||
|
"instr(lower(search.search_text), ?) > 0"
|
||||||
|
}
|
||||||
|
return TextPredicate(predicates.joinToString(" OR ", prefix = "(", postfix = ")"), false, true)
|
||||||
|
}
|
||||||
|
val ftsTerms = if (mode == ChatHistoryMatchMode.PHRASE) {
|
||||||
|
listOf(normalized)
|
||||||
|
} else {
|
||||||
|
terms
|
||||||
|
}
|
||||||
|
val ftsOnly = ftsTerms.filter { it.codePointCount(0, it.length) >= 3 }
|
||||||
|
if (ftsOnly.isEmpty()) {
|
||||||
|
val predicates = terms.map {
|
||||||
|
parameters += it
|
||||||
|
"instr(lower(search.search_text), ?) > 0"
|
||||||
|
}
|
||||||
|
val joiner = if (mode == ChatHistoryMatchMode.ANY) " OR " else " AND "
|
||||||
|
return TextPredicate(predicates.joinToString(joiner, prefix = "(", postfix = ")"), false, true)
|
||||||
|
}
|
||||||
|
val ftsQuery = when (mode) {
|
||||||
|
ChatHistoryMatchMode.PHRASE -> quoteFtsTerm(normalized)
|
||||||
|
ChatHistoryMatchMode.ALL -> ftsOnly.joinToString(" AND ", transform = ::quoteFtsTerm)
|
||||||
|
ChatHistoryMatchMode.ANY -> ftsOnly.joinToString(" OR ", transform = ::quoteFtsTerm)
|
||||||
|
}
|
||||||
|
parameters += ftsQuery
|
||||||
|
val ftsPredicate = "$SEARCH_FTS_TABLE MATCH ?"
|
||||||
|
if (mode != ChatHistoryMatchMode.ALL || !hasShortTerm) {
|
||||||
|
return TextPredicate(ftsPredicate, true, false)
|
||||||
|
}
|
||||||
|
val shortPredicates = terms.filter { it.codePointCount(0, it.length) < 3 }.map {
|
||||||
|
parameters += it
|
||||||
|
"instr(lower(search.search_text), ?) > 0"
|
||||||
|
}
|
||||||
|
return TextPredicate(
|
||||||
|
(listOf(ftsPredicate) + shortPredicates).joinToString(" AND ", prefix = "(", postfix = ")"),
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun quoteFtsTerm(term: String): String =
|
||||||
|
"\"${term.replace("\"", "\"\"")}\""
|
||||||
|
|
||||||
|
private fun loadRecords(
|
||||||
|
connection: Connection,
|
||||||
|
sql: String,
|
||||||
|
parameters: List<Any>,
|
||||||
|
): List<ChatMessageRecord> {
|
||||||
|
return connection.prepareStatement(sql).use { statement ->
|
||||||
|
bindParameters(statement, parameters)
|
||||||
|
statement.executeQuery().use { results ->
|
||||||
|
buildList {
|
||||||
|
while (results.next()) add(results.toRecord())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun bindParameters(
|
||||||
|
statement: java.sql.PreparedStatement,
|
||||||
|
parameters: List<Any>,
|
||||||
|
) {
|
||||||
|
parameters.forEachIndexed { index, value ->
|
||||||
|
when (value) {
|
||||||
|
is Int -> statement.setInt(index + 1, value)
|
||||||
|
is Long -> statement.setLong(index + 1, value)
|
||||||
|
is String -> statement.setString(index + 1, value)
|
||||||
|
else -> error("不支持的查询参数类型 ${value::class}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun hasSearchIndex(connection: Connection): Boolean {
|
||||||
connection.prepareStatement(
|
connection.prepareStatement(
|
||||||
"INSERT INTO chat_history_meta(key, value) VALUES ('schema_version', ?) " +
|
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1"
|
||||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
|
||||||
).use { statement ->
|
).use { statement ->
|
||||||
statement.setString(1, SCHEMA_VERSION.toString())
|
statement.setString(1, SEARCH_FTS_TABLE)
|
||||||
|
statement.executeQuery().use { results ->
|
||||||
|
return results.next()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun readMeta(connection: Connection, key: String): String? {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"SELECT value FROM chat_history_meta WHERE key = ?"
|
||||||
|
).use { statement ->
|
||||||
|
statement.setString(1, key)
|
||||||
|
statement.executeQuery().use { results ->
|
||||||
|
return if (results.next()) results.getString(1) else null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun writeMeta(connection: Connection, key: String, value: String) {
|
||||||
|
connection.prepareStatement(
|
||||||
|
"""
|
||||||
|
INSERT INTO chat_history_meta(key, value) VALUES (?, ?)
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||||
|
""".trimIndent()
|
||||||
|
).use { statement ->
|
||||||
|
statement.setString(1, key)
|
||||||
|
statement.setString(2, value)
|
||||||
statement.executeUpdate()
|
statement.executeUpdate()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun markSearchIndexIncomplete() {
|
||||||
|
runCatching {
|
||||||
|
withWriteConnection { connection ->
|
||||||
|
writeMeta(connection, SEARCH_INDEX_COMPLETE_KEY, "0")
|
||||||
|
}
|
||||||
|
}.onFailure { warn("标记聊天记录全文索引未完成失败", it) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.escapeLike(): String =
|
||||||
|
replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||||
|
|
||||||
|
private fun String.aliasMatchRank(query: String): Int = when {
|
||||||
|
equals(query, ignoreCase = true) -> 0
|
||||||
|
startsWith(query, ignoreCase = true) -> 1
|
||||||
|
contains(query, ignoreCase = true) -> 2
|
||||||
|
else -> Int.MAX_VALUE
|
||||||
|
}
|
||||||
|
|
||||||
private fun ResultSet.toRecord(): ChatMessageRecord {
|
private fun ResultSet.toRecord(): ChatMessageRecord {
|
||||||
val kindOrdinal = getInt("kind")
|
val kindOrdinal = getInt("kind")
|
||||||
val kind = MessageSourceKind.values().getOrNull(kindOrdinal)
|
val kind = MessageSourceKind.values().getOrNull(kindOrdinal)
|
||||||
|
|||||||
@@ -37,15 +37,7 @@ data class ChatMessageRecord(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fun toMessageChain(): MessageChain {
|
fun toMessageChain(): MessageChain {
|
||||||
return try {
|
return decodeMessageCode(code)
|
||||||
MessageChain.deserializeFromJsonString(code)
|
|
||||||
} catch (cause: SerializationException) {
|
|
||||||
try {
|
|
||||||
MiraiCode.deserializeMiraiCode(code)
|
|
||||||
} catch (_: Throwable) {
|
|
||||||
throw cause
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
@@ -69,3 +61,15 @@ data class ChatMessageRecord(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal fun decodeMessageCode(code: String): MessageChain {
|
||||||
|
return try {
|
||||||
|
MessageChain.deserializeFromJsonString(code)
|
||||||
|
} catch (cause: SerializationException) {
|
||||||
|
try {
|
||||||
|
MiraiCode.deserializeMiraiCode(code)
|
||||||
|
} catch (_: Throwable) {
|
||||||
|
throw cause
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package top.jie65535.mirai.tools
|
||||||
|
|
||||||
|
import net.mamoe.mirai.contact.nameCardOrNick
|
||||||
|
import net.mamoe.mirai.event.events.GroupMessageEvent
|
||||||
|
import net.mamoe.mirai.event.events.MessageEvent
|
||||||
|
import top.jie65535.mirai.data.ChatHistorySearchText
|
||||||
|
import top.jie65535.mirai.data.ChatMessageRecord
|
||||||
|
import top.jie65535.mirai.data.ContactSnapshotStore
|
||||||
|
import java.time.Instant
|
||||||
|
import java.time.ZoneId
|
||||||
|
import java.time.format.DateTimeFormatter
|
||||||
|
|
||||||
|
internal object ChatHistoryToolFormatter {
|
||||||
|
private const val SNIPPET_LENGTH = 360
|
||||||
|
private val timeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
||||||
|
|
||||||
|
fun appendRecords(
|
||||||
|
output: StringBuilder,
|
||||||
|
records: List<ChatMessageRecord>,
|
||||||
|
event: MessageEvent,
|
||||||
|
query: String? = null,
|
||||||
|
targetId: Long? = null,
|
||||||
|
) {
|
||||||
|
val group = (event as? GroupMessageEvent)?.group
|
||||||
|
val userIds = records.map(ChatMessageRecord::fromId).distinct()
|
||||||
|
val snapshotNames = runCatching {
|
||||||
|
ContactSnapshotStore.loadDisplayNames(event.bot.id, group?.id, userIds)
|
||||||
|
}.getOrDefault(emptyMap())
|
||||||
|
|
||||||
|
records.forEach { record ->
|
||||||
|
val marker = if (record.id == targetId) ">>> " else ""
|
||||||
|
val sender = when {
|
||||||
|
record.fromId == event.bot.id -> "你"
|
||||||
|
group != null -> group[record.fromId]?.nameCardOrNick
|
||||||
|
?: snapshotNames[record.fromId]
|
||||||
|
?: "未知群员"
|
||||||
|
record.fromId == event.sender.id -> event.senderName
|
||||||
|
else -> snapshotNames[record.fromId] ?: "用户"
|
||||||
|
}
|
||||||
|
val time = timeFormatter.format(
|
||||||
|
Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault())
|
||||||
|
)
|
||||||
|
val content = buildSnippet(ChatHistorySearchText.extract(record.code), query)
|
||||||
|
output.append(marker)
|
||||||
|
.append("[messageId=").append(record.id).append("] ")
|
||||||
|
.append(time).append(' ')
|
||||||
|
.append(sender).append('(').append(record.fromId).append("): ")
|
||||||
|
.appendLine(content.ifEmpty { "[无文本消息]" })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun buildSnippet(content: String, query: String?): String {
|
||||||
|
if (content.length <= SNIPPET_LENGTH) return content
|
||||||
|
val terms = query.orEmpty().split(Regex("\\s+")).filter(String::isNotBlank)
|
||||||
|
val matchIndex = terms.mapNotNull { term ->
|
||||||
|
content.indexOf(term, ignoreCase = true).takeIf { it >= 0 }
|
||||||
|
}.minOrNull() ?: 0
|
||||||
|
val start = (matchIndex - SNIPPET_LENGTH / 3).coerceAtLeast(0)
|
||||||
|
val end = (start + SNIPPET_LENGTH).coerceAtMost(content.length)
|
||||||
|
return buildString {
|
||||||
|
if (start > 0) append("...")
|
||||||
|
append(content, start, end)
|
||||||
|
if (end < content.length) append("...")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package top.jie65535.mirai.tools
|
||||||
|
|
||||||
|
import com.aallam.openai.api.chat.Tool
|
||||||
|
import com.aallam.openai.api.core.Parameters
|
||||||
|
import kotlinx.serialization.json.JsonObject
|
||||||
|
import kotlinx.serialization.json.JsonPrimitive
|
||||||
|
import kotlinx.serialization.json.intOrNull
|
||||||
|
import kotlinx.serialization.json.jsonPrimitive
|
||||||
|
import kotlinx.serialization.json.longOrNull
|
||||||
|
import kotlinx.serialization.json.put
|
||||||
|
import kotlinx.serialization.json.putJsonArray
|
||||||
|
import kotlinx.serialization.json.putJsonObject
|
||||||
|
import net.mamoe.mirai.event.events.MessageEvent
|
||||||
|
import top.jie65535.mirai.JChatGPT
|
||||||
|
import top.jie65535.mirai.data.ChatHistoryStore
|
||||||
|
import top.jie65535.mirai.data.ChatHistorySubject
|
||||||
|
|
||||||
|
class GetChatHistoryContext : BaseAgent(
|
||||||
|
tool = Tool.function(
|
||||||
|
name = "getChatHistoryContext",
|
||||||
|
description = "根据 searchChatHistory 返回的 messageId,读取该消息在当前群聊或私聊中的前后文。" +
|
||||||
|
"结果按时间顺序排列,并用 >>> 标记目标消息。",
|
||||||
|
parameters = Parameters.buildJsonObject {
|
||||||
|
put("type", "object")
|
||||||
|
putJsonObject("properties") {
|
||||||
|
putJsonObject("messageId") {
|
||||||
|
put("type", "integer")
|
||||||
|
put("description", "searchChatHistory 返回的稳定消息 ID")
|
||||||
|
}
|
||||||
|
putJsonObject("before") {
|
||||||
|
put("type", "integer")
|
||||||
|
put("description", "目标消息之前的消息数,默认8,最大15")
|
||||||
|
}
|
||||||
|
putJsonObject("after") {
|
||||||
|
put("type", "integer")
|
||||||
|
put("description", "目标消息之后的消息数,默认8,最大15")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
putJsonArray("required") { add(JsonPrimitive("messageId")) }
|
||||||
|
},
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
override val isEnabled: Boolean
|
||||||
|
get() = JChatGPT.includeHistory
|
||||||
|
|
||||||
|
override val loadingMessage: String
|
||||||
|
get() = "读取聊天记录上下文中..."
|
||||||
|
|
||||||
|
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
|
||||||
|
val parameters = requireNotNull(args)
|
||||||
|
val messageId = parameters["messageId"]?.jsonPrimitive?.longOrNull
|
||||||
|
?: return "缺少有效的 messageId"
|
||||||
|
val before = parameters["before"]?.jsonPrimitive?.intOrNull?.coerceIn(0, MAX_CONTEXT) ?: DEFAULT_CONTEXT
|
||||||
|
val after = parameters["after"]?.jsonPrimitive?.intOrNull?.coerceIn(0, MAX_CONTEXT) ?: DEFAULT_CONTEXT
|
||||||
|
val context = try {
|
||||||
|
ChatHistoryStore.findAround(
|
||||||
|
subject = ChatHistorySubject.from(event.subject),
|
||||||
|
messageId = messageId,
|
||||||
|
before = before,
|
||||||
|
after = after,
|
||||||
|
)
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
JChatGPT.logger.warning("读取聊天记录上下文失败: messageId=$messageId", cause)
|
||||||
|
return "读取聊天记录上下文失败: ${cause.message}"
|
||||||
|
} ?: return "当前会话中不存在 messageId=$messageId 的消息"
|
||||||
|
|
||||||
|
return buildString {
|
||||||
|
appendLine("目标消息及上下文(共 ${context.records.size} 条):")
|
||||||
|
appendLine()
|
||||||
|
ChatHistoryToolFormatter.appendRecords(
|
||||||
|
output = this,
|
||||||
|
records = context.records,
|
||||||
|
event = event,
|
||||||
|
targetId = context.targetId,
|
||||||
|
)
|
||||||
|
}.trimEnd()
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val DEFAULT_CONTEXT = 8
|
||||||
|
private const val MAX_CONTEXT = 15
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,57 +2,69 @@ package top.jie65535.mirai.tools
|
|||||||
|
|
||||||
import com.aallam.openai.api.chat.Tool
|
import com.aallam.openai.api.chat.Tool
|
||||||
import com.aallam.openai.api.core.Parameters
|
import com.aallam.openai.api.core.Parameters
|
||||||
import kotlinx.serialization.json.*
|
import kotlinx.serialization.json.JsonObject
|
||||||
import net.mamoe.mirai.contact.Group
|
import kotlinx.serialization.json.contentOrNull
|
||||||
import net.mamoe.mirai.contact.nameCardOrNick
|
import kotlinx.serialization.json.jsonPrimitive
|
||||||
|
import kotlinx.serialization.json.longOrNull
|
||||||
|
import kotlinx.serialization.json.put
|
||||||
|
import kotlinx.serialization.json.putJsonObject
|
||||||
import net.mamoe.mirai.event.events.GroupMessageEvent
|
import net.mamoe.mirai.event.events.GroupMessageEvent
|
||||||
import net.mamoe.mirai.event.events.MessageEvent
|
import net.mamoe.mirai.event.events.MessageEvent
|
||||||
import net.mamoe.mirai.message.data.Image
|
import net.mamoe.mirai.contact.nameCardOrNick
|
||||||
import net.mamoe.mirai.message.data.Image.Key.queryUrl
|
|
||||||
import net.mamoe.mirai.message.data.SingleMessage
|
|
||||||
import net.mamoe.mirai.message.data.content
|
|
||||||
import top.jie65535.mirai.JChatGPT
|
import top.jie65535.mirai.JChatGPT
|
||||||
import top.jie65535.mirai.config.PluginConfig
|
import top.jie65535.mirai.config.PluginConfig
|
||||||
|
import top.jie65535.mirai.data.ChatHistoryCursor
|
||||||
|
import top.jie65535.mirai.data.ChatHistoryMatchMode
|
||||||
|
import top.jie65535.mirai.data.ChatHistorySearchRequest
|
||||||
|
import top.jie65535.mirai.data.ChatHistorySortOrder
|
||||||
import top.jie65535.mirai.data.ChatHistoryStore
|
import top.jie65535.mirai.data.ChatHistoryStore
|
||||||
import top.jie65535.mirai.data.ChatMessageRecord
|
import top.jie65535.mirai.data.ChatHistorySubject
|
||||||
import java.time.Instant
|
import top.jie65535.mirai.data.ContactNameMatch
|
||||||
|
import top.jie65535.mirai.data.ContactSnapshotStore
|
||||||
|
import java.nio.charset.StandardCharsets
|
||||||
|
import java.time.LocalDate
|
||||||
import java.time.LocalDateTime
|
import java.time.LocalDateTime
|
||||||
import java.time.OffsetDateTime
|
import java.time.OffsetDateTime
|
||||||
import java.time.ZoneId
|
import java.time.ZoneId
|
||||||
import java.time.format.DateTimeFormatter
|
import java.time.format.DateTimeFormatter
|
||||||
import java.time.format.DateTimeParseException
|
import java.time.format.DateTimeParseException
|
||||||
|
import java.util.Base64
|
||||||
|
|
||||||
class SearchChatHistory : BaseAgent(
|
class SearchChatHistory : BaseAgent(
|
||||||
tool = Tool.function(
|
tool = Tool.function(
|
||||||
name = "searchChatHistory",
|
name = "searchChatHistory",
|
||||||
description = "搜索群聊消息历史,可按关键词、发送者、时间范围筛选。用于回溯之前的讨论、查找某人说过的话、统计话题等。" +
|
description = "在当前群聊或私聊的 SQLite 历史中搜索消息,可按内容、发送者和时间筛选。" +
|
||||||
"不指定时间范围时默认搜索最近30天。指定时间时范围不能超过30天,如需更长跨度可分多次查询。" +
|
"默认返回最近匹配并在结果过多时提供 cursor;需要查看命中消息前后讨论时调用 getChatHistoryContext。" +
|
||||||
"可以通过多轮搜索来实现找到某条消息的上下文。",
|
"未指定 from 时默认搜索配置的近期天数,明确指定时间时可以搜索任意历史跨度。",
|
||||||
parameters = Parameters.buildJsonObject {
|
parameters = Parameters.buildJsonObject {
|
||||||
put("type", "object")
|
put("type", "object")
|
||||||
putJsonObject("properties") {
|
putJsonObject("properties") {
|
||||||
putJsonObject("keyword") {
|
putJsonObject("query") {
|
||||||
put("type", "string")
|
put("type", "string")
|
||||||
put("description", "消息内容关键词,人名请用sender")
|
put("description", "要搜索的消息文本;省略时按其他条件浏览时间线")
|
||||||
}
|
}
|
||||||
putJsonObject("sender") {
|
putJsonObject("senderId") {
|
||||||
put("type", "string")
|
|
||||||
put("description", "发送者名称或QQ号,查找某人的发言")
|
|
||||||
}
|
|
||||||
putJsonObject("startTime") {
|
|
||||||
put("type", "string")
|
|
||||||
put("description", "起始时间,格式:yyyy-MM-dd HH:mm,不填则默认为7天前")
|
|
||||||
}
|
|
||||||
putJsonObject("endTime") {
|
|
||||||
put("type", "string")
|
|
||||||
put("description", "结束时间,格式同上,不填则默认到当前时间")
|
|
||||||
}
|
|
||||||
putJsonObject("limit") {
|
|
||||||
put("type", "integer")
|
put("type", "integer")
|
||||||
put("description", "返回消息数量上限,默认20,最大200")
|
put("description", "发送者 QQ 号;仅在用户明确给出或上下文可靠提供 QQ 号时使用,不要根据昵称猜测")
|
||||||
|
}
|
||||||
|
putJsonObject("senderName") {
|
||||||
|
put("type", "string")
|
||||||
|
put("description", "用户用名称指代发送者时优先原样传入群名片、昵称或好友备注;可与 senderId 同时提供,没有 senderId 时解析发送者,有重名时返回候选 QQ 号")
|
||||||
|
}
|
||||||
|
putJsonObject("from") {
|
||||||
|
put("type", "string")
|
||||||
|
put("description", "起始时间,格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd")
|
||||||
|
}
|
||||||
|
putJsonObject("to") {
|
||||||
|
put("type", "string")
|
||||||
|
put("description", "结束时间,格式 yyyy-MM-dd HH:mm 或 yyyy-MM-dd;仅日期表示当天结束")
|
||||||
|
}
|
||||||
|
putJsonObject("cursor") {
|
||||||
|
put("type", "string")
|
||||||
|
put("description", "上一页返回的 nextCursor;继续同一搜索时原样传回")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
override val isEnabled: Boolean
|
override val isEnabled: Boolean
|
||||||
@@ -62,151 +74,190 @@ class SearchChatHistory : BaseAgent(
|
|||||||
get() = "搜索聊天记录中..."
|
get() = "搜索聊天记录中..."
|
||||||
|
|
||||||
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
|
override suspend fun execute(args: JsonObject?, event: MessageEvent): String {
|
||||||
requireNotNull(args)
|
val parameters = args ?: JsonObject(emptyMap())
|
||||||
|
val query = parameters.string("query")?.trim()?.takeIf(String::isNotEmpty)
|
||||||
val keyword = args["keyword"]?.jsonPrimitive?.contentOrNull
|
val matchMode = ChatHistoryMatchMode.ALL
|
||||||
val sender = args["sender"]?.jsonPrimitive?.contentOrNull
|
val sortOrder = ChatHistorySortOrder.NEWEST
|
||||||
|
|
||||||
val maxDays = PluginConfig.searchHistoryMaxDays
|
|
||||||
val now = OffsetDateTime.now()
|
val now = OffsetDateTime.now()
|
||||||
|
val start = parameters.string("from")?.let {
|
||||||
val startTime = args["startTime"]?.jsonPrimitive?.contentOrNull?.let {
|
parseTime(it, endOfDay = false) ?: return "from 格式错误,请使用 yyyy-MM-dd HH:mm 或 yyyy-MM-dd"
|
||||||
parseTime(it) ?: return "startTime 格式错误,请使用 yyyy-MM-dd HH:mm"
|
} ?: now.minusDays(PluginConfig.searchHistoryMaxDays.coerceAtLeast(1).toLong())
|
||||||
} ?: now.minusDays(maxDays.toLong())
|
val end = parameters.string("to")?.let {
|
||||||
|
parseTime(it, endOfDay = true) ?: return "to 格式错误,请使用 yyyy-MM-dd HH:mm 或 yyyy-MM-dd"
|
||||||
val endTime = args["endTime"]?.jsonPrimitive?.contentOrNull?.let {
|
|
||||||
parseTime(it) ?: return "endTime 格式错误,请使用 yyyy-MM-dd HH:mm"
|
|
||||||
} ?: now
|
} ?: now
|
||||||
|
if (start > end) return "起始时间必须早于或等于结束时间"
|
||||||
|
|
||||||
if (startTime >= endTime) {
|
val senderId = parameters["senderId"]?.jsonPrimitive?.longOrNull
|
||||||
return "起始时间必须早于结束时间"
|
val senderName = parameters.string("senderName")?.trim()?.takeIf(String::isNotEmpty)
|
||||||
}
|
val resolvedSenderId = senderId ?: senderName?.let { name ->
|
||||||
|
when (val resolution = resolveSender(name, event)) {
|
||||||
if (java.time.Duration.between(startTime, endTime).toDays() > maxDays) {
|
is SenderResolution.Found -> resolution.userId
|
||||||
return "搜索时间范围不能超过 ${maxDays}天,请缩小范围后重试"
|
is SenderResolution.Ambiguous -> return buildString {
|
||||||
}
|
appendLine("找到多个名称匹配的发送者,请改用 senderId:")
|
||||||
|
resolution.candidates.forEach { candidate ->
|
||||||
val senderQq = resolveSenderQq(sender, event)
|
appendLine("- ${candidate.displayName} (${candidate.userId})")
|
||||||
val startEpoch = startTime.toEpochSecond().toInt()
|
}
|
||||||
val endEpoch = endTime.toEpochSecond().toInt()
|
}.trimEnd()
|
||||||
val maxRecords = PluginConfig.searchHistoryMaxRecords
|
SenderResolution.NotFound -> return "没有找到名称匹配的发送者:$name"
|
||||||
|
|
||||||
val records = try {
|
|
||||||
ChatHistoryStore.query(
|
|
||||||
contact = event.subject,
|
|
||||||
start = startEpoch,
|
|
||||||
end = endEpoch,
|
|
||||||
limit = maxRecords,
|
|
||||||
fromId = senderQq,
|
|
||||||
).sortedWith(compareBy<ChatMessageRecord> { it.time }.thenBy { it.id })
|
|
||||||
} catch (e: Throwable) {
|
|
||||||
JChatGPT.logger.warning("查询消息历史失败", e)
|
|
||||||
return "查询消息历史失败: ${e.message}"
|
|
||||||
}
|
|
||||||
|
|
||||||
var filtered = records
|
|
||||||
|
|
||||||
// 消息内容在数据库中是序列化存储的,关键词只能在内存中过滤
|
|
||||||
if (keyword != null) {
|
|
||||||
filtered = filtered.filter {
|
|
||||||
it.toMessageChain().content.contains(keyword, ignoreCase = true)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (filtered.isEmpty()) {
|
val outputSafePageSize = ((PluginConfig.maxToolOutputLength.coerceAtLeast(1) - OUTPUT_RESERVE_CHARS)
|
||||||
return "未找到匹配的聊天记录"
|
.coerceAtLeast(APPROXIMATE_RECORD_CHARS) / APPROXIMATE_RECORD_CHARS)
|
||||||
|
.coerceIn(1, MAX_PAGE_SIZE)
|
||||||
|
val configuredPageSize = minOf(
|
||||||
|
PluginConfig.searchHistoryMaxRecords.coerceIn(1, MAX_PAGE_SIZE),
|
||||||
|
outputSafePageSize,
|
||||||
|
)
|
||||||
|
val pageSize = DEFAULT_PAGE_SIZE.coerceAtMost(configuredPageSize)
|
||||||
|
val cursor = parameters.string("cursor")?.let {
|
||||||
|
decodeCursor(it, sortOrder) ?: return "cursor 无效,请重新开始搜索"
|
||||||
|
}
|
||||||
|
if (query != null && query.codePointCount(0, query.length) >= 3 &&
|
||||||
|
!ChatHistoryStore.isSearchIndexAvailable
|
||||||
|
) {
|
||||||
|
return "聊天记录全文索引当前不可用,无法执行关键词搜索"
|
||||||
}
|
}
|
||||||
|
|
||||||
val limit = args["limit"]?.jsonPrimitive?.intOrNull?.coerceIn(1, 200) ?: 20
|
val request = ChatHistorySearchRequest(
|
||||||
val total = filtered.size
|
subject = ChatHistorySubject.from(event.subject),
|
||||||
val result = filtered.takeLast(limit)
|
query = query,
|
||||||
|
matchMode = matchMode,
|
||||||
|
fromId = resolvedSenderId,
|
||||||
|
start = start.toEpochSecond().toInt(),
|
||||||
|
end = end.toEpochSecond().toInt(),
|
||||||
|
sortOrder = sortOrder,
|
||||||
|
limit = pageSize,
|
||||||
|
cursor = cursor,
|
||||||
|
)
|
||||||
|
val page = try {
|
||||||
|
ChatHistoryStore.search(request)
|
||||||
|
} catch (cause: Throwable) {
|
||||||
|
JChatGPT.logger.warning("查询消息历史失败", cause)
|
||||||
|
return "查询消息历史失败: ${cause.message}"
|
||||||
|
}
|
||||||
|
|
||||||
|
if (page.records.isEmpty()) {
|
||||||
|
if (query != null && resolvedSenderId != null && cursor == null) {
|
||||||
|
val withoutSender = runCatching {
|
||||||
|
ChatHistoryStore.search(
|
||||||
|
request.copy(
|
||||||
|
fromId = null,
|
||||||
|
limit = DIAGNOSTIC_PAGE_SIZE,
|
||||||
|
cursor = ChatHistoryCursor(request.end ?: Int.MAX_VALUE, Long.MAX_VALUE),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}.getOrNull()
|
||||||
|
if (withoutSender?.records?.isNotEmpty() == true) {
|
||||||
|
return buildString {
|
||||||
|
append("未找到发送者 $resolvedSenderId 的匹配记录。")
|
||||||
|
appendLine("去掉发送者筛选后找到了匹配消息,可能是 QQ 号不正确;示例:")
|
||||||
|
ChatHistoryToolFormatter.appendRecords(this, withoutSender.records, event, query)
|
||||||
|
}.trimEnd()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return if (!ChatHistoryStore.isSearchIndexReady && query != null) {
|
||||||
|
"未找到匹配的聊天记录。历史全文索引仍在后台构建,当前结果可能不完整。"
|
||||||
|
} else if (resolvedSenderId != null) {
|
||||||
|
"未找到发送者 $resolvedSenderId 在当前聊天和时间范围内的匹配记录"
|
||||||
|
} else {
|
||||||
|
"未找到匹配的聊天记录"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return buildString {
|
return buildString {
|
||||||
appendLine("找到 $total 条匹配记录,显示最近 ${result.size} 条:")
|
page.totalMatches?.let { total -> appendLine("找到 $total 条匹配记录,本页 ${page.records.size} 条:") }
|
||||||
|
?: appendLine("继续搜索,本页 ${page.records.size} 条:")
|
||||||
|
page.nextCursor?.let { next ->
|
||||||
|
appendLine("hasMore: true")
|
||||||
|
appendLine("nextCursor: ${encodeCursor(next, sortOrder)}")
|
||||||
|
} ?: appendLine("hasMore: false")
|
||||||
appendLine()
|
appendLine()
|
||||||
appendHistory(this, result, event)
|
ChatHistoryToolFormatter.appendRecords(this, page.records, event, query)
|
||||||
}
|
if (!ChatHistoryStore.isSearchIndexReady && query != null) {
|
||||||
|
appendLine()
|
||||||
|
append("[索引状态] 历史全文索引仍在后台构建,当前结果可能不完整。")
|
||||||
|
}
|
||||||
|
}.trimEnd()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private fun resolveSender(name: String, event: MessageEvent): SenderResolution {
|
||||||
* 将 sender 解析为 QQ 号,优先尝试纯数字,再尝试群成员名称匹配
|
val groupId = (event as? GroupMessageEvent)?.group?.id
|
||||||
*/
|
val matches = runCatching {
|
||||||
private fun resolveSenderQq(sender: String?, event: MessageEvent): Long? {
|
ContactSnapshotStore.findUsersByName(event.bot.id, groupId, name, limit = 6)
|
||||||
if (sender == null) return null
|
}.getOrDefault(emptyList()).toMutableList()
|
||||||
sender.toLongOrNull()?.let { return it }
|
runCatching {
|
||||||
|
ChatHistoryStore.findSenderAliases(ChatHistorySubject.from(event.subject), name, limit = 6)
|
||||||
|
}.getOrDefault(emptyList()).forEach { alias ->
|
||||||
|
if (matches.none { it.userId == alias.userId }) {
|
||||||
|
matches += ContactNameMatch(alias.userId, alias.displayName, alias.matchRank)
|
||||||
|
}
|
||||||
|
}
|
||||||
if (event is GroupMessageEvent) {
|
if (event is GroupMessageEvent) {
|
||||||
return event.group.members.firstOrNull {
|
event.group.members.asSequence()
|
||||||
it.nameCardOrNick.contains(sender, ignoreCase = true)
|
.filter { it.nameCardOrNick.contains(name, ignoreCase = true) }
|
||||||
}?.id
|
.forEach { member ->
|
||||||
}
|
if (matches.none { it.userId == member.id }) {
|
||||||
return null
|
matches += ContactNameMatch(member.id, member.nameCardOrNick, 10)
|
||||||
}
|
|
||||||
|
|
||||||
private suspend fun appendHistory(
|
|
||||||
sb: StringBuilder,
|
|
||||||
records: List<ChatMessageRecord>,
|
|
||||||
event: MessageEvent
|
|
||||||
) {
|
|
||||||
val timeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
|
|
||||||
var lastFromId = 0L
|
|
||||||
|
|
||||||
for (record in records) {
|
|
||||||
val showSender = lastFromId != record.fromId
|
|
||||||
if (showSender) {
|
|
||||||
sb.appendLine()
|
|
||||||
if (event is GroupMessageEvent) {
|
|
||||||
if (event.bot.id == record.fromId) {
|
|
||||||
sb.append("**你** ").append(event.bot.nameCardOrNick)
|
|
||||||
} else {
|
|
||||||
sb.append(getNameCard(event.group, record.fromId))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sb.append(" ")
|
}
|
||||||
.append(timeFormatter.format(
|
val sorted = matches.sortedWith(compareBy({ it.matchRank }, { it.userId }))
|
||||||
Instant.ofEpochSecond(record.time.toLong()).atZone(ZoneId.systemDefault())
|
val first = sorted.firstOrNull() ?: return SenderResolution.NotFound
|
||||||
))
|
val second = sorted.getOrNull(1)
|
||||||
.append(":")
|
return if (second == null || first.matchRank < second.matchRank) {
|
||||||
}
|
SenderResolution.Found(first.userId)
|
||||||
for (msg in record.toMessageChain()) {
|
} else {
|
||||||
sb.append(singleMessageToText(msg, event.subject.id))
|
SenderResolution.Ambiguous(sorted)
|
||||||
}
|
|
||||||
sb.appendLine()
|
|
||||||
lastFromId = record.fromId
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun singleMessageToText(msg: SingleMessage, subjectId: Long): String {
|
private sealed interface SenderResolution {
|
||||||
return when (msg) {
|
data class Found(val userId: Long) : SenderResolution
|
||||||
is Image -> {
|
data class Ambiguous(val candidates: List<ContactNameMatch>) : SenderResolution
|
||||||
try {
|
data object NotFound : SenderResolution
|
||||||
val imageUrl = msg.queryUrl()
|
|
||||||
val index = JChatGPT.registerImage(subjectId, msg.imageId, imageUrl)
|
|
||||||
?: return msg.content
|
|
||||||
"[${if (msg.isEmoji) "表情包" else "图片"}$index]"
|
|
||||||
} catch (_: Throwable) {
|
|
||||||
msg.content
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else -> msg.content
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun getNameCard(group: Group, qq: Long): String {
|
|
||||||
val member = group[qq]
|
|
||||||
return member?.nameCardOrNick ?: "未知群员($qq)"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val timeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")
|
private const val DEFAULT_PAGE_SIZE = 20
|
||||||
|
private const val DIAGNOSTIC_PAGE_SIZE = 6
|
||||||
|
private const val MAX_PAGE_SIZE = 200
|
||||||
|
private const val OUTPUT_RESERVE_CHARS = 1_000
|
||||||
|
private const val APPROXIMATE_RECORD_CHARS = 450
|
||||||
|
private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")
|
||||||
|
private val dateFormatter = DateTimeFormatter.ISO_LOCAL_DATE
|
||||||
|
|
||||||
fun parseTime(text: String): OffsetDateTime? {
|
fun parseTime(text: String, endOfDay: Boolean): OffsetDateTime? {
|
||||||
|
val zone = ZoneId.systemDefault()
|
||||||
return try {
|
return try {
|
||||||
LocalDateTime.parse(text, timeFormatter)
|
LocalDateTime.parse(text, dateTimeFormatter).atZone(zone).toOffsetDateTime()
|
||||||
.atZone(ZoneId.systemDefault())
|
|
||||||
.toOffsetDateTime()
|
|
||||||
} catch (_: DateTimeParseException) {
|
} catch (_: DateTimeParseException) {
|
||||||
null
|
try {
|
||||||
|
val date = LocalDate.parse(text, dateFormatter)
|
||||||
|
val localDateTime = if (endOfDay) date.plusDays(1).atStartOfDay().minusNanos(1) else date.atStartOfDay()
|
||||||
|
localDateTime.atZone(zone).toOffsetDateTime()
|
||||||
|
} catch (_: DateTimeParseException) {
|
||||||
|
null
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private fun encodeCursor(cursor: ChatHistoryCursor, sortOrder: ChatHistorySortOrder): String {
|
||||||
|
val raw = "${sortOrder.name}:${cursor.time}:${cursor.id}"
|
||||||
|
return Base64.getUrlEncoder().withoutPadding()
|
||||||
|
.encodeToString(raw.toByteArray(StandardCharsets.UTF_8))
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun decodeCursor(text: String, sortOrder: ChatHistorySortOrder): ChatHistoryCursor? {
|
||||||
|
return runCatching {
|
||||||
|
val raw = String(Base64.getUrlDecoder().decode(text), StandardCharsets.UTF_8)
|
||||||
|
val parts = raw.split(':')
|
||||||
|
if (parts.size != 3 || parts[0] != sortOrder.name) return null
|
||||||
|
ChatHistoryCursor(parts[1].toInt(), parts[2].toLong())
|
||||||
|
}.getOrNull()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun JsonObject.string(key: String): String? =
|
||||||
|
get(key)?.jsonPrimitive?.contentOrNull
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,14 @@ import java.sql.DriverManager
|
|||||||
import kotlin.io.path.absolutePathString
|
import kotlin.io.path.absolutePathString
|
||||||
import kotlin.test.Test
|
import kotlin.test.Test
|
||||||
import kotlin.test.assertEquals
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertFalse
|
||||||
|
import kotlin.test.assertNotNull
|
||||||
import kotlin.test.assertTrue
|
import kotlin.test.assertTrue
|
||||||
|
import net.mamoe.mirai.message.data.MessageSourceKind
|
||||||
|
import top.jie65535.mirai.data.ChatHistoryMatchMode.PHRASE
|
||||||
|
import kotlinx.serialization.json.buildJsonArray
|
||||||
|
import kotlinx.serialization.json.buildJsonObject
|
||||||
|
import kotlinx.serialization.json.put
|
||||||
|
|
||||||
class ChatHistoryStoreTest {
|
class ChatHistoryStoreTest {
|
||||||
@Test
|
@Test
|
||||||
@@ -55,7 +62,15 @@ class ChatHistoryStoreTest {
|
|||||||
"SELECT value FROM chat_history_meta WHERE key = 'schema_version'"
|
"SELECT value FROM chat_history_meta WHERE key = 'schema_version'"
|
||||||
).use { results ->
|
).use { results ->
|
||||||
assertTrue(results.next())
|
assertTrue(results.next())
|
||||||
assertEquals("2", results.getString(1))
|
assertEquals("3", results.getString(1))
|
||||||
|
}
|
||||||
|
statement.executeQuery(
|
||||||
|
"SELECT COUNT(*) FROM sqlite_master " +
|
||||||
|
"WHERE type = 'table' AND name IN " +
|
||||||
|
"('message_record_search', 'message_record_fts', 'chat_history_sender_alias')"
|
||||||
|
).use { results ->
|
||||||
|
assertTrue(results.next())
|
||||||
|
assertEquals(3, results.getInt(1))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -64,4 +79,204 @@ class ChatHistoryStoreTest {
|
|||||||
directory.toFile().deleteRecursively()
|
directory.toFile().deleteRecursively()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun searchesChineseTextInDatabaseAndSupportsCursorAndContext() {
|
||||||
|
val directory = Files.createTempDirectory("jchatgpt-history-search-test-")
|
||||||
|
try {
|
||||||
|
ChatHistoryStore.init(directory.toFile())
|
||||||
|
val first = insert(directory, 100, 11, "部署失败,需要回滚", senderName = "旧名张三")
|
||||||
|
val second = insert(directory, 101, 12, "部署成功,服务已恢复")
|
||||||
|
val third = insert(directory, 102, 11, "确认部署失败的原因")
|
||||||
|
|
||||||
|
val subject = ChatHistorySubject(1, MessageSourceKind.GROUP, 1000)
|
||||||
|
val firstPage = ChatHistoryStore.search(
|
||||||
|
ChatHistorySearchRequest(
|
||||||
|
subject = subject,
|
||||||
|
query = "部署失败",
|
||||||
|
matchMode = PHRASE,
|
||||||
|
start = 0,
|
||||||
|
end = 200,
|
||||||
|
limit = 1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertEquals(2L, firstPage.totalMatches)
|
||||||
|
assertEquals(third, firstPage.records.single().id)
|
||||||
|
assertNotNull(firstPage.nextCursor)
|
||||||
|
|
||||||
|
val secondPage = ChatHistoryStore.search(
|
||||||
|
ChatHistorySearchRequest(
|
||||||
|
subject = subject,
|
||||||
|
query = "部署失败",
|
||||||
|
matchMode = PHRASE,
|
||||||
|
start = 0,
|
||||||
|
end = 200,
|
||||||
|
limit = 1,
|
||||||
|
cursor = firstPage.nextCursor,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertEquals(null, secondPage.totalMatches)
|
||||||
|
assertEquals(first, secondPage.records.single().id)
|
||||||
|
assertFalse(secondPage.nextCursor != null)
|
||||||
|
|
||||||
|
val context = ChatHistoryStore.findAround(subject, second, before = 1, after = 1)
|
||||||
|
assertNotNull(context)
|
||||||
|
assertEquals(listOf(first, second, third), context.records.map { it.id })
|
||||||
|
|
||||||
|
val shortQuery = ChatHistoryStore.search(
|
||||||
|
ChatHistorySearchRequest(
|
||||||
|
subject = subject,
|
||||||
|
query = "部署",
|
||||||
|
fromId = 11,
|
||||||
|
start = 0,
|
||||||
|
end = 200,
|
||||||
|
sortOrder = ChatHistorySortOrder.OLDEST,
|
||||||
|
limit = 10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertEquals(2L, shortQuery.totalMatches)
|
||||||
|
assertEquals(listOf(first, third), shortQuery.records.map { it.id })
|
||||||
|
|
||||||
|
val aliases = ChatHistoryStore.findSenderAliases(subject, "旧名")
|
||||||
|
assertEquals(listOf(ChatHistorySenderAliasMatch(11, "旧名张三", 1)), aliases)
|
||||||
|
} finally {
|
||||||
|
ChatHistoryStore.close()
|
||||||
|
directory.toFile().deleteRecursively()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun migratesExistingSchemaAndBackfillsSearchTextWithoutReplacingMessageTable() {
|
||||||
|
val directory = Files.createTempDirectory("jchatgpt-history-migration-test-")
|
||||||
|
val database = directory.resolve("chat-history.sqlite")
|
||||||
|
try {
|
||||||
|
Class.forName("org.sqlite.JDBC")
|
||||||
|
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
CREATE TABLE message_record(
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
bot_id INTEGER NOT NULL,
|
||||||
|
from_id INTEGER NOT NULL,
|
||||||
|
target_id INTEGER NOT NULL,
|
||||||
|
ids TEXT,
|
||||||
|
internal_ids TEXT,
|
||||||
|
time INTEGER NOT NULL,
|
||||||
|
kind INTEGER NOT NULL,
|
||||||
|
code TEXT NOT NULL,
|
||||||
|
recalled INTEGER NOT NULL DEFAULT 0
|
||||||
|
)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"CREATE TABLE chat_history_meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"INSERT INTO chat_history_meta(key, value) VALUES ('schema_version', '2')"
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
INSERT INTO message_record(
|
||||||
|
bot_id, from_id, target_id, ids, internal_ids, time, kind, code, recalled
|
||||||
|
) VALUES (1, 11, 1000, '1', '1', 100, ${MessageSourceKind.GROUP.ordinal},
|
||||||
|
'[{"type":"PlainText","content":"旧库部署失败"}]', 0)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
statement.executeUpdate(
|
||||||
|
"""
|
||||||
|
INSERT INTO message_record(
|
||||||
|
bot_id, from_id, target_id, ids, internal_ids, time, kind, code, recalled
|
||||||
|
) VALUES (1, 12, 1000, '2', '2', 101, ${MessageSourceKind.GROUP.ordinal},
|
||||||
|
'[{"type":"PlainText","content":"旧库部署失败后已恢复"}]', 0)
|
||||||
|
""".trimIndent()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ChatHistoryStore.init(directory.toFile())
|
||||||
|
assertTrue(ChatHistoryStore.isSearchIndexAvailable)
|
||||||
|
assertFalse(ChatHistoryStore.isSearchIndexReady)
|
||||||
|
var continueChecks = 0
|
||||||
|
assertEquals(
|
||||||
|
1,
|
||||||
|
ChatHistoryStore.backfillSearchIndex(
|
||||||
|
batchSize = 1,
|
||||||
|
shouldContinue = { continueChecks++ == 0 },
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertFalse(ChatHistoryStore.isSearchIndexReady)
|
||||||
|
ChatHistoryStore.close()
|
||||||
|
|
||||||
|
ChatHistoryStore.init(directory.toFile())
|
||||||
|
assertFalse(ChatHistoryStore.isSearchIndexReady)
|
||||||
|
assertEquals(1, ChatHistoryStore.backfillSearchIndex(batchSize = 1))
|
||||||
|
assertTrue(ChatHistoryStore.isSearchIndexReady)
|
||||||
|
|
||||||
|
val migratedSearch = ChatHistoryStore.search(
|
||||||
|
ChatHistorySearchRequest(
|
||||||
|
subject = ChatHistorySubject(1, MessageSourceKind.GROUP, 1000),
|
||||||
|
query = "旧库部署失败",
|
||||||
|
matchMode = PHRASE,
|
||||||
|
start = 0,
|
||||||
|
end = 200,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assertEquals(2L, migratedSearch.totalMatches)
|
||||||
|
assertEquals(listOf(2L, 1L), migratedSearch.records.map { it.id })
|
||||||
|
|
||||||
|
DriverManager.getConnection("jdbc:sqlite:${database.absolutePathString()}").use { connection ->
|
||||||
|
connection.createStatement().use { statement ->
|
||||||
|
statement.executeQuery("SELECT COUNT(*) FROM message_record").use { results ->
|
||||||
|
assertTrue(results.next())
|
||||||
|
assertEquals(2, results.getInt(1))
|
||||||
|
}
|
||||||
|
statement.executeQuery(
|
||||||
|
"SELECT search_text FROM message_record_search WHERE message_id = 1"
|
||||||
|
).use { results ->
|
||||||
|
assertTrue(results.next())
|
||||||
|
assertEquals("旧库部署失败", results.getString(1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
ChatHistoryStore.close()
|
||||||
|
directory.toFile().deleteRecursively()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun insert(
|
||||||
|
directory: java.nio.file.Path,
|
||||||
|
time: Int,
|
||||||
|
fromId: Long,
|
||||||
|
text: String,
|
||||||
|
senderName: String = "",
|
||||||
|
): Long {
|
||||||
|
val record = ChatMessageRecord(
|
||||||
|
botId = 1,
|
||||||
|
fromId = fromId,
|
||||||
|
targetId = 1000,
|
||||||
|
ids = "$time",
|
||||||
|
internalIds = "$time",
|
||||||
|
time = time,
|
||||||
|
kind = MessageSourceKind.GROUP,
|
||||||
|
code = buildJsonArray {
|
||||||
|
add(buildJsonObject {
|
||||||
|
put("type", "PlainText")
|
||||||
|
put("content", text)
|
||||||
|
})
|
||||||
|
}.toString(),
|
||||||
|
)
|
||||||
|
ChatHistoryStore.insertRecord(record, senderName)
|
||||||
|
DriverManager.getConnection("jdbc:sqlite:${directory.resolve("chat-history.sqlite").absolutePathString()}").use {
|
||||||
|
it.prepareStatement("SELECT id FROM message_record WHERE time = ? AND from_id = ?").use { query ->
|
||||||
|
query.setInt(1, time)
|
||||||
|
query.setLong(2, fromId)
|
||||||
|
query.executeQuery().use { results ->
|
||||||
|
assertTrue(results.next())
|
||||||
|
return results.getLong(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user