From 1c47a6971630f2560437be454da08d2a591d72f1 Mon Sep 17 00:00:00 2001 From: jie65535 Date: Tue, 4 Aug 2026 23:59:28 +0800 Subject: [PATCH] web: support self-hosted Jina Reader --- README.md | 2 + src/main/kotlin/config/PluginConfig.kt | 3 + src/main/kotlin/tools/VisitWeb.kt | 98 +++++++++++++++++++++++--- src/test/kotlin/tools/VisitWebTest.kt | 80 +++++++++++++++++++++ 4 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 src/test/kotlin/tools/VisitWebTest.kt diff --git a/README.md b/README.md index c4a8a81..4f6538d 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,8 @@ imageWatermark: false ttsModel: 'qwen3-tts-instruct-flash' # Jina API Key jinaApiKey: '' +# Jina Reader API 地址；自托管 Docker 服务示例：http://127.0.0.1:4223/ +jinaReaderUrl: 'https://r.jina.ai/' # SearXNG 搜索引擎地址，如 http://127.0.0.1:8080/search 必须启用允许json格式返回 searXngUrl: '' # 在线运行代码 glot.io 的 api token，在官网注册账号即可获取。 diff --git a/src/main/kotlin/config/PluginConfig.kt b/src/main/kotlin/config/PluginConfig.kt index db909d5..eb131eb 100644 --- a/src/main/kotlin/config/PluginConfig.kt +++ b/src/main/kotlin/config/PluginConfig.kt @@ -177,6 +177,9 @@ object PluginConfig : AutoSavePluginConfig("Config") { @ValueDescription("Jina API Key") val jinaApiKey by value("") + @ValueDescription("Jina Reader API 地址，默认使用在线服务；自托管示例：http://127.0.0.1:4223/") + val jinaReaderUrl: String by value("https://r.jina.ai/") + @ValueDescription("SearXNG 搜索引擎地址，如 http://127.0.0.1:8080/search 必须启用允许json格式返回") val searXngUrl: String by value("") diff --git a/src/main/kotlin/tools/VisitWeb.kt b/src/main/kotlin/tools/VisitWeb.kt index c13b242..984cdde 100644 --- a/src/main/kotlin/tools/VisitWeb.kt +++ b/src/main/kotlin/tools/VisitWeb.kt @@ -4,10 +4,17 @@ import com.aallam.openai.api.chat.Tool import com.aallam.openai.api.core.Parameters import io.ktor.client.request.* import io.ktor.client.statement.* +import io.ktor.http.* +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.withContext import kotlinx.serialization.json.* import top.jie65535.mirai.config.PluginConfig +import java.net.InetAddress +import java.net.URI +import java.net.UnknownHostException class VisitWeb : BaseAgent( tool = Tool.function( @@ -36,8 +43,69 @@ class VisitWeb : BaseAgent( ) ) { companion object { - // Visit Tool (Using Jina Reader) - const val JINA_READER_URL_PREFIX = "https://r.jina.ai/" + internal data class ReaderRequest( + val endpoint: String, + val apiKey: String?, + ) + + internal fun createReaderRequest( + rawUrl: String, + readerBaseUrl: String, + apiKey: String, + resolveAddresses: (String) -> List = { + InetAddress.getAllByName(it).toList() + }, + ): ReaderRequest { + val target = parseHttpUrl(rawUrl, "网页地址") + require(target.userInfo == null) { "网页地址不能包含用户凭据" } + + val targetHost = requireNotNull(target.host).removeSurrounding("[", "]").lowercase() + require(targetHost.contains('.') || targetHost.contains(':')) { + "禁止访问单标签或内部主机名" + } + require( + targetHost != "localhost" && + !targetHost.endsWith(".localhost") && + !targetHost.endsWith(".local") && + !targetHost.endsWith(".internal") && + !targetHost.endsWith(".lan") && + !targetHost.endsWith(".home.arpa") + ) { "禁止访问本机或局域网网页地址" } + + val addresses = try { + resolveAddresses(targetHost) + } catch (_: UnknownHostException) { + // 目标可能只能由 Mihomo 的 DNS 解析，交给代理继续处理。 + emptyList() + } + require(addresses.all(VisualImageResolver::isPublicAddress)) { + "网页地址解析到非公网地址，已拒绝访问" + } + + val readerBase = parseHttpUrl(readerBaseUrl, "Jina Reader API 地址") + require(readerBase.query == null && readerBase.fragment == null) { + "Jina Reader API 地址不能包含查询参数或片段" + } + + return ReaderRequest( + endpoint = readerBaseUrl.trim().trimEnd('/') + "/" + + target.toASCIIString().substringBefore('#'), + apiKey = apiKey.trim().takeIf(String::isNotEmpty), + ) + } + + private fun parseHttpUrl(rawUrl: String, label: String): URI { + val uri = try { + URI(rawUrl.trim()) + } catch (e: Exception) { + throw IllegalArgumentException("$label 格式无效", e) + } + require(uri.scheme?.lowercase() in setOf("http", "https")) { + "$label 仅支持 HTTP/HTTPS" + } + require(!uri.host.isNullOrBlank()) { "$label 缺少有效主机名" } + return uri.normalize() + } } override val isEnabled: Boolean @@ -61,12 +129,26 @@ class VisitWeb : BaseAgent( private suspend fun jinaReadPage(url: String): String { return try { - httpClient.get(JINA_READER_URL_PREFIX + url) { - if (PluginConfig.jinaApiKey.isNotEmpty()) { - header("Authorization", "Bearer ${PluginConfig.jinaApiKey}") - } - }.bodyAsText() - } catch (e: Throwable) { + val request = withContext(Dispatchers.IO) { + createReaderRequest( + rawUrl = url, + readerBaseUrl = PluginConfig.jinaReaderUrl, + apiKey = PluginConfig.jinaApiKey, + ) + } + val response = httpClient.get(request.endpoint) { + header(HttpHeaders.Accept, ContentType.Text.Plain) + request.apiKey?.let { header(HttpHeaders.Authorization, "Bearer $it") } + } + val body = response.bodyAsText() + if (response.status.isSuccess()) { + body + } else { + "Error fetching \"$url\": HTTP ${response.status.value} ${body.take(500)}" + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { "Error fetching \"$url\": ${e.message}" } } diff --git a/src/test/kotlin/tools/VisitWebTest.kt b/src/test/kotlin/tools/VisitWebTest.kt new file mode 100644 index 0000000..8b8b8e5 --- /dev/null +++ b/src/test/kotlin/tools/VisitWebTest.kt @@ -0,0 +1,80 @@ +package top.jie65535.mirai.tools + +import java.net.InetAddress +import java.net.UnknownHostException +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class VisitWebTest { + @Test + fun `self hosted reader request uses configured base url`() { + val request = VisitWeb.createReaderRequest( + rawUrl = "https://example.com/docs?q=reader#section", + readerBaseUrl = "http://127.0.0.1:4223/", + apiKey = "", + resolveAddresses = { listOf(InetAddress.getByName("93.184.216.34")) }, + ) + + assertEquals( + "http://127.0.0.1:4223/https://example.com/docs?q=reader", + request.endpoint, + ) + assertEquals(null, request.apiKey) + } + + @Test + fun `hosted reader remains backward compatible`() { + val request = VisitWeb.createReaderRequest( + rawUrl = "https://example.com/", + readerBaseUrl = "https://r.jina.ai/", + apiKey = "token", + resolveAddresses = { listOf(InetAddress.getByName("93.184.216.34")) }, + ) + + assertEquals("https://r.jina.ai/https://example.com/", request.endpoint) + assertEquals("token", request.apiKey) + } + + @Test + fun `private resolved targets are rejected`() { + assertFailsWith { + VisitWeb.createReaderRequest( + rawUrl = "https://example.com/", + readerBaseUrl = "http://127.0.0.1:4223/", + apiKey = "", + resolveAddresses = { listOf(InetAddress.getByName("192.168.1.10")) }, + ) + } + } + + @Test + fun `internal hostnames and credentials are rejected`() { + assertFailsWith { + VisitWeb.createReaderRequest( + rawUrl = "http://redis/", + readerBaseUrl = "http://127.0.0.1:4223/", + apiKey = "", + ) + } + assertFailsWith { + VisitWeb.createReaderRequest( + rawUrl = "https://user:password@example.com/", + readerBaseUrl = "http://127.0.0.1:4223/", + apiKey = "", + ) + } + } + + @Test + fun `unresolved public host may be resolved by reader deployment`() { + val request = VisitWeb.createReaderRequest( + rawUrl = "https://blocked.example/", + readerBaseUrl = "http://127.0.0.1:4223/", + apiKey = "", + resolveAddresses = { throw UnknownHostException("proxy DNS only") }, + ) + + assertEquals("http://127.0.0.1:4223/https://blocked.example/", request.endpoint) + } +}