From 46bf992b6bfdacd92667f9c4c2009ab7a26929ba Mon Sep 17 00:00:00 2001
From: jie65535 <jie65535@qq.com>
Date: Sat, 15 Aug 2026 21:22:45 +0800
Subject: [PATCH] llm: sanitize malformed request Unicode

---
 src/main/kotlin/llm/ModelService.kt     | 29 ++++++++++++++++-
 src/test/kotlin/llm/ModelServiceTest.kt | 43 +++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/src/main/kotlin/llm/ModelService.kt b/src/main/kotlin/llm/ModelService.kt
index 2b8cf2f..f12c487 100644
--- a/src/main/kotlin/llm/ModelService.kt
+++ b/src/main/kotlin/llm/ModelService.kt
@@ -166,7 +166,9 @@ class ModelService(
         extraBody?.forEach { (key, value) ->
             requestJson[key] = value
         }
-        return JsonObject(requestJson).toString()
+        // Ktor's text writer rejects lone UTF-16 surrogates. They can enter chat history through
+        // malformed gateway text, so normalize only invalid code units before encoding the body.
+        return JsonObject(requestJson).toString().replaceUnpairedSurrogates()
     }
 
     internal fun decodeStreamChunk(rawJson: String): ChatCompletionChunk? {
@@ -237,6 +239,31 @@ class ModelService(
             if (normalized.length <= maxChars) normalized else normalized.take(maxChars) + "...[截断]"
         }
 
+    private fun String.replaceUnpairedSurrogates(): String {
+        var output: StringBuilder? = null
+        var index = 0
+        while (index < length) {
+            val current = this[index]
+            when {
+                Character.isHighSurrogate(current) &&
+                        index + 1 < length && Character.isLowSurrogate(this[index + 1]) -> {
+                    output?.append(current)?.append(this[index + 1])
+                    index += 2
+                }
+                Character.isSurrogate(current) -> {
+                    if (output == null) output = StringBuilder(length).append(this, 0, index)
+                    output.append('\uFFFD')
+                    index++
+                }
+                else -> {
+                    output?.append(current)
+                    index++
+                }
+            }
+        }
+        return output?.toString() ?: this
+    }
+
     private fun isSafetyRejection(code: String?, message: String?): Boolean {
         val normalizedCode = code.orEmpty().lowercase()
         val normalizedMessage = message.orEmpty().lowercase()
diff --git a/src/test/kotlin/llm/ModelServiceTest.kt b/src/test/kotlin/llm/ModelServiceTest.kt
index f218a66..3d6b25b 100644
--- a/src/test/kotlin/llm/ModelServiceTest.kt
+++ b/src/test/kotlin/llm/ModelServiceTest.kt
@@ -22,6 +22,7 @@ import kotlinx.serialization.json.jsonPrimitive
 import okhttp3.Protocol
 import java.net.InetSocketAddress
 import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.atomic.AtomicReference
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
@@ -250,6 +251,48 @@ class ModelServiceTest {
         }
     }
 
+    @Test
+    fun sendsRequestsContainingUnpairedSurrogates() = runBlocking {
+        val server = HttpServer.create(InetSocketAddress("127.0.0.1", 0), 0)
+        val receivedBody = AtomicReference<String>()
+        val responseBody = listOf(
+            """data: {"id":"chunk-1","object":"chat.completion.chunk","created":1,"model":"test-model","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}""",
+            "data: [DONE]",
+            "",
+        ).joinToString("\n\n").toByteArray()
+        server.createContext("/v1/chat/completions") { exchange ->
+            receivedBody.set(String(exchange.requestBody.readBytes(), Charsets.UTF_8))
+            exchange.responseHeaders.add("Content-Type", "text/event-stream")
+            exchange.sendResponseHeaders(200, responseBody.size.toLong())
+            exchange.responseBody.use { body -> body.write(responseBody) }
+        }
+        server.start()
+
+        val localService = ModelService(
+            baseUrl = "http://127.0.0.1:${server.address.port}/v1/",
+            token = "test",
+            timeout = 1.seconds,
+            firstChunkTimeout = 1.seconds,
+        )
+        try {
+            val content = StringBuilder()
+            localService.chatCompletions(
+                ChatCompletionRequest(
+                    model = ModelId("test-model"),
+                    messages = listOf(ChatMessage.User("emoji \uD83D\uDE00 before\uD800middle\uDC00after")),
+                )
+            ).collect { chunk ->
+                chunk.choices.firstOrNull()?.delta?.content?.let(content::append)
+            }
+
+            assertEquals("ok", content.toString())
+            assertTrue(receivedBody.get().contains("emoji \uD83D\uDE00 before\uFFFDmiddle\uFFFDafter"))
+        } finally {
+            localService.httpClient.close()
+            server.stop(0)
+        }
+    }
+
     @Test
     fun queuesFlowsBeforeStartingWork() = runBlocking {
         val active = AtomicInteger()