From 5a52dfd2fda86dbb935643bce3d0a235c8995d60 Mon Sep 17 00:00:00 2001
From: tam1m <tam1m@users.noreply.github.com>
Date: Sat, 26 Sep 2026 11:34:00 +0200
Subject: [PATCH] fix: decode extractor responses with declared charset

---
 .../OkHttpExtractorResponseMapper.kt          |  3 +-
 .../OkHttpExtractorResponseMapperTest.kt      | 81 +++++++++++++++++++
 2 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt

diff --git a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt
index 9a47f893..0e770616 100644
--- a/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt
+++ b/server-downloader/src/main/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapper.kt
@@ -4,8 +4,9 @@ import org.schabi.newpipe.extractor.downloader.Response
 
 object OkHttpExtractorResponseMapper {
     fun toExtractorResponse(httpResponse: okhttp3.Response): Response {
+        val contentType = httpResponse.body.contentType()
         val responseBodyBytes = httpResponse.body.bytes()
-        val responseBody = responseBodyBytes.toString(Charsets.UTF_8)
+        val responseBody = responseBodyBytes.toString(contentType?.charset() ?: Charsets.UTF_8)
         return Response(
             httpResponse.code,
             httpResponse.message,
diff --git a/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt b/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt
new file mode 100644
index 00000000..b1bb99c1
--- /dev/null
+++ b/server-downloader/src/test/kotlin/dev/typetype/server/downloader/OkHttpExtractorResponseMapperTest.kt
@@ -0,0 +1,81 @@
+package dev.typetype.server.downloader
+
+import java.nio.charset.Charset
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Protocol
+import okhttp3.Request
+import okhttp3.ResponseBody.Companion.toResponseBody
+import org.junit.jupiter.api.Assertions.assertArrayEquals
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class OkHttpExtractorResponseMapperTest {
+
+    private fun httpResponse(contentType: String?, bytes: ByteArray): okhttp3.Response =
+        okhttp3.Response.Builder()
+            .request(Request.Builder().url("https://example.org/complete/search").build())
+            .protocol(Protocol.HTTP_2)
+            .code(200)
+            .message("OK")
+            .body(bytes.toResponseBody(contentType?.toMediaType()))
+            .build()
+
+    @Test
+    fun `decodes iso-8859-1 accented characters`() {
+        val expected = "ä ö ü ß ë Ë é è ê ç à ñ"
+        val bytes = expected.toByteArray(Charsets.ISO_8859_1)
+
+        val mapped = OkHttpExtractorResponseMapper.toExtractorResponse(
+            httpResponse("text/javascript; charset=ISO-8859-1", bytes),
+        )
+
+        assertEquals(expected, mapped.responseBody())
+        assertArrayEquals(bytes, mapped.rawResponseBody())
+    }
+
+    @Test
+    fun `decodes utf-8 body when charset is declared`() {
+        val expected = "ä ö ü ß 🎵"
+
+        val mapped = OkHttpExtractorResponseMapper.toExtractorResponse(
+            httpResponse("application/json; charset=utf-8", expected.toByteArray(Charsets.UTF_8)),
+        )
+
+        assertEquals(expected, mapped.responseBody())
+    }
+
+    @Test
+    fun `defaults to utf-8 without content type`() {
+        val expected = "ä ö ü ß 🎵"
+
+        val mapped = OkHttpExtractorResponseMapper.toExtractorResponse(
+            httpResponse(null, expected.toByteArray(Charsets.UTF_8)),
+        )
+
+        assertEquals(expected, mapped.responseBody())
+    }
+
+    @Test
+    fun `decodes shift_jis body used for japanese locales`() {
+        val expected = "あいう"
+        val bytes = expected.toByteArray(Charset.forName("Shift_JIS"))
+
+        val mapped = OkHttpExtractorResponseMapper.toExtractorResponse(
+            httpResponse("text/javascript; charset=Shift_JIS", bytes),
+        )
+
+        assertEquals(expected, mapped.responseBody())
+    }
+
+    @Test
+    fun `decodes windows-1251 body used for cyrillic locales`() {
+        val expected = "А Б В"
+        val bytes = expected.toByteArray(Charset.forName("windows-1251"))
+
+        val mapped = OkHttpExtractorResponseMapper.toExtractorResponse(
+            httpResponse("text/javascript; charset=windows-1251", bytes),
+        )
+
+        assertEquals(expected, mapped.responseBody())
+    }
+}