decode: do not split charset decoding across the first read

decode.go

@@ -1,9 +1,11 @@
 package req
 
 import (
-	"github.com/imroc/req/v3/internal/charsets"
+	"bytes"
 	"io"
 	"strings"
+
+	"github.com/imroc/req/v3/internal/charsets"
 )
 
 var textContentTypes = []string{"text", "json", "xml", "html", "java"}
@@ -48,30 +50,20 @@ func (a *autoDecodeReadCloser) peekRead(p []byte) (n int, err error) {
 		return
 	}
 	a.detected = true
-	enc, name := charsets.FindEncoding(p)
+	enc, name := charsets.FindEncoding(p[:n])
 	if enc == nil {
 		return
 	}
 	if a.t.Debugf != nil {
 		a.t.Debugf("charset %s found in body's meta, auto-decode to utf-8", name)
 	}
-	dc := enc.NewDecoder()
-	a.decodeReader = dc.Reader(a.ReadCloser)
-	var pp []byte
-	pp, err = dc.Bytes(p[:n])
-	if err != nil {
-		return
-	}
-	if len(pp) > len(p) {
-		a.peek = make([]byte, len(pp)-len(p))
-		copy(a.peek, pp[len(p):])
-		copy(p, pp[:len(p)])
-		n = len(p)
-		return
-	}
-	copy(p, pp)
-	n = len(p)
-	return
+	// Replay the first chunk through the same decoder as the rest of the
+	// body. Decoding it with Decoder.Bytes treats the chunk as a complete
+	// input, so a multi-byte character split across this read is corrupted.
+	first := make([]byte, n)
+	copy(first, p[:n])
+	a.decodeReader = enc.NewDecoder().Reader(io.MultiReader(bytes.NewReader(first), a.ReadCloser))
+	return a.decodeReader.Read(p)
 }
 
 func (a *autoDecodeReadCloser) peekDrain(p []byte) (n int, err error) {

decode_test.go

@@ -1,8 +1,12 @@
 package req
 
 import (
-	"github.com/imroc/req/v3/internal/tests"
+	"bytes"
+	"io"
 	"testing"
+
+	"github.com/imroc/req/v3/internal/tests"
+	"golang.org/x/text/encoding/simplifiedchinese"
 )
 
 func TestPeekDrain(t *testing.T) {
@@ -15,3 +19,35 @@ func TestPeekDrain(t *testing.T) {
 	tests.AssertEqual(t, 2, n)
 	tests.AssertEqual(t, true, a.peek == nil)
 }
+
+func TestAutoDecodeReadCloserSplitMultibyte(t *testing.T) {
+	gbkChina, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte("中国"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// First io.ReadAll read is 512 bytes. Place 中国 so 国 straddles that boundary:
+	// 中 occupies 509-510, 国 occupies 511-512.
+	const firstRead = 512
+	const titleStart = 509
+	head := []byte(`<!doctype html><html><head><meta charset="gbk"></head><body><title>`)
+	if titleStart < len(head) {
+		t.Fatalf("titleStart %d < head %d", titleStart, len(head))
+	}
+	body := bytes.Repeat([]byte{'x'}, titleStart)
+	copy(body, head)
+	body = append(body, gbkChina...)
+	body = append(body, []byte(`</title></body></html>`)...)
+	if body[firstRead-1] != gbkChina[2] || body[firstRead] != gbkChina[3] {
+		t.Fatalf("国 is not split across the first-read boundary")
+	}
+
+	a := newAutoDecodeReadCloser(io.NopCloser(bytes.NewReader(body)), &Transport{})
+	got, err := io.ReadAll(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Contains(got, []byte("中国")) {
+		t.Fatalf("decoded body missing 中国, got %q", got)
+	}
+}