diff --git a/decode.go b/decode.go index 825cffc6..60547d92 100644 --- a/decode.go +++ b/decode.go @@ -1,9 +1,11 @@ package req import ( - "github.com/imroc/req/v3/internal/charsets" + "bytes" "io" "strings" + + "github.com/imroc/req/v3/internal/charsets" ) var textContentTypes = []string{"text", "json", "xml", "html", "java"} @@ -48,30 +50,20 @@ func (a *autoDecodeReadCloser) peekRead(p []byte) (n int, err error) { return } a.detected = true - enc, name := charsets.FindEncoding(p) + enc, name := charsets.FindEncoding(p[:n]) if enc == nil { return } if a.t.Debugf != nil { a.t.Debugf("charset %s found in body's meta, auto-decode to utf-8", name) } - dc := enc.NewDecoder() - a.decodeReader = dc.Reader(a.ReadCloser) - var pp []byte - pp, err = dc.Bytes(p[:n]) - if err != nil { - return - } - if len(pp) > len(p) { - a.peek = make([]byte, len(pp)-len(p)) - copy(a.peek, pp[len(p):]) - copy(p, pp[:len(p)]) - n = len(p) - return - } - copy(p, pp) - n = len(p) - return + // Replay the first chunk through the same decoder as the rest of the + // body. Decoding it with Decoder.Bytes treats the chunk as a complete + // input, so a multi-byte character split across this read is corrupted. + first := make([]byte, n) + copy(first, p[:n]) + a.decodeReader = enc.NewDecoder().Reader(io.MultiReader(bytes.NewReader(first), a.ReadCloser)) + return a.decodeReader.Read(p) } func (a *autoDecodeReadCloser) peekDrain(p []byte) (n int, err error) { diff --git a/decode_test.go b/decode_test.go index e65a8ea6..ea23198e 100644 --- a/decode_test.go +++ b/decode_test.go @@ -1,8 +1,12 @@ package req import ( - "github.com/imroc/req/v3/internal/tests" + "bytes" + "io" "testing" + + "github.com/imroc/req/v3/internal/tests" + "golang.org/x/text/encoding/simplifiedchinese" ) func TestPeekDrain(t *testing.T) { @@ -15,3 +19,35 @@ func TestPeekDrain(t *testing.T) { tests.AssertEqual(t, 2, n) tests.AssertEqual(t, true, a.peek == nil) } + +func TestAutoDecodeReadCloserSplitMultibyte(t *testing.T) { + gbkChina, err := simplifiedchinese.GBK.NewEncoder().Bytes([]byte("中国")) + if err != nil { + t.Fatal(err) + } + + // First io.ReadAll read is 512 bytes. Place 中国 so 国 straddles that boundary: + // 中 occupies 509-510, 国 occupies 511-512. + const firstRead = 512 + const titleStart = 509 + head := []byte(``) + if titleStart < len(head) { + t.Fatalf("titleStart %d < head %d", titleStart, len(head)) + } + body := bytes.Repeat([]byte{'x'}, titleStart) + copy(body, head) + body = append(body, gbkChina...) + body = append(body, []byte(``)...) + if body[firstRead-1] != gbkChina[2] || body[firstRead] != gbkChina[3] { + t.Fatalf("国 is not split across the first-read boundary") + } + + a := newAutoDecodeReadCloser(io.NopCloser(bytes.NewReader(body)), &Transport{}) + got, err := io.ReadAll(a) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(got, []byte("中国")) { + t.Fatalf("decoded body missing 中国, got %q", got) + } +}