From 89d899a231f7e18aff31339c36636d0c9a16fba3 Mon Sep 17 00:00:00 2001 From: pikann22 Date: Tue, 3 Nov 2026 10:19:53 +0000 Subject: [PATCH 1/2] feat(plugin): implement chunked body fetching and error handling for oversized responses --- fetch_chunks.go | 70 +++++++++++++++++++++++++++++ fetch_chunks_test.go | 88 ++++++++++++++++++++++++++++++++++++ host_limits.go | 36 +++++++++++++++ host_limits_test.go | 39 ++++++++++++++++ native_backends.go | 1 + wasm_backends.go | 104 +++++++++++++++++++++++++++++++++---------- wasm_exports.go | 49 +++++++++++--------- wasm_result.go | 21 +++++++++ wasm_result_test.go | 30 +++++++++++++ 9 files changed, 395 insertions(+), 43 deletions(-) create mode 100644 fetch_chunks.go create mode 100644 fetch_chunks_test.go create mode 100644 host_limits.go create mode 100644 host_limits_test.go create mode 100644 wasm_result.go create mode 100644 wasm_result_test.go diff --git a/fetch_chunks.go b/fetch_chunks.go new file mode 100644 index 0000000..44d6f15 --- /dev/null +++ b/fetch_chunks.go @@ -0,0 +1,70 @@ +package plugin + +import ( + "errors" + "fmt" +) + +// A fetch response whose envelope does not fit the plugin's host buffer is +// delivered in pieces: the host replies with status/headers plus a +// body_handle, and the plugin pulls the body with {"body_handle","offset", +// "limit"} requests, each chunk carried as base64 in the JSON envelope. This +// file holds the transport-independent half so it can be unit tested without +// a WASM runtime; wasm_backends.go supplies the real host call. + +const ( + // fetchChunkSize is the raw bytes requested per chunk (~350 KiB once + // base64-encoded), small enough to fit a fresh 1 MiB buffer with room to + // spare. + fetchChunkSize = 256 * 1024 + // minFetchChunkSize is where retries stop shrinking the chunk. + minFetchChunkSize = 4 * 1024 +) + +// fetchEnvelope is the paca.fetch response: a [FetchResponse], plus the +// chunking fields set when the body was parked host-side. +type fetchEnvelope struct { + FetchResponse + BodyHandle string `json:"body_handle"` + BodySize int `json:"body_size"` +} + +// fetchChunk is the response to a body_handle read. +type fetchChunk struct { + Chunk []byte `json:"chunk"` // base64 on the wire + BodySize int `json:"body_size"` + Done bool `json:"done"` + Error string `json:"error"` + Code string `json:"code"` +} + +// chunkReader performs one body_handle read against the host. +type chunkReader func(handle string, offset, limit int) (*fetchChunk, error) + +// readBodyChunks drains a parked body of size bytes. If a chunk does not fit +// the host buffer it halves the chunk size and retries, down to +// minFetchChunkSize, so a plugin whose buffer is partly used still succeeds. +func readBodyChunks(handle string, size int, read chunkReader) (string, error) { + body := make([]byte, 0, size) + limit := fetchChunkSize + for { + c, err := read(handle, len(body), limit) + if err != nil { + if errors.Is(err, ErrResponseTooLarge) && limit > minFetchChunkSize { + limit /= 2 + continue + } + return "", err + } + body = append(body, c.Chunk...) + if c.Done { + if len(body) != size { + return "", fmt.Errorf("plugin: fetch: body is %d bytes, host announced %d", len(body), size) + } + return string(body), nil + } + if len(c.Chunk) == 0 { + return "", fmt.Errorf("plugin: fetch: host returned an empty chunk at offset %d of %d", len(body), size) + } + } +} diff --git a/fetch_chunks_test.go b/fetch_chunks_test.go new file mode 100644 index 0000000..4ed687f --- /dev/null +++ b/fetch_chunks_test.go @@ -0,0 +1,88 @@ +package plugin + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +// fakeHost serves body in limit-capped chunks and can reject chunks above +// maxFit with ErrResponseTooLarge, like a plugin whose buffer is partly full. +type fakeHost struct { + body string + maxFit int + calls []int // limits requested +} + +func (f *fakeHost) read(_ string, offset, limit int) (*fetchChunk, error) { + f.calls = append(f.calls, limit) + if f.maxFit > 0 && limit > f.maxFit { + return nil, hostResultError("fetch", "does not fit", hostCodeResponseTooLarge) + } + end := min(offset+limit, len(f.body)) + return &fetchChunk{Chunk: []byte(f.body[offset:end]), BodySize: len(f.body), Done: end == len(f.body)}, nil +} + +func TestReadBodyChunks_Reassembles(t *testing.T) { + body := strings.Repeat("0123456789", 100_000) // 1 MB, > several chunks + h := &fakeHost{body: body} + got, err := readBodyChunks("h", len(body), h.read) + if err != nil || got != body { + t.Fatalf("err=%v, equal=%v", err, got == body) + } + if len(h.calls) != 4 { // ceil(1_000_000 / 262_144) + t.Errorf("calls = %d, want 4", len(h.calls)) + } +} + +func TestReadBodyChunks_ShrinksOnTooLarge(t *testing.T) { + body := strings.Repeat("x", 100_000) + h := &fakeHost{body: body, maxFit: 20_000} + got, err := readBodyChunks("h", len(body), h.read) + if err != nil || got != body { + t.Fatalf("err=%v, equal=%v", err, got == body) + } + if h.calls[0] != fetchChunkSize || h.calls[len(h.calls)-1] > 20_000 { + t.Errorf("limits did not shrink as expected: %v", h.calls) + } +} + +func TestReadBodyChunks_GivesUpAtMinimum(t *testing.T) { + h := &fakeHost{body: strings.Repeat("x", 100_000), maxFit: 100} + _, err := readBodyChunks("h", 100_000, h.read) + if !errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("err = %v, want ErrResponseTooLarge", err) + } +} + +func TestReadBodyChunks_Errors(t *testing.T) { + boom := errors.New("boom") + if _, err := readBodyChunks("h", 5, func(string, int, int) (*fetchChunk, error) { return nil, boom }); !errors.Is(err, boom) { + t.Errorf("host error not propagated: %v", err) + } + empty := func(string, int, int) (*fetchChunk, error) { return &fetchChunk{}, nil } + if _, err := readBodyChunks("h", 5, empty); err == nil || !strings.Contains(err.Error(), "empty chunk") { + t.Errorf("empty chunk must not loop forever: %v", err) + } + short := func(string, int, int) (*fetchChunk, error) { return &fetchChunk{Chunk: []byte("ab"), Done: true}, nil } + if _, err := readBodyChunks("h", 5, short); err == nil || !strings.Contains(err.Error(), "announced") { + t.Errorf("size mismatch must be reported: %v", err) + } +} + +// The wire shape must match the host: chunk is base64, the meta envelope +// flattens FetchResponse next to body_handle. +func TestFetchWireShapes(t *testing.T) { + var c fetchChunk + if err := json.Unmarshal([]byte(`{"chunk":"aGVsbG8=","body_size":5,"done":true}`), &c); err != nil || string(c.Chunk) != "hello" || !c.Done { + t.Fatalf("chunk decode: %v %+v", err, c) + } + var e fetchEnvelope + if err := json.Unmarshal([]byte(`{"status":200,"headers":{"A":"b"},"body_handle":"h1","body_size":42}`), &e); err != nil { + t.Fatal(err) + } + if e.Status != 200 || e.Headers["A"] != "b" || e.BodyHandle != "h1" || e.BodySize != 42 { + t.Fatalf("envelope decode: %+v", e) + } +} diff --git a/host_limits.go b/host_limits.go new file mode 100644 index 0000000..8a3ac2d --- /dev/null +++ b/host_limits.go @@ -0,0 +1,36 @@ +package plugin + +import ( + "errors" + "fmt" +) + +// ErrResponseTooLarge is returned (wrapped) when the host produced a result +// but it could not be delivered because it does not fit in the plugin's +// fixed host-allocation buffer. It is a size limit, not a network failure: +// retry with less data (e.g. a smaller page size), not with backoff. +// +// if errors.Is(err, plugin.ErrResponseTooLarge) { ... } +var ErrResponseTooLarge = errors.New("response too large") + +// hostCodeResponseTooLarge is the "code" value hosts put in an error result +// that maps to ErrResponseTooLarge. Keep in sync with the host's +// errCodeResponseTooLarge (services/api/internal/platform/plugin/runtime.go). +const hostCodeResponseTooLarge = "response_too_large" + +// hostResultError builds the error for a host result envelope that carried +// an error message (and optionally a machine-readable code). +func hostResultError(op, msg, code string) error { + if code == hostCodeResponseTooLarge { + return fmt.Errorf("plugin: %s: %w: %s", op, ErrResponseTooLarge, msg) + } + return fmt.Errorf("plugin: %s: %s", op, msg) +} + +// errEmptyHostResponse is returned when a host call delivered no data at all. +// Newer hosts always send an explicit error envelope instead, so reaching +// this means an older host, an exhausted buffer, or a host-side failure -- +// not necessarily an unreachable network. +func errEmptyHostResponse(op string) error { + return fmt.Errorf("plugin: %s: host returned no data (the result may not have fit in the plugin's host buffer, or the host call failed; this does not necessarily mean the network is unreachable)", op) +} diff --git a/host_limits_test.go b/host_limits_test.go new file mode 100644 index 0000000..ce14783 --- /dev/null +++ b/host_limits_test.go @@ -0,0 +1,39 @@ +package plugin + +import ( + "errors" + "strings" + "testing" +) + +func TestHostResultError_ResponseTooLarge(t *testing.T) { + err := hostResultError("fetch", "fetch: response of 1238310 bytes does not fit", hostCodeResponseTooLarge) + if !errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("errors.Is(err, ErrResponseTooLarge) = false for %v", err) + } + for _, want := range []string{"plugin: fetch:", "response too large", "1238310"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q missing %q", err, want) + } + } +} + +func TestHostResultError_PlainError(t *testing.T) { + err := hostResultError("fetch", "fetch: domain not permitted by plugin manifest", "") + if errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("plain host error must not match ErrResponseTooLarge: %v", err) + } + if got, want := err.Error(), "plugin: fetch: fetch: domain not permitted by plugin manifest"; got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestErrEmptyHostResponse_NotMisleading(t *testing.T) { + msg := errEmptyHostResponse("fetch").Error() + if strings.Contains(msg, "empty response from host") { + t.Errorf("message should not use the old ambiguous wording: %q", msg) + } + if !strings.Contains(msg, "does not necessarily mean the network is unreachable") { + t.Errorf("message should disclaim a network failure: %q", msg) + } +} diff --git a/native_backends.go b/native_backends.go index ca4270b..2ca6c28 100644 --- a/native_backends.go +++ b/native_backends.go @@ -71,6 +71,7 @@ type FetchResponse struct { Body string `json:"body"` Headers map[string]string `json:"headers"` Error string `json:"error"` + Code string `json:"code,omitempty"` } // Fetch always returns errNotWASM outside a WASM module. diff --git a/wasm_backends.go b/wasm_backends.go index 29c7808..336fb39 100644 --- a/wasm_backends.go +++ b/wasm_backends.go @@ -11,12 +11,12 @@ import ( // ── Memory management ───────────────────────────────────────────────────────── -// mallocBuffer is a pre-allocated buffer for host-managed memory allocations. -// The host writes request data into this buffer, and the response is later -// packed into the same space — never both at once: wasm_exports.go resets -// the allocator between reading the request and allocating the response, so -// each phase gets the buffer's full capacity rather than splitting it. Sized -// to match services/api's DefaultResourceLimits.MaxRequestBodyBytes +// mallocBuffer is a pre-allocated buffer for host-managed memory allocations: +// the host writes request data and host-function results into it. Results the +// plugin returns are not placed here (see packWASMResult), so this only has to +// hold what the host sends in, not what the plugin sends back. wasm_exports.go +// resets the allocator between reading the request and handling it. Sized to +// match services/api's DefaultResourceLimits.MaxRequestBodyBytes // (paca/services/api/internal/platform/plugin/runtime.go) — the host already // rejects any payload larger than that before ever attempting to write it // here, so this only needs to cover the same ceiling, not more. This buffer @@ -84,15 +84,19 @@ func (b *wasmDBBackend) Query(sql string, params []any) (*DBQueryResult, error) ) resPtr, resLen, errPtr, errLen := decodeQuery2Output(outputBuf) if errLen > 0 { - return nil, &hostError{string(wasmSlice(errPtr, errLen))} + msg := string(wasmSlice(errPtr, errLen)) + wasmResetAllocator() + return nil, &hostError{msg} } if resLen == 0 { return &DBQueryResult{}, nil } - resBytes := wasmSlice(resPtr, resLen) - var result DBQueryResult - if err := json.Unmarshal(resBytes, &result); err != nil { + err = json.Unmarshal(wasmSlice(resPtr, resLen), &result) + // Unmarshal copies what it keeps, so release the host allocation now; + // otherwise results accumulate in the shared arena across calls. + wasmResetAllocator() + if err != nil { return nil, err } return &result, nil @@ -115,7 +119,9 @@ func (b *wasmDBBackend) Exec(sql string, params []any) (int64, error) { errPtr := int32(uint32(outputBuf[8]) | uint32(outputBuf[9])<<8 | uint32(outputBuf[10])<<16 | uint32(outputBuf[11])<<24) errLen := int32(uint32(outputBuf[12]) | uint32(outputBuf[13])<<8 | uint32(outputBuf[14])<<16 | uint32(outputBuf[15])<<24) if errLen > 0 { - return 0, &hostError{string(wasmSlice(errPtr, errLen))} + msg := string(wasmSlice(errPtr, errLen)) + wasmResetAllocator() + return 0, &hostError{msg} } return rowsAffected, nil } @@ -138,7 +144,9 @@ func (b *wasmKVBackend) Get(key string) (string, bool) { if valLen == 0 { return "", false } - return string(wasmSlice(valPtr, valLen)), true + val := string(wasmSlice(valPtr, valLen)) + wasmResetAllocator() + return val, true } func (b *wasmKVBackend) Set(key, value string) { @@ -170,7 +178,9 @@ func (b *wasmCacheBackend) Get(key string) (string, bool) { if valLen == 0 { return "", false } - return string(wasmSlice(valPtr, valLen)), true + val := string(wasmSlice(valPtr, valLen)) + wasmResetAllocator() + return val, true } func (b *wasmCacheBackend) Set(key, value string, ttl time.Duration) { @@ -216,7 +226,9 @@ func (b *wasmConfigBackend) Get(key string) (string, bool) { if valLen == 0 { return "", false } - return string(wasmSlice(valPtr, valLen)), true + val := string(wasmSlice(valPtr, valLen)) + wasmResetAllocator() + return val, true } // ── WASM Permission backend ─────────────────────────────────────────────────── @@ -280,6 +292,9 @@ type FetchResponse struct { Body string `json:"body"` Headers map[string]string `json:"headers"` Error string `json:"error"` + // Code is a machine-readable error classification set by newer hosts + // (e.g. "response_too_large"). Fetch maps it to a typed error. + Code string `json:"code,omitempty"` } // Fetch makes an outbound HTTP request via the paca.fetch host function. @@ -290,16 +305,45 @@ func Fetch(method, rawURL string, headers map[string]string, body string) (*Fetc URL string `json:"url"` Headers map[string]string `json:"headers"` Body string `json:"body"` + // Chunked tells the host it may return a body_handle (read via + // readBodyChunks) instead of response_too_large. + Chunked bool `json:"chunked"` }{ Method: method, URL: rawURL, Headers: headers, Body: body, + Chunked: true, } reqJSON, err := json.Marshal(req) if err != nil { return nil, err } + resBytes, err := callFetchHost(reqJSON) + if err != nil { + return nil, err + } + var env fetchEnvelope + if err := json.Unmarshal(resBytes, &env); err != nil { + return nil, fmt.Errorf("plugin: fetch: decode response: %w", err) + } + if env.Error != "" { + return nil, hostResultError("fetch", env.Error, env.Code) + } + if env.BodyHandle != "" { + // Too big for one envelope: the host parked the body; pull it in pieces. + body, err := readBodyChunks(env.BodyHandle, env.BodySize, readFetchChunk) + if err != nil { + return nil, err + } + env.Body = body + } + return &env.FetchResponse, nil +} + +// callFetchHost sends one request to paca.fetch and returns the response +// bytes, copied out of the host arena (which is then released). +func callFetchHost(reqJSON []byte) ([]byte, error) { outputBuf := make([]byte, 8) hostFetch( int64(ptrOf(reqJSON)), int64(len(reqJSON)), @@ -308,18 +352,31 @@ func Fetch(method, rawURL string, headers map[string]string, body string) (*Fetc resPtr := int32(uint32(outputBuf[0]) | uint32(outputBuf[1])<<8 | uint32(outputBuf[2])<<16 | uint32(outputBuf[3])<<24) resLen := int32(uint32(outputBuf[4]) | uint32(outputBuf[5])<<8 | uint32(outputBuf[6])<<16 | uint32(outputBuf[7])<<24) if resLen == 0 { - return nil, fmt.Errorf("plugin: fetch: empty response from host") + return nil, errEmptyHostResponse("fetch") } resBytes := append([]byte(nil), wasmSlice(resPtr, resLen)...) wasmResetAllocator() - var resp FetchResponse - if err := json.Unmarshal(resBytes, &resp); err != nil { - return nil, fmt.Errorf("plugin: fetch: decode response: %w", err) + return resBytes, nil +} + +// readFetchChunk reads one piece of a body the host parked for handle. +func readFetchChunk(handle string, offset, limit int) (*fetchChunk, error) { + reqJSON, err := json.Marshal(map[string]any{"body_handle": handle, "offset": offset, "limit": limit}) + if err != nil { + return nil, err + } + resBytes, err := callFetchHost(reqJSON) + if err != nil { + return nil, err + } + var c fetchChunk + if err := json.Unmarshal(resBytes, &c); err != nil { + return nil, fmt.Errorf("plugin: fetch: decode body chunk: %w", err) } - if resp.Error != "" { - return nil, fmt.Errorf("plugin: fetch: %s", resp.Error) + if c.Error != "" { + return nil, hostResultError("fetch", c.Error, c.Code) } - return &resp, nil + return &c, nil } // ── GetBranding ─────────────────────────────────────────────────────────────── @@ -337,6 +394,7 @@ type BrandingInfo struct { type brandingResult struct { BrandingInfo Error string `json:"error"` + Code string `json:"code"` } // GetBranding reads the workspace's current branding (logo, brand name, @@ -349,7 +407,7 @@ func GetBranding() (*BrandingInfo, error) { resPtr := int32(uint32(outputBuf[0]) | uint32(outputBuf[1])<<8 | uint32(outputBuf[2])<<16 | uint32(outputBuf[3])<<24) resLen := int32(uint32(outputBuf[4]) | uint32(outputBuf[5])<<8 | uint32(outputBuf[6])<<16 | uint32(outputBuf[7])<<24) if resLen == 0 { - return nil, fmt.Errorf("plugin: settings_get: empty response from host") + return nil, errEmptyHostResponse("settings_get") } resBytes := append([]byte(nil), wasmSlice(resPtr, resLen)...) wasmResetAllocator() @@ -358,7 +416,7 @@ func GetBranding() (*BrandingInfo, error) { return nil, fmt.Errorf("plugin: settings_get: decode response: %w", err) } if res.Error != "" { - return nil, fmt.Errorf("plugin: settings_get: %s", res.Error) + return nil, hostResultError("settings_get", res.Error, res.Code) } return &res.BrandingInfo, nil } diff --git a/wasm_exports.go b/wasm_exports.go index 13ee8e2..795a8c5 100644 --- a/wasm_exports.go +++ b/wasm_exports.go @@ -7,6 +7,8 @@ // paca_free for host-managed memory allocation. package plugin +import "unsafe" + // ── Memory management ───────────────────────────────────────────────────────── // Exported as paca_malloc/paca_free, not malloc/free: TinyGo's own bundled @@ -54,7 +56,7 @@ func HandleRequest(ptr, length int32) int64 { // stacking the response after the request — otherwise a request near // the arena's capacity would leave no room to allocate the response. wasmResetAllocator() - return packWASMResult(result) + return packWASMResult(result, func(msg string) []byte { return errorResponse(500, msg) }) } //go:wasmexport ResetAllocator @@ -70,7 +72,9 @@ func EvaluateCondition(ptr, length int32) int64 { payload := wasmSlice(ptr, length) result := globalDispatcher.evaluateCondition(payload) wasmResetAllocator() // see HandleRequest: payload is fully consumed by this point - return packWASMResult(result) + return packWASMResult(result, func(msg string) []byte { + return marshalConditionResult(ConditionResult{Matched: false, Error: msg}) + }) } //go:wasmexport RunAction @@ -81,28 +85,30 @@ func RunAction(ptr, length int32) int64 { payload := wasmSlice(ptr, length) result := globalDispatcher.runAction(payload) wasmResetAllocator() // see HandleRequest: payload is fully consumed by this point - return packWASMResult(result) + return packWASMResult(result, func(msg string) []byte { + return marshalActionResult(ActionResult{Applied: false, Error: msg}) + }) } -// packWASMResult allocates space in mallocBuffer for result, copies it in, -// and returns the packed (ptr<<32)|len combined offset+length the host -// expects — the same packing HandleRequest, EvaluateCondition, and RunAction -// all use. NOTE: Host MUST copy out the response before calling -// ResetAllocator, which is called after each export call completes. -func packWASMResult(result []byte) int64 { +// resultBuf keeps the last result alive, outside mallocBuffer, until the next +// export call replaces it. The host reads a result from whatever linear-memory +// address it is given, so a result does not have to fit the fixed arena the host +// allocates request data in; the Go heap does not move objects, so the address +// stays valid until then. +var resultBuf []byte + +// packWASMResult returns result as the packed (ptr<<32)|len pair the host +// expects. A result over maxResultBytes is replaced by what tooLarge builds, so +// the host gets an explanation instead of an empty result. NOTE: the host reads +// the result before the plugin's next export call. +func packWASMResult(result []byte, tooLarge func(msg string) []byte) int64 { + resultBuf = nil // release the previous call's result + result = boundResult(result, tooLarge) if len(result) == 0 { return 0 } - outPtr := wasmMalloc(int32(len(result))) - if outPtr == 0 { - return 0 - } - out := wasmSlice(outPtr, int32(len(result))) - if len(out) != len(result) { - return 0 - } - copy(out, result) - return (int64(outPtr) << 32) | int64(len(result)) + resultBuf = result + return (int64(uintptr(unsafe.Pointer(&resultBuf[0]))) << 32) | int64(len(resultBuf)) } //go:wasmexport HandleEvent @@ -111,7 +117,10 @@ func HandleEvent(topicPtr, topicLen, payloadPtr, payloadLen int32) { return } topic := string(wasmSlice(topicPtr, topicLen)) - payload := wasmSlice(payloadPtr, payloadLen) + // Copy: the handler may call host functions (KV, DB, Fetch, ...) that + // reset the shared arena once they have copied their own results out, + // which would otherwise overwrite an Event.Payload still aliasing it. + payload := append([]byte(nil), wasmSlice(payloadPtr, payloadLen)...) globalDispatcher.handleEvent(topic, payload) } diff --git a/wasm_result.go b/wasm_result.go new file mode 100644 index 0000000..7d1c092 --- /dev/null +++ b/wasm_result.go @@ -0,0 +1,21 @@ +package plugin + +import "fmt" + +// maxResultBytes is the largest result (a response, a condition or an action +// result) a plugin hands back to the host. Results are returned from the +// guest's own heap, not from the fixed host-allocation arena, so this is a +// resource limit rather than a buffer size; it stays well below the host's +// linear-memory cap so building the result cannot exhaust it. +const maxResultBytes = 16 << 20 + +// boundResult returns result when it may be handed to the host. A larger one is +// replaced by what tooLarge builds for its size, so the caller learns why +// instead of receiving an empty result. Kept in a non-wasip1-tagged file so it +// can be unit tested on the host build; wasm_exports.go is the real caller. +func boundResult(result []byte, tooLarge func(msg string) []byte) []byte { + if len(result) <= maxResultBytes { + return result + } + return tooLarge(fmt.Sprintf("plugin: result of %d bytes exceeds the limit of %d bytes", len(result), maxResultBytes)) +} diff --git a/wasm_result_test.go b/wasm_result_test.go new file mode 100644 index 0000000..d0cb2a2 --- /dev/null +++ b/wasm_result_test.go @@ -0,0 +1,30 @@ +package plugin + +import ( + "strings" + "testing" +) + +func TestBoundResult_PassesResultsUpToTheLimit(t *testing.T) { + small := []byte("ok") + if got := boundResult(small, func(string) []byte { t.Fatal("must not build a fallback"); return nil }); string(got) != "ok" { + t.Fatalf("got %q", got) + } + atLimit := make([]byte, maxResultBytes) + if got := boundResult(atLimit, func(string) []byte { t.Fatal("limit is inclusive"); return nil }); len(got) != maxResultBytes { + t.Fatalf("len = %d", len(got)) + } +} + +func TestBoundResult_ReplacesAnOversizedResultWithAnExplicitError(t *testing.T) { + var msg string + got := boundResult(make([]byte, maxResultBytes+1), func(m string) []byte { msg = m; return []byte("err") }) + if string(got) != "err" { + t.Fatalf("got %q", got) + } + for _, want := range []string{"exceeds the limit", "16777217"} { + if !strings.Contains(msg, want) { + t.Errorf("message %q missing %q", msg, want) + } + } +} From dc57388d80415c40cd9c3b6dd11acbebe7da9cf3 Mon Sep 17 00:00:00 2001 From: pikann Date: Tue, 3 Nov 2026 17:29:43 +0700 Subject: [PATCH 2/2] fix(plugin): harden chunked fetch against bad host sizes Validate announced body_size, reject overruns, build the body without a second copy, retry chunk reads on empty host replies, drop the doubled op prefix in host errors, and reuse errEmptyHostResponse in CallHostFunction. --- custom_host_call.go | 2 +- fetch_chunks.go | 30 ++++++++++++++++++++++-------- fetch_chunks_test.go | 33 +++++++++++++++++++++++++++++++++ host_limits.go | 10 +++++++++- host_limits_test.go | 2 +- wasm_exports.go | 3 ++- 6 files changed, 68 insertions(+), 12 deletions(-) diff --git a/custom_host_call.go b/custom_host_call.go index 6aa117b..09c64ea 100644 --- a/custom_host_call.go +++ b/custom_host_call.go @@ -45,7 +45,7 @@ func CallHostFunction(hostFn CustomHostImport, request any, response any) error resPtr := int32(uint32(outputBuf[0]) | uint32(outputBuf[1])<<8 | uint32(outputBuf[2])<<16 | uint32(outputBuf[3])<<24) resLen := int32(uint32(outputBuf[4]) | uint32(outputBuf[5])<<8 | uint32(outputBuf[6])<<16 | uint32(outputBuf[7])<<24) if resLen == 0 { - return fmt.Errorf("plugin: custom host call: empty response from host") + return errEmptyHostResponse("custom host call") } resBytes := append([]byte(nil), wasmSlice(resPtr, resLen)...) wasmResetAllocator() diff --git a/fetch_chunks.go b/fetch_chunks.go index 44d6f15..7be020e 100644 --- a/fetch_chunks.go +++ b/fetch_chunks.go @@ -3,6 +3,7 @@ package plugin import ( "errors" "fmt" + "strings" ) // A fetch response whose envelope does not fit the plugin's host buffer is @@ -19,6 +20,10 @@ const ( fetchChunkSize = 256 * 1024 // minFetchChunkSize is where retries stop shrinking the chunk. minFetchChunkSize = 4 * 1024 + // maxFetchBodyBytes caps the body size a host may announce, so a bad + // body_size becomes an error instead of an allocation that traps the + // guest. It sits above the host's own 50 MiB per-response cap. + maxFetchBodyBytes = 64 << 20 ) // fetchEnvelope is the paca.fetch response: a [FetchResponse], plus the @@ -45,26 +50,35 @@ type chunkReader func(handle string, offset, limit int) (*fetchChunk, error) // the host buffer it halves the chunk size and retries, down to // minFetchChunkSize, so a plugin whose buffer is partly used still succeeds. func readBodyChunks(handle string, size int, read chunkReader) (string, error) { - body := make([]byte, 0, size) + if size < 0 || size > maxFetchBodyBytes { + return "", fmt.Errorf("plugin: fetch: %w: host announced a body of %d bytes (limit %d)", ErrResponseTooLarge, size, maxFetchBodyBytes) + } + var body strings.Builder + body.Grow(size) limit := fetchChunkSize for { - c, err := read(handle, len(body), limit) + c, err := read(handle, body.Len(), limit) if err != nil { - if errors.Is(err, ErrResponseTooLarge) && limit > minFetchChunkSize { + // A reply that did not fit, or no data at all (an exhausted + // buffer on a host that cannot say so), may succeed smaller. + if (errors.Is(err, ErrResponseTooLarge) || errors.Is(err, errHostNoData)) && limit > minFetchChunkSize { limit /= 2 continue } return "", err } - body = append(body, c.Chunk...) + if body.Len()+len(c.Chunk) > size { + return "", fmt.Errorf("plugin: fetch: host sent more than the announced %d bytes", size) + } + body.Write(c.Chunk) if c.Done { - if len(body) != size { - return "", fmt.Errorf("plugin: fetch: body is %d bytes, host announced %d", len(body), size) + if body.Len() != size { + return "", fmt.Errorf("plugin: fetch: body is %d bytes, host announced %d", body.Len(), size) } - return string(body), nil + return body.String(), nil } if len(c.Chunk) == 0 { - return "", fmt.Errorf("plugin: fetch: host returned an empty chunk at offset %d of %d", len(body), size) + return "", fmt.Errorf("plugin: fetch: host returned an empty chunk at offset %d of %d", body.Len(), size) } } } diff --git a/fetch_chunks_test.go b/fetch_chunks_test.go index 4ed687f..5581541 100644 --- a/fetch_chunks_test.go +++ b/fetch_chunks_test.go @@ -86,3 +86,36 @@ func TestFetchWireShapes(t *testing.T) { t.Fatalf("envelope decode: %+v", e) } } + +func TestReadBodyChunks_RejectsBadAnnouncedSize(t *testing.T) { + never := func(string, int, int) (*fetchChunk, error) { t.Fatal("must not read"); return nil, nil } + for _, size := range []int{-1, maxFetchBodyBytes + 1} { + if _, err := readBodyChunks("h", size, never); !errors.Is(err, ErrResponseTooLarge) { + t.Errorf("size %d: err = %v, want ErrResponseTooLarge", size, err) + } + } +} + +func TestReadBodyChunks_RejectsOverrun(t *testing.T) { + over := func(string, int, int) (*fetchChunk, error) { return &fetchChunk{Chunk: []byte("abcdef")}, nil } + if _, err := readBodyChunks("h", 5, over); err == nil || !strings.Contains(err.Error(), "more than the announced") { + t.Errorf("overrun must be reported: %v", err) + } +} + +func TestReadBodyChunks_ShrinksOnNoData(t *testing.T) { + body := strings.Repeat("x", 50_000) + var calls []int + read := func(_ string, offset, limit int) (*fetchChunk, error) { + calls = append(calls, limit) + if limit > 20_000 { + return nil, errEmptyHostResponse("fetch") + } + end := min(offset+limit, len(body)) + return &fetchChunk{Chunk: []byte(body[offset:end]), Done: end == len(body)}, nil + } + got, err := readBodyChunks("h", len(body), read) + if err != nil || got != body { + t.Fatalf("err=%v, equal=%v, calls=%v", err, got == body, calls) + } +} diff --git a/host_limits.go b/host_limits.go index 8a3ac2d..0f17c21 100644 --- a/host_limits.go +++ b/host_limits.go @@ -3,6 +3,7 @@ package plugin import ( "errors" "fmt" + "strings" ) // ErrResponseTooLarge is returned (wrapped) when the host produced a result @@ -21,16 +22,23 @@ const hostCodeResponseTooLarge = "response_too_large" // hostResultError builds the error for a host result envelope that carried // an error message (and optionally a machine-readable code). func hostResultError(op, msg, code string) error { + // Host messages already carry the operation prefix ("fetch: ..."); drop it + // rather than printing "plugin: fetch: fetch: ...". + msg = strings.TrimPrefix(msg, op+": ") if code == hostCodeResponseTooLarge { return fmt.Errorf("plugin: %s: %w: %s", op, ErrResponseTooLarge, msg) } return fmt.Errorf("plugin: %s: %s", op, msg) } +// errHostNoData is wrapped by [errEmptyHostResponse] so internal callers can +// tell "no data" apart from other failures. +var errHostNoData = errors.New("host returned no data") + // errEmptyHostResponse is returned when a host call delivered no data at all. // Newer hosts always send an explicit error envelope instead, so reaching // this means an older host, an exhausted buffer, or a host-side failure -- // not necessarily an unreachable network. func errEmptyHostResponse(op string) error { - return fmt.Errorf("plugin: %s: host returned no data (the result may not have fit in the plugin's host buffer, or the host call failed; this does not necessarily mean the network is unreachable)", op) + return fmt.Errorf("plugin: %s: %w (the result may not have fit in the plugin's host buffer, or the host call failed; this does not necessarily mean the network is unreachable)", op, errHostNoData) } diff --git a/host_limits_test.go b/host_limits_test.go index ce14783..8e5bf6d 100644 --- a/host_limits_test.go +++ b/host_limits_test.go @@ -23,7 +23,7 @@ func TestHostResultError_PlainError(t *testing.T) { if errors.Is(err, ErrResponseTooLarge) { t.Fatalf("plain host error must not match ErrResponseTooLarge: %v", err) } - if got, want := err.Error(), "plugin: fetch: fetch: domain not permitted by plugin manifest"; got != want { + if got, want := err.Error(), "plugin: fetch: domain not permitted by plugin manifest"; got != want { t.Errorf("got %q, want %q", got, want) } } diff --git a/wasm_exports.go b/wasm_exports.go index 795a8c5..0aec4fc 100644 --- a/wasm_exports.go +++ b/wasm_exports.go @@ -94,7 +94,8 @@ func RunAction(ptr, length int32) int64 { // export call replaces it. The host reads a result from whatever linear-memory // address it is given, so a result does not have to fit the fixed arena the host // allocates request data in; the Go heap does not move objects, so the address -// stays valid until then. +// stays valid until then. This relies on the runtime's GC being non-moving (true +// of Go and TinyGo); a moving collector would invalidate the address handed out. var resultBuf []byte // packWASMResult returns result as the packed (ptr<<32)|len pair the host