From 2899388794267cab53329e190b6b6d2995da2ec1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 27 Sep 2026 01:26:54 +0800 Subject: [PATCH] review: sandbox on encoding/json/v2 Every module now calls encoding/json/v2 and jsontext. The one v1 import supplies DefaultOptionsV1 for the two digests. Kept as before: - ClusterDigest and the pools.json seed hash marshal with the v1 option set, so a rolling upgrade keeps the mesh digest and the warm pools; golden tests pin the v1 bytes. - Request bodies and MCP input match member names case-insensitively. - Guest bytes that are not valid UTF-8 come out as U+FFFD in sandboxd responses, the usage and audit journals, and MCP replies; MCP input with invalid UTF-8 or a lone surrogate is decoded the same way instead of being dropped without a reply. - MCP replies sort map keys, so tools/list is byte-stable across calls and sessions. - A renew without a body takes the node default. - A nil stdin or upload chunk encodes as "", now the v2 default, so wire.B64 is gone. - The MCP ping reply keeps its empty result object (omitzero). Changed on purpose: - Nil slices and maps encode as [] and {}. silkd's Vec fields accept both; a git add or replace with an empty list is now a no-op instead of bad_request, as an explicit empty list already was. - Map keys are no longer sorted in sandboxd responses, journals and gossip; the digests and MCP replies still sort them. - A request body with invalid UTF-8 is 400, and so is one with data after the value, except renew's optional body, which stops at the first value. - Case-insensitive matching also ignores - and _, as DecodeStrictJSON already did. - JSON responses no longer end with a newline. - sdk/go returns an error for an argv or path that is not valid UTF-8 instead of sending U+FFFD. Request bodies decode through UnmarshalRead and responses encode through MarshalWrite: a claim body takes 670 ns and 6 allocs (v1 1020 ns, 11), a response 342 ns and 2 allocs (v1 323 ns, 2). --- e2e/cmd/browsersmoke/main.go | 2 +- e2e/cmd/envdsmoke/main.go | 5 +- e2e/cmd/qaab/main.go | 2 +- e2e/cmd/smoke/main.go | 13 ++-- e2e/e2e_test.go | 2 +- mcp/node_info_test.go | 4 +- mcp/server.go | 33 +++++----- mcp/server_test.go | 66 ++++++++++++++++--- mcp/tools.go | 59 +++++++++-------- protocol/wire/frame.go | 53 +++++++--------- protocol/wire/frame_bench_test.go | 2 +- protocol/wire/frame_test.go | 2 +- sandboxd/config/config.go | 11 ++-- sandboxd/config/config_test.go | 21 +++++++ sandboxd/engine/engine.go | 2 +- sandboxd/engine/installca_test.go | 2 +- sandboxd/engine/portconn.go | 2 +- sandboxd/engine/portconn_test.go | 4 +- sandboxd/log_test.go | 4 +- sandboxd/mesh/mesh.go | 2 +- sandboxd/pool/checkpoint.go | 2 +- sandboxd/pool/checkpoint_heal_test.go | 2 +- sandboxd/pool/checkpoint_test.go | 2 +- sandboxd/pool/claims.go | 2 +- sandboxd/pool/journal.go | 5 +- sandboxd/pool/journal_test.go | 14 +++++ sandboxd/pool/pool.go | 2 +- sandboxd/pool/poolstore.go | 4 +- sandboxd/pool/poolstore_test.go | 15 ++++- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/telemetry.go | 2 +- sandboxd/pool/telemetry_test.go | 2 +- sandboxd/pool/template.go | 2 +- sandboxd/pool/volume_rw_test.go | 2 +- sandboxd/pool/volume_test.go | 2 +- sandboxd/server/client_endpoint_test.go | 5 +- sandboxd/server/exec_test.go | 25 +++++++- sandboxd/server/preview.go | 2 +- sandboxd/server/server_http.go | 9 +-- sandboxd/server/server_test.go | 84 +++++++++++++++++-------- sandboxd/utils/utils.go | 8 ++- sdk/go/checkpoint_test.go | 24 +++---- sdk/go/client.go | 8 +-- sdk/go/client_test.go | 58 ++++++++--------- sdk/go/drain_test.go | 4 +- sdk/go/info_test.go | 4 +- sdk/go/pools_test.go | 14 ++--- sdk/go/proc_test.go | 2 +- sdk/go/template_test.go | 4 +- sdk/go/volumes_test.go | 34 +++++----- 50 files changed, 395 insertions(+), 241 deletions(-) diff --git a/e2e/cmd/browsersmoke/main.go b/e2e/cmd/browsersmoke/main.go index be53fab5..fd8405da 100644 --- a/e2e/cmd/browsersmoke/main.go +++ b/e2e/cmd/browsersmoke/main.go @@ -5,7 +5,7 @@ package main import ( "context" - "encoding/json" + "encoding/json/v2" "flag" "fmt" "os" diff --git a/e2e/cmd/envdsmoke/main.go b/e2e/cmd/envdsmoke/main.go index fa363685..4d052038 100644 --- a/e2e/cmd/envdsmoke/main.go +++ b/e2e/cmd/envdsmoke/main.go @@ -13,7 +13,8 @@ import ( "context" "encoding/base64" "encoding/binary" - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "flag" "fmt" "io" @@ -274,7 +275,7 @@ func readEnvelopes(r io.Reader) ([]string, error) { continue } var end struct { - Error json.RawMessage `json:"error"` + Error jsontext.Value `json:"error"` } if err := json.Unmarshal(body, &end); err != nil { return out, fmt.Errorf("parse end-of-stream envelope %q: %w", body, err) diff --git a/e2e/cmd/qaab/main.go b/e2e/cmd/qaab/main.go index 4304038d..2e0ea159 100644 --- a/e2e/cmd/qaab/main.go +++ b/e2e/cmd/qaab/main.go @@ -4,7 +4,7 @@ package main import ( "context" - "encoding/json" + "encoding/json/v2" "flag" "fmt" "os" diff --git a/e2e/cmd/smoke/main.go b/e2e/cmd/smoke/main.go index 84d61ee1..daba46e1 100644 --- a/e2e/cmd/smoke/main.go +++ b/e2e/cmd/smoke/main.go @@ -9,7 +9,8 @@ import ( "bufio" "bytes" "context" - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "errors" "flag" "fmt" @@ -709,7 +710,7 @@ func lspWrite(w io.Writer, body string) error { // lspReadResponse reads framed server messages until the response carrying // id arrives (server-initiated requests also carry ids but have a method; // notifications have no id — both are skipped), returning its result. -func lspReadResponse(r *bufio.Reader, id int) (json.RawMessage, error) { +func lspReadResponse(r *bufio.Reader, id int) (jsontext.Value, error) { tp := textproto.NewReader(r) for { hdr, err := tp.ReadMIMEHeader() @@ -725,10 +726,10 @@ func lspReadResponse(r *bufio.Reader, id int) (json.RawMessage, error) { return nil, err } var msg struct { - ID *int `json:"id"` - Method string `json:"method"` - Result json.RawMessage `json:"result"` - Error json.RawMessage `json:"error"` + ID *int `json:"id"` + Method string `json:"method"` + Result jsontext.Value `json:"result"` + Error jsontext.Value `json:"error"` } if err := json.Unmarshal(body, &msg); err != nil { return nil, fmt.Errorf("lsp frame %q: %w", body, err) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 40819efa..6eafd4ba 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -5,7 +5,7 @@ package e2e import ( - "encoding/json" + "encoding/json/v2" "errors" "fmt" "io" diff --git a/mcp/node_info_test.go b/mcp/node_info_test.go index 9aac61c3..562a41af 100644 --- a/mcp/node_info_test.go +++ b/mcp/node_info_test.go @@ -1,7 +1,7 @@ package main import ( - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "strings" @@ -10,7 +10,7 @@ import ( func TestNodeInfoReportsCapacityState(t *testing.T) { node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(map[string]any{ + _ = json.MarshalWrite(w, map[string]any{ "pools": []any{}, "claimed": 2, "hibernated": 1, diff --git a/mcp/server.go b/mcp/server.go index 3e682aa8..50115d8a 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -3,7 +3,8 @@ package main import ( "bufio" "context" - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "errors" "fmt" "io" @@ -23,16 +24,16 @@ const ( ) type rpcRequest struct { - ID json.RawMessage `json:"id"` - Method string `json:"method"` - Params json.RawMessage `json:"params"` + ID jsontext.Value `json:"id"` + Method string `json:"method"` + Params jsontext.Value `json:"params"` } type rpcResponse struct { - JSONRPC string `json:"jsonrpc"` - ID json.RawMessage `json:"id"` - Result any `json:"result,omitempty"` - Error *rpcErrorBody `json:"error,omitempty"` + JSONRPC string `json:"jsonrpc"` + ID jsontext.Value `json:"id"` + Result any `json:"result,omitzero"` + Error *rpcErrorBody `json:"error,omitempty"` } type rpcErrorBody struct { @@ -85,14 +86,14 @@ func (s *server) serve(ctx context.Context, r *bufio.Reader, w io.Writer) error return err } var req rpcRequest - if unmarshalErr := json.Unmarshal(line, &req); unmarshalErr != nil || req.Method == "" { + if unmarshalErr := unmarshalInput(line, &req); unmarshalErr != nil || req.Method == "" { continue } if req.ID == nil { continue // notification (e.g. notifications/initialized) } resp := s.dispatch(ctx, &req) - out, err := json.Marshal(resp) + out, err := json.Marshal(resp, jsontext.AllowInvalidUTF8(true), json.Deterministic(true)) if err != nil { return err } @@ -114,10 +115,10 @@ func (s *server) dispatch(ctx context.Context, req *rpcRequest) rpcResponse { return result(req.ID, map[string]any{"tools": toolSpecs()}) case "tools/call": var call struct { - Name string `json:"name"` - Arguments json.RawMessage `json:"arguments"` + Name string `json:"name"` + Arguments jsontext.Value `json:"arguments"` } - if err := json.Unmarshal(req.Params, &call); err != nil { + if err := unmarshalInput(req.Params, &call); err != nil { return rpcError(req.ID, -32602, "invalid params") } text, err := s.callTool(ctx, call.Name, call.Arguments) @@ -140,7 +141,7 @@ func (s *server) dispatch(ctx context.Context, req *rpcRequest) rpcResponse { } } -func (s *server) callTool(ctx context.Context, name string, args json.RawMessage) (string, error) { +func (s *server) callTool(ctx context.Context, name string, args jsontext.Value) (string, error) { ctx, cancel := context.WithTimeout(ctx, execTimeout) defer cancel() i := slices.IndexFunc(tools, func(t tool) bool { return t.name == name }) @@ -215,10 +216,10 @@ func (s *server) dropCkpt(id string) { delete(s.ckpts, id) } -func result(id json.RawMessage, v any) rpcResponse { +func result(id jsontext.Value, v any) rpcResponse { return rpcResponse{JSONRPC: "2.0", ID: id, Result: v} } -func rpcError(id json.RawMessage, code int, msg string) rpcResponse { +func rpcError(id jsontext.Value, code int, msg string) rpcResponse { return rpcResponse{JSONRPC: "2.0", ID: id, Error: &rpcErrorBody{Code: code, Message: msg}} } diff --git a/mcp/server_test.go b/mcp/server_test.go index 6a399231..b391a7ff 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -4,11 +4,12 @@ import ( "bufio" "bytes" "encoding/base64" - "encoding/json" + "encoding/json/v2" "fmt" "io" "net/http" "net/http/httptest" + "slices" "strings" "sync" "sync/atomic" @@ -32,7 +33,7 @@ func TestServeSpeaksMCP(t *testing.T) { if !strings.HasSuffix(r.URL.Path, "/checkpoint") { return false } - _ = json.NewEncoder(w).Encode(map[string]any{ + _ = json.MarshalWrite(w, map[string]any{ "checkpoint": map[string]any{"id": "ck_0011223344556677", "name": "s1", "sandbox_id": "sb_1"}, }) return true @@ -45,9 +46,11 @@ func TestServeSpeaksMCP(t *testing.T) { `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"checkpoint","arguments":{"sandbox_id":"sb_1","name":"s1"}}}`, `{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"release","arguments":{"sandbox_id":"sb_1"}}}`, `{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"exec","arguments":{"sandbox_id":"sb_1","command":"true"}}}`, + `{"jsonrpc":"2.0","id":7,"method":"ping"}`, + `{"jsonrpc":"2.0","id":8,"method":"ping","params":{"note":"\ud800"}}`, ) - if len(replies) != 6 { - t.Fatalf("got %d replies, want 6 (notification unanswered)", len(replies)) + if len(replies) != 8 { + t.Fatalf("got %d replies, want 8 (notification unanswered)", len(replies)) } initResult := replies[1]["result"].(map[string]any) @@ -77,6 +80,27 @@ func TestServeSpeaksMCP(t *testing.T) { if !strings.Contains(toolText(t, replies[6]), "sb_1") { t.Errorf("exec error should name the released id: %q", toolText(t, replies[6])) } + if pong, ok := replies[7]["result"].(map[string]any); !ok || len(pong) != 0 { + t.Errorf("ping reply %+v, want an empty result object", replies[7]) + } + if _, ok := replies[8]["result"].(map[string]any); !ok { + t.Errorf("ping with a lone surrogate %+v, want a reply", replies[8]) + } +} + +func TestRepliesEncodeMapsInSortedOrder(t *testing.T) { + var out bytes.Buffer + in := bufio.NewReader(strings.NewReader(strings.Repeat(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}`+"\n", 20))) + if err := newTestServer(t, nil).serve(t.Context(), in, &out); err != nil { + t.Fatalf("serve: %v", err) + } + lines := slices.Collect(bytes.Lines(out.Bytes())) + if len(lines) != 20 || slices.ContainsFunc(lines, func(l []byte) bool { return !bytes.Equal(l, lines[0]) }) { + t.Errorf("20 tools/list replies are not byte-identical (%d lines)", len(lines)) + } + if got := jsonText(map[string]int{"b": 2, "a": 1, "d": 4, "c": 3, "f": 6, "e": 5, "h": 8, "g": 7}); got != `{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7,"h":8}` { + t.Errorf("jsonText = %s, want sorted keys", got) + } } func TestExecKeepsOutputWhenTheGuestDrops(t *testing.T) { @@ -125,6 +149,31 @@ func TestLogsKeepsAtMostTheCap(t *testing.T) { } } +func TestToolRepliesReplaceInvalidUTF8(t *testing.T) { + lossy := base64.StdEncoding.EncodeToString([]byte("a\xffb")) + srv := newTestServer(t, agentRoute(t, func(req string) (string, bool) { + switch { + case strings.Contains(req, `"op":"fs_stat"`): + return `{"type":"stat","info":{"kind":"file","size":3}}` + "\n", false + case strings.Contains(req, `"op":"logs"`): + return `{"type":"stdout","data":"` + lossy + `"}` + "\n" + `{"type":"done"}` + "\n", false + } + return `{"type":"data","data":"` + lossy + `"}` + "\n" + `{"type":"done"}` + "\n", false + })) + replies := serveLines(t, srv, + `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"create_sandbox","arguments":{}}}`, + `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"read_file","arguments":{"sandbox_id":"sb_1","path":"/bin/x"}}}`, + `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"logs","arguments":{"sandbox_id":"sb_1","pid":7}}}`, + ) + if text := toolText(t, replies[2]); text != "a\uFFFDb" { + t.Errorf("read_file answered %q, want the invalid byte replaced", text) + } + var logs map[string]any + if err := json.Unmarshal([]byte(toolText(t, replies[3])), &logs); err != nil || logs["stdout"] != "a\uFFFDb" { + t.Errorf("logs answered %v (%v), want JSON with the invalid byte replaced", logs, err) + } +} + func TestTrackBoxAfterCloseReleasesTheClaim(t *testing.T) { released := make(chan string, 1) node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -230,7 +279,7 @@ func newTestServer(t *testing.T, extra func(http.ResponseWriter, *http.Request) node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/v1/claim": - _ = json.NewEncoder(w).Encode(map[string]any{"id": "sb_1", "token": "tok"}) + _ = json.MarshalWrite(w, map[string]any{"id": "sb_1", "token": "tok"}) case strings.HasSuffix(r.URL.Path, "/release"): w.WriteHeader(http.StatusNoContent) case extra != nil && extra(w, r): @@ -283,11 +332,10 @@ func serveLines(t *testing.T, srv *server, lines ...string) map[int]map[string]a t.Fatalf("serve: %v", err) } replies := map[int]map[string]any{} - dec := json.NewDecoder(&out) - for dec.More() { + for line := range bytes.Lines(out.Bytes()) { var resp map[string]any - if err := dec.Decode(&resp); err != nil { - t.Fatalf("decode reply: %v", err) + if err := json.Unmarshal(line, &resp); err != nil { + t.Fatalf("decode reply %q: %v", line, err) } replies[int(resp["id"].(float64))] = resp } diff --git a/mcp/tools.go b/mcp/tools.go index 039898de..6397d190 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -3,7 +3,8 @@ package main import ( "cmp" "context" - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "errors" "fmt" "strings" @@ -85,7 +86,7 @@ type tool struct { name string description string schema map[string]any - handler func(context.Context, *server, json.RawMessage) (string, error) + handler func(context.Context, *server, jsontext.Value) (string, error) } func toolSpecs() []map[string]any { @@ -96,7 +97,7 @@ func toolSpecs() []map[string]any { return specs } -func toolCreateSandbox(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolCreateSandbox(ctx context.Context, s *server, raw jsontext.Value) (string, error) { var args struct { Template string `json:"template"` Net string `json:"net"` @@ -135,7 +136,7 @@ func (a sandboxArg) id() string { return a.SandboxID } type sandboxIDer interface{ id() string } // parseAndBox decodes a tool's arguments and resolves their sandbox_id to a live handle, the prologue every sandbox-scoped tool shares. -func parseAndBox[T sandboxIDer](s *server, raw json.RawMessage) (T, *sandbox.Sandbox, error) { +func parseAndBox[T sandboxIDer](s *server, raw jsontext.Value) (T, *sandbox.Sandbox, error) { var args T if err := parse(raw, &args); err != nil { return args, nil, err @@ -150,7 +151,7 @@ type cmdArgs struct { Cwd string `json:"cwd"` } -func parseCommand(s *server, raw json.RawMessage) (cmdArgs, *sandbox.Sandbox, error) { +func parseCommand(s *server, raw jsontext.Value) (cmdArgs, *sandbox.Sandbox, error) { args, sb, err := parseAndBox[cmdArgs](s, raw) if err == nil && args.Command == "" { return args, nil, errors.New("command must not be empty") @@ -177,7 +178,7 @@ func (o *cappedOutput) Write(p []byte) (int, error) { return n, nil } -func toolExec(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolExec(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseCommand(s, raw) if err != nil { return "", err @@ -199,7 +200,7 @@ func toolExec(ctx context.Context, s *server, raw json.RawMessage) (string, erro return jsonText(out), err } -func toolSpawn(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolSpawn(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseCommand(s, raw) if err != nil { return "", err @@ -211,7 +212,7 @@ func toolSpawn(ctx context.Context, s *server, raw json.RawMessage) (string, err return jsonText(map[string]uint32{"pid": pid}), nil } -func toolPs(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolPs(ctx context.Context, s *server, raw jsontext.Value) (string, error) { sb, err := s.boxArg(raw) if err != nil { return "", err @@ -229,7 +230,7 @@ type killArgs struct { Signal int32 `json:"signal"` } -func toolKill(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolKill(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[killArgs](s, raw) if err != nil { return "", err @@ -245,7 +246,7 @@ type logsArgs struct { PID uint32 `json:"pid"` } -func toolLogs(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolLogs(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[logsArgs](s, raw) if err != nil { return "", err @@ -268,7 +269,7 @@ type writeFileArgs struct { Content string `json:"content"` } -func toolWriteFile(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolWriteFile(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[writeFileArgs](s, raw) if err != nil { return "", err @@ -284,7 +285,7 @@ type pathArgs struct { Path string `json:"path"` } -func toolReadFile(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolReadFile(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[pathArgs](s, raw) if err != nil { return "", err @@ -306,7 +307,7 @@ func toolReadFile(ctx context.Context, s *server, raw json.RawMessage) (string, return out.String(), nil } -func toolListDir(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolListDir(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[pathArgs](s, raw) if err != nil { return "", err @@ -323,7 +324,7 @@ type forkArgs struct { Count int `json:"count"` } -func toolFork(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolFork(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[forkArgs](s, raw) if err != nil { return "", err @@ -345,7 +346,7 @@ type checkpointArgs struct { Name string `json:"name"` } -func toolCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolCheckpoint(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[checkpointArgs](s, raw) if err != nil { return "", err @@ -358,7 +359,7 @@ func toolCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string return jsonText(map[string]any{"checkpoint_id": ckpt.ID, "name": ckpt.Name}), nil } -func toolBranchCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolBranchCheckpoint(ctx context.Context, s *server, raw jsontext.Value) (string, error) { ckpt, err := s.checkpointArg(raw) if err != nil { return "", err @@ -371,7 +372,7 @@ func toolBranchCheckpoint(ctx context.Context, s *server, raw json.RawMessage) ( return jsonText(map[string]any{"sandbox_id": sb.ID}), nil } -func toolListCheckpoints(ctx context.Context, s *server, _ json.RawMessage) (string, error) { +func toolListCheckpoints(ctx context.Context, s *server, _ jsontext.Value) (string, error) { ckpts, err := s.client.Checkpoints(ctx) if err != nil { return "", err @@ -383,7 +384,7 @@ func toolListCheckpoints(ctx context.Context, s *server, _ json.RawMessage) (str return jsonText(out), nil } -func toolDeleteCheckpoint(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolDeleteCheckpoint(ctx context.Context, s *server, raw jsontext.Value) (string, error) { ckpt, err := s.checkpointArg(raw) if err != nil { return "", err @@ -395,7 +396,7 @@ func toolDeleteCheckpoint(ctx context.Context, s *server, raw json.RawMessage) ( return "deleted", nil } -func toolHibernate(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolHibernate(ctx context.Context, s *server, raw jsontext.Value) (string, error) { sb, err := s.boxArg(raw) if err != nil { return "", err @@ -411,7 +412,7 @@ type promoteArgs struct { TemplateName string `json:"template_name"` } -func toolPromote(ctx context.Context, s *server, raw json.RawMessage) (string, error) { +func toolPromote(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseAndBox[promoteArgs](s, raw) if err != nil { return "", err @@ -423,7 +424,7 @@ func toolPromote(ctx context.Context, s *server, raw json.RawMessage) (string, e return jsonText(map[string]any{"template": tpl.Name}), nil } -func toolRelease(_ context.Context, s *server, raw json.RawMessage) (string, error) { +func toolRelease(_ context.Context, s *server, raw jsontext.Value) (string, error) { sb, err := s.boxArg(raw) if err != nil { return "", err @@ -435,7 +436,7 @@ func toolRelease(_ context.Context, s *server, raw json.RawMessage) (string, err return "released", nil } -func toolNodeInfo(ctx context.Context, s *server, _ json.RawMessage) (string, error) { +func toolNodeInfo(ctx context.Context, s *server, _ jsontext.Value) (string, error) { info, err := s.client.Info(ctx) if err != nil { return "", err @@ -443,13 +444,13 @@ func toolNodeInfo(ctx context.Context, s *server, _ json.RawMessage) (string, er return jsonText(info), nil } -func (s *server) boxArg(raw json.RawMessage) (*sandbox.Sandbox, error) { +func (s *server) boxArg(raw jsontext.Value) (*sandbox.Sandbox, error) { _, sb, err := parseAndBox[sandboxArg](s, raw) return sb, err } // checkpointArg resolves a checkpoint_id argument: a handle minted in this session when available, else a fresh one, because checkpoints outlive sessions. -func (s *server) checkpointArg(raw json.RawMessage) (*sandbox.Checkpoint, error) { +func (s *server) checkpointArg(raw jsontext.Value) (*sandbox.Checkpoint, error) { var args struct { CheckpointID string `json:"checkpoint_id"` } @@ -465,18 +466,22 @@ func (s *server) checkpointArg(raw json.RawMessage) (*sandbox.Checkpoint, error) return s.client.Checkpoint(args.CheckpointID), nil } -func parse(raw json.RawMessage, v any) error { +func parse(raw jsontext.Value, v any) error { if len(raw) == 0 { return nil } - if err := json.Unmarshal(raw, v); err != nil { + if err := unmarshalInput(raw, v); err != nil { return fmt.Errorf("invalid tool arguments: %w", err) } return nil } +func unmarshalInput(raw []byte, v any) error { + return json.Unmarshal(raw, v, json.MatchCaseInsensitiveNames(true), jsontext.AllowInvalidUTF8(true)) +} + func jsonText(v any) string { - out, err := json.Marshal(v) + out, err := json.Marshal(v, jsontext.AllowInvalidUTF8(true), json.Deterministic(true)) if err != nil { return fmt.Sprintf("%+v", v) } diff --git a/protocol/wire/frame.go b/protocol/wire/frame.go index c6e1e549..783338dd 100644 --- a/protocol/wire/frame.go +++ b/protocol/wire/frame.go @@ -10,7 +10,8 @@ import ( "bufio" "bytes" "encoding/base64" - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "fmt" "io" "strconv" @@ -149,16 +150,6 @@ type respPtr[T any] interface { Response } -// B64 carries request payload bytes; a nil slice marshals as "" because silkd rejects null. -type B64 []byte - -func (b B64) MarshalJSON() ([]byte, error) { - if b == nil { - b = B64{} - } - return json.Marshal([]byte(b)) -} - // Exec starts a process; with Session set it runs inside that persistent // shell instead. Detach is emitted even when false — it is part of the // fixture corpus shape. @@ -228,7 +219,7 @@ func (SessionRm) Op() string { return "session_rm" } // Stdin carries a chunk of exec stdin. type Stdin struct { - Data B64 `json:"data"` + Data []byte `json:"data"` } func (Stdin) Op() string { return "stdin" } @@ -450,7 +441,7 @@ func (LspStop) Op() string { return "lsp_stop" } // Data carries one chunk of an upload stream (FsWrite/FsPush payloads). type Data struct { - Data B64 `json:"data"` + Data []byte `json:"data"` } func (Data) Op() string { return "data" } @@ -773,27 +764,26 @@ func frameTag(line, head []byte, key string) ([]byte, error) { // scanTag walks the top-level keys without decoding the frame; a missing tag returns "" for the callers' unknown-tag error. func scanTag(line []byte, key string) (string, error) { - dec := json.NewDecoder(bytes.NewReader(line)) - tok, err := dec.Token() + dec := jsontext.NewDecoder(bytes.NewReader(line)) + tok, err := dec.ReadToken() if err != nil { return "", err } - if d, ok := tok.(json.Delim); !ok || d != '{' { + if tok.Kind() != '{' { return "", fmt.Errorf("frame is not an object") } - for dec.More() { - if tok, err = dec.Token(); err != nil { + for dec.PeekKind() != '}' { + if tok, err = dec.ReadToken(); err != nil { return "", err } - if k, ok := tok.(string); ok && k == key { - if tok, err = dec.Token(); err != nil { + if tok.String() == key { + if tok, err = dec.ReadToken(); err != nil { return "", err } - s, ok := tok.(string) - if !ok { + if tok.Kind() != '"' { return "", fmt.Errorf("%s tag is not a string", key) } - return s, nil + return tok.String(), nil } if err = skipValue(dec); err != nil { return "", err @@ -803,21 +793,20 @@ func scanTag(line []byte, key string) (string, error) { } // skipValue consumes one JSON value; a guest cannot grow the host stack with nesting. -func skipValue(dec *json.Decoder) error { +func skipValue(dec *jsontext.Decoder) error { depth := 0 for { - tok, err := dec.Token() + tok, err := dec.ReadToken() if err != nil { return err } - if d, ok := tok.(json.Delim); ok { - if d == '{' || d == '[' { - if depth++; depth > maxSkipDepth { - return fmt.Errorf("value nested deeper than %d", maxSkipDepth) - } - } else { - depth-- + switch tok.Kind() { + case '{', '[': + if depth++; depth > maxSkipDepth { + return fmt.Errorf("value nested deeper than %d", maxSkipDepth) } + case '}', ']': + depth-- } if depth == 0 { return nil diff --git a/protocol/wire/frame_bench_test.go b/protocol/wire/frame_bench_test.go index 470104d1..d3017289 100644 --- a/protocol/wire/frame_bench_test.go +++ b/protocol/wire/frame_bench_test.go @@ -1,7 +1,7 @@ package wire import ( - "encoding/json" + "encoding/json/v2" "fmt" "testing" ) diff --git a/protocol/wire/frame_test.go b/protocol/wire/frame_test.go index b7df54d2..d45c1141 100644 --- a/protocol/wire/frame_test.go +++ b/protocol/wire/frame_test.go @@ -2,7 +2,7 @@ package wire import ( "bytes" - "encoding/json" + "encoding/json/v2" "os" "path/filepath" "reflect" diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index b7c4046b..ff007336 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -7,9 +7,8 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" - "encoding/json" "encoding/json/jsontext" - jsonv2 "encoding/json/v2" + "encoding/json/v2" "fmt" "net" "net/netip" @@ -60,13 +59,13 @@ func (s *PoolSpec) UnmarshalJSONFrom(dec *jsontext.Decoder) error { return err } type plain PoolSpec - if err := jsonv2.Unmarshal(raw, (*plain)(s), dec.Options()); err != nil { + if err := json.Unmarshal(raw, (*plain)(s), dec.Options()); err != nil { return err } var probe struct { Warm *int `json:"warm"` } - _ = jsonv2.Unmarshal(raw, &probe, jsonv2.MatchCaseInsensitiveNames(true)) + _ = json.Unmarshal(raw, &probe, json.MatchCaseInsensitiveNames(true)) s.warmSet = probe.Warm != nil return nil } @@ -235,13 +234,13 @@ func (c *Config) ClusterDigest(caFingerprint string) string { tenants[i] = auth{t.Name, t.Token} } slices.SortFunc(tenants, func(a, b auth) int { return strings.Compare(a.Name, b.Name) }) - raw, _ := json.Marshal([]any{c.APIToken, c.PreviewSecret, caFingerprint, tenants, c.CheckpointTTLHours}) + raw, _ := utils.DigestJSON([]any{c.APIToken, c.PreviewSecret, caFingerprint, tenants, c.CheckpointTTLHours}) mac := hmac.New(sha256.New, key) mac.Write(raw) return hex.EncodeToString(mac.Sum(nil)) } } - raw, _ := json.Marshal([]any{caFingerprint, names, c.CheckpointTTLHours}) + raw, _ := utils.DigestJSON([]any{caFingerprint, names, c.CheckpointTTLHours}) sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:]) } diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 304955de..bbe6905d 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -41,6 +41,27 @@ func TestClusterDigest(t *testing.T) { } } +func TestClusterDigestMatchesTheV1Bytes(t *testing.T) { + key := base64.StdEncoding.EncodeToString(make([]byte, 32)) + tenants := []TenantSpec{{Name: "acme", Token: "t<1>&\u2028"}, {Name: "beta", Token: "t2"}} + for _, tt := range []struct { + name string + cfg *Config + fp string + want string + }{ + {"keyless", &Config{APIToken: "tk", PreviewSecret: "p&s", Tenants: tenants, CheckpointTTLHours: 24}, "ca<&>\u2028fp", "a1945e2f38cf6e61638bc08896a292792b1705fb860b405d8117abbf13e1c760"}, + {"keyed", &Config{APIToken: "tk&\u2029", PreviewSecret: "p&s<", Tenants: tenants, CheckpointTTLHours: 24, Mesh: &MeshConfig{ClusterKey: key}}, "ca<&>fp", "5146d3799789afbd1d1bff606d238ea75374dec8f11f5b69edd6b9fcc40164b7"}, + {"no tenants", &Config{}, "fp", "38fc616b12f612c2c5c3f83af9d4c6caa771e1d3ffca39a6c835e522d3db49f8"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := tt.cfg.ClusterDigest(tt.fp); got != tt.want { + t.Errorf("digest %s, want the encoding/json v1 digest %s", got, tt.want) + } + }) + } +} + func TestLoadAppliesDefaults(t *testing.T) { path := writeConfig(t, `{"pools":[{"template":"rt:24.04","net":"none","size":"small"}]}`) cfg, err := Load(path) diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 27d4e1ec..a509038a 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -7,7 +7,7 @@ import ( "bufio" "bytes" "context" - "encoding/json" + "encoding/json/v2" "fmt" "hash/fnv" "io" diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index c9589499..307dfef7 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -3,7 +3,7 @@ package engine import ( "bufio" "bytes" - "encoding/json" + "encoding/json/v2" "fmt" "io" "net" diff --git a/sandboxd/engine/portconn.go b/sandboxd/engine/portconn.go index ca88ec55..59122937 100644 --- a/sandboxd/engine/portconn.go +++ b/sandboxd/engine/portconn.go @@ -4,7 +4,7 @@ import ( "bufio" "bytes" "encoding/base64" - "encoding/json" + "encoding/json/v2" "fmt" "io" "net" diff --git a/sandboxd/engine/portconn_test.go b/sandboxd/engine/portconn_test.go index b277a41d..2a8f8822 100644 --- a/sandboxd/engine/portconn_test.go +++ b/sandboxd/engine/portconn_test.go @@ -4,7 +4,7 @@ import ( "bufio" "bytes" "encoding/base64" - "encoding/json" + "encoding/json/v2" "io" "net" "os" @@ -18,7 +18,7 @@ func TestGuestPortConnFramesBothWays(t *testing.T) { silk, gc := newTestGuestPortConn(t) go func() { - frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("HTTP/1.1 200 OK"))}) + frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("HTTP/1.1 200 OK"))}, json.Deterministic(true)) _, _ = silk.Write(append(frame, '\n')) _, _ = silk.Write([]byte(`{"type":"done"}` + "\n")) }() diff --git a/sandboxd/log_test.go b/sandboxd/log_test.go index 77054e6c..b1fbe593 100644 --- a/sandboxd/log_test.go +++ b/sandboxd/log_test.go @@ -2,7 +2,7 @@ package main import ( "bytes" - "encoding/json" + "encoding/json/jsontext" "io" "strings" "testing" @@ -67,7 +67,7 @@ func TestJournalWriterKeepsJSONRecordsBehindThePrefix(t *testing.T) { logger := zerolog.New(newJournalWriter(counter, true)) logger.Error().Str("k", "v").Msg("boom") line := counter.buf.String() - if counter.writes != 1 || !strings.HasPrefix(line, syslogErr+"{") || !json.Valid([]byte(line[len(syslogErr):])) { + if counter.writes != 1 || !strings.HasPrefix(line, syslogErr+"{") || !jsontext.Value(line[len(syslogErr):]).IsValid() { t.Errorf("writes = %d, line = %q, want one write of %q and one JSON record", counter.writes, line, syslogErr) } } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 8a99d243..4b351e30 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -4,7 +4,7 @@ package mesh import ( "cmp" "context" - "encoding/json" + "encoding/json/v2" "fmt" "maps" "math/rand/v2" diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index da1e9f72..279ea98b 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "errors" "fmt" "maps" diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index 8f8f8e29..95d178ca 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "errors" "os" "path/filepath" diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index 15042051..73c26482 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "errors" "os" "path/filepath" diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index f9d75be9..de8ecb17 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -1,7 +1,7 @@ package pool import ( - "encoding/json" + "encoding/json/v2" "errors" "fmt" "io/fs" diff --git a/sandboxd/pool/journal.go b/sandboxd/pool/journal.go index fbbb31c4..5654d18c 100644 --- a/sandboxd/pool/journal.go +++ b/sandboxd/pool/journal.go @@ -1,7 +1,8 @@ package pool import ( - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "errors" "os" "sync" @@ -47,7 +48,7 @@ func newJournal(path string) (*journal, error) { } func (j *journal) append(v any) error { - line, err := json.Marshal(v) + line, err := json.Marshal(v, jsontext.AllowInvalidUTF8(true)) if err != nil { return err } diff --git a/sandboxd/pool/journal_test.go b/sandboxd/pool/journal_test.go index 21dbe347..017034f2 100644 --- a/sandboxd/pool/journal_test.go +++ b/sandboxd/pool/journal_test.go @@ -52,6 +52,20 @@ func TestJournalRotateFailureDegradesAndRecovers(t *testing.T) { } } +func TestJournalKeepsAnEventWithInvalidUTF8(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.jsonl") + j, err := newJournal(path) + if err != nil { + t.Fatalf("new journal: %v", err) + } + if err = j.append(map[string]string{"dest": "a\xffb"}); err != nil { + t.Fatalf("append: %v", err) + } + if got := tail(t, path); !strings.Contains(got, `"dest":"a`+"\uFFFD"+`b"`) { + t.Errorf("journal %q, want the event with the invalid byte as U+FFFD", got) + } +} + func TestJournalRotateReopenFailureKeepsWriting(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "usage.jsonl") diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 38077073..97eed437 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -8,7 +8,7 @@ import ( "context" "crypto/rand" "encoding/hex" - "encoding/json" + "encoding/json/v2" "errors" "fmt" "maps" diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index acd6a053..2bcf4fd7 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -4,7 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" + "encoding/json/v2" "errors" "fmt" "io/fs" @@ -112,7 +112,7 @@ func poolSeedHash(specs []config.PoolSpec) string { shaped[i].Warmup = nil } slices.SortFunc(shaped, func(a, b config.PoolSpec) int { return strings.Compare(a.Hash(), b.Hash()) }) - raw, _ := json.Marshal(shaped) + raw, _ := utils.DigestJSON(shaped) sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:]) } diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index e6fe4b79..18963502 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -1,7 +1,7 @@ package pool import ( - "encoding/json" + "encoding/json/v2" "os" "path/filepath" "sync" @@ -55,6 +55,19 @@ func TestDeletingPoolsFileRestoresConfigSeed(t *testing.T) { } } +func TestPoolSeedHashMatchesTheV1Bytes(t *testing.T) { + specs := []config.PoolSpec{ + {Template: "rt:24.04&", Net: types.NetNone, Size: "small", Warm: 2}, + {Template: "python:3.12", Net: types.NetEgress, Size: "large", WarmMax: 4}, + } + if got, want := poolSeedHash(nil), "74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b"; got != want { + t.Errorf("seed of no pools %s, want the encoding/json v1 seed %s", got, want) + } + if got, want := poolSeedHash(specs), "8f01958ec3e7ca20e71032d3c857073614887c045887d53a8e93776a2826aad7"; got != want { + t.Errorf("seed %s, want the encoding/json v1 seed %s", got, want) + } +} + func TestConcurrentSetPoolsPersistLatest(t *testing.T) { dir := t.TempDir() m := newTestManagerAt(t, newFakeEngine(), dir, config.PoolSpec{PoolKey: seedKey, Warm: 1}) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index bf4e83ed..a080764c 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "fmt" "os" "path/filepath" diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index b2e4ca59..4513179b 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "sync/atomic" "time" diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index 9ab3ac62..27a06ce6 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -1,7 +1,7 @@ package pool import ( - "encoding/json" + "encoding/json/v2" "errors" "os" "path/filepath" diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 273fe3b8..376206d4 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "errors" "fmt" "os" diff --git a/sandboxd/pool/volume_rw_test.go b/sandboxd/pool/volume_rw_test.go index 3ea6e5ee..a2ea2ecb 100644 --- a/sandboxd/pool/volume_rw_test.go +++ b/sandboxd/pool/volume_rw_test.go @@ -1,7 +1,7 @@ package pool import ( - "encoding/json" + "encoding/json/v2" "errors" "os" "path/filepath" diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 7a29e081..a323e20f 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -2,7 +2,7 @@ package pool import ( "context" - "encoding/json" + "encoding/json/v2" "errors" "os" "path/filepath" diff --git a/sandboxd/server/client_endpoint_test.go b/sandboxd/server/client_endpoint_test.go index 65aea71c..2927be52 100644 --- a/sandboxd/server/client_endpoint_test.go +++ b/sandboxd/server/client_endpoint_test.go @@ -1,7 +1,8 @@ package server import ( - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "net/http/httptest" "strings" "testing" @@ -23,7 +24,7 @@ func TestClientAddressesInResponses(t *testing.T) { r := httptest.NewRequest(tt.method, tt.path, strings.NewReader(tt.body)) w := httptest.NewRecorder() s.Handler().ServeHTTP(w, r) - var body map[string]json.RawMessage + var body map[string]jsontext.Value if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatal(err) } diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go index e54ae03f..ed6389de 100644 --- a/sandboxd/server/exec_test.go +++ b/sandboxd/server/exec_test.go @@ -5,7 +5,7 @@ import ( "bytes" "context" "encoding/base64" - "encoding/json" + "encoding/json/v2" "io" "net" "net/http" @@ -58,6 +58,29 @@ func TestExecReturnsBufferedOutput(t *testing.T) { } } +func TestExecKeepsInvalidUTF8OutputAsReplacementCharacters(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + _, _ = r.ReadBytes('\n') + _, _ = r.ReadBytes('\n') + _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, `{"type":"stdout","data":"`+base64.StdEncoding.EncodeToString([]byte("ok\xff\n"))+`"}`+"\n") + _, _ = io.WriteString(conn, `{"type":"exit","code":0}`+"\n") + }) + status, body := postExec(t, ts, `{"argv":["cat","blob"],"timeout_seconds":5}`) + if status != http.StatusOK { + t.Fatalf("status %d, want 200: %s", status, body) + } + var out ExecResponse + if err := json.Unmarshal(body, &out); err != nil { + t.Fatalf("decode %q: %v", body, err) + } + if out.Stdout != "ok\uFFFD\n" { + t.Errorf("stdout %q, want the invalid byte as U+FFFD", out.Stdout) + } +} + func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { ts, srv := newRelayServer(t, func(conn net.Conn) { defer conn.Close() diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index e408f456..11f7e45d 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -5,7 +5,7 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/base64" - "encoding/json" + "encoding/json/v2" "fmt" "net" "net/http" diff --git a/sandboxd/server/server_http.go b/sandboxd/server/server_http.go index 2778d33d..4b1e065f 100644 --- a/sandboxd/server/server_http.go +++ b/sandboxd/server/server_http.go @@ -4,7 +4,8 @@ import ( "bufio" "cmp" "context" - "encoding/json" + "encoding/json/jsontext" + "encoding/json/v2" "errors" "io" "net" @@ -90,7 +91,7 @@ func hijackClient(ctx context.Context, w http.ResponseWriter, guest net.Conn) (n func decodeBody[T any](w http.ResponseWriter, r *http.Request) (T, bool) { var v T - if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(&v); err != nil { + if err := json.UnmarshalRead(http.MaxBytesReader(w, r.Body, maxBodyBytes), &v, json.MatchCaseInsensitiveNames(true)); err != nil { writeErr(w, http.StatusBadRequest, "invalid request body") return v, false } @@ -100,7 +101,7 @@ func decodeBody[T any](w http.ResponseWriter, r *http.Request) (T, bool) { // decodeOptionalBody reads an absent body as the zero value, for verbs whose fields all default. func decodeOptionalBody[T any](w http.ResponseWriter, r *http.Request) (T, bool) { var v T - err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)).Decode(&v) + err := json.UnmarshalDecode(jsontext.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)), &v, json.MatchCaseInsensitiveNames(true)) if errors.Is(err, io.EOF) { return v, true } @@ -152,7 +153,7 @@ func writeResult(w http.ResponseWriter, r *http.Request, op, id, failMsg string, func writeJSON(w http.ResponseWriter, code int, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(code) - _ = json.NewEncoder(w).Encode(v) + _ = json.MarshalWrite(w, v, jsontext.AllowInvalidUTF8(true)) } func writeErr(w http.ResponseWriter, code int, msg string) { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index d49298b0..52a7daca 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -3,7 +3,7 @@ package server import ( "bytes" "context" - "encoding/json" + "encoding/json/v2" "errors" "fmt" "io" @@ -56,7 +56,7 @@ func TestClaimHappyPath(t *testing.T) { t.Fatalf("status %d, want 200", resp.StatusCode) } var cr types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { + if err := json.UnmarshalRead(resp.Body, &cr); err != nil { t.Fatalf("decode: %v", err) } if cr.ID != "sb_1" || cr.Token != "tok" || !cr.Deadline.Equal(time.Unix(42, 0)) { @@ -145,7 +145,7 @@ func TestRenewGrantsAndReportsTheDeadline(t *testing.T) { t.Fatalf("status %d, want 200", resp.StatusCode) } var got types.RenewResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if !got.Deadline.Equal(want) { @@ -175,6 +175,36 @@ func TestRenewWithoutABodyTakesTheNodeDefault(t *testing.T) { } } +func TestRenewMatchesBodyKeysCaseInsensitively(t *testing.T) { + var gotTTL time.Duration + mgr := &fakeManager{renew: func(_, _ string, ttl time.Duration) (time.Time, error) { + gotTTL = ttl + return time.Now(), nil + }} + ts := newTestServer(t, "", mgr, nil) + + resp := postJSON(t, ts.URL+"/v1/sandboxes/sb_1/renew", "tok", `{"TTL_Seconds":3600}`) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK || gotTTL != time.Hour { + t.Errorf("status %d ttl %v, want 200 and 1h", resp.StatusCode, gotTTL) + } +} + +func TestClaimMatchesBodyKeysCaseInsensitively(t *testing.T) { + var gotKey types.PoolKey + mgr := &fakeManager{claim: func(_ context.Context, key types.PoolKey, _ time.Duration) (*types.Sandbox, error) { + gotKey = key + return &types.Sandbox{ID: "sb_1"}, nil + }} + ts := newTestServer(t, "", mgr, nil) + + resp := postJSON(t, ts.URL+"/v1/claim", "", `{"Template":"rt:24.04"}`) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK || gotKey.Template != "rt:24.04" { + t.Errorf("status %d template %q, want 200 and rt:24.04", resp.StatusCode, gotKey.Template) + } +} + func TestRenewRejectsWrongToken(t *testing.T) { mgr := &fakeManager{renew: func(string, string, time.Duration) (time.Time, error) { return time.Time{}, pool.ErrUnknownSandbox @@ -266,7 +296,7 @@ func TestVolumeCatalogReturnsScopedFleetProjection(t *testing.T) { } defer resp.Body.Close() var got types.VolumeListResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } want := []types.VolumeInfo{{ @@ -298,7 +328,7 @@ func TestSandboxIndexReturnsScopedAppliedVolumes(t *testing.T) { } defer resp.Body.Close() var got SandboxListResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } want := []types.Volume{{Name: "imagenet", Mount: "/datasets/imagenet"}} @@ -324,7 +354,7 @@ func TestSandboxesReportClaimedAt(t *testing.T) { } defer resp.Body.Close() var got SandboxListResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Sandboxes) != 1 || !got.Sandboxes[0].ClaimedAt.Equal(claimed) { @@ -357,7 +387,7 @@ func TestSandboxByIDCarriesTheTokenOnlyForTheRootCredential(t *testing.T) { } defer resp.Body.Close() var got pool.SandboxSummary - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if resp.StatusCode != http.StatusOK || got.Token != tt.want { @@ -519,7 +549,7 @@ func TestDrainEndpoints(t *testing.T) { t.Fatalf("do: %v", err) } var info InfoResponse - if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + if err := json.UnmarshalRead(resp.Body, &info); err != nil { t.Fatalf("decode: %v", err) } resp.Body.Close() @@ -563,7 +593,7 @@ func TestPutPoolsUpdatesTargets(t *testing.T) { t.Fatalf("SetPools got %+v", got) } var out InfoResponse - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + if err := json.UnmarshalRead(resp.Body, &out); err != nil { t.Fatalf("decode: %v", err) } if len(out.Pools) != 1 || out.Pools[0].Target != 2 || !out.Pools[0].Golden { @@ -777,7 +807,7 @@ func TestForkFlow(t *testing.T) { t.Fatalf("status %d, want 200", resp.StatusCode) } var fr types.ForkResponse - if err := json.NewDecoder(resp.Body).Decode(&fr); err != nil { + if err := json.UnmarshalRead(resp.Body, &fr); err != nil { t.Fatalf("decode: %v", err) } if len(fr.Children) != 2 || fr.Children[0].ID != "sb_c0" || fr.Children[0].OwnerAddr != "node:7777" { @@ -841,7 +871,7 @@ func TestPromoteAndDeleteTemplateFlow(t *testing.T) { defer resp.Body.Close() var result types.PromoteResponse if resp.StatusCode == http.StatusOK { - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + if err := json.UnmarshalRead(resp.Body, &result); err != nil { t.Fatalf("decode promote response: %v", err) } } @@ -935,7 +965,7 @@ func TestCheckpointFlow(t *testing.T) { resp := post("/v1/sandboxes/sb_1/checkpoint", `{"token":"tok","name":"step-1"}`) defer resp.Body.Close() var cr types.CheckpointResponse - if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil || cr.Checkpoint.ID != "ck_0011223344556677" { + if err := json.UnmarshalRead(resp.Body, &cr); err != nil || cr.Checkpoint.ID != "ck_0011223344556677" { t.Fatalf("checkpoint: status %d, %+v, %v", resp.StatusCode, cr, err) } if cr.Checkpoint.Name != "step-1" { @@ -945,7 +975,7 @@ func TestCheckpointFlow(t *testing.T) { resp2 := post("/v1/checkpoints/"+cr.Checkpoint.ID+"/claim", `{}`) defer resp2.Body.Close() var claim types.ClaimResponse - if err := json.NewDecoder(resp2.Body).Decode(&claim); err != nil || claim.ID != "sb_branch" { + if err := json.UnmarshalRead(resp2.Body, &claim); err != nil || claim.ID != "sb_branch" { t.Fatalf("claim from checkpoint: status %d, %+v, %v", resp2.StatusCode, claim, err) } @@ -1060,7 +1090,7 @@ func TestClaimRedirectsOnWarmMiss(t *testing.T) { } defer resp.Body.Close() var cr types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { + if err := json.UnmarshalRead(resp.Body, &cr); err != nil { t.Fatalf("decode: %v", err) } if len(cr.Redirect) != 2 || cr.Redirect[0] != "node-b:7777" { @@ -1102,7 +1132,7 @@ func TestClaimRedirectsToTemplateOwner(t *testing.T) { } defer resp.Body.Close() var cr types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { + if err := json.UnmarshalRead(resp.Body, &cr); err != nil { t.Fatalf("decode: %v", err) } gotRedirect := len(cr.Redirect) > 0 @@ -1135,7 +1165,7 @@ func TestDeleteTemplateRedirectsToOwner(t *testing.T) { t.Fatalf("status %d, want 200 redirect", resp.StatusCode) } var cr types.ClaimResponse - if decodeErr := json.NewDecoder(resp.Body).Decode(&cr); decodeErr != nil { + if decodeErr := json.UnmarshalRead(resp.Body, &cr); decodeErr != nil { t.Fatalf("decode: %v", decodeErr) } if len(cr.Redirect) != 1 || cr.Redirect[0] != "node-b:7777" { @@ -1186,7 +1216,7 @@ func TestClaimProvisionsWhenNoCandidate(t *testing.T) { } defer resp.Body.Close() var cr types.ClaimResponse - _ = json.NewDecoder(resp.Body).Decode(&cr) + _ = json.UnmarshalRead(resp.Body, &cr) if cr.ID != "sb_local" || len(cr.Redirect) != 0 { t.Errorf("got %+v, want local sandbox", cr) } @@ -1240,7 +1270,7 @@ func TestVolumeClaimUsesWarmBeforeRedirectOrProvision(t *testing.T) { } defer resp.Body.Close() var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if resp.StatusCode != http.StatusOK || got.ID != tt.wantID || !slices.Equal(got.Redirect, tt.wantRedirect) { @@ -1499,7 +1529,7 @@ func TestForeignTemplateGossipNeverEscalates(t *testing.T) { t.Fatalf("status=%d, want 200", resp.StatusCode) } var cr types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&cr); err != nil { + if err := json.UnmarshalRead(resp.Body, &cr); err != nil { t.Fatalf("decode: %v", err) } if got := len(cr.Redirect) > 0; got != tt.wantRedirect { @@ -1627,7 +1657,7 @@ func TestOwnerEndpoint(t *testing.T) { var body struct { OwnerAddr string `json:"owner_addr"` } - _ = json.NewDecoder(resp.Body).Decode(&body) + _ = json.UnmarshalRead(resp.Body, &body) if body.OwnerAddr != "node-b:7777" { t.Errorf("owner %q, want node-b:7777", body.OwnerAddr) } @@ -1663,7 +1693,7 @@ func TestPreviewHandlerZeroDeadlineMintsLiveToken(t *testing.T) { t.Fatalf("status %d, want 200", resp.StatusCode) } var out types.PreviewResponse - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + if err := json.UnmarshalRead(resp.Body, &out); err != nil { t.Fatalf("decode: %v", err) } _, token, _ := strings.Cut(out.URL, "/p/") @@ -1689,7 +1719,7 @@ func TestCheckpointClaimRedirectsToOwner(t *testing.T) { t.Fatalf("status = %d, want 200 carrying a redirect (the mesh redirect is a 200, not a 3xx)", resp.StatusCode) } var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Redirect) != 1 || got.Redirect[0] != "owner-a:7777" { @@ -1739,7 +1769,7 @@ func TestCheckpointClaimRedirectBeatsHeal(t *testing.T) { t.Fatalf("status = %d, want 200 carrying a redirect", resp.StatusCode) } var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Redirect) != 1 || got.Redirect[0] != "owner-a:7777" { @@ -1765,7 +1795,7 @@ func TestCheckpointClaimFallsBackToHealWhenNoOwner(t *testing.T) { t.Fatalf("status = %d, want 200", resp.StatusCode) } var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if got.ID != "sb_healed" { @@ -1792,7 +1822,7 @@ func TestCheckpointClaimNoRedirectGoesStraightToHeal(t *testing.T) { t.Fatalf("status = %d, want 200", resp.StatusCode) } var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Redirect) != 0 { @@ -1966,7 +1996,7 @@ func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { t.Fatalf("status = %d, want 200 carrying a redirect", resp.StatusCode) } var got types.ClaimResponse - if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(resp.Body, &got); err != nil { t.Fatalf("decode: %v", err) } if len(got.Redirect) != 1 || got.Redirect[0] != tsB.URL { @@ -1977,7 +2007,7 @@ func TestCheckpointClaimProbesRealPeerAndRedirects(t *testing.T) { func assertVolumeClaimResponse(t *testing.T, body io.Reader, wantRedirect string, wantRequirePromoted bool) { t.Helper() var got types.ClaimResponse - if err := json.NewDecoder(body).Decode(&got); err != nil { + if err := json.UnmarshalRead(body, &got); err != nil { t.Fatalf("decode: %v", err) } if wantRedirect != "" { diff --git a/sandboxd/utils/utils.go b/sandboxd/utils/utils.go index b1c66b93..28092516 100644 --- a/sandboxd/utils/utils.go +++ b/sandboxd/utils/utils.go @@ -2,7 +2,8 @@ package utils import ( - json "encoding/json/v2" + jsonv1 "encoding/json" + "encoding/json/v2" "fmt" "net" "os" @@ -86,3 +87,8 @@ func RemoveDirEntries(dir string, match func(name string) bool) error { func DecodeStrictJSON(raw []byte, v any) error { return json.Unmarshal(raw, v, json.RejectUnknownMembers(true), json.MatchCaseInsensitiveNames(true)) } + +// DigestJSON encodes v with v1 semantics so a digest keeps its bytes across builds. +func DigestJSON(v any) ([]byte, error) { + return json.Marshal(v, jsonv1.DefaultOptionsV1()) +} diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go index 0ff3b90a..7c0185b2 100644 --- a/sdk/go/checkpoint_test.go +++ b/sdk/go/checkpoint_test.go @@ -1,7 +1,7 @@ package sandbox import ( - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "strings" @@ -12,17 +12,17 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { var nodeACalls int nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req checkpointClaimRequest - _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.UnmarshalRead(r.Body, &req) if !req.NoRedirect { t.Error("retry did not carry no_redirect") } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_ck1", Token: "tok"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_ck1", Token: "tok"}) })) t.Cleanup(nodeB.Close) nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { nodeACalls++ - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, }) })) @@ -78,13 +78,13 @@ func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(broken.URL, "http://")}, }) return } w.WriteHeader(http.StatusConflict) - _ = json.NewEncoder(w).Encode(errorResponse{Error: "origin also failed"}) + _ = json.MarshalWrite(w, errorResponse{Error: "origin also failed"}) })) t.Cleanup(entry.Close) @@ -115,15 +115,15 @@ func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(busy.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(busy.URL, "http://")}}) return } var req checkpointClaimRequest - _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.UnmarshalRead(r.Body, &req) if !req.NoRedirect { t.Error("origin fallback did not carry no_redirect") } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_healed", Token: "t"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_healed", Token: "t"}) })) t.Cleanup(entry.Close) @@ -143,7 +143,7 @@ func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{"127.0.0.1:1"}}) })) t.Cleanup(entry.Close) @@ -160,12 +160,12 @@ func TestCheckpointNewRedirectNeverYieldsEmptyID(t *testing.T) { func TestCheckpointNewSecondLevelRedirectFails(t *testing.T) { nodeB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{"127.0.0.1:1"}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{"127.0.0.1:1"}}) })) t.Cleanup(nodeB.Close) nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, }) })) diff --git a/sdk/go/client.go b/sdk/go/client.go index db105497..bcbe7f86 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -6,7 +6,7 @@ import ( "cmp" "context" "crypto/tls" - "encoding/json" + "encoding/json/v2" "errors" "fmt" "io" @@ -267,7 +267,7 @@ func (c *Client) deleteTemplates(ctx context.Context, addr string, u url.Values) return nil, nil case http.StatusOK: var body claimResponse - if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + if err := json.UnmarshalRead(resp.Body, &body); err != nil { return nil, fmt.Errorf("decode delete redirect: %w", err) } return body.Redirect, nil @@ -317,7 +317,7 @@ func doJSON[T any](ctx context.Context, c *Client, method, addr, path string, bo if resp.StatusCode != http.StatusOK { return out, apiError(verb, resp) } - if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + if err := json.UnmarshalRead(resp.Body, &out); err != nil { return out, fmt.Errorf("decode %s response: %w", verb, err) } return out, nil @@ -438,7 +438,7 @@ func encodeBody(verb string, v any) ([]byte, error) { func apiError(verb string, resp *http.Response) error { var er errorResponse - _ = json.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&er) + _ = json.UnmarshalRead(io.LimitReader(resp.Body, 4096), &er) _, _ = io.Copy(io.Discard, resp.Body) return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 9ca36a9c..5bf7cba4 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -1,7 +1,7 @@ package sandbox import ( - "encoding/json" + "encoding/json/v2" "errors" "io" "net/http" @@ -45,10 +45,10 @@ func TestNewSendsClaim(t *testing.T) { t.Errorf("got %s %s", r.Method, r.URL.Path) } gotAuth = r.Header.Get("Authorization") - if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + if err := json.UnmarshalRead(r.Body, &gotBody); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ ID: "sb_1", Token: "tok", Deadline: time.Unix(42, 0), TemplateDigest: "sha256:template", }) })) @@ -78,7 +78,7 @@ func TestNewSendsClaim(t *testing.T) { func TestNewSurfacesServerError(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusConflict) - _ = json.NewEncoder(w).Encode(errorResponse{Error: "node has no egress attachment"}) + _ = json.MarshalWrite(w, errorResponse{Error: "node has no egress attachment"}) })) t.Cleanup(ts.Close) @@ -124,14 +124,14 @@ func TestCloseReleases(t *testing.T) { func TestNewFollowsRedirect(t *testing.T) { var nodeB *httptest.Server nodeB = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ ID: "sb_2", Token: "tok", OwnerAddr: strings.TrimPrefix(nodeB.URL, "http://"), }) })) t.Cleanup(nodeB.Close) nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, }) })) @@ -163,7 +163,7 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { t.Cleanup(nodeB.Close) nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, }) })) @@ -195,7 +195,7 @@ func TestDeleteTemplateRetryPolicy(t *testing.T) { addrs[i] = strings.TrimPrefix(c.URL, "http://") } s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: addrs}) + _ = json.MarshalWrite(w, claimResponse{Redirect: addrs}) })) t.Cleanup(s.Close) return s @@ -230,14 +230,14 @@ func TestRedirectSetsNoRedirect(t *testing.T) { var gotNoRedirect bool target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var req claimRequest - _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.UnmarshalRead(r.Body, &req) gotNoRedirect = req.NoRedirect - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_3", Token: "t"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_3", Token: "t"}) })) t.Cleanup(target.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, }) })) @@ -257,12 +257,12 @@ func TestRedirectSetsNoRedirect(t *testing.T) { func TestRedirectTriesAllCandidates(t *testing.T) { good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_4", Token: "t"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_4", Token: "t"}) })) t.Cleanup(good.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{"127.0.0.1:1", strings.TrimPrefix(good.URL, "http://")}, }) })) @@ -284,12 +284,12 @@ func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { t.Cleanup(forbidden.Close) good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_5", Token: "t"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_5", Token: "t"}) })) t.Cleanup(good.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(forbidden.URL, "http://"), strings.TrimPrefix(good.URL, "http://")}, }) })) @@ -316,15 +316,15 @@ func TestRedirectFallbackHeals(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) return } var req claimRequest - _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.UnmarshalRead(r.Body, &req) if !req.NoRedirect { t.Error("origin fallback did not carry no_redirect") } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_healed", Token: "t"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_healed", Token: "t"}) })) t.Cleanup(entry.Close) @@ -351,7 +351,7 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fullCalls++ w.WriteHeader(http.StatusTooManyRequests) - _ = json.NewEncoder(w).Encode(errorResponse{Error: "candidate full"}) + _ = json.MarshalWrite(w, errorResponse{Error: "candidate full"}) })) t.Cleanup(full.Close) @@ -359,11 +359,11 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) return } w.WriteHeader(http.StatusConflict) - _ = json.NewEncoder(w).Encode(errorResponse{Error: "origin also unavailable"}) + _ = json.MarshalWrite(w, errorResponse{Error: "origin also unavailable"}) })) t.Cleanup(entry.Close) @@ -402,14 +402,14 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { t.Run(tt.name, func(t *testing.T) { bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(tt.code) - _ = json.NewEncoder(w).Encode(errorResponse{Error: tt.want}) + _ = json.MarshalWrite(w, errorResponse{Error: tt.want}) })) t.Cleanup(bad.Close) var entryCalls int entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(bad.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(bad.URL, "http://")}}) })) t.Cleanup(entry.Close) @@ -429,7 +429,7 @@ func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { stale := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { staleCalls++ w.WriteHeader(http.StatusUnauthorized) - _ = json.NewEncoder(w).Encode(errorResponse{Error: "invalid api token"}) + _ = json.MarshalWrite(w, errorResponse{Error: "invalid api token"}) })) t.Cleanup(stale.Close) @@ -437,15 +437,15 @@ func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.NewEncoder(w).Encode(claimResponse{Redirect: []string{strings.TrimPrefix(stale.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(stale.URL, "http://")}}) return } var req claimRequest - _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.UnmarshalRead(r.Body, &req) if !req.NoRedirect { t.Error("origin fallback did not carry no_redirect") } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_local", Token: "t"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_local", Token: "t"}) })) t.Cleanup(entry.Close) @@ -465,7 +465,7 @@ func TestLookupScatter(t *testing.T) { var owner *httptest.Server owner = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/sandboxes/sb_1/owner" && r.Header.Get("Authorization") == "Bearer tok" { - _ = json.NewEncoder(w).Encode(map[string]string{"owner_addr": strings.TrimPrefix(owner.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(owner.URL, "http://")}) return } w.WriteHeader(http.StatusNotFound) @@ -477,7 +477,7 @@ func TestLookupScatter(t *testing.T) { case "/v1/sandboxes/sb_1/owner": w.WriteHeader(http.StatusNotFound) case "/v1/peers": - _ = json.NewEncoder(w).Encode(map[string]any{"peers": []string{strings.TrimPrefix(owner.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(owner.URL, "http://")}}) } })) t.Cleanup(entry.Close) diff --git a/sdk/go/drain_test.go b/sdk/go/drain_test.go index 45988faf..c3bc1f3c 100644 --- a/sdk/go/drain_test.go +++ b/sdk/go/drain_test.go @@ -1,7 +1,7 @@ package sandbox import ( - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "testing" @@ -17,7 +17,7 @@ func TestDrainAndUncordon(t *testing.T) { t.Errorf("auth = %q, want Bearer root", auth) } methods = append(methods, r.Method) - _ = json.NewEncoder(w).Encode(NodeInfo{Claimed: 3, Draining: r.Method == http.MethodPost}) + _ = json.MarshalWrite(w, NodeInfo{Claimed: 3, Draining: r.Method == http.MethodPost}) })) defer ts.Close() diff --git a/sdk/go/info_test.go b/sdk/go/info_test.go index 82fc2a8d..7a432d93 100644 --- a/sdk/go/info_test.go +++ b/sdk/go/info_test.go @@ -1,7 +1,7 @@ package sandbox import ( - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "testing" @@ -9,7 +9,7 @@ import ( func TestInfoReportsCapacityState(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(map[string]any{ + _ = json.MarshalWrite(w, map[string]any{ "pools": []any{}, "claimed": 2, "hibernated": 1, diff --git a/sdk/go/pools_test.go b/sdk/go/pools_test.go index b76e0c66..1046da77 100644 --- a/sdk/go/pools_test.go +++ b/sdk/go/pools_test.go @@ -1,7 +1,7 @@ package sandbox import ( - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "strings" @@ -18,8 +18,8 @@ func TestSetPools(t *testing.T) { if auth := r.Header.Get("Authorization"); auth != "Bearer root" { t.Errorf("auth = %q, want Bearer root", auth) } - _ = json.NewDecoder(r.Body).Decode(&got) - _ = json.NewEncoder(w).Encode(NodeInfo{Claimed: 2}) + _ = json.UnmarshalRead(r.Body, &got) + _ = json.MarshalWrite(w, NodeInfo{Claimed: 2}) })) defer ts.Close() @@ -41,11 +41,11 @@ func TestSetPoolsClusterFansOut(t *testing.T) { applied := map[string]int{} pools := func(w http.ResponseWriter, r *http.Request) { var body poolUpdate - _ = json.NewDecoder(r.Body).Decode(&body) + _ = json.UnmarshalRead(r.Body, &body) mu.Lock() applied[r.Host] = body.Pools[0].Warm mu.Unlock() - _ = json.NewEncoder(w).Encode(NodeInfo{}) + _ = json.MarshalWrite(w, NodeInfo{}) } peer := httptest.NewServer(http.HandlerFunc(pools)) defer peer.Close() @@ -53,7 +53,7 @@ func TestSetPoolsClusterFansOut(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { - _ = json.NewEncoder(w).Encode(map[string][]string{"peers": {peerAddr}}) + _ = json.MarshalWrite(w, map[string][]string{"peers": {peerAddr}}) return } pools(w, r) @@ -93,7 +93,7 @@ func TestSetPoolsClusterPeerDiscoveryFails(t *testing.T) { return } applied++ - _ = json.NewEncoder(w).Encode(NodeInfo{}) + _ = json.MarshalWrite(w, NodeInfo{}) })) defer entry.Close() diff --git a/sdk/go/proc_test.go b/sdk/go/proc_test.go index 3a12d2cf..6688dcad 100644 --- a/sdk/go/proc_test.go +++ b/sdk/go/proc_test.go @@ -3,7 +3,7 @@ package sandbox import ( "bufio" "bytes" - "encoding/json" + "encoding/json/v2" "net" "strings" "testing" diff --git a/sdk/go/template_test.go b/sdk/go/template_test.go index 09d10fbe..ce5e42f6 100644 --- a/sdk/go/template_test.go +++ b/sdk/go/template_test.go @@ -1,7 +1,7 @@ package sandbox import ( - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "testing" @@ -12,7 +12,7 @@ func TestPromoteReturnsContentDigest(t *testing.T) { if r.Method != http.MethodPost || r.URL.Path != "/v1/sandboxes/sb_1/promote" { t.Errorf("got %s %s", r.Method, r.URL.Path) } - _ = json.NewEncoder(w).Encode(map[string]any{ + _ = json.MarshalWrite(w, map[string]any{ "key": map[string]string{"template": "task:v1", "net": "none", "size": "small"}, "content_digest": "sha256:promoted", }) diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index 2c102dc9..d8a2c14e 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -2,7 +2,7 @@ package sandbox import ( "context" - "encoding/json" + "encoding/json/v2" "net/http" "net/http/httptest" "slices" @@ -16,10 +16,10 @@ func TestClientNewSendsVolumes(t *testing.T) { Volumes []Volume `json:"volumes"` } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(r.Body, &got); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ ID: "sb_1", Token: "tok", Volumes: []Volume{{Name: "weights-llama", Mount: "/models"}}, }) })) @@ -52,10 +52,10 @@ func TestTemplateNewSendsVolumes(t *testing.T) { NoRedirect bool `json:"no_redirect"` } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(r.Body, &got); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ ID: "sb_2", Token: "tok", Volumes: []Volume{{Name: "imagenet", Mount: "/datasets/imagenet"}}, }) })) @@ -85,15 +85,15 @@ func TestTemplateNewVolumeClaimFollowsRedirect(t *testing.T) { var originAuth, targetAuth string target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetAuth = r.Header.Get("Authorization") - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(r.Body, &got); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_3", Token: "tok", Volumes: want}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_3", Token: "tok", Volumes: want}) })) t.Cleanup(target.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { originAuth = r.Header.Get("Authorization") - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, RequirePromoted: true, }) })) @@ -135,7 +135,7 @@ func TestClientVolumes(t *testing.T) { if got := r.Header.Get("Authorization"); got != "Bearer sekret" { t.Errorf("authorization = %q", got) } - _ = json.NewEncoder(w).Encode(volumeListResponse{Volumes: tt.want}) + _ = json.MarshalWrite(w, volumeListResponse{Volumes: tt.want}) })) t.Cleanup(ts.Close) @@ -154,14 +154,14 @@ func TestVolumeClaimRedirectPreservesEntries(t *testing.T) { want := []Volume{{Name: "imagenet", Mount: "/datasets/imagenet"}} var got claimRequest target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + if err := json.UnmarshalRead(r.Body, &got); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_3", Token: "tok", Volumes: want}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_3", Token: "tok", Volumes: want}) })) t.Cleanup(target.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ + _ = json.MarshalWrite(w, claimResponse{ Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, }) })) @@ -205,10 +205,10 @@ func TestWithVolumesEncodesMode(t *testing.T) { Volumes []map[string]any `json:"volumes"` } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&raw); err != nil { + if err := json.UnmarshalRead(r.Body, &raw); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_1", Token: "tok"}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok"}) })) t.Cleanup(ts.Close) if _, err := testClient(t, ts).New(t.Context(), "rt:24.04", WithVolumes(Volume{Name: "a", Mode: tt.mode})); err != nil { @@ -233,10 +233,10 @@ func TestWithVolumesAttachOnlySendsFlagAndDecodesMountlessEcho(t *testing.T) { } echo := []Volume{{Name: "imagenet"}, {Name: "scratch", Mode: "rw"}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if err := json.NewDecoder(r.Body).Decode(&raw); err != nil { + if err := json.UnmarshalRead(r.Body, &raw); err != nil { t.Errorf("decode body: %v", err) } - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_1", Token: "tok", Volumes: echo}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: echo}) })) t.Cleanup(ts.Close) @@ -303,7 +303,7 @@ func TestRejectsInvalidVolumeModeLocally(t *testing.T) { func TestSandboxVolumesEchoMode(t *testing.T) { want := []Volume{{Name: "imagenet"}, {Name: "scratch", Mode: "rw"}} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(claimResponse{ID: "sb_1", Token: "tok", Volumes: want}) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_1", Token: "tok", Volumes: want}) })) t.Cleanup(ts.Close)