From a208c0a468bdd0396fd705f7cbc9f0c51ec29511 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sun, 18 Oct 2026 15:57:41 +0200 Subject: [PATCH 1/2] feat(agents): expose scoped diagnostic API reads Signed-off-by: Fredrik Ahlgren --- .changeset/mcp-diagnostic-api-reads.md | 5 + contract/registry.yaml | 1 + go/internal/api/api_agent_reads.go | 124 ++++++++++++++++++ go/internal/api/api_agent_reads_test.go | 113 ++++++++++++++++ go/internal/appproto/agentapi.go | 96 ++++++++++++++ go/internal/appproto/agentapi_test.go | 89 +++++++++++++ go/internal/appproto/agents.go | 2 + .../appproto/gencontract/gencontract.go | 6 + go/internal/appproto/passthrough.go | 24 ++++ go/internal/redact/data.go | 102 ++++++++++++++ go/internal/redact/data_test.go | 43 ++++++ go/protocol/agentapi.go | 66 ++++++++++ go/protocol/agentapi_test.go | 23 ++++ go/protocol/contract_gen.go | 4 + 14 files changed, 698 insertions(+) create mode 100644 .changeset/mcp-diagnostic-api-reads.md create mode 100644 go/internal/api/api_agent_reads.go create mode 100644 go/internal/api/api_agent_reads_test.go create mode 100644 go/internal/appproto/agentapi.go create mode 100644 go/internal/appproto/agentapi_test.go create mode 100644 go/internal/redact/data.go create mode 100644 go/internal/redact/data_test.go create mode 100644 go/protocol/agentapi.go create mode 100644 go/protocol/agentapi_test.go diff --git a/.changeset/mcp-diagnostic-api-reads.md b/.changeset/mcp-diagnostic-api-reads.md new file mode 100644 index 000000000..256f440d3 --- /dev/null +++ b/.changeset/mcp-diagnostic-api-reads.md @@ -0,0 +1,5 @@ +--- +"ftw": minor +--- + +Let authorized assistants read the energy ledger, measured series, forecasts, models and planner snapshots through MCP. The box checks each data scope, redacts identifying fields and stops reads when access ends. diff --git a/contract/registry.yaml b/contract/registry.yaml index ca5bdb6cc..22a78b80b 100644 --- a/contract/registry.yaml +++ b/contract/registry.yaml @@ -445,6 +445,7 @@ reads: - { msg: logs.get, scope: ftw.logs.read, desc: A log excerpt the box redacts } - { msg: config.get, scope: ftw.config.read, desc: A configuration section the box redacts } - { msg: diag.run, scope: ftw.diag.run, desc: A check that changes nothing } + - { msg: api.read, const: MsgAPIRead, scope: ftw.diag.run, desc: Read a diagnostic API route within its data scope } # --------------------------------------------------------------------------- # Cars. A car belongs to its owner, not to the home, and visits chargers. The diff --git a/go/internal/api/api_agent_reads.go b/go/internal/api/api_agent_reads.go new file mode 100644 index 000000000..ed4afab2a --- /dev/null +++ b/go/internal/api/api_agent_reads.go @@ -0,0 +1,124 @@ +package api + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "strconv" + + "github.com/srcfl/ftw/go/internal/apiauth" + "github.com/srcfl/ftw/go/internal/redact" + "github.com/srcfl/ftw/go/protocol" +) + +// ServeAgentRead runs only a catalogued JSON read through the usual HTTP +// handler. It buffers and redacts the whole answer on the box before any +// byte leaves for FTW Cloud. The caller comes from the Noise session. +func (s *Server) ServeAgentRead(w http.ResponseWriter, request *http.Request, redactor *redact.Redactor) { + caller, named := apiauth.FromRequest(request) + route, known := protocol.AgentAPIPath(request.URL.Path) + facts := s.Route(request) + if !named || redactor == nil || !known || request.Method != http.MethodGet || facts.Static || facts.Tier != apiauth.TierRead || + !caller.Scopes.Has(protocol.ScopeDiagRun) || !caller.Scopes.Has(route.Scope) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "this diagnostic read is not granted"}) + return + } + query := request.URL.Query() + for name, values := range query { + if _, ok := route.Query[name]; !ok || len(values) != 1 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "unknown or repeated query parameter"}) + return + } + } + // Resolve only ids issued by this box's redactor. A real hardware id or + // driver name is never accepted in an agent query. + for _, field := range []string{"asset_id", "driver"} { + id := query.Get(field) + if id == "" { + continue + } + name, ok := s.agentReadID(field, id, redactor) + if !ok { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "unknown opaque id; read the asset or series catalogue"}) + return + } + query.Set(field, name) + } + request = request.Clone(request.Context()) + request.URL.RawQuery = query.Encode() + if request.URL.Path == "/api/series" && query.Get("points") == "" { + query.Set("points", "500") + request.URL.RawQuery = query.Encode() + } + if raw := query.Get("points"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > 5000 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "points must be between 1 and 5000"}) + return + } + } + answer := &agentReadBuffer{header: make(http.Header), status: http.StatusOK} + s.ServeHTTP(answer, request) + if answer.large { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "answer is too large; narrow the time range, points or limit"}) + return + } + var body any + dec := json.NewDecoder(bytes.NewReader(answer.body.Bytes())) + dec.UseNumber() + if !json.Valid(answer.body.Bytes()) || dec.Decode(&body) != nil { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": "the diagnostic route did not return JSON"}) + return + } + writeJSON(w, answer.status, redactor.Data(body)) +} + +func (s *Server) agentReadID(field, id string, redactor *redact.Redactor) (string, bool) { + if s.deps.State == nil { + return "", false + } + if field == "asset_id" { + assets, err := s.deps.State.EnergyAssets() + if err == nil { + for _, asset := range assets { + if redactor.ID(asset.AssetID) == id { + return asset.AssetID, true + } + } + } + } else { + names, err := s.deps.State.DriverNames() + if err == nil { + for _, name := range names { + if redactor.ID(name) == id { + return name, true + } + } + } + } + return "", false +} + +type agentReadBuffer struct { + header http.Header + status int + body bytes.Buffer + large bool + wrote bool +} + +func (b *agentReadBuffer) Header() http.Header { return b.header } +func (b *agentReadBuffer) WriteHeader(status int) { + if !b.wrote { + b.status, b.wrote = status, true + } +} +func (b *agentReadBuffer) Write(body []byte) (int, error) { + b.WriteHeader(http.StatusOK) + if b.large || b.body.Len()+len(body) > protocol.AgentAPIMaxBytes { + b.large = true + return 0, errors.New("diagnostic answer too large") + } + return b.body.Write(body) +} diff --git a/go/internal/api/api_agent_reads_test.go b/go/internal/api/api_agent_reads_test.go new file mode 100644 index 000000000..7aff7fcad --- /dev/null +++ b/go/internal/api/api_agent_reads_test.go @@ -0,0 +1,113 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/srcfl/ftw/go/internal/apiauth" + "github.com/srcfl/ftw/go/internal/redact" + "github.com/srcfl/ftw/go/protocol" +) + +func agentReadRequest(path string, scopes ...string) *http.Request { + r := httptest.NewRequest(http.MethodGet, path, nil) + return r.WithContext(apiauth.WithCaller(r.Context(), apiauth.Caller{Kind: apiauth.KindAgent, Role: apiauth.RoleAgent, Subject: "agent:test", Scopes: apiauth.NewScopeSet(scopes...)})) +} + +func TestAgentReadsTheEnergyLedgerByItsOpaqueAssetID(t *testing.T) { + st, base, asset := seedAPIEnergyHistory(t) + t.Cleanup(func() { _ = st.Close() }) + srv := New(&Deps{State: st}) + rd := redact.New([]byte("agent-key"), []string{"Site meter"}) + catalog := httptest.NewRecorder() + srv.ServeAgentRead(catalog, agentReadRequest("/api/energy/assets", protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) + if catalog.Code != http.StatusOK || !strings.Contains(catalog.Body.String(), rd.ID(asset)) || strings.Contains(catalog.Body.String(), "Site meter") || strings.Contains(catalog.Body.String(), "maker:api-meter") { + t.Fatalf("asset catalogue: %d %s", catalog.Code, catalog.Body.String()) + } + q := url.Values{"scope": {"asset"}, "asset_id": {rd.ID(asset)}, "since": {formatInt64(base)}, "until": {formatInt64(base + 300_000)}, "bucket": {"5m"}} + answer := httptest.NewRecorder() + srv.ServeAgentRead(answer, agentReadRequest("/api/energy/history?"+q.Encode(), protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) + var body struct { + Points []struct { + AssetID string `json:"asset_id"` + Energy float64 `json:"energy_wh"` + Quality string `json:"quality"` + Source string `json:"source"` + Provenance string `json:"provenance"` + } `json:"points"` + } + if answer.Code != http.StatusOK || json.Unmarshal(answer.Body.Bytes(), &body) != nil || len(body.Points) == 0 { + t.Fatalf("energy ledger: %d %s", answer.Code, answer.Body.String()) + } + var measured float64 + for _, p := range body.Points { + if p.AssetID != rd.ID(asset) { + t.Fatal("ledger exposed another id") + } + if p.Quality == "measured" { + measured += p.Energy + if p.Source != "hardware_counter" || p.Provenance != "counter" { + t.Fatalf("measurement provenance changed: %+v", p) + } + } + } + if measured != 12 { + t.Fatalf("measured energy = %v, want 12 Wh", measured) + } + q.Set("asset_id", asset) + refused := httptest.NewRecorder() + srv.ServeAgentRead(refused, agentReadRequest("/api/energy/history?"+q.Encode(), protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) + if refused.Code != http.StatusNotFound { + t.Fatalf("raw asset id accepted: %d", refused.Code) + } +} + +func TestDiagnosticReadsRequireBothScopesAndTheReadTier(t *testing.T) { + srv := New(&Deps{}) + rd := redact.New([]byte("agent-key"), nil) + for _, tc := range []struct { + path string + scopes []string + }{ + {"/api/energy/history", []string{protocol.ScopeDiagRun}}, + {"/api/energy/history", []string{protocol.ScopeHistoryRead}}, + {"/api/config", []string{protocol.ScopeDiagRun, protocol.ScopeConfigRead}}, + {"/api/scan", []string{protocol.ScopeDiagRun, protocol.ScopeHealthRead}}, + {"/api/oauth/myuplink/start", []string{protocol.ScopeDiagRun, protocol.ScopeConfigRead}}, + } { + answer := httptest.NewRecorder() + srv.ServeAgentRead(answer, agentReadRequest(tc.path, tc.scopes...), rd) + if answer.Code != http.StatusForbidden { + t.Errorf("%s: %d", tc.path, answer.Code) + } + } + for _, route := range protocol.AgentAPIRoutes { + facts := srv.Route(httptest.NewRequest(http.MethodGet, route.Path, nil)) + if facts.Tier != apiauth.TierRead || facts.Static { + t.Errorf("catalogue entry %s is not a Read route: %+v", route.Path, facts) + } + } + answer := httptest.NewRecorder() + srv.ServeAgentRead(answer, agentReadRequest("/api/series?format=csv", protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) + if answer.Code != http.StatusBadRequest { + t.Fatal("an unlisted query parameter was accepted") + } +} + +func TestDiagnosticAPIDoesNotPublishAnOversizedAnswer(t *testing.T) { + srv := New(&Deps{}) + srv.mux = http.NewServeMux() + srv.marks = make(map[string]routeMark) + srv.handle("GET /api/status", Read, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"text":"` + strings.Repeat("x", protocol.AgentAPIMaxBytes) + `"}`)) + }) + answer := httptest.NewRecorder() + srv.ServeAgentRead(answer, agentReadRequest("/api/status", protocol.ScopeDiagRun, protocol.ScopeLiveRead), redact.New([]byte("agent-key"), nil)) + if answer.Code != http.StatusRequestEntityTooLarge || strings.Contains(answer.Body.String(), "xxxxxxxx") { + t.Fatalf("oversized read: %d %s", answer.Code, answer.Body.String()) + } +} diff --git a/go/internal/appproto/agentapi.go b/go/internal/appproto/agentapi.go new file mode 100644 index 000000000..1fc18f1bf --- /dev/null +++ b/go/internal/appproto/agentapi.go @@ -0,0 +1,96 @@ +package appproto + +import ( + "context" + "net/http" + "sync" + + "github.com/srcfl/ftw/go/internal/apiauth" + "github.com/srcfl/ftw/go/internal/redact" + "github.com/srcfl/ftw/go/protocol" +) + +type agentReadGateway interface { + ServeAgentRead(http.ResponseWriter, *http.Request, *redact.Redactor) +} + +func (h *Handler) onAPIRead(env Envelope) error { + if env.ID == nil { + return nil + } + read, err := protocol.DecodeAPIRead(env.B) + if err != nil { + return h.refuse(env.ID, ErrUnknownOp, map[string]any{"t": env.T}) + } + route, known := protocol.AgentAPIPath(read.Path) + request := APIReq{Method: APIGet, Path: read.Path, Query: read.Query, MaxBytes: read.MaxBytes} + if !known || badAPIRequest(request) != "" || read.MaxBytes < 0 || read.MaxBytes > protocol.AgentAPIMaxBytes { + return h.refuse(env.ID, ErrUnknownOp, map[string]any{"t": env.T}) + } + gateway, available := h.cfg.API.(agentReadGateway) + if !available || h.cfg.Redactor == nil { + return h.refuse(env.ID, ErrUnavailable, map[string]any{"subsystem": "api"}) + } + if !h.apiBusy.CompareAndSwap(false, true) { + return h.refuse(env.ID, ErrUnavailable, map[string]any{"reason": "busy"}) + } + id := *env.ID + go func() { + var once sync.Once + free := func() { once.Do(func() { h.apiBusy.Store(false) }) } + defer free() + ctx, cancel := context.WithTimeout(h.ctx, apiHandlerTimeout) + defer cancel() + if h.isAgent() { + _, refusal, end := h.agentGate() + if refusal != nil { + _ = h.sendError(&id, *refusal) + if end != "" { + _ = h.Terminate(end) + } + return + } + } + caller, enrolled := h.liveCaller() + if !enrolled || !caller.Scopes.Has(protocol.ScopeDiagRun) || !caller.Scopes.Has(route.Scope) { + _ = h.refuse(&id, ErrScopeDenied, map[string]any{"needScope": route.Scope}) + return + } + req := buildAPIRequest(ctx, caller, request) + facts := h.cfg.API.Route(req) + if facts.Static || facts.Tier != apiauth.TierRead { + _ = h.refuse(&id, ErrScopeDenied, map[string]any{"t": env.T}) + return + } + max := int64(protocol.AgentAPIMaxBytes) + if read.MaxBytes > 0 { + max = read.MaxBytes + } + writer := &apiWriter{h: h, id: id, ctx: ctx, max: max, free: free, header: http.Header{}, buf: make([]byte, 0, APIChunkBytes)} + writer.recheck = func() *ErrorBody { + if h.isAgent() { + if _, refusal, _ := h.agentGate(); refusal != nil { + return refusal + } + } + current, ok := h.liveCaller() + if !ok || !current.Scopes.Has(protocol.ScopeDiagRun) || !current.Scopes.Has(route.Scope) { + return &ErrorBody{Code: ErrScopeDenied, Args: map[string]any{"needScope": route.Scope}} + } + return nil + } + func() { + defer func() { + if recover() != nil { + writer.WriteHeader(http.StatusInternalServerError) + _, _ = writer.Write([]byte(`{"error":"diagnostic read failed"}`)) + } + }() + gateway.ServeAgentRead(writer, req, h.cfg.Redactor()) + }() + if err := writer.finish(); err != nil { + h.log.Warn("diagnostic API read failed", "err", err) + } + }() + return nil +} diff --git a/go/internal/appproto/agentapi_test.go b/go/internal/appproto/agentapi_test.go new file mode 100644 index 000000000..18aa4aad2 --- /dev/null +++ b/go/internal/appproto/agentapi_test.go @@ -0,0 +1,89 @@ +package appproto + +import ( + "net/http" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/apiauth" + "github.com/srcfl/ftw/go/internal/redact" + "github.com/srcfl/ftw/go/protocol" +) + +type diagnosticGateway struct { + stubAPI + read func(http.ResponseWriter, *http.Request) +} + +func (g *diagnosticGateway) ServeAgentRead(w http.ResponseWriter, r *http.Request, _ *redact.Redactor) { + g.read(w, r) +} + +func TestAnAgentDiagnosticReadRequiresItsDataScope(t *testing.T) { + for _, hasHistory := range []bool{false, true} { + t.Run(map[bool]string{false: "missing history", true: "granted"}[hasHistory], func(t *testing.T) { + r := newAgentRig(t) + scopes := []string{protocol.ScopeDiagRun} + if hasHistory { + scopes = append(scopes, protocol.ScopeHistoryRead) + } + row, _ := r.grant(apiauth.RoleAgent, scopes, nil, protocol.Caps{}, time.Hour, protocol.Principal{Client: "ChatGPT"}) + h, rec := r.agentSession(row) + t.Cleanup(h.Close) + called := make(chan struct{}, 1) + h.cfg.API = &diagnosticGateway{stubAPI: stubAPI{facts: apiauth.RouteFacts{Tier: apiauth.TierRead}}, read: func(w http.ResponseWriter, request *http.Request) { + called <- struct{}{} + if request.Method != http.MethodGet || request.URL.Path != "/api/energy/history" { + t.Error("wrong diagnostic request") + } + _, _ = w.Write([]byte(`{"energy_wh":12}`)) + }} + deliver(t, h, protocol.MsgAPIRead, ptrU32(1), protocol.APIRead{Path: "/api/energy/history"}) + if !hasHistory { + if got := body[ErrorBody](t, waitFor(t, rec, MsgError)); got.Code != ErrScopeDenied { + t.Fatalf("missing scope: %+v", got) + } + select { + case <-called: + t.Fatal("a denied read reached its handler") + default: + } + return + } + end := body[APIEnd](t, waitFor(t, rec, MsgAPIEnd)) + if end.Truncated || string(apiChunks(rec)) != `{"energy_wh":12}` { + t.Fatalf("diagnostic stream: %+v %s", end, apiChunks(rec)) + } + }) + } +} + +func TestRevocationStopsADiagnosticReadBeforeItsAnswer(t *testing.T) { + r := newAgentRig(t) + row, _ := r.grant(apiauth.RoleAgent, []string{protocol.ScopeDiagRun, protocol.ScopeHistoryRead}, nil, protocol.Caps{}, time.Hour, protocol.Principal{}) + h, rec := r.agentSession(row) + t.Cleanup(h.Close) + started, release := make(chan struct{}), make(chan struct{}) + h.cfg.API = &diagnosticGateway{stubAPI: stubAPI{facts: apiauth.RouteFacts{Tier: apiauth.TierRead}}, read: func(w http.ResponseWriter, _ *http.Request) { + close(started) + <-release + _, _ = w.Write([]byte(`{"must_not_leave":123}`)) + }} + deliver(t, h, protocol.MsgAPIRead, ptrU32(1), protocol.APIRead{Path: "/api/energy/history"}) + select { + case <-started: + case <-time.After(2 * time.Second): + close(release) + t.Fatal("diagnostic handler did not start") + } + if err := r.access.RevokeAgent(row.ID); err != nil { + t.Fatal(err) + } + close(release) + if got := body[ErrorBody](t, waitFor(t, rec, MsgError)); got.Code != ErrGrantRevoked { + t.Fatalf("revoked read: %+v", got) + } + if rec.has(MsgAPIHead) || len(apiChunks(rec)) != 0 { + t.Fatal("a revoked read returned data") + } +} diff --git a/go/internal/appproto/agents.go b/go/internal/appproto/agents.go index 7bba69b26..fddb94646 100644 --- a/go/internal/appproto/agents.go +++ b/go/internal/appproto/agents.go @@ -786,6 +786,8 @@ func (h *Handler) onRead(env Envelope) error { } } switch env.T { + case protocol.MsgAPIRead: + return h.onAPIRead(env) case protocol.MsgHealthGet: return h.onHealthGet(env) case protocol.MsgDriversGet: diff --git a/go/internal/appproto/gencontract/gencontract.go b/go/internal/appproto/gencontract/gencontract.go index ca88d1d8a..de1d8d4a8 100644 --- a/go/internal/appproto/gencontract/gencontract.go +++ b/go/internal/appproto/gencontract/gencontract.go @@ -76,6 +76,7 @@ type Registry struct { } `yaml:"agent_args"` Reads []struct { Msg string `yaml:"msg"` + Const string `yaml:"const"` Scope string `yaml:"scope"` Desc string `yaml:"desc"` } `yaml:"reads"` @@ -419,6 +420,11 @@ func writeAgentArgs(b *bytes.Buffer, reg Registry) { // writeReads emits each read an agent may send with the scope it needs. func writeReads(b *bytes.Buffer, reg Registry) { + for _, r := range reg.Reads { + if r.Const != "" { + fmt.Fprintf(b, "const %s = %q\n\n", r.Const, r.Msg) + } + } b.WriteString("// RegistryReads maps each read an agent may send to the scope it needs.\n") b.WriteString("var RegistryReads = map[string]string{\n") for _, r := range reg.Reads { diff --git a/go/internal/appproto/passthrough.go b/go/internal/appproto/passthrough.go index a79c4e771..79fab9039 100644 --- a/go/internal/appproto/passthrough.go +++ b/go/internal/appproto/passthrough.go @@ -475,6 +475,9 @@ type apiWriter struct { ctx context.Context max int64 free func() + // recheck is set only for a scoped diagnostic read. It runs before + // each response frame so a narrowed grant stops a read in flight. + recheck func() *ErrorBody header http.Header buf []byte @@ -493,6 +496,9 @@ func (w *apiWriter) WriteHeader(status int) { if w.wroteHead || w.refused { return } + if w.checkGrant() != nil { + return + } if w.ctx.Err() != nil { // Out of time, or the session is gone. Either way the status must not // go out: once one has, the app is committed to it, and finish is @@ -573,6 +579,9 @@ func (w *apiWriter) Write(p []byte) (int, error) { } func (w *apiWriter) flush(n int) error { + if err := w.checkGrant(); err != nil { + return err + } chunk := APIChunk{Seq: w.seq, Data: w.buf[:n]} w.seq++ if err := w.h.sendBulk(MsgAPIChunk, &w.id, chunk); err != nil { @@ -583,6 +592,21 @@ func (w *apiWriter) flush(n int) error { return nil } +func (w *apiWriter) checkGrant() error { + if w.recheck == nil { + return nil + } + if refusal := w.recheck(); refusal != nil { + w.refused = true + w.sendErr = w.h.sendError(&w.id, *refusal) + if w.sendErr != nil { + return w.sendErr + } + return errAPIStopped + } + return nil +} + // finish closes the answer. func (w *apiWriter) finish() error { if w.h.ctx.Err() != nil { diff --git a/go/internal/redact/data.go b/go/internal/redact/data.go new file mode 100644 index 000000000..98ab9127a --- /dev/null +++ b/go/internal/redact/data.go @@ -0,0 +1,102 @@ +package redact + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "strings" + "time" + "unicode" +) + +// Data scrubs diagnostic JSON while keeping measured quantities, nulls, +// timestamps and coverage. Config's numeric allowlist is for settings and +// would drop sample counts and other evidence from these reads. +func (r *Redactor) Data(doc any) any { return r.data(doc, "") } + +var dataPath = regexp.MustCompile(`(?:[A-Za-z]:\\|/)[^\s"'<>]*[/\\][^\s"'<>]*`) +var dataMetric = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.-]{0,127}$`) + +func dataSource(value string) bool { + switch value { + case "hardware_counter", "power_integrated", "power_integration", "mixed", "unknown", "ntp", "rtc", "system", "manual", "estimated", "vehicle", "charger", "telemetry", "forecast": + return true + } + return false +} + +func dataKey(key string) string { + var b strings.Builder + letters := []rune(key) + for i, c := range letters { + if unicode.IsUpper(c) && i > 0 && (unicode.IsLower(letters[i-1]) || unicode.IsDigit(letters[i-1]) || i+1 < len(letters) && unicode.IsLower(letters[i+1])) { + b.WriteByte('_') + } + b.WriteRune(unicode.ToLower(c)) + } + return b.String() +} + +func dataID(key string) bool { + return key == "id" || strings.HasSuffix(key, "_id") || key == "src" || key == "driver" || key == "drivers" || + key == "name" || key == "names" || key == "label" || key == "title" || key == "alias" || strings.HasSuffix(key, "_name") +} + +func (r *Redactor) data(doc any, key string) any { + switch v := doc.(type) { + case map[string]any: + out := make(map[string]any, len(v)) + for k, value := range v { + field := dataKey(k) + if secretKey(field) || removedKey(field) || field == "vin" || field == "imei" || field == "serial_number" || field == "path" || field == "file" || field == "directory" || field == "home_id" || field == "box_id" { + continue + } + name := k + if r.names != nil && r.names.MatchString(k) || reURL.MatchString(k) || reIPv4.MatchString(k) || reEmail.MatchString(k) || reMAC.MatchString(k) || reHexRun.MatchString(k) || reHost.MatchString(k) || strings.Count(k, ":") >= 3 && reIPv6.MatchString(k) || strings.Contains(k, "::") { + name = r.ID(k) + } + out[name] = r.data(value, field) + } + return out + case []any: + out := make([]any, len(v)) + for i, value := range v { + out[i] = r.data(value, key) + } + return out + case string: + if v == "" { + return v + } + // Metric names are queryable schema, not free-text device values. + // The log scrubber would turn names such as pv_mppt1_w into [id]. + if (key == "metric" || key == "metrics") && dataMetric.MatchString(v) { + return v + } + if dataID(key) || key == "source" && !dataSource(v) { + return r.ID(v) + } + if _, err := time.Parse(time.RFC3339Nano, v); err == nil { + return v + } + // Recorded planner inputs can themselves be JSON strings. + if strings.HasPrefix(v, "{") || strings.HasPrefix(v, "[") { + var nested any + dec := json.NewDecoder(bytes.NewBufferString(v)) + dec.UseNumber() + if dec.Decode(&nested) == nil && json.Valid([]byte(v)) { + body, err := json.Marshal(r.data(nested, "")) + if err == nil { + return string(body) + } + } + } + return dataPath.ReplaceAllString(r.Text(v), "[path]") + default: + if dataID(key) && isNumber(doc) { + return r.ID(fmt.Sprint(doc)) + } + return doc + } +} diff --git a/go/internal/redact/data_test.go b/go/internal/redact/data_test.go new file mode 100644 index 000000000..1b385b13d --- /dev/null +++ b/go/internal/redact/data_test.go @@ -0,0 +1,43 @@ +package redact + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestDiagnosticDataKeepsEvidenceAndScrubsIdentity(t *testing.T) { + r := New([]byte("box-agent-key"), []string{"Garage meter"}) + var doc any + input := `{"asset_id":"hardware:secret-device","label":"Garage meter","source":"hardware_counter","quality":"measured","provenance":"counter","energy_wh":12.25,"sample_count":7,"at":"2026-10-18T15:00:00Z","first_seen_ms":1792328400000,"missing":null,"driver":"Garage meter","VIN":"ABC12345678901234","apiToken":"do-not-publish","endpoint":"http://192.168.1.7","input_json":"{\"power_w\":1234,\"password\":\"secret\"}"}` + dec := json.NewDecoder(strings.NewReader(input)) + dec.UseNumber() + if err := dec.Decode(&doc); err != nil { + t.Fatal(err) + } + got := r.Data(doc).(map[string]any) + if got["energy_wh"] != json.Number("12.25") || got["sample_count"] != json.Number("7") || got["first_seen_ms"] != json.Number("1792328400000") || got["at"] != "2026-10-18T15:00:00Z" || got["missing"] != nil { + t.Fatalf("evidence changed: %+v", got) + } + if got["asset_id"] != r.ID("hardware:secret-device") || got["driver"] != r.ID("Garage meter") || got["source"] != "hardware_counter" || got["provenance"] != "counter" { + t.Fatalf("identity or provenance: %+v", got) + } + body, _ := json.Marshal(got) + for _, secret := range []string{"hardware:secret-device", "Garage meter", "ABC12345678901234", "do-not-publish", "192.168.1.7", "password", "secret"} { + if strings.Contains(string(body), secret) { + t.Errorf("diagnostic answer contains %q: %s", secret, body) + } + } + if !strings.Contains(got["input_json"].(string), `"power_w":1234`) { + t.Fatalf("nested planner input lost its values: %s", body) + } + private := r.Data(map[string]any{"error": "read /private/house/config.yaml: failed", "192.168.2.7": 12, "clock": map[string]any{"source": "ntp"}, "SN": 123456789, "VIN": 987654321, "IPAddress": "10.0.0.1"}).(map[string]any) + encoded, _ := json.Marshal(private) + if strings.Contains(string(encoded), "/private") || strings.Contains(string(encoded), "192.168.2.7") || strings.Contains(string(encoded), "123456789") || strings.Contains(string(encoded), "987654321") || strings.Contains(string(encoded), "10.0.0.1") || private["clock"].(map[string]any)["source"] != "ntp" { + t.Fatalf("network identity, path or clock source: %s", encoded) + } + metrics := r.Data(map[string]any{"metrics": []any{"pv_mppt1_w", "battery_2_soc"}, "metric": "hp_phase1_current_a"}).(map[string]any) + if metrics["metric"] != "hp_phase1_current_a" || metrics["metrics"].([]any)[0] != "pv_mppt1_w" { + t.Fatalf("metric names can no longer be queried: %+v", metrics) + } +} diff --git a/go/protocol/agentapi.go b/go/protocol/agentapi.go new file mode 100644 index 000000000..fa4280891 --- /dev/null +++ b/go/protocol/agentapi.go @@ -0,0 +1,66 @@ +package protocol + +// APIRead carries a read from the diagnostic catalogue. It has no method, +// body, headers or step-up field. api.req remains closed to agents. +// The answer uses APIHeadMsg, APIChunk and APIEnd under the request id. +type APIRead struct { + Path string `cbor:"path"` + Query map[string]string `cbor:"query,omitempty"` + MaxBytes int64 `cbor:"maxBytes,omitempty"` +} + +func DecodeAPIRead(body []byte) (APIRead, error) { + var read APIRead + err := strictDec.Unmarshal(body, &read) + return read, err +} + +const AgentAPIMaxBytes = 1 << 20 + +// AgentAPIRoute is a diagnostic read. The box checks both the api.read +// scope and this route's scope, and its HTTP router must still call it Read. +type AgentAPIRoute struct { + Path string `json:"path"` + Scope string `json:"scope"` + Description string `json:"description"` + Query map[string]string `json:"query,omitempty"` +} + +// AgentAPIRoutes is the catalogue the box serves and MCP lists. Paths are +// exact; query keys not listed here are refused. Responses stay JSON. +var AgentAPIRoutes = []AgentAPIRoute{ + {Path: "/api/status", Scope: ScopeLiveRead, Description: "Current measurements, control state and freshness"}, + {Path: "/api/health", Scope: ScopeHealthRead, Description: "Box health"}, + {Path: "/api/system/info", Scope: ScopeHealthRead, Description: "Runtime and host diagnostics, with identifying fields removed"}, + {Path: "/api/storage/inventory", Scope: ScopeHealthRead, Description: "Storage use and retention"}, + {Path: "/api/components", Scope: ScopeHealthRead, Description: "Component versions and state"}, + {Path: "/api/components/history", Scope: ScopeHealthRead, Description: "Component changes"}, + {Path: "/api/drivers", Scope: ScopeHealthRead, Description: "Driver diagnostics by opaque id"}, + {Path: "/api/data-sources", Scope: ScopeHealthRead, Description: "Available data sources and freshness"}, + {Path: "/api/battery_models", Scope: ScopePlanRead, Description: "Learned battery model and efficiency"}, + {Path: "/api/loadmodel", Scope: ScopePlanRead, Description: "Load forecast model and learning"}, + {Path: "/api/pvmodel", Scope: ScopePlanRead, Description: "Solar forecast model and learning"}, + {Path: "/api/forecast", Scope: ScopePlanRead, Description: "Load and solar forecasts", Query: map[string]string{"range": "Time ahead, such as 24h"}}, + {Path: "/api/prices", Scope: ScopePlanRead, Description: "Stored import and export prices", Query: map[string]string{"range": "Time ahead, such as 24h", "since_ms": "Unix milliseconds", "until_ms": "Unix milliseconds"}}, + {Path: "/api/planner/prefs", Scope: ScopeConfigRead, Description: "Planner preferences"}, + {Path: "/api/mpc/plan", Scope: ScopePlanRead, Description: "Full current plan and per-asset slots"}, + {Path: "/api/mpc/diagnose", Scope: ScopePlanRead, Description: "Current planner inputs, constraints and result"}, + {Path: "/api/mpc/diagnose/history", Scope: ScopePlanRead, Description: "Recorded planning decisions", Query: map[string]string{"since": "Unix milliseconds", "until": "Unix milliseconds", "limit": "Maximum snapshots, up to 5000"}}, + {Path: "/api/mpc/diagnose/at", Scope: ScopePlanRead, Description: "The planning snapshot active at an incident", Query: map[string]string{"ts": "Required incident time, Unix milliseconds"}}, + {Path: "/api/history", Scope: ScopeHistoryRead, Description: "Recorded site measurements", Query: map[string]string{"range": "Time back, such as 1h", "points": "Maximum rows"}}, + {Path: "/api/energy/daily", Scope: ScopeHistoryRead, Description: "Daily measured energy and costs", Query: map[string]string{"days": "Number of days"}}, + {Path: "/api/energy/assets", Scope: ScopeHistoryRead, Description: "Energy-ledger assets by opaque id"}, + {Path: "/api/energy/history", Scope: ScopeHistoryRead, Description: "Energy-ledger entries in Wh, including flow, source, quality, provenance and sample count", Query: map[string]string{"scope": "system or asset", "asset_id": "Opaque id from /api/energy/assets", "since": "Unix milliseconds", "until": "Unix milliseconds", "bucket": "Bucket duration, at least 5m", "limit": "Maximum buckets, up to 5000"}}, + {Path: "/api/savings/daily", Scope: ScopeHistoryRead, Description: "Daily cost comparison and coverage", Query: map[string]string{"days": "Number of days"}}, + {Path: "/api/series/catalog", Scope: ScopeHistoryRead, Description: "Stored drivers, metric names and units"}, + {Path: "/api/series", Scope: ScopeHistoryRead, Description: "One driver's measured series with min, max, sample count, resolution and bounds", Query: map[string]string{"driver": "Required opaque driver id from /api/series/catalog", "metric": "Required metric name or comma-separated names from the catalogue", "range": "Time back, such as 6h", "since": "Unix milliseconds", "until": "Unix milliseconds", "points": "Maximum points per metric"}}, +} + +func AgentAPIPath(path string) (AgentAPIRoute, bool) { + for _, route := range AgentAPIRoutes { + if route.Path == path { + return route, true + } + } + return AgentAPIRoute{}, false +} diff --git a/go/protocol/agentapi_test.go b/go/protocol/agentapi_test.go new file mode 100644 index 000000000..cd6420c29 --- /dev/null +++ b/go/protocol/agentapi_test.go @@ -0,0 +1,23 @@ +package protocol + +import ( + "encoding/hex" + "testing" +) + +func TestDiagnosticAPIReadVector(t *testing.T) { + // CBOR map(1), text(4) path, text(19) /api/energy/history. + body, err := hex.DecodeString("a16470617468732f6170692f656e657267792f686973746f7279") + if err != nil { + t.Fatal(err) + } + read, err := DecodeAPIRead(body) + if err != nil || read.Path != "/api/energy/history" || read.MaxBytes != 0 { + t.Fatalf("read vector: %+v %v", read, err) + } + // A method field cannot add a write path to this read. + bad, _ := hex.DecodeString("a26470617468732f6170692f656e657267792f686973746f7279666d6574686f6464504f5354") + if _, err := DecodeAPIRead(bad); err == nil { + t.Fatal("unknown method field accepted") + } +} diff --git a/go/protocol/contract_gen.go b/go/protocol/contract_gen.go index 6f5d07a89..fcbcaf2e7 100644 --- a/go/protocol/contract_gen.go +++ b/go/protocol/contract_gen.go @@ -348,6 +348,8 @@ var RegistryAgentArgs = map[string][]AgentArg{ }, } +const MsgAPIRead = "api.read" + // RegistryReads maps each read an agent may send to the scope it needs. var RegistryReads = map[string]string{ // live.get — Readings and freshness now. @@ -374,6 +376,8 @@ var RegistryReads = map[string]string{ "config.get": "ftw.config.read", // diag.run — A check that changes nothing. "diag.run": "ftw.diag.run", + // api.read — Read a diagnostic API route within its data scope. + "api.read": "ftw.diag.run", } // Error codes. The box sends the code and machine-readable args; the app From 94890d1a15734617738a3f72a302860ff899a9cc Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sun, 18 Oct 2026 16:19:52 +0200 Subject: [PATCH 2/2] Expose the full technical read API and debugging context --- .changeset/mcp-diagnostic-api-reads.md | 2 +- contract/registry.yaml | 4 +- go/internal/api/api_agent_archive.go | 161 +++++++++++++++ go/internal/api/api_agent_catalog.go | 156 ++++++++++++++ go/internal/api/api_agent_reads.go | 219 ++++++++++++-------- go/internal/api/api_agent_reads_test.go | 264 +++++++++++++++++++----- go/internal/appproto/agentapi.go | 31 +-- go/internal/appproto/agentapi_test.go | 17 +- go/internal/redact/data.go | 102 --------- go/internal/redact/data_test.go | 43 ---- go/protocol/agentapi.go | 73 +++---- go/protocol/agentapi_test.go | 23 +++ go/protocol/contract_gen.go | 4 +- 13 files changed, 736 insertions(+), 363 deletions(-) create mode 100644 go/internal/api/api_agent_archive.go create mode 100644 go/internal/api/api_agent_catalog.go delete mode 100644 go/internal/redact/data.go delete mode 100644 go/internal/redact/data_test.go diff --git a/.changeset/mcp-diagnostic-api-reads.md b/.changeset/mcp-diagnostic-api-reads.md index 256f440d3..87a4cfb3d 100644 --- a/.changeset/mcp-diagnostic-api-reads.md +++ b/.changeset/mcp-diagnostic-api-reads.md @@ -2,4 +2,4 @@ "ftw": minor --- -Let authorized assistants read the energy ledger, measured series, forecasts, models and planner snapshots through MCP. The box checks each data scope, redacts identifying fields and stops reads when access ends. +Give authorized MCP sessions the full diagnostic read API, including the energy ledger, raw series, logs, configuration, driver source and support exports. Keep technical names and addresses, list the connected box's routes, and page large answers without a total size limit. Stop reads when access ends. diff --git a/contract/registry.yaml b/contract/registry.yaml index 22a78b80b..0ae5357af 100644 --- a/contract/registry.yaml +++ b/contract/registry.yaml @@ -141,7 +141,7 @@ scopes: - { name: ftw.members.write, desc: Invite and revoke } - { name: ftw.logs.read, desc: Log excerpts the box redacts } - { name: ftw.config.read, desc: Configuration the box redacts } - - { name: ftw.diag.run, desc: Checks that change nothing } + - { name: ftw.diag.run, desc: Full technical diagnostics and read API } - { name: ftw.dispatch.hold, desc: Hold energy assets for a capped time } - { name: ftw.goals.request, desc: Ask an owner to change a durable goal } # Reserved: no op asks for an update yet, and no role but owner carries it. @@ -445,7 +445,7 @@ reads: - { msg: logs.get, scope: ftw.logs.read, desc: A log excerpt the box redacts } - { msg: config.get, scope: ftw.config.read, desc: A configuration section the box redacts } - { msg: diag.run, scope: ftw.diag.run, desc: A check that changes nothing } - - { msg: api.read, const: MsgAPIRead, scope: ftw.diag.run, desc: Read a diagnostic API route within its data scope } + - { msg: api.read, const: MsgAPIRead, scope: ftw.diag.run, desc: Read the full technical diagnostic API } # --------------------------------------------------------------------------- # Cars. A car belongs to its owner, not to the home, and visits chargers. The diff --git a/go/internal/api/api_agent_archive.go b/go/internal/api/api_agent_archive.go new file mode 100644 index 000000000..c15976dee --- /dev/null +++ b/go/internal/api/api_agent_archive.go @@ -0,0 +1,161 @@ +package api + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "context" + "io" + "net/http" + "os" + "strings" +) + +// Archive reads stay on the box. Spool a generated export to a private temp +// file, inspect entries without extracting paths, and remove it on return. +// The existing page writer keeps only the requested member's byte page. +func (s *Server) serveAgentArchive(w *agentReadBuffer, r *http.Request, member string) { + file, err := os.CreateTemp("", "ftw-diagnostic-*") + if err != nil { + writeJSON(w, 503, map[string]string{"error": "could not open diagnostic export"}) + return + } + defer os.Remove(file.Name()) + defer file.Close() + spool := &agentArchiveSpool{ctx: r.Context(), file: file, header: make(http.Header), status: 200} + s.serveAgentHTTP(spool, r) + if spool.err != nil || r.Context().Err() != nil { + writeJSON(w, 503, map[string]string{"error": "could not finish diagnostic export"}) + return + } + if _, err = file.Seek(0, io.SeekStart); err != nil { + writeJSON(w, 500, map[string]string{"error": "could not read diagnostic export"}) + return + } + // Preserve the handler's error answer rather than trying to open it as an archive. + if spool.status != 200 { + w.Header().Set("Content-Type", spool.header.Get("Content-Type")) + w.WriteHeader(spool.status) + _, _ = io.Copy(w, file) + return + } + var files []map[string]any + found := false + relative := func(name string) string { + _, rest, ok := strings.Cut(name, "/") + if ok && rest != "" { + return rest + } + return name + } + send := func(name string, size int64, open func() (io.ReadCloser, error)) error { + name = relative(name) + if member == "." { + files = append(files, map[string]any{"name": name, "bytes": size}) + return nil + } + if name != member { + return nil + } + body, err := open() + if err != nil { + return err + } + defer body.Close() + found = true + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, err = io.Copy(w, body) + return err + } + switch strings.Split(spool.header.Get("Content-Type"), ";")[0] { + case "application/zip": + stat, e := file.Stat() + if e != nil { + err = e + break + } + archive, e := zip.NewReader(file, stat.Size()) + if e != nil { + err = e + break + } + for _, entry := range archive.File { + if entry.FileInfo().IsDir() { + continue + } + entry := entry + err = send(entry.Name, int64(entry.UncompressedSize64), entry.Open) + if err != nil || found { + break + } + } + case "application/gzip", "application/x-gzip": + gz, e := gzip.NewReader(file) + if e != nil { + err = e + break + } + defer gz.Close() + archive := tar.NewReader(gz) + for { + header, e := archive.Next() + if e == io.EOF { + break + } + if e != nil { + err = e + break + } + if header.Typeflag != tar.TypeReg { + continue + } + err = send(header.Name, header.Size, func() (io.ReadCloser, error) { return io.NopCloser(archive), nil }) + if err != nil || found { + break + } + } + default: + writeJSON(w, 415, map[string]string{"error": "archive_member requires a ZIP or tar.gz export; omit it to read the original bytes"}) + return + } + if err != nil { + w.failed = err + return + } + if member == "." { + writeJSON(w, 200, map[string]any{"files": files}) + return + } + if !found { + writeJSON(w, 404, map[string]string{"error": "archive member not found; use archive_member '.' to list files"}) + } +} + +type agentArchiveSpool struct { + ctx context.Context + file *os.File + header http.Header + status int + wrote bool + err error +} + +func (s *agentArchiveSpool) Header() http.Header { return s.header } +func (s *agentArchiveSpool) WriteHeader(status int) { + if !s.wrote { + s.status, s.wrote = status, true + } +} +func (s *agentArchiveSpool) Write(body []byte) (int, error) { + if s.err != nil { + return 0, s.err + } + if err := s.ctx.Err(); err != nil { + s.err = err + return 0, err + } + s.WriteHeader(200) + n, err := s.file.Write(body) + s.err = err + return n, err +} diff --git a/go/internal/api/api_agent_catalog.go b/go/internal/api/api_agent_catalog.go new file mode 100644 index 000000000..c5d0e6e9c --- /dev/null +++ b/go/internal/api/api_agent_catalog.go @@ -0,0 +1,156 @@ +package api + +import ( + "net/http" + "net/url" + "regexp" + "runtime/debug" + "sort" + "strings" + + "github.com/srcfl/ftw/go/internal/apiauth" + "github.com/srcfl/ftw/go/protocol" +) + +// The router owns eligibility. These hints only describe queries; they never +// decide which Read routes a connected assistant can reach. +var agentAPIHints = []protocol.AgentAPIRoute{ + {Path: "/api/status", Description: "Current measurements, control state and freshness"}, + {Path: "/api/health", Description: "Box health"}, + {Path: "/api/system/info", Description: "Runtime and host diagnostics, including host details"}, + {Path: "/api/storage/inventory", Description: "Storage use and retention"}, + {Path: "/api/components", Description: "Component versions and state"}, + {Path: "/api/components/history", Description: "Component changes"}, + {Path: "/api/drivers", Description: "Driver diagnostics by configured name"}, + {Path: "/api/data-sources", Description: "Available data sources and freshness"}, + {Path: "/api/battery_models", Description: "Learned battery model and efficiency"}, + {Path: "/api/loadmodel", Description: "Load forecast model and learning"}, + {Path: "/api/pvmodel", Description: "Solar forecast model and learning"}, + {Path: "/api/forecast", Description: "Load and solar forecasts", Query: map[string]string{"range": "Time ahead, such as 24h"}}, + {Path: "/api/prices", Description: "Stored import and export prices", Query: map[string]string{"range": "Time ahead, such as 24h", "since_ms": "Unix milliseconds", "until_ms": "Unix milliseconds"}}, + {Path: "/api/planner/prefs", Description: "Planner preferences"}, + {Path: "/api/mpc/plan", Description: "Full current plan and per-asset slots"}, + {Path: "/api/mpc/diagnose", Description: "Current planner inputs, constraints and result"}, + {Path: "/api/mpc/diagnose/history", Description: "Recorded planning decisions", Query: map[string]string{"since": "Unix milliseconds", "until": "Unix milliseconds", "limit": "Maximum snapshots, up to 5000"}}, + {Path: "/api/mpc/diagnose/at", Description: "The planning snapshot active at an incident", Query: map[string]string{"ts": "Required incident time, Unix milliseconds"}}, + {Path: "/api/history", Description: "Recorded site measurements", Query: map[string]string{"range": "Time back, such as 1h", "points": "Maximum rows"}}, + {Path: "/api/energy/daily", Description: "Daily measured energy and costs", Query: map[string]string{"days": "Number of days"}}, + {Path: "/api/energy/assets", Description: "Energy-ledger assets by configured name"}, + {Path: "/api/energy/history", Description: "Energy-ledger entries in Wh, including flow, source, quality, provenance and sample count", Query: map[string]string{"scope": "system or asset", "asset_id": "Asset id from /api/energy/assets", "since": "Unix milliseconds", "until": "Unix milliseconds", "bucket": "Bucket duration, at least 5m", "limit": "Maximum buckets, up to 5000"}}, + {Path: "/api/savings/daily", Description: "Daily cost comparison and coverage", Query: map[string]string{"days": "Number of days"}}, + {Path: "/api/series/catalog", Description: "Stored drivers, metric names and units"}, + {Path: "/api/series", Description: "One driver's measured series with min, max, sample count, resolution and bounds", Query: map[string]string{"driver": "Required driver name from /api/series/catalog", "metric": "Required metric name or comma-separated names from the catalogue", "range": "Time back, such as 6h", "since": "Unix milliseconds", "until": "Unix milliseconds", "points": "Maximum points per metric"}}, +} + +// These GET handlers return diagnostic data despite their Local tier for +// phone passthrough. OAuth GETs start or complete a credential exchange and +// are not reads. Configure and Actuate handlers keep their command path. +var agentLocalReads = map[string]bool{ + "/api/config": true, + "/api/drivers/{id}/source": true, + "/api/drivers/{name}/logs": true, + "/api/logs": true, + "/api/support/dump": true, + "/api/support/report": true, + "/api/ocpp/chargers": true, + "/api/backups/{id}": true, + "/api/agents": true, + "/api/agents/audit": true, +} + +func agentReadMark(pattern string, mark routeMark) bool { + fields := strings.Fields(pattern) + return len(fields) == 2 && fields[0] == http.MethodGet && !mark.static && + (mark.tier == apiauth.TierRead || mark.tier == apiauth.TierLocal && agentLocalReads[fields[1]]) +} + +func (s *Server) agentReadAllowed(r *http.Request) bool { + if r.Method != http.MethodGet { + return false + } + _, pattern := s.mux.Handler(r) + mark, ok := s.marks[pattern] + return ok && agentReadMark(pattern, mark) +} + +func (s *Server) agentReadCatalog() []protocol.AgentAPIRoute { + routes := make([]protocol.AgentAPIRoute, 0, len(s.marks)) + for pattern, mark := range s.marks { + if !agentReadMark(pattern, mark) { + continue + } + path := strings.Fields(pattern)[1] + route := protocol.AgentAPIRoute{Path: path, Scope: protocol.ScopeDiagRun, Description: "Read " + path} + for _, hint := range agentAPIHints { + if hint.Path == path { + route.Description, route.Query = hint.Description, hint.Query + break + } + } + if query, ok := agentAPIQueryHints[path]; ok { + route.Query = query + } + routes = append(routes, route) + } + sort.Slice(routes, func(i, j int) bool { return routes[i].Path < routes[j].Path }) + return routes +} + +var agentAPIQueryHints = map[string]map[string]string{ + "/api/components/history": {"id": "Component id", "kind": "Component kind", "limit": "Maximum changes"}, + "/api/data-sources": {"lat": "Latitude", "lon": "Longitude"}, + "/api/device_repository/catalog": {"channel": "Driver channel"}, + "/api/notifications/history": {"limit": "Maximum notifications"}, + "/api/logs": {"limit": "Maximum log entries, up to 5000"}, + "/api/drivers/{name}/logs": {"limit": "Maximum log entries, up to 2000"}, + "/api/ev/status": {"driver": "Configured driver name"}, + "/api/research/load/dump": {"days": "Days of original observations, 1 to 365; tar.gz export"}, + "/api/agents/audit": {"before": "Audit sequence before which to page", "n": "Maximum audit rows"}, + "/api/series": {"driver": "Configured driver name", "metric": "Metric or comma-separated metrics from /api/series/catalog", "range": "Time back, such as 6h", "since": "Unix milliseconds", "until": "Unix milliseconds", "points": "Maximum points per metric; 0 returns stored rows", "format": "json (default) or csv"}, + "/api/energy/history.csv": {"scope": "system or asset", "asset_id": "Asset id from /api/energy/assets", "since": "Unix milliseconds", "until": "Unix milliseconds", "bucket": "Bucket duration, at least 5m", "limit": "Maximum buckets, up to 5000"}, +} + +// This context is requested for technical debugging, not for routine status, +// history or control. Source links name a recorded build or a declared +// release; a default-branch fallback says explicitly that it may differ. +func (s *Server) agentDebugContext() map[string]any { + ref, kind, revision, modified := "master", "default branch; may differ from the running build", "", false + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" { + revision = setting.Value + } + if setting.Key == "vcs.modified" { + modified = setting.Value == "true" + } + } + } + if revision != "" { + ref, kind = revision, "recorded build revision" + } + if regexp.MustCompile(`^v?0\.[0-9]+\.[0-9]+(?:-beta\.[0-9]+)?$`).MatchString(s.deps.Version) { + ref, kind = "v"+strings.TrimPrefix(s.deps.Version, "v"), "declared release tag" + } + tree := "https://github.com/srcfl/ftw/tree/" + url.PathEscape(ref) + return map[string]any{ + "core": map[string]any{"version": s.deps.Version, "revision": revision, "modified": modified, "source_ref_kind": kind, + "repository": "https://github.com/srcfl/ftw", "source": tree, + "api": tree + "/go/internal/api", "energy_history": tree + "/go/internal/state", "planner": tree + "/go/internal/mpc", + "driver_host": tree + "/go/internal/drivers", "bundled_driver_pin": tree + "/drivers/BUNDLED_SOURCE.json", "issues": "https://github.com/srcfl/ftw/issues"}, + "related_sources": []map[string]string{ + {"part": "Editable Lua drivers", "repository": "https://github.com/srcfl/device-drivers", "running_source": "Read /api/drivers/catalog, /api/drivers/{id}/source and /api/device_repository/drivers/{id}/versions for installed code and hashes"}, + {"part": "FTW Cloud and MCP", "repository": "https://github.com/srcfl/ftw-cloud", "running_source": "https://cloud.ftw.energy/source", "revision": "The Cloud HTTP response names X-Source-Revision"}, + {"part": "Web client", "repository": "https://github.com/srcfl/ftw-webapp"}, + {"part": "Native apps", "repository": "https://github.com/srcfl/ftw-app"}, + {"part": "Compiled Energyplan solver", "repository": "https://github.com/srcfl/energyplan", "access": "Private repository; Core carries binaries and validates their plans"}, + }, + "workflow": []string{ + "Record the symptom, expected behavior, incident time and timezone. Separate measured facts from guesses.", + "Read /api/health, /api/support/report and /api/components for versions, clock, freshness and failures.", + "Use /api/energy/history for Wh and provenance, /api/series for original measurements, and /api/mpc/diagnose/history then /api/mpc/diagnose/at for the plan active at the incident.", + "Inspect configuration, driver details, driver source and logs as needed. Compare with source for the running version; default-branch code may differ.", + "For a bug report, include the Core version and source reference, equipment and driver versions, reproduction steps, expected and observed results, incident time, exact API queries and relevant logs or snapshots. Mark data gaps and whether evidence came from real equipment, a simulator or a test.", + "Use the existing control and goal tools when an action is needed. Reading a diagnostic view does not issue a command. Prepare a bug report for the user; do not publish it without their request.", + }, + } +} diff --git a/go/internal/api/api_agent_reads.go b/go/internal/api/api_agent_reads.go index ed4afab2a..7f790d6e7 100644 --- a/go/internal/api/api_agent_reads.go +++ b/go/internal/api/api_agent_reads.go @@ -2,110 +2,153 @@ package api import ( "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" - "errors" + "hash" "net/http" "strconv" + "unicode/utf8" "github.com/srcfl/ftw/go/internal/apiauth" - "github.com/srcfl/ftw/go/internal/redact" "github.com/srcfl/ftw/go/protocol" ) -// ServeAgentRead runs only a catalogued JSON read through the usual HTTP -// handler. It buffers and redacts the whole answer on the box before any -// byte leaves for FTW Cloud. The caller comes from the Noise session. -func (s *Server) ServeAgentRead(w http.ResponseWriter, request *http.Request, redactor *redact.Redactor) { +// ServeAgentRead gives a granted diagnostic session all technical reads. +// It keeps names, addresses, source code and measured data as the handlers +// return them. Existing handlers still mask saved credentials where they do +// so on the box page. api.req and all write handlers remain separate. +func (s *Server) ServeAgentRead(w http.ResponseWriter, request *http.Request, read protocol.APIRead) { caller, named := apiauth.FromRequest(request) - route, known := protocol.AgentAPIPath(request.URL.Path) - facts := s.Route(request) - if !named || redactor == nil || !known || request.Method != http.MethodGet || facts.Static || facts.Tier != apiauth.TierRead || - !caller.Scopes.Has(protocol.ScopeDiagRun) || !caller.Scopes.Has(route.Scope) { + if !named || !apiauth.AgentRole(caller.Role) || !caller.Scopes.Has(protocol.ScopeDiagRun) || request.Method != http.MethodGet { writeJSON(w, http.StatusForbidden, map[string]string{"error": "this diagnostic read is not granted"}) return } - query := request.URL.Query() - for name, values := range query { - if _, ok := route.Query[name]; !ok || len(values) != 1 { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "unknown or repeated query parameter"}) - return - } + if request.URL.Path == protocol.AgentAPICatalogPath { + writeJSON(w, http.StatusOK, map[string]any{ + "routes": s.agentReadCatalog(), "requiredScope": protocol.ScopeDiagRun, + "maxPageBytes": protocol.AgentAPIPageBytes, + "troubleshooting": s.agentDebugContext(), + "queryPolicy": "Use the existing API's parameters. Query hints are not a restriction.", + "timeUnit": "Time query parameters since, until, since_ms, until_ms and ts use Unix milliseconds", + "pagination": "Large answers use byte pages. Pass next_offset as offset and sha256 as if_sha256 to detect a changed answer.", + }) + return } - // Resolve only ids issued by this box's redactor. A real hardware id or - // driver name is never accepted in an agent query. - for _, field := range []string{"asset_id", "driver"} { - id := query.Get(field) - if id == "" { - continue - } - name, ok := s.agentReadID(field, id, redactor) - if !ok { - writeJSON(w, http.StatusNotFound, map[string]string{"error": "unknown opaque id; read the asset or series catalogue"}) - return - } - query.Set(field, name) - } - request = request.Clone(request.Context()) - request.URL.RawQuery = query.Encode() - if request.URL.Path == "/api/series" && query.Get("points") == "" { - query.Set("points", "500") - request.URL.RawQuery = query.Encode() - } - if raw := query.Get("points"); raw != "" { - n, err := strconv.Atoi(raw) - if err != nil || n < 1 || n > 5000 { - writeJSON(w, http.StatusBadRequest, map[string]string{"error": "points must be between 1 and 5000"}) - return - } + if !s.agentReadAllowed(request) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "this route is not a diagnostic read; settings and control use their command tools"}) + return + } + if read.Offset < 0 || read.MaxBytes < 0 || read.MaxBytes > protocol.AgentAPIPageBytes { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid diagnostic page"}) + return + } + pageBytes := read.MaxBytes + if pageBytes == 0 { + pageBytes = 64 << 10 + } + answer := &agentReadBuffer{ctx: request.Context(), header: make(http.Header), status: http.StatusOK, + offset: read.Offset, limit: pageBytes, digest: sha256.New()} + + // DiagRun covers all diagnostic reads, including pairing and vehicle + // status. Add only these read scopes to the handler context; this does + // not change the stored grant, role, caller or any command authority. + scopes := append(caller.Scopes.Names(), protocol.ScopeMembersRead, protocol.ScopeCarsRead) + caller.Scopes = apiauth.NewScopeSet(scopes...) + request = request.WithContext(apiauth.WithCaller(request.Context(), caller)) + if read.ArchiveMember != "" { + s.serveAgentArchive(answer, request, read.ArchiveMember) + } else { + s.serveAgentHTTP(answer, request) + } + if answer.failed != nil { + writeJSON(w, http.StatusBadGateway, map[string]string{"error": "could not read the archive member completely"}) + return + } + if request.Context().Err() != nil { + return } - answer := &agentReadBuffer{header: make(http.Header), status: http.StatusOK} - s.ServeHTTP(answer, request) - if answer.large { - writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "answer is too large; narrow the time range, points or limit"}) + digest := hex.EncodeToString(answer.digest.Sum(nil)) + if read.IfSHA256 != "" && read.IfSHA256 != digest { + writeJSON(w, http.StatusConflict, map[string]string{"error": "the answer changed between pages; start again or narrow the query"}) return } - var body any - dec := json.NewDecoder(bytes.NewReader(answer.body.Bytes())) - dec.UseNumber() - if !json.Valid(answer.body.Bytes()) || dec.Decode(&body) != nil { - writeJSON(w, http.StatusBadGateway, map[string]string{"error": "the diagnostic route did not return JSON"}) + if read.Offset > answer.total { + writeJSON(w, http.StatusRequestedRangeNotSatisfiable, map[string]string{"error": "offset exceeds the answer length"}) return } - writeJSON(w, answer.status, redactor.Data(body)) + body := answer.body.Bytes() + more := read.Offset+int64(len(body)) < answer.total + complete := read.Offset == 0 && !more + encoding := "base64" + var data any = base64.StdEncoding.EncodeToString(body) + if complete && json.Valid(body) { + encoding, data = "json", json.RawMessage(body) + } else if utf8.Valid(body) && !agentBinaryMedia(answer.header.Get("Content-Type")) { + encoding, data = "utf-8", string(body) + } + result := map[string]any{"content_type": answer.header.Get("Content-Type"), "encoding": encoding, "body": data, + "offset": read.Offset, "total_bytes": answer.total, "sha256": digest, "complete": complete, "more": more} + if more { + result["next_offset"] = read.Offset + int64(len(body)) + } + writeJSON(w, answer.status, result) } -func (s *Server) agentReadID(field, id string, redactor *redact.Redactor) (string, bool) { - if s.deps.State == nil { - return "", false - } - if field == "asset_id" { - assets, err := s.deps.State.EnergyAssets() - if err == nil { - for _, asset := range assets { - if redactor.ID(asset.AssetID) == id { - return asset.AssetID, true - } - } +// The LAN administration handlers have presence checks as well as data +// reads. Project their existing read results here without claiming that this +// remote caller is on the LAN or is an owner, and without admitting mutations. +func (s *Server) serveAgentHTTP(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/agents", "/api/agents/audit": + if s.deps.Agents == nil { + writeJSON(w, 503, map[string]string{"error": "agents are not available on this box"}) + return } - } else { - names, err := s.deps.State.DriverNames() - if err == nil { - for _, name := range names { - if redactor.ID(name) == id { - return name, true - } + a := s.deps.Agents + if r.URL.Path == "/api/agents/audit" { + before, _ := strconv.ParseInt(r.URL.Query().Get("before"), 10, 64) + n, _ := strconv.Atoi(r.URL.Query().Get("n")) + rows, more, err := a.Audit(before, n) + if err != nil { + agentError(w, err) + return } + writeJSON(w, 200, protocol.AuditPage{Rows: nonNil(rows), More: more}) + return } + facts := a.Facts() + facts.Widen = false + writeJSON(w, 200, agentsPage{AgentList: protocol.AgentList{Agents: nonNil(a.Agents()), Requests: nonNil(a.Requests()), Keys: nonNil(a.ApprovalKeys())}, Box: facts, Registry: registryForPage()}) + case "/api/cloud", "/api/cloud/preview": + link, ok := s.cloudLink(w) + if !ok { + return + } + if r.URL.Path == "/api/cloud" { + writeJSON(w, 200, link.Status()) + } else { + writeJSON(w, 200, link.Preview()) + } + default: + s.ServeHTTP(w, r) } - return "", false } +// Retain one byte page while hashing and counting the whole answer. A large +// history export or backup has no total-byte cutoff. Context cancellation +// stops a handler that keeps writing after the session ends. type agentReadBuffer struct { - header http.Header - status int - body bytes.Buffer - large bool - wrote bool + ctx context.Context + header http.Header + status int + body bytes.Buffer + wrote bool + offset, limit, total int64 + digest hash.Hash + failed error } func (b *agentReadBuffer) Header() http.Header { return b.header } @@ -115,10 +158,24 @@ func (b *agentReadBuffer) WriteHeader(status int) { } } func (b *agentReadBuffer) Write(body []byte) (int, error) { + if err := b.ctx.Err(); err != nil { + return 0, err + } b.WriteHeader(http.StatusOK) - if b.large || b.body.Len()+len(body) > protocol.AgentAPIMaxBytes { - b.large = true - return 0, errors.New("diagnostic answer too large") + _, _ = b.digest.Write(body) + start := min(int64(len(body)), max(int64(0), b.offset-b.total)) + count := min(int64(len(body))-start, b.limit-int64(b.body.Len())) + if count > 0 { + _, _ = b.body.Write(body[start : start+count]) + } + b.total += int64(len(body)) + return len(body), nil +} + +func agentBinaryMedia(media string) bool { + switch media { + case "application/zip", "application/gzip", "application/x-gzip", "application/octet-stream": + return true } - return b.body.Write(body) + return false } diff --git a/go/internal/api/api_agent_reads_test.go b/go/internal/api/api_agent_reads_test.go index 7aff7fcad..507d1b816 100644 --- a/go/internal/api/api_agent_reads_test.go +++ b/go/internal/api/api_agent_reads_test.go @@ -1,6 +1,13 @@ package api import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" "net/http" "net/http/httptest" @@ -9,7 +16,6 @@ import ( "testing" "github.com/srcfl/ftw/go/internal/apiauth" - "github.com/srcfl/ftw/go/internal/redact" "github.com/srcfl/ftw/go/protocol" ) @@ -17,20 +23,26 @@ func agentReadRequest(path string, scopes ...string) *http.Request { r := httptest.NewRequest(http.MethodGet, path, nil) return r.WithContext(apiauth.WithCaller(r.Context(), apiauth.Caller{Kind: apiauth.KindAgent, Role: apiauth.RoleAgent, Subject: "agent:test", Scopes: apiauth.NewScopeSet(scopes...)})) } - -func TestAgentReadsTheEnergyLedgerByItsOpaqueAssetID(t *testing.T) { +func diagnosticRead(t *testing.T, srv *Server, path string, read protocol.APIRead) (*httptest.ResponseRecorder, map[string]json.RawMessage) { + t.Helper() + answer := httptest.NewRecorder() + srv.ServeAgentRead(answer, agentReadRequest(path, protocol.ScopeDiagRun), read) + var doc map[string]json.RawMessage + if err := json.Unmarshal(answer.Body.Bytes(), &doc); err != nil { + t.Fatalf("invalid answer: %d %s", answer.Code, answer.Body.String()) + } + return answer, doc +} +func TestAgentReadsEnergyEvidenceWithTechnicalIDs(t *testing.T) { st, base, asset := seedAPIEnergyHistory(t) t.Cleanup(func() { _ = st.Close() }) srv := New(&Deps{State: st}) - rd := redact.New([]byte("agent-key"), []string{"Site meter"}) - catalog := httptest.NewRecorder() - srv.ServeAgentRead(catalog, agentReadRequest("/api/energy/assets", protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) - if catalog.Code != http.StatusOK || !strings.Contains(catalog.Body.String(), rd.ID(asset)) || strings.Contains(catalog.Body.String(), "Site meter") || strings.Contains(catalog.Body.String(), "maker:api-meter") { - t.Fatalf("asset catalogue: %d %s", catalog.Code, catalog.Body.String()) + catalog, _ := diagnosticRead(t, srv, "/api/energy/assets", protocol.APIRead{}) + if catalog.Code != 200 || !strings.Contains(catalog.Body.String(), "Site meter") || !strings.Contains(catalog.Body.String(), "maker:api-meter") { + t.Fatalf("technical data lost: %s", catalog.Body.String()) } - q := url.Values{"scope": {"asset"}, "asset_id": {rd.ID(asset)}, "since": {formatInt64(base)}, "until": {formatInt64(base + 300_000)}, "bucket": {"5m"}} - answer := httptest.NewRecorder() - srv.ServeAgentRead(answer, agentReadRequest("/api/energy/history?"+q.Encode(), protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) + q := url.Values{"scope": {"asset"}, "asset_id": {asset}, "since": {formatInt64(base)}, "until": {formatInt64(base + 300_000)}, "bucket": {"5m"}} + answer, doc := diagnosticRead(t, srv, "/api/energy/history?"+q.Encode(), protocol.APIRead{}) var body struct { Points []struct { AssetID string `json:"asset_id"` @@ -40,74 +52,214 @@ func TestAgentReadsTheEnergyLedgerByItsOpaqueAssetID(t *testing.T) { Provenance string `json:"provenance"` } `json:"points"` } - if answer.Code != http.StatusOK || json.Unmarshal(answer.Body.Bytes(), &body) != nil || len(body.Points) == 0 { - t.Fatalf("energy ledger: %d %s", answer.Code, answer.Body.String()) + if answer.Code != 200 || json.Unmarshal(doc["body"], &body) != nil || len(body.Points) == 0 { + t.Fatalf("ledger: %s", answer.Body.String()) } var measured float64 for _, p := range body.Points { - if p.AssetID != rd.ID(asset) { - t.Fatal("ledger exposed another id") + if p.AssetID != asset { + t.Fatal("id changed") } if p.Quality == "measured" { measured += p.Energy if p.Source != "hardware_counter" || p.Provenance != "counter" { - t.Fatalf("measurement provenance changed: %+v", p) + t.Fatalf("provenance changed: %+v", p) } } } if measured != 12 { - t.Fatalf("measured energy = %v, want 12 Wh", measured) + t.Fatalf("energy=%v, want 12 Wh", measured) } - q.Set("asset_id", asset) - refused := httptest.NewRecorder() - srv.ServeAgentRead(refused, agentReadRequest("/api/energy/history?"+q.Encode(), protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) - if refused.Code != http.StatusNotFound { - t.Fatalf("raw asset id accepted: %d", refused.Code) + csv, doc := diagnosticRead(t, srv, "/api/energy/history.csv?"+q.Encode(), protocol.APIRead{}) + if csv.Code != 200 || !bytes.Contains(doc["body"], []byte("provenance")) || !bytes.Contains(doc["encoding"], []byte("utf-8")) { + t.Fatalf("CSV: %s", csv.Body.String()) } } - -func TestDiagnosticReadsRequireBothScopesAndTheReadTier(t *testing.T) { +func TestDiagnosticCatalogueCoversAllReadAndLocalDiagnosticViews(t *testing.T) { srv := New(&Deps{}) - rd := redact.New([]byte("agent-key"), nil) - for _, tc := range []struct { - path string - scopes []string - }{ - {"/api/energy/history", []string{protocol.ScopeDiagRun}}, - {"/api/energy/history", []string{protocol.ScopeHistoryRead}}, - {"/api/config", []string{protocol.ScopeDiagRun, protocol.ScopeConfigRead}}, - {"/api/scan", []string{protocol.ScopeDiagRun, protocol.ScopeHealthRead}}, - {"/api/oauth/myuplink/start", []string{protocol.ScopeDiagRun, protocol.ScopeConfigRead}}, - } { - answer := httptest.NewRecorder() - srv.ServeAgentRead(answer, agentReadRequest(tc.path, tc.scopes...), rd) - if answer.Code != http.StatusForbidden { - t.Errorf("%s: %d", tc.path, answer.Code) + catalog := map[string]bool{} + for _, route := range srv.agentReadCatalog() { + catalog[route.Path] = true + } + for pattern, mark := range srv.marks { + fields := strings.Fields(pattern) + if len(fields) != 2 || fields[0] != "GET" || mark.static { + continue + } + path := fields[1] + if mark.tier == Read && !catalog[path] { + t.Errorf("missing Read route %s", path) + } + if mark.tier == Local && !catalog[path] && !strings.HasPrefix(path, "/api/oauth/") { + t.Errorf("unclassified Local GET %s", path) } } - for _, route := range protocol.AgentAPIRoutes { - facts := srv.Route(httptest.NewRequest(http.MethodGet, route.Path, nil)) - if facts.Tier != apiauth.TierRead || facts.Static { - t.Errorf("catalogue entry %s is not a Read route: %+v", route.Path, facts) + for _, path := range []string{"/api/config", "/api/logs", "/api/drivers/{id}/source", "/api/support/dump", "/api/support/report", "/api/agents/audit", "/api/backups/{id}", "/api/loadpoints/{id}/battery_boost", "/api/research/load/dump"} { + if !catalog[path] { + t.Errorf("missing %s", path) } } - answer := httptest.NewRecorder() - srv.ServeAgentRead(answer, agentReadRequest("/api/series?format=csv", protocol.ScopeDiagRun, protocol.ScopeHistoryRead), rd) - if answer.Code != http.StatusBadRequest { - t.Fatal("an unlisted query parameter was accepted") + for _, path := range []string{"/api/scan", "/api/version/check", "/api/oauth/myuplink/start", "/api/oauth/myuplink/callback"} { + if catalog[path] { + t.Errorf("side effect in catalogue: %s", path) + } + answer, _ := diagnosticRead(t, srv, path, protocol.APIRead{}) + if answer.Code != 403 { + t.Errorf("side-effect route %s accepted: %d", path, answer.Code) + } + } + srv.handle("GET /api/future/{id}", Read, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, 200, map[string]string{"id": r.PathValue("id"), "query": r.URL.Query().Get("new_parameter")}) + }) + answer, _ := diagnosticRead(t, srv, "/api/future/charger-192.168.1.9?new_parameter=yes", protocol.APIRead{}) + if answer.Code != 200 || !strings.Contains(answer.Body.String(), "charger-192.168.1.9") || !strings.Contains(answer.Body.String(), "yes") { + t.Fatalf("future read lost data: %s", answer.Body.String()) + } + for _, scope := range []string{"", protocol.ScopeHistoryRead} { + denied := httptest.NewRecorder() + srv.ServeAgentRead(denied, agentReadRequest("/api/future/x", scope), protocol.APIRead{}) + if denied.Code != 403 { + t.Fatal("diagnostic grant not checked") + } + } + mutation := agentReadRequest("/api/config", protocol.ScopeDiagRun) + mutation.Method = http.MethodPost + denied := httptest.NewRecorder() + srv.ServeAgentRead(denied, mutation, protocol.APIRead{}) + if denied.Code != 403 { + t.Fatal("write accepted") + } + t.Logf("Core exposes %d diagnostic routes", len(catalog)) +} +func TestDiagnosticPagesRetainEveryByteOfLargeAnswers(t *testing.T) { + srv := New(&Deps{}) + original := []byte(strings.Repeat("charger 192.168.1.9 9007199254740993\n", 40000) + "\x00\xff") + current := original + srv.handle("GET /api/test/export", Read, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(current) + }) + sum := sha256.Sum256(original) + digest := hex.EncodeToString(sum[:]) + var joined []byte + var offset int64 + for { + answer, doc := diagnosticRead(t, srv, "/api/test/export", protocol.APIRead{Offset: offset, MaxBytes: protocol.AgentAPIPageBytes, IfSHA256: digest}) + if answer.Code != 200 { + t.Fatalf("page: %s", answer.Body.String()) + } + var encoding, text string + _ = json.Unmarshal(doc["encoding"], &encoding) + _ = json.Unmarshal(doc["body"], &text) + part := []byte(text) + if encoding == "base64" { + var err error + part, err = base64.StdEncoding.DecodeString(text) + if err != nil { + t.Fatal(err) + } + } + joined = append(joined, part...) + var more bool + _ = json.Unmarshal(doc["more"], &more) + if !more { + break + } + _ = json.Unmarshal(doc["next_offset"], &offset) + } + if !bytes.Equal(joined, original) { + t.Fatal("paged bytes changed") + } + current = []byte("changed") + answer, doc := diagnosticRead(t, srv, "/api/test/export", protocol.APIRead{Offset: 1, IfSHA256: digest}) + if answer.Code != 409 || doc["body"] != nil { + t.Fatal("changed pages accepted") + } +} + +func TestDiagnosticStartIncludesVersionedSourceAndBugEvidence(t *testing.T) { + srv := New(&Deps{Version: "0.151.0-beta.1"}) + answer, _ := diagnosticRead(t, srv, protocol.AgentAPICatalogPath, protocol.APIRead{}) + for _, text := range []string{"https://github.com/srcfl/ftw/tree/v0.151.0-beta.1", "https://github.com/srcfl/device-drivers", "https://cloud.ftw.energy/source", "reproduction steps", "timezone", "real equipment", "/api/support/report"} { + if !strings.Contains(answer.Body.String(), text) { + t.Errorf("missing debugging context %q", text) + } } } -func TestDiagnosticAPIDoesNotPublishAnOversizedAnswer(t *testing.T) { +func TestDiagnosticArchiveMembersCanBeReadWithoutExternalTools(t *testing.T) { + for _, kind := range []string{"zip", "tar.gz"} { + t.Run(kind, func(t *testing.T) { + var archive bytes.Buffer + evidence := []byte("driver charger: 192.168.1.9; counter=9007199254740993\n") + media := "application/zip" + if kind == "zip" { + writer := zip.NewWriter(&archive) + file, err := writer.Create("ftw-support-stamp/logs/global.log") + if err != nil { + t.Fatal(err) + } + _, _ = file.Write(evidence) + if err = writer.Close(); err != nil { + t.Fatal(err) + } + } else { + media = "application/gzip" + gz := gzip.NewWriter(&archive) + writer := tar.NewWriter(gz) + if err := writer.WriteHeader(&tar.Header{Name: "ftw-research-stamp/logs/global.log", Size: int64(len(evidence)), Mode: 0600}); err != nil { + t.Fatal(err) + } + _, _ = writer.Write(evidence) + _ = writer.Close() + _ = gz.Close() + } + srv := New(&Deps{}) + srv.handle("GET /api/test/archive", Read, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", media) + _, _ = w.Write(archive.Bytes()) + }) + list, _ := diagnosticRead(t, srv, "/api/test/archive", protocol.APIRead{ArchiveMember: "."}) + if list.Code != 200 || !strings.Contains(list.Body.String(), "logs/global.log") { + t.Fatalf("archive list: %s", list.Body.String()) + } + answer, doc := diagnosticRead(t, srv, "/api/test/archive", protocol.APIRead{ArchiveMember: "logs/global.log"}) + var text string + _ = json.Unmarshal(doc["body"], &text) + if answer.Code != 200 || text != string(evidence) { + t.Fatalf("archive evidence changed: %s", answer.Body.String()) + } + absent, _ := diagnosticRead(t, srv, "/api/test/archive", protocol.APIRead{ArchiveMember: "../not-a-file"}) + if absent.Code != 404 { + t.Fatal("archive member lookup escaped its file list") + } + }) + } +} + +func TestDiagnosticArchiveCorruptionReturnsNoPartialMember(t *testing.T) { + var archive bytes.Buffer + writer := zip.NewWriter(&archive) + file, err := writer.CreateHeader(&zip.FileHeader{Name: "root/evidence.txt", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + evidence := []byte("original evidence must not leave as a successful partial read") + _, _ = file.Write(evidence) + _ = writer.Close() + corrupt := append([]byte(nil), archive.Bytes()...) + at := bytes.Index(corrupt, evidence) + if at < 0 { + t.Fatal("test member not found") + } + corrupt[at] ^= 0xff srv := New(&Deps{}) - srv.mux = http.NewServeMux() - srv.marks = make(map[string]routeMark) - srv.handle("GET /api/status", Read, func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(`{"text":"` + strings.Repeat("x", protocol.AgentAPIMaxBytes) + `"}`)) + srv.handle("GET /api/test/corrupt", Read, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(corrupt) }) - answer := httptest.NewRecorder() - srv.ServeAgentRead(answer, agentReadRequest("/api/status", protocol.ScopeDiagRun, protocol.ScopeLiveRead), redact.New([]byte("agent-key"), nil)) - if answer.Code != http.StatusRequestEntityTooLarge || strings.Contains(answer.Body.String(), "xxxxxxxx") { - t.Fatalf("oversized read: %d %s", answer.Code, answer.Body.String()) + answer, doc := diagnosticRead(t, srv, "/api/test/corrupt", protocol.APIRead{ArchiveMember: "evidence.txt"}) + if answer.Code != 502 || doc["body"] != nil { + t.Fatalf("corrupt member returned data: %s", answer.Body.String()) } } diff --git a/go/internal/appproto/agentapi.go b/go/internal/appproto/agentapi.go index 1fc18f1bf..80ca8c669 100644 --- a/go/internal/appproto/agentapi.go +++ b/go/internal/appproto/agentapi.go @@ -5,13 +5,11 @@ import ( "net/http" "sync" - "github.com/srcfl/ftw/go/internal/apiauth" - "github.com/srcfl/ftw/go/internal/redact" "github.com/srcfl/ftw/go/protocol" ) type agentReadGateway interface { - ServeAgentRead(http.ResponseWriter, *http.Request, *redact.Redactor) + ServeAgentRead(http.ResponseWriter, *http.Request, protocol.APIRead) } func (h *Handler) onAPIRead(env Envelope) error { @@ -22,13 +20,13 @@ func (h *Handler) onAPIRead(env Envelope) error { if err != nil { return h.refuse(env.ID, ErrUnknownOp, map[string]any{"t": env.T}) } - route, known := protocol.AgentAPIPath(read.Path) + known := protocol.ValidAgentAPIPath(read.Path) request := APIReq{Method: APIGet, Path: read.Path, Query: read.Query, MaxBytes: read.MaxBytes} - if !known || badAPIRequest(request) != "" || read.MaxBytes < 0 || read.MaxBytes > protocol.AgentAPIMaxBytes { + if !known || badAPIRequest(request) != "" || read.MaxBytes < 0 || read.MaxBytes > protocol.AgentAPIPageBytes || read.Offset < 0 { return h.refuse(env.ID, ErrUnknownOp, map[string]any{"t": env.T}) } gateway, available := h.cfg.API.(agentReadGateway) - if !available || h.cfg.Redactor == nil { + if !available { return h.refuse(env.ID, ErrUnavailable, map[string]any{"subsystem": "api"}) } if !h.apiBusy.CompareAndSwap(false, true) { @@ -52,21 +50,12 @@ func (h *Handler) onAPIRead(env Envelope) error { } } caller, enrolled := h.liveCaller() - if !enrolled || !caller.Scopes.Has(protocol.ScopeDiagRun) || !caller.Scopes.Has(route.Scope) { - _ = h.refuse(&id, ErrScopeDenied, map[string]any{"needScope": route.Scope}) + if !enrolled || !caller.Scopes.Has(protocol.ScopeDiagRun) { + _ = h.refuse(&id, ErrScopeDenied, map[string]any{"needScope": protocol.ScopeDiagRun}) return } req := buildAPIRequest(ctx, caller, request) - facts := h.cfg.API.Route(req) - if facts.Static || facts.Tier != apiauth.TierRead { - _ = h.refuse(&id, ErrScopeDenied, map[string]any{"t": env.T}) - return - } - max := int64(protocol.AgentAPIMaxBytes) - if read.MaxBytes > 0 { - max = read.MaxBytes - } - writer := &apiWriter{h: h, id: id, ctx: ctx, max: max, free: free, header: http.Header{}, buf: make([]byte, 0, APIChunkBytes)} + writer := &apiWriter{h: h, id: id, ctx: ctx, max: protocol.AgentAPIMaxBytes, free: free, header: http.Header{}, buf: make([]byte, 0, APIChunkBytes)} writer.recheck = func() *ErrorBody { if h.isAgent() { if _, refusal, _ := h.agentGate(); refusal != nil { @@ -74,8 +63,8 @@ func (h *Handler) onAPIRead(env Envelope) error { } } current, ok := h.liveCaller() - if !ok || !current.Scopes.Has(protocol.ScopeDiagRun) || !current.Scopes.Has(route.Scope) { - return &ErrorBody{Code: ErrScopeDenied, Args: map[string]any{"needScope": route.Scope}} + if !ok || !current.Scopes.Has(protocol.ScopeDiagRun) { + return &ErrorBody{Code: ErrScopeDenied, Args: map[string]any{"needScope": protocol.ScopeDiagRun}} } return nil } @@ -86,7 +75,7 @@ func (h *Handler) onAPIRead(env Envelope) error { _, _ = writer.Write([]byte(`{"error":"diagnostic read failed"}`)) } }() - gateway.ServeAgentRead(writer, req, h.cfg.Redactor()) + gateway.ServeAgentRead(writer, req, read) }() if err := writer.finish(); err != nil { h.log.Warn("diagnostic API read failed", "err", err) diff --git a/go/internal/appproto/agentapi_test.go b/go/internal/appproto/agentapi_test.go index 18aa4aad2..fda30d823 100644 --- a/go/internal/appproto/agentapi_test.go +++ b/go/internal/appproto/agentapi_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/srcfl/ftw/go/internal/apiauth" - "github.com/srcfl/ftw/go/internal/redact" "github.com/srcfl/ftw/go/protocol" ) @@ -15,17 +14,17 @@ type diagnosticGateway struct { read func(http.ResponseWriter, *http.Request) } -func (g *diagnosticGateway) ServeAgentRead(w http.ResponseWriter, r *http.Request, _ *redact.Redactor) { +func (g *diagnosticGateway) ServeAgentRead(w http.ResponseWriter, r *http.Request, _ protocol.APIRead) { g.read(w, r) } -func TestAnAgentDiagnosticReadRequiresItsDataScope(t *testing.T) { - for _, hasHistory := range []bool{false, true} { - t.Run(map[bool]string{false: "missing history", true: "granted"}[hasHistory], func(t *testing.T) { +func TestAnAgentDiagnosticReadNeedsOnlyItsDiagnosticGrant(t *testing.T) { + for _, hasDiagnostic := range []bool{false, true} { + t.Run(map[bool]string{false: "missing diagnostic grant", true: "granted"}[hasDiagnostic], func(t *testing.T) { r := newAgentRig(t) - scopes := []string{protocol.ScopeDiagRun} - if hasHistory { - scopes = append(scopes, protocol.ScopeHistoryRead) + scopes := []string{protocol.ScopeHistoryRead} + if hasDiagnostic { + scopes = []string{protocol.ScopeDiagRun} } row, _ := r.grant(apiauth.RoleAgent, scopes, nil, protocol.Caps{}, time.Hour, protocol.Principal{Client: "ChatGPT"}) h, rec := r.agentSession(row) @@ -39,7 +38,7 @@ func TestAnAgentDiagnosticReadRequiresItsDataScope(t *testing.T) { _, _ = w.Write([]byte(`{"energy_wh":12}`)) }} deliver(t, h, protocol.MsgAPIRead, ptrU32(1), protocol.APIRead{Path: "/api/energy/history"}) - if !hasHistory { + if !hasDiagnostic { if got := body[ErrorBody](t, waitFor(t, rec, MsgError)); got.Code != ErrScopeDenied { t.Fatalf("missing scope: %+v", got) } diff --git a/go/internal/redact/data.go b/go/internal/redact/data.go deleted file mode 100644 index 98ab9127a..000000000 --- a/go/internal/redact/data.go +++ /dev/null @@ -1,102 +0,0 @@ -package redact - -import ( - "bytes" - "encoding/json" - "fmt" - "regexp" - "strings" - "time" - "unicode" -) - -// Data scrubs diagnostic JSON while keeping measured quantities, nulls, -// timestamps and coverage. Config's numeric allowlist is for settings and -// would drop sample counts and other evidence from these reads. -func (r *Redactor) Data(doc any) any { return r.data(doc, "") } - -var dataPath = regexp.MustCompile(`(?:[A-Za-z]:\\|/)[^\s"'<>]*[/\\][^\s"'<>]*`) -var dataMetric = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_.-]{0,127}$`) - -func dataSource(value string) bool { - switch value { - case "hardware_counter", "power_integrated", "power_integration", "mixed", "unknown", "ntp", "rtc", "system", "manual", "estimated", "vehicle", "charger", "telemetry", "forecast": - return true - } - return false -} - -func dataKey(key string) string { - var b strings.Builder - letters := []rune(key) - for i, c := range letters { - if unicode.IsUpper(c) && i > 0 && (unicode.IsLower(letters[i-1]) || unicode.IsDigit(letters[i-1]) || i+1 < len(letters) && unicode.IsLower(letters[i+1])) { - b.WriteByte('_') - } - b.WriteRune(unicode.ToLower(c)) - } - return b.String() -} - -func dataID(key string) bool { - return key == "id" || strings.HasSuffix(key, "_id") || key == "src" || key == "driver" || key == "drivers" || - key == "name" || key == "names" || key == "label" || key == "title" || key == "alias" || strings.HasSuffix(key, "_name") -} - -func (r *Redactor) data(doc any, key string) any { - switch v := doc.(type) { - case map[string]any: - out := make(map[string]any, len(v)) - for k, value := range v { - field := dataKey(k) - if secretKey(field) || removedKey(field) || field == "vin" || field == "imei" || field == "serial_number" || field == "path" || field == "file" || field == "directory" || field == "home_id" || field == "box_id" { - continue - } - name := k - if r.names != nil && r.names.MatchString(k) || reURL.MatchString(k) || reIPv4.MatchString(k) || reEmail.MatchString(k) || reMAC.MatchString(k) || reHexRun.MatchString(k) || reHost.MatchString(k) || strings.Count(k, ":") >= 3 && reIPv6.MatchString(k) || strings.Contains(k, "::") { - name = r.ID(k) - } - out[name] = r.data(value, field) - } - return out - case []any: - out := make([]any, len(v)) - for i, value := range v { - out[i] = r.data(value, key) - } - return out - case string: - if v == "" { - return v - } - // Metric names are queryable schema, not free-text device values. - // The log scrubber would turn names such as pv_mppt1_w into [id]. - if (key == "metric" || key == "metrics") && dataMetric.MatchString(v) { - return v - } - if dataID(key) || key == "source" && !dataSource(v) { - return r.ID(v) - } - if _, err := time.Parse(time.RFC3339Nano, v); err == nil { - return v - } - // Recorded planner inputs can themselves be JSON strings. - if strings.HasPrefix(v, "{") || strings.HasPrefix(v, "[") { - var nested any - dec := json.NewDecoder(bytes.NewBufferString(v)) - dec.UseNumber() - if dec.Decode(&nested) == nil && json.Valid([]byte(v)) { - body, err := json.Marshal(r.data(nested, "")) - if err == nil { - return string(body) - } - } - } - return dataPath.ReplaceAllString(r.Text(v), "[path]") - default: - if dataID(key) && isNumber(doc) { - return r.ID(fmt.Sprint(doc)) - } - return doc - } -} diff --git a/go/internal/redact/data_test.go b/go/internal/redact/data_test.go deleted file mode 100644 index 1b385b13d..000000000 --- a/go/internal/redact/data_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package redact - -import ( - "encoding/json" - "strings" - "testing" -) - -func TestDiagnosticDataKeepsEvidenceAndScrubsIdentity(t *testing.T) { - r := New([]byte("box-agent-key"), []string{"Garage meter"}) - var doc any - input := `{"asset_id":"hardware:secret-device","label":"Garage meter","source":"hardware_counter","quality":"measured","provenance":"counter","energy_wh":12.25,"sample_count":7,"at":"2026-10-18T15:00:00Z","first_seen_ms":1792328400000,"missing":null,"driver":"Garage meter","VIN":"ABC12345678901234","apiToken":"do-not-publish","endpoint":"http://192.168.1.7","input_json":"{\"power_w\":1234,\"password\":\"secret\"}"}` - dec := json.NewDecoder(strings.NewReader(input)) - dec.UseNumber() - if err := dec.Decode(&doc); err != nil { - t.Fatal(err) - } - got := r.Data(doc).(map[string]any) - if got["energy_wh"] != json.Number("12.25") || got["sample_count"] != json.Number("7") || got["first_seen_ms"] != json.Number("1792328400000") || got["at"] != "2026-10-18T15:00:00Z" || got["missing"] != nil { - t.Fatalf("evidence changed: %+v", got) - } - if got["asset_id"] != r.ID("hardware:secret-device") || got["driver"] != r.ID("Garage meter") || got["source"] != "hardware_counter" || got["provenance"] != "counter" { - t.Fatalf("identity or provenance: %+v", got) - } - body, _ := json.Marshal(got) - for _, secret := range []string{"hardware:secret-device", "Garage meter", "ABC12345678901234", "do-not-publish", "192.168.1.7", "password", "secret"} { - if strings.Contains(string(body), secret) { - t.Errorf("diagnostic answer contains %q: %s", secret, body) - } - } - if !strings.Contains(got["input_json"].(string), `"power_w":1234`) { - t.Fatalf("nested planner input lost its values: %s", body) - } - private := r.Data(map[string]any{"error": "read /private/house/config.yaml: failed", "192.168.2.7": 12, "clock": map[string]any{"source": "ntp"}, "SN": 123456789, "VIN": 987654321, "IPAddress": "10.0.0.1"}).(map[string]any) - encoded, _ := json.Marshal(private) - if strings.Contains(string(encoded), "/private") || strings.Contains(string(encoded), "192.168.2.7") || strings.Contains(string(encoded), "123456789") || strings.Contains(string(encoded), "987654321") || strings.Contains(string(encoded), "10.0.0.1") || private["clock"].(map[string]any)["source"] != "ntp" { - t.Fatalf("network identity, path or clock source: %s", encoded) - } - metrics := r.Data(map[string]any{"metrics": []any{"pv_mppt1_w", "battery_2_soc"}, "metric": "hp_phase1_current_a"}).(map[string]any) - if metrics["metric"] != "hp_phase1_current_a" || metrics["metrics"].([]any)[0] != "pv_mppt1_w" { - t.Fatalf("metric names can no longer be queried: %+v", metrics) - } -} diff --git a/go/protocol/agentapi.go b/go/protocol/agentapi.go index fa4280891..9a070da47 100644 --- a/go/protocol/agentapi.go +++ b/go/protocol/agentapi.go @@ -1,12 +1,21 @@ package protocol -// APIRead carries a read from the diagnostic catalogue. It has no method, -// body, headers or step-up field. api.req remains closed to agents. -// The answer uses APIHeadMsg, APIChunk and APIEnd under the request id. +import ( + "path" + "strings" + "unicode" +) + +// APIRead is a diagnostic GET, with no method, body, headers or write path. +// Offset and MaxBytes select a byte page. IfSHA256 prevents joining pages +// from different answers. The answer uses api.head/chunk/end, carrying JSON. type APIRead struct { - Path string `cbor:"path"` - Query map[string]string `cbor:"query,omitempty"` - MaxBytes int64 `cbor:"maxBytes,omitempty"` + Path string `cbor:"path"` + Query map[string]string `cbor:"query,omitempty"` + MaxBytes int64 `cbor:"maxBytes,omitempty"` + Offset int64 `cbor:"offset,omitempty"` + IfSHA256 string `cbor:"ifSHA256,omitempty"` + ArchiveMember string `cbor:"archiveMember,omitempty"` } func DecodeAPIRead(body []byte) (APIRead, error) { @@ -15,10 +24,14 @@ func DecodeAPIRead(body []byte) (APIRead, error) { return read, err } -const AgentAPIMaxBytes = 1 << 20 +const ( + AgentAPIMaxBytes = 1 << 20 // Complete JSON wire answer, including escaped text. + AgentAPIPageBytes = 128 << 10 // Source bytes per page; not a total data limit. + AgentAPICatalogPath = "/api/diagnostics/catalog" +) -// AgentAPIRoute is a diagnostic read. The box checks both the api.read -// scope and this route's scope, and its HTTP router must still call it Read. +// AgentAPIRoute describes a route registered by the connected Core version. +// Query is guidance; the existing HTTP handler owns parameter validation. type AgentAPIRoute struct { Path string `json:"path"` Scope string `json:"scope"` @@ -26,41 +39,9 @@ type AgentAPIRoute struct { Query map[string]string `json:"query,omitempty"` } -// AgentAPIRoutes is the catalogue the box serves and MCP lists. Paths are -// exact; query keys not listed here are refused. Responses stay JSON. -var AgentAPIRoutes = []AgentAPIRoute{ - {Path: "/api/status", Scope: ScopeLiveRead, Description: "Current measurements, control state and freshness"}, - {Path: "/api/health", Scope: ScopeHealthRead, Description: "Box health"}, - {Path: "/api/system/info", Scope: ScopeHealthRead, Description: "Runtime and host diagnostics, with identifying fields removed"}, - {Path: "/api/storage/inventory", Scope: ScopeHealthRead, Description: "Storage use and retention"}, - {Path: "/api/components", Scope: ScopeHealthRead, Description: "Component versions and state"}, - {Path: "/api/components/history", Scope: ScopeHealthRead, Description: "Component changes"}, - {Path: "/api/drivers", Scope: ScopeHealthRead, Description: "Driver diagnostics by opaque id"}, - {Path: "/api/data-sources", Scope: ScopeHealthRead, Description: "Available data sources and freshness"}, - {Path: "/api/battery_models", Scope: ScopePlanRead, Description: "Learned battery model and efficiency"}, - {Path: "/api/loadmodel", Scope: ScopePlanRead, Description: "Load forecast model and learning"}, - {Path: "/api/pvmodel", Scope: ScopePlanRead, Description: "Solar forecast model and learning"}, - {Path: "/api/forecast", Scope: ScopePlanRead, Description: "Load and solar forecasts", Query: map[string]string{"range": "Time ahead, such as 24h"}}, - {Path: "/api/prices", Scope: ScopePlanRead, Description: "Stored import and export prices", Query: map[string]string{"range": "Time ahead, such as 24h", "since_ms": "Unix milliseconds", "until_ms": "Unix milliseconds"}}, - {Path: "/api/planner/prefs", Scope: ScopeConfigRead, Description: "Planner preferences"}, - {Path: "/api/mpc/plan", Scope: ScopePlanRead, Description: "Full current plan and per-asset slots"}, - {Path: "/api/mpc/diagnose", Scope: ScopePlanRead, Description: "Current planner inputs, constraints and result"}, - {Path: "/api/mpc/diagnose/history", Scope: ScopePlanRead, Description: "Recorded planning decisions", Query: map[string]string{"since": "Unix milliseconds", "until": "Unix milliseconds", "limit": "Maximum snapshots, up to 5000"}}, - {Path: "/api/mpc/diagnose/at", Scope: ScopePlanRead, Description: "The planning snapshot active at an incident", Query: map[string]string{"ts": "Required incident time, Unix milliseconds"}}, - {Path: "/api/history", Scope: ScopeHistoryRead, Description: "Recorded site measurements", Query: map[string]string{"range": "Time back, such as 1h", "points": "Maximum rows"}}, - {Path: "/api/energy/daily", Scope: ScopeHistoryRead, Description: "Daily measured energy and costs", Query: map[string]string{"days": "Number of days"}}, - {Path: "/api/energy/assets", Scope: ScopeHistoryRead, Description: "Energy-ledger assets by opaque id"}, - {Path: "/api/energy/history", Scope: ScopeHistoryRead, Description: "Energy-ledger entries in Wh, including flow, source, quality, provenance and sample count", Query: map[string]string{"scope": "system or asset", "asset_id": "Opaque id from /api/energy/assets", "since": "Unix milliseconds", "until": "Unix milliseconds", "bucket": "Bucket duration, at least 5m", "limit": "Maximum buckets, up to 5000"}}, - {Path: "/api/savings/daily", Scope: ScopeHistoryRead, Description: "Daily cost comparison and coverage", Query: map[string]string{"days": "Number of days"}}, - {Path: "/api/series/catalog", Scope: ScopeHistoryRead, Description: "Stored drivers, metric names and units"}, - {Path: "/api/series", Scope: ScopeHistoryRead, Description: "One driver's measured series with min, max, sample count, resolution and bounds", Query: map[string]string{"driver": "Required opaque driver id from /api/series/catalog", "metric": "Required metric name or comma-separated names from the catalogue", "range": "Time back, such as 6h", "since": "Unix milliseconds", "until": "Unix milliseconds", "points": "Maximum points per metric"}}, -} - -func AgentAPIPath(path string) (AgentAPIRoute, bool) { - for _, route := range AgentAPIRoutes { - if route.Path == path { - return route, true - } - } - return AgentAPIRoute{}, false +// ValidAgentAPIPath validates syntax only. Core's router decides whether the +// path is a read; Cloud does not keep a second list of the box's API. +func ValidAgentAPIPath(value string) bool { + return strings.HasPrefix(value, "/api/") && path.Clean(value) == value && + !strings.ContainsAny(value, "?%#\\") && strings.IndexFunc(value, unicode.IsControl) < 0 } diff --git a/go/protocol/agentapi_test.go b/go/protocol/agentapi_test.go index cd6420c29..934dd9982 100644 --- a/go/protocol/agentapi_test.go +++ b/go/protocol/agentapi_test.go @@ -21,3 +21,26 @@ func TestDiagnosticAPIReadVector(t *testing.T) { t.Fatal("unknown method field accepted") } } + +func TestDiagnosticPageVector(t *testing.T) { + body, err := hex.DecodeString("a46470617468732f6170692f656e657267792f686973746f7279666f6666736574187b686d617842797465731a000100006d617263686976654d656d626572612e") + if err != nil { + t.Fatal(err) + } + read, err := DecodeAPIRead(body) + if err != nil || read.Offset != 123 || read.MaxBytes != 65536 || read.ArchiveMember != "." { + t.Fatalf("page vector: %+v %v", read, err) + } +} +func TestDiagnosticPathSyntaxDoesNotKeepAnAPIList(t *testing.T) { + for _, value := range []string{"/api/future/read", "/api/drivers/charger-192.168.1.9/source"} { + if !ValidAgentAPIPath(value) { + t.Errorf("valid path refused: %s", value) + } + } + for _, value := range []string{"/status", "/api/../config", "/api/status?x=1", "/api/status#x", "/api/a%2fb", "/api/a\\b", "/api/a\n"} { + if ValidAgentAPIPath(value) { + t.Errorf("invalid path accepted: %q", value) + } + } +} diff --git a/go/protocol/contract_gen.go b/go/protocol/contract_gen.go index fcbcaf2e7..549cf6760 100644 --- a/go/protocol/contract_gen.go +++ b/go/protocol/contract_gen.go @@ -183,7 +183,7 @@ const ( ScopeLogsRead = "ftw.logs.read" // ScopeConfigRead — Configuration the box redacts. ScopeConfigRead = "ftw.config.read" - // ScopeDiagRun — Checks that change nothing. + // ScopeDiagRun — Full technical diagnostics and read API. ScopeDiagRun = "ftw.diag.run" // ScopeDispatchHold — Hold energy assets for a capped time. ScopeDispatchHold = "ftw.dispatch.hold" @@ -376,7 +376,7 @@ var RegistryReads = map[string]string{ "config.get": "ftw.config.read", // diag.run — A check that changes nothing. "diag.run": "ftw.diag.run", - // api.read — Read a diagnostic API route within its data scope. + // api.read — Read the full technical diagnostic API. "api.read": "ftw.diag.run", }