From 92f044f86027cc86557ab72beb691a42d384303d Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 17 Oct 2026 11:15:47 +0200 Subject: [PATCH 1/3] feat(stats): publish consented installation and energy aggregates --- CLAUDE.md | 6 +- README.md | 29 +++++ cmd/ftwcloudd/main.go | 8 +- go.mod | 2 +- go.sum | 6 +- internal/registry/statistics.go | 179 +++++++++++++++++++++++++++ internal/registry/statistics_test.go | 126 +++++++++++++++++++ internal/server/server.go | 21 +++- internal/server/server_test.go | 6 + internal/statistics/statistics.go | 52 ++++++++ internal/uplink/statistics_test.go | 64 ++++++++++ internal/uplink/uplink.go | 26 +++- 12 files changed, 508 insertions(+), 17 deletions(-) create mode 100644 internal/registry/statistics.go create mode 100644 internal/registry/statistics_test.go create mode 100644 internal/statistics/statistics.go create mode 100644 internal/uplink/statistics_test.go diff --git a/CLAUDE.md b/CLAUDE.md index aeda985..b18aa05 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,11 @@ maintains it. never from a body field. Every later box message but `tun` must verify with that key. `tun` is unsigned: its `d` is a phone's ciphertext, which the service routes by `dev` and `sid` and never reads. -- **No readings on disk.** Memory holds readings, and only while the roster's +- **Live readings stay in memory.** The separate, versioned project statistics + consent permits rounded closed-hour energy and configured capacities on + disk for 13 months. Only group cells pass `/v1/stats`; no staff route reads + a home. Withdrawal and home purge delete its unpublished statistics. + Memory holds live readings, and only while the roster's grant serves a copy; SQLite holds only the home id, the box key, the signed roster, owners' suspensions, tombstones, APNs registrations, the hashes of extension tokens when each away box's alert is due, browser notice sent markers and diff --git a/README.md b/README.md index ddc22db..2f8a437 100644 --- a/README.md +++ b/README.md @@ -380,3 +380,32 @@ events scope and suspension before sending. A readable-copy grant is not needed. The roster holds the delivery endpoints and ciphertext; a separate per-home row holds only the current outage's sent ids. Purge removes both. The sender refuses redirects, proxies and local destinations. + +## Project usage statistics + +`GET /v1/stats` publishes only group cells under +`ftw-project-stats-2026-10-v1`. Boxes send signed `stats` hours only after the +owner accepts that policy. The previous statistics policy sends no new +fields. Reporting installations in the last 24 closed hours are not people +or total FTW installs. Power and capacity are configured; energy sums +recorded hours, including partial hours. Missing data is unknown. + +Cloud stores consented hourly values for 13 months, upserts reconnects, and +deletes household rows on a signed withdrawal or purge. Cells need 20 homes +and no home above 30 percent of the sum. The stats Worker polls this public +view; it receives no home id or per-home value. One instance supplies its +local shard only; public multi-shard totals need an explicit safe collector. + +Owner acceptance uses the existing box API. On the box's loopback interface, +read `GET /api/cloud/stats-policy`, accept its terms, then +`POST /api/cloud/stats` with `{"policy":"ftw-project-stats-2026-10-v1"}`. +The readable Cloud grant must already be on. `DELETE /api/cloud/stats` +withdraws this choice. Do not enable it for someone else or reuse an older +consent. Native apps keep the old scope until their consent screen adopts +this policy. The first contributing hour starts after consent; the sender +does not upload earlier ledger history. + +Deploying this ingestion scope still needs the signed C8 DPIA and the +updated consent text described by Core's ADR 0008. This change does not +claim that sign-off, change an owner's saved choice, or release a Core +update. The existing app statistics switch does not accept the new policy. diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index 5082cd9..3ddfa5e 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -47,6 +47,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/registry" "github.com/srcfl/ftw-cloud/internal/server" "github.com/srcfl/ftw-cloud/internal/shard" + "github.com/srcfl/ftw-cloud/internal/statistics" "github.com/srcfl/ftw-cloud/internal/store" "github.com/srcfl/ftw-cloud/internal/token" "github.com/srcfl/ftw-cloud/internal/uplink" @@ -139,6 +140,11 @@ func serve(args []string) error { st.UseSuspensions(reg.Suspended) up := uplink.New(st, reg, key, log) up.Shards = shards + usageSvc, err := statistics.New(reg) + if err != nil { + return err + } + up.Statistics = true // A restart is a deploy, and commands ride through here: phones are // admitted by the rosters their boxes signed before, without waiting // for every box to reconnect. @@ -154,7 +160,7 @@ func serve(args []string) error { } tokens := token.New(st, reg, log) tokens.Shards = shards - srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Log: log} + srv := &server.Server{Store: st, Registry: reg, Uplink: up, Tokens: tokens, Fleet: fleetSvc, Statistics: usageSvc, Log: log} // Assistants: the authorization server, with agent keys in ftwkeysd. agentKeys := keys.Dial(*keysSocket) diff --git a/go.mod b/go.mod index af38224..dea0f31 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261017070253-cbdd8a587105 + github.com/srcfl/ftw/go/protocol v0.0.0-20261017091326-b05efd4e0568 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index a605893..db3daac 100644 --- a/go.sum +++ b/go.sum @@ -52,10 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261016102129-7fb84d6ed573 h1:1OOC4RdeERT9r6cLMTlcmTjg8+ML2kU6hCvhx/x+8xI= -github.com/srcfl/ftw/go/protocol v0.0.0-20261016102129-7fb84d6ed573/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= -github.com/srcfl/ftw/go/protocol v0.0.0-20261017070253-cbdd8a587105 h1:56AF3FQxXwfTVacD+AtJQM96TdzntTeQy1f4yiN54ic= -github.com/srcfl/ftw/go/protocol v0.0.0-20261017070253-cbdd8a587105/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261017091326-b05efd4e0568 h1:9ZRgYviK0JWzMgeIk7XKZnsV7lNjtbpvJEsoUwMOcnk= +github.com/srcfl/ftw/go/protocol v0.0.0-20261017091326-b05efd4e0568/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/registry/statistics.go b/internal/registry/statistics.go new file mode 100644 index 0000000..0322e1e --- /dev/null +++ b/internal/registry/statistics.go @@ -0,0 +1,179 @@ +package registry + +import ( + "bytes" + "database/sql" + "encoding/json" + "errors" + "slices" + "time" + + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/cloudwire" +) + +// Statistics tables contain only separately consented, closed-hour values. +// They have no staff read route, and leave with the home's purge. +func (r *Registry) StatisticsInit() error { + _, err := r.db.Exec(`CREATE TABLE IF NOT EXISTS statistics_grants ( + home TEXT PRIMARY KEY REFERENCES homes(home) ON DELETE CASCADE, + stats_id BLOB NOT NULL, since_ms INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS statistics_hours ( + home TEXT NOT NULL REFERENCES homes(home) ON DELETE CASCADE, + hour_ms INTEGER NOT NULL, metrics BLOB NOT NULL, PRIMARY KEY(home,hour_ms)); + CREATE INDEX IF NOT EXISTS statistics_hours_time ON statistics_hours(hour_ms);`) + return err +} + +// StatisticsGrant clears old consent rows before a changed grant may send. +func (r *Registry) StatisticsGrant(home string, g cloudwire.Grant) error { + tx, err := r.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + var old []byte + err = tx.QueryRow(`SELECT stats_id FROM statistics_grants WHERE home=?`, home).Scan(&old) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + on := g.Controller == cloudwire.ControllerSourceful && g.StatsPolicy == cloudwire.StatsPolicy && + len(g.StatsID) == 16 && g.StatsSinceMs > 0 && slices.Contains(g.Purposes, protocol.CloudPurposeStats) && + slices.Contains(g.Purposes, protocol.CloudPurposeServe) + if !on || !bytes.Equal(old, g.StatsID) { + if _, err := tx.Exec(`DELETE FROM statistics_hours WHERE home=?`, home); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM statistics_grants WHERE home=?`, home); err != nil { + return err + } + } + if on { + if _, err := tx.Exec(`INSERT INTO statistics_grants(home,stats_id,since_ms) VALUES(?,?,?) + ON CONFLICT(home) DO UPDATE SET stats_id=excluded.stats_id,since_ms=excluded.since_ms`, home, g.StatsID, g.StatsSinceMs); err != nil { + return err + } + } + return tx.Commit() +} + +func (r *Registry) StatisticsPut(home string, report cloudwire.Stats, now time.Time) error { + if !report.Valid(now.UnixMilli()) { + return errors.New("statistics: invalid closed hour") + } + data, err := json.Marshal(report.Metrics) + if err != nil { + return err + } + // Consent and the write are one SQL statement: a withdrawn grant cannot + // race a pending report back into storage. Reconnects replace one hour. + _, err = r.db.Exec(`INSERT INTO statistics_hours(home,hour_ms,metrics) + SELECT home,?,? FROM statistics_grants WHERE home=? AND stats_id=? AND since_ms<=? + ON CONFLICT(home,hour_ms) DO UPDATE SET metrics=excluded.metrics`, report.HourMS, data, home, report.StatsID, report.HourMS) + return err +} + +func (r *Registry) StatisticsPrune(now time.Time) error { + _, err := r.db.Exec(`DELETE FROM statistics_hours WHERE hour_ms 0 && biggest*10 > sum*3 { + c.State = "dominated" + return c + } + c.State, c.Value = "visible", &sum + return c +} + +// StatisticsView returns cells only. Neither the home nor statsId leaves it. +// Capacities use each home's latest report; energy sums its recorded hours. +func (r *Registry) StatisticsView(now time.Time) (StatisticsView, error) { + end := now.UTC().Truncate(time.Hour) + start := end.Add(-24 * time.Hour) + v := StatisticsView{Schema: "ftw.project-usage/1", ObservedAt: now.UTC().Format(time.RFC3339), + WindowStart: start.Format(time.RFC3339), WindowEnd: end.Format(time.RFC3339), Minimum: cloudwire.StatsMinimumHomes, Metrics: map[string]StatisticsCell{}} + rows, err := r.rd.Query(`SELECT h.home,h.hour_ms,h.metrics FROM statistics_hours h + JOIN statistics_grants g ON g.home=h.home WHERE h.hour_ms>=? AND h.hour_ms= cloudwire.StatsMinimumHomes { + value := int64(n) + v.Installations = StatisticsCell{State: "visible", Value: &value, Homes: &n} + } + for _, key := range cloudwire.StatsMetricKeys { + values := []int64{} + for _, metrics := range homes { + if value, ok := metrics[key]; ok { + values = append(values, value) + } + } + v.Metrics[key] = cell(values) + } + return v, nil +} diff --git a/internal/registry/statistics_test.go b/internal/registry/statistics_test.go new file mode 100644 index 0000000..3620aae --- /dev/null +++ b/internal/registry/statistics_test.go @@ -0,0 +1,126 @@ +package registry + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/cloudwire" + "strings" + "testing" + "time" +) + +func statsFixture(t *testing.T) (*Registry, time.Time, cloudwire.Grant) { + t.Helper() + r := open(t) + if err := r.StatisticsInit(); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 10, 17, 12, 30, 0, 0, time.UTC) + g := cloudwire.Grant{Controller: cloudwire.ControllerSourceful, Purposes: []string{protocol.CloudPurposeServe, protocol.CloudPurposeStats}, StatsID: bytes.Repeat([]byte{1}, 16), StatsPolicy: cloudwire.StatsPolicy, StatsSinceMs: now.Add(-25 * time.Hour).UnixMilli()} + return r, now, g +} +func statsHome(t *testing.T, r *Registry, now time.Time, g cloudwire.Grant, home string, value int64) cloudwire.Stats { + t.Helper() + if err := r.Claim(home, []byte("key"), now); err != nil { + t.Fatal(err) + } + if err := r.StatisticsGrant(home, g); err != nil { + t.Fatal(err) + } + report := cloudwire.Stats{StatsID: g.StatsID, Policy: g.StatsPolicy, HourMS: now.Truncate(time.Hour).Add(-time.Hour).UnixMilli(), Metrics: map[string]int64{"flexible_power_w": value, "pv_generation_wh": 100}} + if err := r.StatisticsPut(home, report, now); err != nil { + t.Fatal(err) + } + return report +} +func TestStatisticsThresholdDedupUnknownAndDominance(t *testing.T) { + r, now, g := statsFixture(t) + for i := range 19 { + statsHome(t, r, now, g, fmt.Sprint("home", i), 1000) + } + v, err := r.StatisticsView(now) + if err != nil { + t.Fatal(err) + } + if v.Installations.Value != nil || v.Metrics["flexible_power_w"].Value != nil { + t.Fatal("small cohort exposed") + } + report := statsHome(t, r, now, g, "home19", 1000) + if err := r.StatisticsPut("home19", report, now); err != nil { + t.Fatal(err) + } + v, err = r.StatisticsView(now) + if err != nil { + t.Fatal(err) + } + if *v.Installations.Value != 20 || *v.Metrics["flexible_power_w"].Value != 20000 || *v.Metrics["pv_generation_wh"].Value != 2000 { + t.Fatalf("wrong totals %+v", v) + } + if v.Metrics["pv_rated_w"].Value != nil { + t.Fatal("unknown became zero") + } + report.Metrics["flexible_power_w"] = 100000 + if err := r.StatisticsPut("home19", report, now); err != nil { + t.Fatal(err) + } + v, _ = r.StatisticsView(now) + if v.Metrics["flexible_power_w"].State != "dominated" || v.Metrics["flexible_power_w"].Value != nil { + t.Fatal("dominant home exposed") + } + raw, _ := json.Marshal(v) + if strings.Contains(string(raw), "home19") || strings.Contains(string(raw), "statsId") { + t.Fatal("identifier exposed") + } +} +func TestStatisticsWithdrawalPolicyRotationPurgeAndConsentWindow(t *testing.T) { + r, now, g := statsFixture(t) + report := statsHome(t, r, now, g, "home", 1000) + old := g + g.StatsPolicy = "old-policy" + if err := r.StatisticsGrant("home", g); err != nil { + t.Fatal(err) + } + if err := r.StatisticsPut("home", report, now); err != nil { + t.Fatal(err) + } + var n int + _ = r.rd.QueryRow(`SELECT COUNT(*) FROM statistics_hours`).Scan(&n) + if n != 0 { + t.Fatal("withdrawn report stored") + } + g = old + g.StatsSinceMs = now.UnixMilli() + if err := r.StatisticsGrant("home", g); err != nil { + t.Fatal(err) + } + _ = r.StatisticsPut("home", report, now) + _ = r.rd.QueryRow(`SELECT COUNT(*) FROM statistics_hours`).Scan(&n) + if n != 0 { + t.Fatal("pre-consent hour stored") + } + statsHome(t, r, now, old, "home", 1000) + if err := r.Purge("home", nil, now); err != nil { + t.Fatal(err) + } + _ = r.rd.QueryRow(`SELECT COUNT(*) FROM statistics_hours`).Scan(&n) + if n != 0 { + t.Fatal("purge left household rows") + } +} +func TestStatisticsRetention(t *testing.T) { + r, now, g := statsFixture(t) + statsHome(t, r, now, g, "home", 1000) + if _, err := r.db.Exec(`UPDATE statistics_hours SET hour_ms=?`, now.AddDate(-2, 0, 0).UnixMilli()); err != nil { + t.Fatal(err) + } + if err := r.StatisticsPrune(now); err != nil { + t.Fatal(err) + } + var n int + _ = r.rd.QueryRow(`SELECT COUNT(*) FROM statistics_hours`).Scan(&n) + if n != 0 { + t.Fatal("old rows remain") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index bcbdda5..1c87650 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -37,7 +37,7 @@ import ( const SourceArchive = "/usr/share/ftw-cloud/source.tar.gz" // Routes are every path the service answers. -var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/source", "/healthz", "/mcp"}, oauth.Routes...) +var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/v1/stats", "/source", "/healthz", "/mcp"}, oauth.Routes...) // devRoutes adds a local inspection route in binaries built with // -tags ftwclouddev. It is nil in every other build. @@ -56,10 +56,11 @@ type Server struct { Tokens *token.Service // OAuth is the authorization server for assistants, and MCP the server // they call. Nil serves none. - OAuth *oauth.Server - Fleet http.Handler - MCP http.Handler - Log *slog.Logger + OAuth *oauth.Server + Fleet http.Handler + Statistics http.Handler + MCP http.Handler + Log *slog.Logger // AltSvc, when set, is the Alt-Svc header that points TCP clients at // HTTP/3 (package quicdoor). It goes on plain responses only: a client // that learns HTTP/3 from a WebSocket upgrade might try the socket over @@ -94,6 +95,11 @@ func (s *Server) Handler() http.Handler { } else { mux.HandleFunc("/v1/fleet", notYet) } + if s.Statistics != nil { + mux.Handle("/v1/stats", s.Statistics) + } else { + mux.HandleFunc("/v1/stats", notYet) + } mux.HandleFunc("/source", source) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") @@ -186,6 +192,11 @@ func (s *Server) Upkeep(ctx context.Context, every time.Duration) { s.Log.Warn("fleet upkeep failed") } } + if f, ok := s.Statistics.(interface{ Prune(time.Time) error }); ok { + if err := f.Prune(time.Now()); err != nil { + s.Log.Warn("statistics upkeep failed") + } + } if err := s.Registry.TokenExpire(time.Now()); err != nil { s.Log.Warn("token upkeep failed", "err", err) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 7390fec..51f49f2 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -16,6 +16,7 @@ import ( "github.com/srcfl/ftw-cloud/internal/appsession" "github.com/srcfl/ftw-cloud/internal/fleet" "github.com/srcfl/ftw-cloud/internal/registry" + "github.com/srcfl/ftw-cloud/internal/statistics" "github.com/srcfl/ftw-cloud/internal/store" "github.com/srcfl/ftw-cloud/internal/token" "github.com/srcfl/ftw-cloud/internal/uplink" @@ -42,6 +43,11 @@ func newServer(t *testing.T) (*Server, string) { if err != nil { t.Fatal(err) } + s.Statistics, err = statistics.New(reg) + if err != nil { + t.Fatal(err) + } + up.Statistics = true // A home with readings in it, held the way a box's uplink holds them. const home = "canaryhomecanaryhomecanary" box, _ := st.For(store.Principal{Kind: store.Box, Home: home}) diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go new file mode 100644 index 0000000..5d6a874 --- /dev/null +++ b/internal/statistics/statistics.go @@ -0,0 +1,52 @@ +// Package statistics publishes only cells that meet the consent's group rules. +package statistics + +import ( + "encoding/json" + "net/http" + "sync" + "time" + + "github.com/srcfl/ftw-cloud/internal/registry" +) + +type Service struct { + Reg *registry.Registry + mu sync.Mutex + cached []byte + at time.Time +} + +func New(reg *registry.Registry) (*Service, error) { + if err := reg.StatisticsInit(); err != nil { + return nil, err + } + return &Service{Reg: reg}, nil +} + +func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "public, max-age=60") + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.cached == nil || time.Since(s.at) > time.Minute { + view, err := s.Reg.StatisticsView(time.Now()) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + data, err := json.Marshal(view) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + s.cached, s.at = data, time.Now() + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(s.cached) +} + +func (s *Service) Prune(now time.Time) error { return s.Reg.StatisticsPrune(now) } diff --git a/internal/uplink/statistics_test.go b/internal/uplink/statistics_test.go new file mode 100644 index 0000000..00753f0 --- /dev/null +++ b/internal/uplink/statistics_test.go @@ -0,0 +1,64 @@ +package uplink + +import ( + "bytes" + + "github.com/srcfl/ftw/go/protocol" + "github.com/srcfl/ftw/go/protocol/cloudwire" + "testing" + "time" +) + +func TestSignedStatisticsRequireCurrentConsentAndNeverForward(t *testing.T) { + s := newService(t) + if err := s.reg.StatisticsInit(); err != nil { + t.Fatal(err) + } + s.up.Statistics = true + now := time.Now() + b := newBox(t) + if _, err := b.connect(s, nil); err != nil { + t.Fatal(err) + } + r := b.roster(protocol.ScopeLiveRead) + r.Grant.Purposes = append(r.Grant.Purposes, protocol.CloudPurposeStats) + r.Grant.StatsID = bytes.Repeat([]byte{9}, 16) + r.Grant.StatsPolicy = cloudwire.StatsPolicy + r.Grant.StatsSinceMs = now.Add(-2 * time.Hour).UnixMilli() + b.send(cloudwire.MsgRoster, r) + report := cloudwire.Stats{Header: b.header(), StatsID: r.Grant.StatsID, Policy: cloudwire.StatsPolicy, HourMS: now.Truncate(time.Hour).Add(-time.Hour).UnixMilli(), Metrics: map[string]int64{"pv_generation_wh": 120}} + b.send(cloudwire.MsgStats, report) + wait(t, "statistics accepted", func() bool { + v, err := s.reg.StatisticsView(now) + return err == nil && v.Installations.State == "withheld" + }) + for _, raw := range snapshot(t, s, b.home) { + env, err := cloudwire.DecodeEnvelope(raw) + if err != nil { + t.Fatal(err) + } + if env.T == cloudwire.MsgStats { + t.Fatal("statistics reached phone") + } + } + r.Header = b.header() + r.RosterSeq++ + r.Grant.Purposes = []string{protocol.CloudPurposeServe} + r.Grant.StatsID = nil + r.Grant.StatsPolicy = "" + r.Grant.StatsSinceMs = 0 + b.send(cloudwire.MsgRoster, r) + wait(t, "withdrawal deleted rows", func() bool { + v, err := s.reg.StatisticsView(now) + return err == nil && v.Installations.State == "empty" + }) + report.Header = b.header() + b.send(cloudwire.MsgStats, report) + // A following live frame proves the service processed the stale stats. + b.send(cloudwire.MsgLive, cloudwire.Live{Header: b.header(), Fields: map[string]int64{"2": 1200}}) + wait(t, "following live frame", func() bool { return len(snapshot(t, s, b.home)) > 0 }) + v, err := s.reg.StatisticsView(now) + if err != nil || v.Installations.State != "empty" { + t.Fatal("stale consent restored rows") + } +} diff --git a/internal/uplink/uplink.go b/internal/uplink/uplink.go index 4a2b729..a722ded 100644 --- a/internal/uplink/uplink.go +++ b/internal/uplink/uplink.go @@ -101,11 +101,13 @@ func TunDropped() int64 { return dropTunUp.Load() + dropTunDown.Load() } // Uplink is the /v1/box handler. type Uplink struct { - store *store.Store - registry *registry.Registry - key *ecdsa.PrivateKey - log *slog.Logger - now func() time.Time + // Statistics is set before serving, after the consent tables exist. + Statistics bool + store *store.Store + registry *registry.Registry + key *ecdsa.PrivateKey + log *slog.Logger + now func() time.Time mu sync.Mutex conns map[string]*boxConn @@ -752,6 +754,15 @@ func (c *boxConn) apply(env cloudwire.Envelope) (stop bool, err error) { switch env.T { case cloudwire.MsgRoster: return false, c.roster(env) + case cloudwire.MsgStats: + if !c.u.Statistics { + return false, nil + } + var report cloudwire.Stats + if err := cloudwire.Unmarshal(env.B, &report); err != nil { + return true, err + } + return false, c.u.registry.StatisticsPut(c.home, report, c.u.now()) case cloudwire.MsgBye: var b cloudwire.Bye if err := cloudwire.Unmarshal(env.B, &b); err != nil { @@ -896,6 +907,11 @@ func (c *boxConn) roster(env cloudwire.Envelope) error { if !c.handle.SetRoster(&r) { return nil // older than what is held: ignored } + if c.u.Statistics { + if err := c.u.registry.StatisticsGrant(c.home, g); err != nil { + return err + } + } if withdrawn { if err := c.dropCopy(g.ID); err != nil { return err From e41b70db41e258233f508f25aa367744c4d08b8e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 17 Oct 2026 16:01:57 +0200 Subject: [PATCH 2/3] feat(stats): gate collection for review and export public daily counts --- README.md | 18 +++- cmd/ftwcloudd/main.go | 8 +- deploy/compose.yml | 2 + go.mod | 2 +- go.sum | 4 +- internal/fleet/public.go | 46 ++++++++++ internal/fleet/public_test.go | 57 ++++++++++++ internal/server/server.go | 7 +- internal/server/server_test.go | 1 + internal/statistics/statistics.go | 15 ++- internal/statistics/statistics_test.go | 32 +++++++ internal/uplink/statistics_test.go | 54 ++++++++++- internal/uplink/uplink.go | 21 +++-- privacy/project-statistics-dpia.md | 122 +++++++++++++++++++++++++ 14 files changed, 369 insertions(+), 20 deletions(-) create mode 100644 internal/fleet/public.go create mode 100644 internal/fleet/public_test.go create mode 100644 internal/statistics/statistics_test.go create mode 100644 privacy/project-statistics-dpia.md diff --git a/README.md b/README.md index 2f8a437..e93e677 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,13 @@ The sender refuses redirects, proxies and local destinations. `GET /v1/stats` publishes only group cells under `ftw-project-stats-2026-10-v1`. Boxes send signed `stats` hours only after the -owner accepts that policy. The previous statistics policy sends no new +owner accepts that policy and the service advertises it in signed `hello_ok`. +The service starts with collection off. After the signed C8 DPIA review, +operators can set `FTW_CLOUD_STATISTICS_POLICY` to that exact policy in the +host's compose environment; empty keeps it off. An unapproved policy makes +`/v1/stats` return 503 with `FTW-Statistics-State: disabled`. Withdrawal, +purge and retention remain active while collection is off. The review draft +is in [privacy/project-statistics-dpia.md](privacy/project-statistics-dpia.md). The previous statistics policy sends no new fields. Reporting installations in the last 24 closed hours are not people or total FTW installs. Power and capacity are configured; energy sums recorded hours, including partial hours. Missing data is unknown. @@ -408,4 +414,12 @@ does not upload earlier ledger history. Deploying this ingestion scope still needs the signed C8 DPIA and the updated consent text described by Core's ADR 0008. This change does not claim that sign-off, change an owner's saved choice, or release a Core -update. The existing app statistics switch does not accept the new policy. +update. The paired native consent change requires a fresh choice for this policy. + + +The separate anonymous daily-report feed is `GET /v1/fleet/public`. It exposes +only daily counts and dimension labels with at least 10 reports, for at most +90 days. It contains imported relay history and new reports received by Cloud; +consumers replace the old archive rather than adding both. It needs no hourly +energy policy. Averaging seven complete UTC days gives an estimate of active +reporting installations, never people or all installations ever made. diff --git a/cmd/ftwcloudd/main.go b/cmd/ftwcloudd/main.go index 3ddfa5e..ce65868 100644 --- a/cmd/ftwcloudd/main.go +++ b/cmd/ftwcloudd/main.go @@ -99,6 +99,7 @@ func serve(args []string) error { devCapture := fs.String("dev-apns-capture", "", "write each alert to this directory instead of sending it to Apple (development builds only)") keysSocket := fs.String("keys-socket", "/run/ftw-cloud-keys/keys.sock", "ftwkeysd's socket; assistants can connect only while it answers") shardsPath := fs.String("shards", "", "shard directory, JSON; without it this instance holds every home") + statsPolicy := fs.String("statistics-policy", "", "approved statistics policy; empty keeps hourly collection off pending C8 sign-off") metricsAddr := fs.String("metrics-listen", defaultMetricsAddr, "loopback address for /metrics; empty turns it off") restartRate := fs.Int("restart-rate", defaultRestartRate, "sockets a second the next instance can take after a restart; sets the spread of the redial") restartMin := fs.Duration("restart-spread-min", 15*time.Second, "the shortest spread: longer than the port stays closed while the next container starts") @@ -121,6 +122,9 @@ func serve(args []string) error { } else if err != nil { return err } + if *statsPolicy != "" && *statsPolicy != cloudwire.StatsPolicy { + return fmt.Errorf("unknown statistics policy") + } shards := shard.Single(*domain) if *shardsPath != "" { if shards, err = shard.Load(*shardsPath); err != nil { @@ -144,7 +148,9 @@ func serve(args []string) error { if err != nil { return err } - up.Statistics = true + up.StatisticsStore = true + up.Statistics = *statsPolicy == cloudwire.StatsPolicy + usageSvc.Enabled = up.Statistics // A restart is a deploy, and commands ride through here: phones are // admitted by the rosters their boxes signed before, without waiting // for every box to reconnect. diff --git a/deploy/compose.yml b/deploy/compose.yml index 62e071f..f1ef77e 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -19,6 +19,8 @@ services: - '443:443/udp' command: - -listen=:443 + # Set only after the C8 DPIA and policy text have been signed off. + - -statistics-policy=${FTW_CLOUD_STATISTICS_POLICY:-} - -h3=${FTW_CLOUD_H3:-true} - -quic-sessions=${FTW_CLOUD_QUIC_SESSIONS:-true} - -domain=cloud.ftw.energy diff --git a/go.mod b/go.mod index dea0f31..9791c01 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261017091326-b05efd4e0568 + github.com/srcfl/ftw/go/protocol v0.0.0-20261017135611-2757af66d806 golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index db3daac..85beec9 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261017091326-b05efd4e0568 h1:9ZRgYviK0JWzMgeIk7XKZnsV7lNjtbpvJEsoUwMOcnk= -github.com/srcfl/ftw/go/protocol v0.0.0-20261017091326-b05efd4e0568/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261017135611-2757af66d806 h1:/xzFcozvsvszpd+dnZlYzyUaokVf5MTnHGLytfJduco= +github.com/srcfl/ftw/go/protocol v0.0.0-20261017135611-2757af66d806/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/internal/fleet/public.go b/internal/fleet/public.go new file mode 100644 index 0000000..650b4c8 --- /dev/null +++ b/internal/fleet/public.go @@ -0,0 +1,46 @@ +package fleet + +import ( + "encoding/json" + "net/http" + "time" + + "github.com/srcfl/ftw/go/protocol/fleetwire" +) + +// Public exports only cells already large enough for the project dashboard. +// A day below the minimum has a null count and no dimension labels. +func (s *Service) Public(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "public, max-age=60") + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + view, err := s.reg.FleetView(s.now()) + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + const minimum = 10 + days := make([]map[string]any, 0, len(view.Days)) + for _, d := range view.Days { + row := map[string]any{"date": d.Date, "reports": nil} + for key, values := range map[string]map[string]int{"ftw_versions": d.Versions, "channels": d.Channels, "drivers": d.Drivers, "battery_kwh": d.Battery, "price_zones": d.Zones, "install_age": d.Age} { + safe := map[string]int{} + if d.Reports >= minimum { + for label, n := range values { + if n >= minimum { + safe[label] = n + } + } + } + row[key] = safe + } + if d.Reports >= minimum { + row["reports"] = d.Reports + } + days = append(days, row) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"schema": "ftw.fleet-public/1", "meaning": fleetwire.Meaning, "observed_at": s.now().UTC().Format(time.RFC3339), "minimum": minimum, "days": days}) +} diff --git a/internal/fleet/public_test.go b/internal/fleet/public_test.go new file mode 100644 index 0000000..03a76bf --- /dev/null +++ b/internal/fleet/public_test.go @@ -0,0 +1,57 @@ +package fleet + +import ( + "encoding/json" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/srcfl/ftw-cloud/internal/registry" + "github.com/srcfl/ftw/go/protocol/fleetwire" +) + +func TestPublicFleetHidesSmallDaysAndEachSmallLabel(t *testing.T) { + reg, err := registry.Open(filepath.Join(t.TempDir(), "db")) + if err != nil { + t.Fatal(err) + } + defer reg.Close() + svc, err := New(reg) + if err != nil { + t.Fatal(err) + } + now := time.Now() + svc.now = func() time.Time { return now } + report := fleetwire.Report{Schema: fleetwire.Schema, Version: "v0.147.0", Channel: "stable", Drivers: []string{}, Battery: "none", Zone: "SE4", Age: "unknown"} + for range 9 { + _, err = reg.FleetPut(report, now) + if err != nil { + t.Fatal(err) + } + } + get := func() map[string]any { + w := httptest.NewRecorder() + svc.Public(w, httptest.NewRequest("GET", "/v1/fleet/public", nil)) + var v map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil { + t.Fatal(err) + } + return v + } + v := get() + d := v["days"].([]any)[0].(map[string]any) + if d["reports"] != nil || len(d["ftw_versions"].(map[string]any)) != 0 { + t.Fatal("small day exposed") + } + report.Version = "v0.147.1" + _, _ = reg.FleetPut(report, now) + v = get() + d = v["days"].([]any)[0].(map[string]any) + if d["reports"] != float64(10) || len(d["ftw_versions"].(map[string]any)) != 0 { + t.Fatal("small version exposed") + } + if d["channels"].(map[string]any)["stable"] != float64(10) { + t.Fatal("safe cell hidden") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 1c87650..7b5fcb8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -37,7 +37,7 @@ import ( const SourceArchive = "/usr/share/ftw-cloud/source.tar.gz" // Routes are every path the service answers. -var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/v1/stats", "/source", "/healthz", "/mcp"}, oauth.Routes...) +var Routes = append([]string{"/v1/box", "/v1/app", "/v1/app/once", "/v1/now", "/v1/fleet", "/v1/fleet/public", "/v1/stats", "/source", "/healthz", "/mcp"}, oauth.Routes...) // devRoutes adds a local inspection route in binaries built with // -tags ftwclouddev. It is nil in every other build. @@ -92,6 +92,11 @@ func (s *Server) Handler() http.Handler { } if s.Fleet != nil { mux.Handle("/v1/fleet", s.Fleet) + if f, ok := s.Fleet.(interface { + Public(http.ResponseWriter, *http.Request) + }); ok { + mux.HandleFunc("/v1/fleet/public", f.Public) + } } else { mux.HandleFunc("/v1/fleet", notYet) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 51f49f2..c3ec50c 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -48,6 +48,7 @@ func newServer(t *testing.T) (*Server, string) { t.Fatal(err) } up.Statistics = true + s.Statistics.(*statistics.Service).Enabled = true // A home with readings in it, held the way a box's uplink holds them. const home = "canaryhomecanaryhomecanary" box, _ := st.For(store.Principal{Kind: store.Box, Home: home}) diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go index 5d6a874..97033cb 100644 --- a/internal/statistics/statistics.go +++ b/internal/statistics/statistics.go @@ -11,10 +11,11 @@ import ( ) type Service struct { - Reg *registry.Registry - mu sync.Mutex - cached []byte - at time.Time + Reg *registry.Registry + Enabled bool + mu sync.Mutex + cached []byte + at time.Time } func New(reg *registry.Registry) (*Service, error) { @@ -26,6 +27,12 @@ func New(reg *registry.Registry) (*Service, error) { func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "public, max-age=60") + if !s.Enabled { + w.Header().Set("Retry-After", "3600") + w.Header().Set("FTW-Statistics-State", "disabled") + w.WriteHeader(http.StatusServiceUnavailable) + return + } if r.Method != http.MethodGet { w.WriteHeader(http.StatusMethodNotAllowed) return diff --git a/internal/statistics/statistics_test.go b/internal/statistics/statistics_test.go new file mode 100644 index 0000000..b8af7b6 --- /dev/null +++ b/internal/statistics/statistics_test.go @@ -0,0 +1,32 @@ +package statistics + +import ( + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/srcfl/ftw-cloud/internal/registry" +) + +func TestCollectionOffByDefaultCannotLookLikeZeroUsage(t *testing.T) { + r, err := registry.Open(filepath.Join(t.TempDir(), "db")) + if err != nil { + t.Fatal(err) + } + defer r.Close() + s, err := New(r) + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + s.ServeHTTP(w, httptest.NewRequest("GET", "/v1/stats", nil)) + if w.Code != 503 || w.Header().Get("FTW-Statistics-State") != "disabled" { + t.Fatalf("disabled collection pretended to be zero usage: %d", w.Code) + } + s.Enabled = true + w = httptest.NewRecorder() + s.ServeHTTP(w, httptest.NewRequest("GET", "/v1/stats", nil)) + if w.Code != 200 { + t.Fatal(w.Code) + } +} diff --git a/internal/uplink/statistics_test.go b/internal/uplink/statistics_test.go index 00753f0..6a66031 100644 --- a/internal/uplink/statistics_test.go +++ b/internal/uplink/statistics_test.go @@ -17,8 +17,10 @@ func TestSignedStatisticsRequireCurrentConsentAndNeverForward(t *testing.T) { s.up.Statistics = true now := time.Now() b := newBox(t) - if _, err := b.connect(s, nil); err != nil { + if ok, err := b.connect(s, nil); err != nil { t.Fatal(err) + } else if ok.StatsPolicy != cloudwire.StatsPolicy { + t.Fatal("approved policy not advertised") } r := b.roster(protocol.ScopeLiveRead) r.Grant.Purposes = append(r.Grant.Purposes, protocol.CloudPurposeStats) @@ -62,3 +64,53 @@ func TestSignedStatisticsRequireCurrentConsentAndNeverForward(t *testing.T) { t.Fatal("stale consent restored rows") } } + +func TestPausedStatisticsIgnoreReportsButStillDeleteOnWithdrawal(t *testing.T) { + s := newService(t) + if err := s.reg.StatisticsInit(); err != nil { + t.Fatal(err) + } + s.up.StatisticsStore = true + now := time.Now() + b := newBox(t) + ok, err := b.connect(s, nil) + if err != nil { + t.Fatal(err) + } + if ok.StatsPolicy != "" { + t.Fatal("paused service advertised a policy") + } + r := b.roster(protocol.ScopeLiveRead) + r.Grant.Purposes = append(r.Grant.Purposes, protocol.CloudPurposeStats) + r.Grant.StatsID = bytes.Repeat([]byte{9}, 16) + r.Grant.StatsPolicy = cloudwire.StatsPolicy + r.Grant.StatsSinceMs = now.Add(-2 * time.Hour).UnixMilli() + b.send(cloudwire.MsgRoster, r) + report := cloudwire.Stats{Header: b.header(), StatsID: r.Grant.StatsID, Policy: cloudwire.StatsPolicy, HourMS: now.Truncate(time.Hour).Add(-time.Hour).UnixMilli(), Metrics: map[string]int64{"pv_generation_wh": 120}} + b.send(cloudwire.MsgStats, report) + b.send(cloudwire.MsgLive, cloudwire.Live{Header: b.header(), Fields: map[string]int64{"2": 1200}}) + wait(t, "paused report processed", func() bool { return len(snapshot(t, s, b.home)) > 0 }) + v, err := s.reg.StatisticsView(now) + if err != nil || v.Installations.State != "empty" { + t.Fatal("paused service stored a report") + } + // A row from before the pause must still be erased by the next roster. + if err := s.reg.StatisticsPut(b.home, report, now); err != nil { + t.Fatal(err) + } + v, err = s.reg.StatisticsView(now) + if err != nil || v.Installations.State != "withheld" { + t.Fatal("fixture did not store the earlier row") + } + r.Header = b.header() + r.RosterSeq++ + r.Grant.Purposes = []string{protocol.CloudPurposeServe} + r.Grant.StatsID = nil + r.Grant.StatsPolicy = "" + r.Grant.StatsSinceMs = 0 + b.send(cloudwire.MsgRoster, r) + wait(t, "paused service erased withdrawn rows", func() bool { + v, err := s.reg.StatisticsView(now) + return err == nil && v.Installations.State == "empty" + }) +} diff --git a/internal/uplink/uplink.go b/internal/uplink/uplink.go index a722ded..4fc6e90 100644 --- a/internal/uplink/uplink.go +++ b/internal/uplink/uplink.go @@ -102,12 +102,13 @@ func TunDropped() int64 { return dropTunUp.Load() + dropTunDown.Load() } // Uplink is the /v1/box handler. type Uplink struct { // Statistics is set before serving, after the consent tables exist. - Statistics bool - store *store.Store - registry *registry.Registry - key *ecdsa.PrivateKey - log *slog.Logger - now func() time.Time + Statistics bool + StatisticsStore bool + store *store.Store + registry *registry.Registry + key *ecdsa.PrivateKey + log *slog.Logger + now func() time.Time mu sync.Mutex conns map[string]*boxConn @@ -559,7 +560,11 @@ func (c *boxConn) run() error { return err } } - if err := c.send(helloOK{BoxNonce: hello.BoxNonce, EKM: c.ekm, Have: map[string]string{}, RosterSeq: c.rosterSeq()}); err != nil { + statsPolicy := "" + if c.u.Statistics { + statsPolicy = cloudwire.StatsPolicy + } + if err := c.send(helloOK{StatsPolicy: statsPolicy, BoxNonce: hello.BoxNonce, EKM: c.ekm, Have: map[string]string{}, RosterSeq: c.rosterSeq()}); err != nil { return err } boxHandshake.Since(c.started) @@ -907,7 +912,7 @@ func (c *boxConn) roster(env cloudwire.Envelope) error { if !c.handle.SetRoster(&r) { return nil // older than what is held: ignored } - if c.u.Statistics { + if c.u.Statistics || c.u.StatisticsStore { if err := c.u.registry.StatisticsGrant(c.home, g); err != nil { return err } diff --git a/privacy/project-statistics-dpia.md b/privacy/project-statistics-dpia.md new file mode 100644 index 0000000..22aabba --- /dev/null +++ b/privacy/project-statistics-dpia.md @@ -0,0 +1,122 @@ +# FTW projektstatistik – underlag för konsekvensbedömning + +Datum: 17 oktober 2026. Status: utkast för granskning. Inte signerat. + +Detta underlag beskriver den avgränsade behandlingen under policyn +`ftw-project-stats-2026-10-v1`. Det ger inget tillstånd att starta insamlingen. +ADR 0008, C8, kräver signerad konsekvensbedömning. Ansvarig företrädare för +Sourceful Labs AB ska ta ställning till underlaget, öppna frågor och kvarvarande +risker före aktivering. Juridisk granskning och dataskyddsombudets råd, om ett +ombud finns, ska dokumenteras. Utkastet avgör inte om artikel 35 i sig kräver +en bedömning i just detta fall; projektets eget krav gäller tills vidare. + +## Ändamål och omfattning + +Sourceful vill förstå hur de hem som frivilligt bidrar använder och flyttar +energi, förbättra FTW:s planering och publicera gruppsummor på stats.ftw.energy. +Ingen reklam, spårning mellan tjänster, individuell profilering, beslut om ett +hushåll eller styrning av utrustning ingår i denna behandling. En assistents +åtkomst är en annan behandling med ett annat tillstånd. + +Personuppgiftsansvarig: Sourceful Labs AB, org.nr 559382-0458, Kalmar. +Kontakt i den befintliga integritetstexten: support@sourceful.energy. +Berörda: de hushåll där en behörig ägare väljer att bidra. Andra boende kan +beröras av vad energimönstren avslöjar. Partnerstyrda installationer, inklusive +aktiverad Nova, bidrar inte genom denna kodväg. Appen fungerar utan statistik. + +## Uppgifter och flöde + +| Steg | Uppgifter och hantering | +|---|---| +| Lokal box | Läser senaste avslutade UTC-timmen ur energiboken och konfigurationen. Inget tidigare än det nya samtycket laddas upp. Saknade värden utelämnas. | +| Rapport | Slumpmässigt 128-bitars statistik-id, policyversion, timstart, registrerad nätimport/export, solproduktion, batteriladdning/urladdning, billaddning/urladdning, inställd batterikapacitet, batteri-/laddareffekt och solkapacitet. Avrundning till 10 Wh eller W. Inga namn, adresser, koordinater, enhets-id:n, fordons-id:n, priser, friare text eller loggar. | +| FTW Cloud | Verifierar boxens signatur och aktuellt samtycke. Kopplar raden till hemmets Cloud-id för radering. En rad per hem och timme; omsändning ersätter raden. Stockholm, AWS eu-north-1 enligt driftsbeskrivningen. | +| Lagring | SQLite på Cloud-värdens beständiga disk. Timrader gallras efter 13 månader. Återkallat samtycke och hemradering tar bort opublicerade rader när Cloud tar emot begäran. Offline återkallelse når Cloud vid nästa anslutning. | +| Publicering | Senaste 24 avslutade timmarna. Kapacitet från respektive hems senaste rapport; energi från dess registrerade timmar. Minst 20 bidragande hem per värde, inget hem över 30 procent av summan. Worker i Cloudflare får bara redan publika gruppvärden och sparar publicerade ögonblicksbilder i 90 dagar. | + +Raderna i Cloud är personuppgifter, inte anonym data: tjänsten kan knyta dem +till ett hem. TLS skyddar transporten men Sourceful kan läsa rapporten. +Den som har full kontroll över värden kan läsa databas och minne. Det finns +ingen route där personal kan läsa ett enskilt hems timrader. Detta hindrar inte +teknisk åtkomst för en administratör med fulla värdrättigheter. + +IP-adresser och anslutningstider kan synas vid transporten. Appkoden avser inte +att logga adresser. Infrastrukturens loggar, säkerhetskopior, underbiträden och +överföringar utanför EES behöver verifieras före beslut. Slutlig gallring i +säkerhetskopior kan inte påstås vara verifierad av databasens radtest. + +## Nödvändighet, proportionalitet och samtycke + +Den föreslagna rättsliga grunden är frivilligt, särskilt och informerat samtycke. +Ansvarig och juridisk granskare ska bekräfta grunden, inklusive information till +andra boende och eventuella regler om åtkomst till terminalutrustning. + +Ägaren får svensk eller engelsk information om fälten, ändamålet, Sourcefuls +åtkomst, lagring, gruppregler och radering. Valet är separat och av som standard. +Äldre val och sparade kryssrutor godkänner inte den nya policyn. Appen skickar +policyns exakta id till boxen vid en ny bekräftelse. Boxen sparar id, tid och +vem som gjorde valet. En ny policy ger nytt statistik-id och nytt startdatum. +Avstängning använder samma inställningsyta. Ett byte av hem är inte samtycke. + +Alternativ som prövats: GitHub-nedladdningar säger inte om något installerats; +anonyma dagsrapporter kan uppskatta rapporterande aktiva installationer men +kan inte summera timenergi eller deduplicera hem. Endast publika gruppvärden +på boxen fungerar inte utan en gemensam sammanställning. Högre upplösning, +liveeffekt, adress, plats och hårdvaruid behövs inte och skickas inte. + +13 månader är den föreslagna perioden för årsvariation. Ansvarig ska bekräfta +att årsanalysen faktiskt behövs; den publika 24-timmarsvyn behöver i sig inte +13 månaders hushållsrader. Om nyttan inte visas ska perioden kortas före start. + +## Risker och skydd + +| Risk för de boende | Skydd som koden ger | Kvarvarande fråga | +|---|---|---| +| Närvaro och vanor framgår av timenergi | Frivilligt val, avslutade timmar, få fält, avrundning, inga platsfält | Timenergi kan ändå avslöja vanor. Ansvarig ska bedöma om risken är rimlig och hur andra boende informeras. | +| En ny policy följer med ett gammalt kryss | Nytt policy-id, ingen automatisk återanvändning, box och Cloud kontrollerar policyn | Granska de renderade svenska och engelska skärmarna samt versionsflödet. | +| Återidentifiering ur små eller dominerade grupper | Minst 20 hem, högst 30 procent per hem, inga hushållsrader till Worker | Upprepade offentliga summor kan möjliggöra differensanalys. Trösklarna är inte ett bevis på anonymitet. Bedöm större eller fastare grupper före start om risken kvarstår. | +| Intrång eller missbruk av värdåtkomst | Signerade rapporter, snäva fält, ingen personalroute, SSH-fri SSM-drift, separata behörigheter för agentnycklar | Full administratör kan läsa data. Verifiera åtkomstlista, loggning, diskkryptering, patchning och incidentväg. | +| Felaktiga slutsatser eller dubbla rapporter | Upsert per hem/timme, partnerfilter, okända värden utelämnas, konfigurering märks som sådan | Delvisa timmar och självrapporterad konfiguration bevisar inte fysisk styrning eller full dygnsproduktion. | +| Uppgifter lever kvar efter avstängning | Radering vid mottaget signerat återkallande, även när insamlingen är pausad; gallring efter 13 månader | Säkerhetskopiors livstid och radering efter förlorad box ska dokumenteras och provas. | + +Risknivåer och acceptans är ännu inte fastställda av ansvarig. Höga kvarvarande +risker får inte döljas med en sammanlagd poäng. Bedöm behovet av ytterligare +åtgärder och eventuellt förhandssamråd innan start. + +## Bevis och återstående beslut + +Kod: srcfl/ftw#1693, srcfl/ftw-cloud#31, samtyckesändringen i srcfl/ftw-app +samt integritetstexten i srcfl/ftw-web. Relevanta tester täcker fält och +signering, äldre samtycke, serverns avstängda läge, gruppstorlek, dominans, +dubbletter, okända värden, återkallande, radering och gallring. En lokal testsvit +bevisar inte alla produktionskontroller eller ett giltigt rättsligt samtycke. + +Före aktivering ska ansvarig fylla i eller länka bevis för: + +1. Granskad svensk och engelsk information och samtyckesskärm. +2. Rättslig grund, information till berörda boende och behovsbedömning. +3. AWS personuppgiftsbiträdesavtal, underbiträden och överföringsbedömning; + Cloudflare får bara gruppvärden, vars återidentifieringsrisk ändå ska bedömas. +4. Diskkryptering, operatörsbehörigheter, loggar och säkerhetskopiors gallring. +5. Praktiskt prov på radering och rättighetsbegäran, inklusive en förlorad box + och utlämnande/rättelse av lagrade rader. De vägarna är inte verifierade här. +6. Bedömd risk efter skydd, motiverad lagringstid och dokumenterat beslut om + riskerna kan godtas. Eventuellt dataskyddsombuds råd och hur de följts. + +Aktivering görs först efter beslut genom den exakta policyn i +`FTW_CLOUD_STATISTICS_POLICY`. Tomt värde är standard. Cloud annonserar då +ingen godkänd statistikpolicy och boxen skickar inga timrapporter. Rollback av +aktiveringen stoppar nya rapporter; radering och gallring fortsätter. + +Beslut: väntar. Beslutsfattare: ej angiven. Datum och underskrift: saknas. +Nästa översyn: före start, vid ändrat ändamål/fält, fler delade mottagare eller +shards, incident eller ny risk; därefter minst årligen. + +## Vägledning som underlaget följer + +- [IMY: hur en konsekvensbedömning ska genomföras](https://www.imy.se/verksamhet/dataskydd/det-har-galler-enligt-gdpr/konsekvensbedomning/hur-ska-en-konsekvensbedomning-genomforas/). +- [IMY: när en konsekvensbedömning behövs](https://www.imy.se/verksamhet/dataskydd/det-har-galler-enligt-gdpr/konsekvensbedomning/nar-ska-en-konsekvensbedomning-genomforas/). +- [EDPB: riktlinjer 05/2020 om samtycke](https://www.edpb.europa.eu/sites/default/files/files/file1/edpb_guidelines_202005_consent_en.pdf). + +Underlaget skiljer tekniskt verifierade egenskaper från beslut som ansvarig +behöver fatta. Källorna ersätter inte granskning av just denna behandling. From 146e3ec107ee46616860ad589ddb18dcdf121e40 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 17 Oct 2026 16:09:28 +0200 Subject: [PATCH 3/3] chore(protocol): pin the merged Core statistics contract --- README.md | 8 ++++---- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e93e677..6336941 100644 --- a/README.md +++ b/README.md @@ -411,10 +411,10 @@ consent. Native apps keep the old scope until their consent screen adopts this policy. The first contributing hour starts after consent; the sender does not upload earlier ledger history. -Deploying this ingestion scope still needs the signed C8 DPIA and the -updated consent text described by Core's ADR 0008. This change does not -claim that sign-off, change an owner's saved choice, or release a Core -update. The paired native consent change requires a fresh choice for this policy. +Enabling this collection scope needs the signed C8 DPIA and the updated +consent text described by Core's ADR 0008. A deployment with collection off +does not claim that sign-off or change an owner's saved choice. The paired +native consent change requires a fresh choice for this policy. The separate anonymous daily-report feed is `GET /v1/fleet/public`. It exposes diff --git a/go.mod b/go.mod index 9791c01..108f2b0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.4 github.com/gorilla/websocket v1.5.3 github.com/quic-go/quic-go v0.63.0 - github.com/srcfl/ftw/go/protocol v0.0.0-20261017135611-2757af66d806 + github.com/srcfl/ftw/go/protocol v0.0.0-20261017140802-6335dbcea60e golang.org/x/crypto v0.57.0 golang.org/x/net v0.58.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 85beec9..4868045 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,8 @@ github.com/quic-go/quic-go v0.63.0 h1:LIFGHI4PFUhhw2dDD1ARHdCff143ffMHwZtbnbuJ78 github.com/quic-go/quic-go v0.63.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/srcfl/ftw/go/protocol v0.0.0-20261017135611-2757af66d806 h1:/xzFcozvsvszpd+dnZlYzyUaokVf5MTnHGLytfJduco= -github.com/srcfl/ftw/go/protocol v0.0.0-20261017135611-2757af66d806/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= +github.com/srcfl/ftw/go/protocol v0.0.0-20261017140802-6335dbcea60e h1:9J9k985aF1/Drp5GHOZFkLmVq2ImuWRPIzUlXiPyoqY= +github.com/srcfl/ftw/go/protocol v0.0.0-20261017140802-6335dbcea60e/go.mod h1:VEjr/o0yZijJ1nApuJK+nak6FxJ/fubAUBNOunqNrUU= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=