egress: write the host's lane verdict into every NIC-bearing guest

docs/desktop.md

@@ -26,9 +26,9 @@ up a few seconds later — poll `GET /screenshot` until it returns 200.
   relay. Thunderbird's mail policy is pinned to the SOCKS5 door, so a pool that
   runs mail tasks sets `"socks5": true` in its policy, and so does the tenant
   policy when the claim carries one. `net=egress` on a
-  guarded bridge gives the desktop no web: the NIC is locked and the session's
-  proxy setup does not follow the lock marker (a CNI-backed lane still routes
-  directly).
+  guarded bridge works the same way: the session's proxy setup waits for the
+  host's lane verdict in `/etc/silkd-lane` and follows it, so the locked NIC
+  costs the desktop nothing (a CNI-backed lane is told `direct` and routes).
 - **Size**: `2xlarge` (8 CPU / 16G) — the t3.xlarge class the OSWorld AWS
   image runs on; the idle session is ~0.5 GB anonymous memory with
   gnome-shell around 290 MB RSS, and the headroom is for the apps.
@@ -78,9 +78,9 @@ Configure it with `SANDBOXD_ADDR`, `SANDBOXD_TOKEN`, `COCOON_TEMPLATE`
 - `chromium` is a second launcher onto the same engine, for an agent that
   needs a browser the tasks do not pkill and rebind. It takes the caller's
   profile and debugging port.
-- On a guest with no NIC, `guest-proxy.service` points the session's browsers
-  and `osworld-server`'s commands at silkd's loopback relay; a guest with a
-  NIC routes directly and the unit does nothing. The unit tests NIC presence
-  only, so it does not follow the locked-NIC marker silkd honors on the bridge
-  egress lane; run desktop pools on the none lane.
+- `guest-proxy.service` points the session's browsers and `osworld-server`'s
+  commands at silkd's loopback relay on a guest with no NIC, and on a guest
+  whose NIC the host reports as `relay` in `/etc/silkd-lane`; it waits up to
+  30 s for that verdict before the session starts. A NIC the host calls
+  `direct`, or a host that writes nothing, leaves the unit idle.
 - x86_64 only.

docs/egress.md

@@ -75,11 +75,13 @@ remove keeps an existing lock in place; the next restart retries the remove). A
 lock that never applied plus a failed remove leaves the VM unguarded until a
 later remove succeeds.
 
-Because the lock is invisible from inside the guest, sandboxd also tells silkd
-about it: a marker written into the guest before the golden snapshot (or on a
-cold boot) makes silkd hand the proxy variables to every exec on this lane
-exactly as on the none lane, while git keeps running as on any lane with a NIC.
-A golden built before the marker existed is rebuilt, not adopted.
+Because the lock is invisible from inside the guest, sandboxd writes its
+verdict into `/etc/silkd-lane` before the golden snapshot (or on a cold boot):
+`relay` on a locked bridge lane, `direct` on a CNI lane. Where silkd reads
+`relay` it hands the proxy variables to every exec exactly as on the none lane,
+and the image's units wait for the same file instead of guessing from NIC
+presence; git keeps running as on any lane with a NIC. A golden built under
+another verdict is rebuilt, not adopted.
 
 Egress-lane sandboxes do not hibernate, archive, fork, checkpoint, or promote:
 cocoon resumes a guest before its fresh tap can be re-locked, so any resume from

docs/silkd.md

@@ -68,11 +68,11 @@ the host's [guarded-egress](egress.md) proxy over vsock (`CID2:2049`), and
 `127.0.0.1:1080` to its SOCKS5 door (`CID2:2050`) for clients whose protocol
 has no HTTP-proxy form; when the host wired no policy the per-connection dial
 is refused, so the ports are inert.
-Where nothing routes directly — the no-network lane, or a NIC the host has
-nft-locked and marked as such via `/etc/silkd-nic-locked` — silkd forwards the
-image-baked proxy variables (`http_proxy` and friends) into every exec, so
-unconfigured clients use the relay without being told; a lane with its own
-routed network never gets them.
+Where nothing routes directly — the no-network lane, or a NIC whose host
+verdict in `/etc/silkd-lane` reads `relay` because it is nft-locked — silkd
+forwards the image-baked proxy variables (`http_proxy` and friends) into every
+exec, so unconfigured clients use the relay without being told; a lane with its
+own routed network never gets them.
 
 ## Limits
 

e2e/cmd/desktopsmoke/main.go

@@ -1,6 +1,7 @@
-// desktopsmoke is the desktop-flavor acceptance: claim (none/2xlarge) → the
-// OSWorld guest server over the relay (screenshot, AT-SPI tree, a PyAutoGUI
-// action echoed by the cursor) → checkpoint/branch of the warmed desktop.
+// desktopsmoke is the desktop-flavor acceptance: claim (2xlarge) → the OSWorld
+// guest server over the relay (screenshot, AT-SPI tree, a PyAutoGUI action
+// echoed by the cursor) → on the none lane a checkpoint/branch of the warmed
+// desktop, on the egress lane a fetch through the session's proxy environment.
 package main
 
 import (
@@ -31,21 +32,23 @@ func main() {
 	addr := flag.String("addr", "127.0.0.1:7777", "sandboxd address")
 	token := flag.String("token", "", "node api token")
 	template := flag.String("template", "ghcr.io/cocoonstack/sandbox/desktop:24.04", "desktop template ref")
+	lane := flag.String("net", "none", "lane: none, or egress for a guarded bridge pool with a policy")
+	probe := flag.String("probe", "http://postman-echo.com/get", "URL the egress policy allows, fetched from the session's environment on the egress lane")
 	flag.Parse()
 
-	if err := run(*addr, *token, *template); err != nil {
+	if err := run(*addr, *token, *template, sandbox.NetShape(*lane), *probe); err != nil {
 		fmt.Fprintln(os.Stderr, "desktopsmoke:", err)
 		os.Exit(1)
 	}
 	fmt.Println("DESKTOPSMOKE PASS")
 }
 
-func run(addr, token, template string) error {
+func run(addr, token, template string, lane sandbox.NetShape, probe string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
 	defer cancel()
 	start := time.Now()
 	_, sb, err := harness.Claim(ctx, addr, token, template,
-		sandbox.WithNetwork(sandbox.NetNone), sandbox.WithSize(sandbox.XXLarge),
+		sandbox.WithNetwork(lane), sandbox.WithSize(sandbox.XXLarge),
 		sandbox.WithTimeout(30*time.Minute))
 	if err != nil {
 		return err
@@ -62,6 +65,9 @@ func run(addr, token, template string) error {
 	if err = clickAndReadCursor(ctx, sb); err != nil {
 		return err
 	}
+	if lane == sandbox.NetEgress {
+		return proxyReachesSession(ctx, sb, probe)
+	}
 
 	ckpt, err := sb.Checkpoint(ctx, "desktop-warmed")
 	if err != nil {
@@ -119,23 +125,10 @@ func accessibilityTree(ctx context.Context, sb *sandbox.Sandbox) error {
 // clickAndReadCursor proves /execute reaches the X display, not just the shell.
 func clickAndReadCursor(ctx context.Context, sb *sandbox.Sandbox) error {
 	action := fmt.Sprintf("import pyautogui; pyautogui.moveTo(%d, %d); pyautogui.click()", clickX, clickY)
-	req, _ := json.Marshal(map[string]any{"command": []string{"python", "-c", action}, "shell": false, "timeout": 15})
-	body, err := harness.HTTPOverPort(ctx, sb, serverPort, "POST", "/execute", req)
-	if err != nil {
-		return fmt.Errorf("execute: %w", err)
-	}
-	var exec struct {
-		Status     string `json:"status"`
-		ReturnCode int    `json:"returncode"`
-		Error      string `json:"error"`
-	}
-	if err = json.Unmarshal(body, &exec); err != nil {
-		return fmt.Errorf("execute: %w", err)
-	}
-	if exec.Status != "success" || exec.ReturnCode != 0 {
-		return fmt.Errorf("execute: %s rc=%d: %s", exec.Status, exec.ReturnCode, strings.TrimSpace(exec.Error))
+	if _, err := execute(ctx, sb, []string{"python", "-c", action}, 15); err != nil {
+		return err
 	}
-	body, err = harness.HTTPOverPort(ctx, sb, serverPort, "GET", "/cursor_position", nil)
+	body, err := harness.HTTPOverPort(ctx, sb, serverPort, "GET", "/cursor_position", nil)
 	if err != nil {
 		return fmt.Errorf("cursor_position: %w", err)
 	}
@@ -149,3 +142,39 @@ func clickAndReadCursor(ctx context.Context, sb *sandbox.Sandbox) error {
 	fmt.Printf("  server: /execute pyautogui click → /cursor_position %v\n", pos)
 	return nil
 }
+
+// proxyReachesSession proves the session's environment carries the relay and the relay reaches an allowed origin.
+func proxyReachesSession(ctx context.Context, sb *sandbox.Sandbox, probe string) error {
+	script := `test -n "$http_proxy" && curl -sS -m 20 -o /dev/null -w '%{http_code}' ` + probe
+	out, err := execute(ctx, sb, []string{"sh", "-c", script}, 30)
+	if err != nil {
+		return fmt.Errorf("proxy leg: %w", err)
+	}
+	if strings.TrimSpace(out) != "200" {
+		return fmt.Errorf("proxy leg: fetch through the session's proxy returned %q, want 200", strings.TrimSpace(out))
+	}
+	fmt.Println("  proxy: the session's environment carries the relay and an allowed origin answers 200")
+	return nil
+}
+
+// execute runs one command through osworld-server and returns its output; a non-zero exit is an error.
+func execute(ctx context.Context, sb *sandbox.Sandbox, command []string, timeout int) (string, error) {
+	req, _ := json.Marshal(map[string]any{"command": command, "shell": false, "timeout": timeout})
+	body, err := harness.HTTPOverPort(ctx, sb, serverPort, "POST", "/execute", req)
+	if err != nil {
+		return "", fmt.Errorf("execute: %w", err)
+	}
+	var reply struct {
+		Status     string `json:"status"`
+		Output     string `json:"output"`
+		ReturnCode int    `json:"returncode"`
+		Error      string `json:"error"`
+	}
+	if err = json.Unmarshal(body, &reply); err != nil {
+		return "", fmt.Errorf("execute: %w", err)
+	}
+	if reply.Status != "success" || reply.ReturnCode != 0 {
+		return "", fmt.Errorf("execute: %s rc=%d: %s", reply.Status, reply.ReturnCode, strings.TrimSpace(reply.Error))
+	}
+	return reply.Output, nil
+}

e2e/fakeengine_test.go

@@ -110,7 +110,7 @@ func (f *fakeEngine) DialGuestPort(context.Context, string, uint16) (net.Conn, e
 
 func (f *fakeEngine) InstallCACert(context.Context, string, []byte) error { return nil }
 
-func (f *fakeEngine) MarkNICLocked(context.Context, string) error { return nil }
+func (f *fakeEngine) MarkLane(context.Context, string, engine.Lane) error { return nil }
 
 func (f *fakeEngine) Warmup(context.Context, string, []string) error { return nil }
 

os-image/desktop/24.04/Dockerfile

@@ -151,11 +151,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
       'host='"'"'127.0.0.1'"'"'' \
       'port=1080' \
       > /usr/local/share/guest-proxy.dconf \
+    # a NIC alone says nothing about routing: the host writes /etc/silkd-lane once silkd answers, so a NIC-bearing guest waits for the verdict
     && printf '%s\n' \
       '#!/bin/sh' \
       'set -e' \
       ': > /run/guest-proxy.env' \
-      'for n in /sys/class/net/*; do case ${n##*/} in lo|sit0) ;; *) exit 0;; esac; done' \
+      'nic=0' \
+      'for n in /sys/class/net/*; do case ${n##*/} in lo|sit0) ;; *) nic=1;; esac; done' \
+      'if [ $nic = 1 ]; then' \
+      '  i=0' \
+      '  while [ ! -e /etc/silkd-lane ] && [ $i -lt 300 ]; do sleep 0.1; i=$((i+1)); done' \
+      '  if [ ! -e /etc/silkd-lane ]; then echo "guest-proxy: no lane verdict from the host; routing directly" >&2; exit 0; fi' \
+      '  read -r lane < /etc/silkd-lane' \
+      '  [ "$lane" = relay ] || exit 0' \
+      'fi' \
       'cat /usr/local/share/guest-proxy.env > /run/guest-proxy.env' \
       'cat /usr/local/share/guest-proxy.dconf > /etc/dconf/db/local.d/01-proxy' \
       'dconf update' \

sandboxd/engine/lane.go

@@ -0,0 +1,20 @@
+package engine
+
+import "context"
+
+const (
+	laneFile = "/etc/silkd-lane"
+
+	LaneRelay  Lane = "relay"
+	LaneDirect Lane = "direct"
+)
+
+// Lane is the host's verdict on how a guest's traffic leaves: through the relay when its NIC is nft-locked, directly when the NIC routes.
+type Lane string
+
+// MarkLane writes the verdict into the guest, where silkd and the image's units read it; the nft lock is invisible from inside.
+func (e *Engine) MarkLane(ctx context.Context, vsockSocket string, lane Lane) error {
+	ctx, cancel := context.WithTimeout(ctx, cmdTimeout)
+	defer cancel()
+	return e.silkdWriteFile(ctx, vsockSocket, laneFile, 0o644, []byte(lane+"\n"))
+}

sandboxd/pool/egress.go

@@ -94,17 +94,30 @@ func (m *Manager) lockEgressNIC(ctx context.Context, sb *types.Sandbox) error {
 	return nil
 }
 
-// markLockedNIC tells the guest its NIC is nft-locked; silkd cannot see the hook from inside.
-func (m *Manager) markLockedNIC(ctx context.Context, key types.PoolKey, sock string) error {
-	if !m.locksNIC(key) {
+// markLane tells the guest how its traffic leaves; the nft lock is invisible from inside.
+func (m *Manager) markLane(ctx context.Context, key types.PoolKey, sock string) error {
+	lane := m.laneOf(key)
+	if lane == "" {
 		return nil
 	}
-	if err := m.eng.MarkNICLocked(ctx, sock); err != nil {
-		return fmt.Errorf("mark nic locked: %w", err)
+	if err := m.eng.MarkLane(ctx, sock, lane); err != nil {
+		return fmt.Errorf("mark lane: %w", err)
 	}
 	return nil
 }
 
+// laneOf is the verdict a guest receives; the none lane has no NIC to route through and gets none.
+func (m *Manager) laneOf(key types.PoolKey) engine.Lane {
+	switch {
+	case key.Net != types.NetEgress:
+		return ""
+	case m.locksNIC(key):
+		return engine.LaneRelay
+	default:
+		return engine.LaneDirect
+	}
+}
+
 func (m *Manager) locksNIC(key types.PoolKey) bool {
 	return m.lockEgress && key.Net == types.NetEgress
 }

sandboxd/pool/egress_test.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -103,22 +104,25 @@ func TestArmEgressBindsSocksOnlyWhenOptedIn(t *testing.T) {
 	}
 }
 
-func TestLockedLaneMarksGuestOnColdBoots(t *testing.T) {
+func TestLaneVerdictReachesGuestOnColdBoots(t *testing.T) {
 	tests := []struct {
-		name string
-		key  types.PoolKey
-		cold bool
-		want int
+		name   string
+		key    types.PoolKey
+		locked bool
+		cold   bool
+		want   []string
 	}{
-		{"golden build on the egress lane", egKey, false, 1},
-		{"golden build on the none lane", testKey, false, 0},
-		{"cold provision on the egress lane", egKey, true, 1},
-		{"cold provision on the none lane", testKey, true, 0},
+		{"golden build on a locked egress lane", egKey, true, false, []string{"relay"}},
+		{"golden build on an unlocked egress lane", egKey, false, false, []string{"direct"}},
+		{"golden build on the none lane", testKey, true, false, nil},
+		{"cold provision on a locked egress lane", egKey, true, true, []string{"relay"}},
+		{"cold provision on the none lane", testKey, true, true, nil},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			eng := newFakeEngine()
 			m := egressManager(t, eng, config.PoolSpec{PoolKey: tt.key, Warm: 1, Egress: egPolicy})
+			m.lockEgress = tt.locked
 			if tt.cold {
 				sb, err := m.provision(t.Context(), tt.key, "")
 				if err != nil {
@@ -128,8 +132,8 @@ func TestLockedLaneMarksGuestOnColdBoots(t *testing.T) {
 			} else if err := m.buildGoldenSteps(t.Context(), tt.key, "sbx-gb", "snap", filepath.Join(m.goldensDir(), tt.key.Hash())); err != nil {
 				t.Fatalf("buildGoldenSteps: %v", err)
 			}
-			if got := len(eng.nicMarks); got != tt.want {
-				t.Errorf("MarkNICLocked calls = %d, want %d", got, tt.want)
+			if !slices.Equal(eng.laneMarks, tt.want) {
+				t.Errorf("MarkLane verdicts = %q, want %q", eng.laneMarks, tt.want)
 			}
 		})
 	}
@@ -146,7 +150,7 @@ func TestGoldenNICSidecarGatesAdoption(t *testing.T) {
 	if p.goldenDir != "" {
 		t.Error("adopted an egress-lane golden that never marked its guest; want rebuild")
 	}
-	if err := writeGoldenSidecar(g+nicSidecarSuffix, nicStamp(true)); err != nil {
+	if err := writeGoldenSidecar(g+nicSidecarSuffix, string(engine.LaneRelay)); err != nil {
 		t.Fatalf("write sidecar: %v", err)
 	}
 	m.adoptGolden(p)

sandboxd/pool/pool.go

@@ -117,7 +117,7 @@ type Engine interface {
 	Probe(ctx context.Context, vsockSocket string, timeout time.Duration) error
 	DialGuestPort(ctx context.Context, vsockSocket string, port uint16) (net.Conn, error)
 	InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error
-	MarkNICLocked(ctx context.Context, vsockSocket string) error
+	MarkLane(ctx context.Context, vsockSocket string, lane engine.Lane) error
 	Warmup(ctx context.Context, vsockSocket string, argv []string) error
 	DiskAttach(ctx context.Context, vmName string, spec engine.VolumeSpec) error
 	MountVolume(ctx context.Context, vsockSocket, name, mount string, rw bool) error

sandboxd/pool/pool_test.go

@@ -813,7 +813,7 @@ type fakeEngine struct {
 	removeSeenOps   map[string][]string
 	removeSeenDirty map[string][]string
 	dirtyPaths      []string
-	nicMarks        []string
+	laneMarks       []string
 	syncs           []string
 
 	hibernates, restores, snapRemoves []string
@@ -1060,10 +1060,10 @@ func (f *fakeEngine) InstallCACert(_ context.Context, vsockSocket string, _ []by
 	return f.installCAErr
 }
 
-func (f *fakeEngine) MarkNICLocked(_ context.Context, vsockSocket string) error {
+func (f *fakeEngine) MarkLane(_ context.Context, _ string, lane engine.Lane) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	f.nicMarks = append(f.nicMarks, vsockSocket)
+	f.laneMarks = append(f.laneMarks, string(lane))
 	return nil
 }
 

sandboxd/pool/reconcile.go

@@ -177,7 +177,7 @@ func (m *Manager) resyncEgress(ctx context.Context, live map[string]types.VMReco
 				err = netfilter.Lock(tap)
 			}
 			if err == nil && !locked[tap] {
-				err = m.markLockedNIC(ctx, sb.Key, sb.VsockSocket)
+				err = m.markLane(ctx, sb.Key, sb.VsockSocket)
 			}
 			if err != nil {
 				logger.Errorf(ctx, err, "ensure egress lock %s; quarantining", sb.ID)