From af9423454c1c45e69f95c89d5d90653b670d4cd8 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Wed, 7 Oct 2026 10:32:59 +0800 Subject: [PATCH 1/3] fix: reconcile owned CNI configurations across upgrades --- COMPATIBILITY.md | 16 ++ README.md | 19 +- cniconf/files.go | 295 ++++++++++++++++++++++++++ cniconf/files_test.go | 468 ++++++++++++++++++++++++++++++++++++++++++ cniconf/watcher.go | 66 +++--- 5 files changed, 830 insertions(+), 34 deletions(-) create mode 100644 cniconf/files.go create mode 100644 cniconf/files_test.go diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index fc2d4c7..cc8b344 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -44,6 +44,22 @@ binary path remains only as a compatibility fallback for older providers. Wrappers are atomically installed as regular mode-0700 files; content, type, and permission drift causes the selected wrappers to be restored. +The manager owns materializing Metadata's CNI configuration, including the +known IPsec filename transition. Only an unrequested `10-rancher.conf` with +the exact `rancher-cni-network` / `rancher-bridge` / `rancher-cni-ipam` contract +is retired when the desired `10-pasturestack.conf` declares +`pasturestack-cni-network` / `pasture-bridge` / `metadata-cni-ipam`. All desired +files must validate and complete atomic mode-0600 writes first. Retirement +preserves the original bytes under a `.pasturestack-retired` suffix that the +CNI loader does not execute. An existing backup must be byte-identical; +different contents, ambiguous ownership, malformed configs, symlinks, or +unsafe paths fail reconciliation without retiring the old config. Rolling +back to the exact legacy contract retires only an unrequested native file +with its complete known contract. A config still requested by Metadata is +never retired. Changes to `managed.d` stage a new symlink before replacing +the old pointer. Other administrator files are preserved; no directory-wide +cleanup or inferred ownership is used. + For a host-port container whose Metadata primary IP has not converged, the manager may read only that running container's network namespace. The Docker PID must remain identical across the read, the network must already expose a diff --git a/README.md b/README.md index 01fae8b..c58d4e4 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,23 @@ reselect a same-labelled container at invocation time. Binary names are strictly validated, wrappers are installed atomically as regular mode-0700 files, and content, type, or permission drift is repaired. Older drivers that do not yet contain a private bundle retain the existing shared-binary fallback. -The driver still owns its CNI data plane; Network Plugin Manager continues to -own only host NAT, forwarding, and host-port reconciliation. +The driver still owns its CNI data plane; Network Plugin Manager owns host +NAT, forwarding, host-port reconciliation, and materializing the CNI +configuration supplied by Metadata. + +CNI configuration upgrades also reconcile the known historical IPsec file. +When current Metadata requests `10-pasturestack.conf` with the native bridge +and metadata IPAM contract, the manager validates and atomically writes all +desired configs before retiring an unrequested `10-rancher.conf` whose +network name, bridge type, and IPAM type exactly match the legacy platform +contract. Its unchanged contents remain in +`10-rancher.conf.pasturestack-retired`, outside the active `.conf`/`.json` +set. Rolling back to the exact legacy Metadata contract retires the known +native counterpart in the same way, so both configs cannot run together. +Legacy Metadata remains supported; unrelated administrator files remain +untouched. Malformed or ambiguous old files, symlinks, and conflicting backup +contents fail reconciliation explicitly. This is a bounded config migration, +not a claim that every host or firewall upgrade is safe. The current preflight inspects already loaded legacy tables using an independent iptables-legacy executable. Active old platform or Docker hooks diff --git a/cniconf/files.go b/cniconf/files.go new file mode 100644 index 0000000..777b572 --- /dev/null +++ b/cniconf/files.go @@ -0,0 +1,295 @@ +package cniconf + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/containernetworking/cni/libcni" +) + +const ( + legacyConfigName = "10-rancher.conf" + currentConfigName = "10-pasturestack.conf" + retiredSuffix = ".pasturestack-retired" +) + +type configFile struct { + path string + content []byte +} + +// Metadata filenames and technical network names are single path components. +// The allowed characters match CNI's network-name character set; UI display +// names are not used as paths here. +func validConfigName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + for _, character := range name { + if (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || + (character >= '0' && character <= '9') || character == '.' || character == '-' || character == '_' { + continue + } + return false + } + return true +} + +func checkConfigDirectory(path string) error { + for current := filepath.Clean(path); ; current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err == nil && (!info.IsDir() || info.Mode()&os.ModeSymlink != 0) { + return fmt.Errorf("CNI directory is not a regular directory: %s", current) + } + if err != nil && !os.IsNotExist(err) { + return err + } + if filepath.Dir(current) == current { + return nil + } + } +} + +func ensureConfigDirectory(path string) error { + if err := checkConfigDirectory(path); err != nil { + return err + } + return os.MkdirAll(path, 0700) +} + +func checkConfigFile(path string) error { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("CNI file is not a regular file: %s", path) + } + return nil +} + +// Validate and serialize every desired file before changing any active config. +func prepareConfigFiles(directory string, configs map[string]interface{}) ([]configFile, error) { + if err := checkConfigDirectory(directory); err != nil { + return nil, err + } + files := make([]configFile, 0, len(configs)) + for name, config := range configs { + if !validConfigName(name) { + return nil, fmt.Errorf("invalid CNI config filename %q", name) + } + path := filepath.Join(directory, name) + if err := checkConfigFile(path); err != nil { + return nil, err + } + content, err := json.MarshalIndent(config, "", " ") + if err != nil { + return nil, fmt.Errorf("encode CNI config %q: %w", name, err) + } + if _, err := libcni.NetworkPluginConfFromBytes(content); err != nil { + return nil, fmt.Errorf("validate CNI config %q: %w", name, err) + } + files = append(files, configFile{path: path, content: content}) + } + sort.Slice(files, func(i, j int) bool { return files[i].path < files[j].path }) + return files, nil +} + +func writeConfigAtomic(path string, content []byte) (err error) { + if err := checkConfigFile(path); err != nil { + return err + } + temporary, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-*") + if err != nil { + return fmt.Errorf("create CNI config temporary file: %w", err) + } + temporaryPath := temporary.Name() + defer func() { + _ = temporary.Close() + _ = os.Remove(temporaryPath) + }() + if _, err := temporary.Write(content); err != nil { + return fmt.Errorf("write CNI config temporary file: %w", err) + } + if err := temporary.Chmod(0600); err != nil { + return err + } + if err := temporary.Sync(); err != nil { + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := checkConfigFile(path); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("install CNI config: %w", err) + } + return nil +} + +type bridgeContract struct { + Name string `json:"name"` + Type string `json:"type"` + IPAM struct { + Type string `json:"type"` + } `json:"ipam"` +} + +type legacyRetirement struct { + path string + backup string + content []byte +} + +// Only the known platform IPsec filename transition is reconciled, in either +// upgrade or rollback direction. Other files have no provable ownership. +func prepareLegacyRetirement(directory string, files []configFile) (*legacyRetirement, error) { + var current, legacy *configFile + for i := range files { + switch filepath.Base(files[i].path) { + case legacyConfigName: + legacy = &files[i] + case currentConfigName: + current = &files[i] + } + } + if (current == nil && legacy == nil) || (current != nil && legacy != nil) { + // Never retire a config still explicitly requested by Metadata. + return nil, nil + } + currentContract := [3]string{"pasturestack-cni-network", "pasture-bridge", "metadata-cni-ipam"} + legacyContract := [3]string{"rancher-cni-network", "rancher-bridge", "rancher-cni-ipam"} + successor, expected, oldName, oldExpected := current, currentContract, legacyConfigName, legacyContract + if legacy != nil { + successor, expected, oldName, oldExpected = legacy, legacyContract, currentConfigName, currentContract + } + var desired bridgeContract + if err := json.Unmarshal(successor.content, &desired); err != nil || matchingContractFields(desired, expected) != 3 { + return nil, nil + } + path := filepath.Join(directory, oldName) + if err := checkConfigFile(path); err != nil { + return nil, err + } + content, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var old bridgeContract + if _, err := libcni.NetworkPluginConfFromBytes(content); err != nil { + return nil, fmt.Errorf("cannot classify legacy CNI config %s: %w", path, err) + } + if err := json.Unmarshal(content, &old); err != nil { + return nil, fmt.Errorf("cannot classify legacy CNI config %s: %w", path, err) + } + ownedFields := matchingContractFields(old, oldExpected) + if ownedFields == 0 { + return nil, nil + } + if ownedFields != 3 { + return nil, fmt.Errorf("ambiguous platform ownership of legacy CNI config %s", path) + } + retirement := &legacyRetirement{path: path, backup: path + retiredSuffix, content: content} + if err := retirement.checkBackup(); err != nil { + return nil, err + } + return retirement, nil +} + +func matchingContractFields(config bridgeContract, expected [3]string) int { + matched := 0 + for i, actual := range [3]string{config.Name, config.Type, config.IPAM.Type} { + if actual == expected[i] { + matched++ + } + } + return matched +} + +func (r *legacyRetirement) checkBackup() error { + if err := checkConfigFile(r.backup); err != nil { + return err + } + backup, err := os.ReadFile(r.backup) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !bytes.Equal(backup, r.content) { + return fmt.Errorf("retired CNI backup differs from active legacy config: %s", r.backup) + } + return nil +} + +func (r *legacyRetirement) retire() error { + if r == nil { + return nil + } + if err := checkConfigFile(r.path); err != nil { + return err + } + content, err := os.ReadFile(r.path) + if err != nil { + return err + } + if !bytes.Equal(content, r.content) { + return fmt.Errorf("legacy CNI config changed during migration: %s", r.path) + } + if err := r.checkBackup(); err != nil { + return err + } + if _, err := os.Lstat(r.backup); err == nil { + // An identical recoverable copy already exists; never overwrite it. + return os.Remove(r.path) + } else if !os.IsNotExist(err) { + return err + } + return os.Rename(r.path, r.backup) +} + +// Stage the new link before replacing an existing pointer, so a failure cannot +// remove the previously working managed network link. +func replaceManagedSymlink(path, target string) error { + return replaceManagedSymlinkWithOps(path, target, os.Symlink, os.Rename) +} + +func replaceManagedSymlinkWithOps(path, target string, symlink, rename func(string, string) error) error { + if info, err := os.Lstat(path); err == nil { + if info.Mode()&os.ModeSymlink == 0 { + return fmt.Errorf("managed CNI path is not a symlink: %s", path) + } + } else if !os.IsNotExist(err) { + return err + } + staged, err := os.CreateTemp(filepath.Dir(path), ".managed.d.tmp-*") + if err != nil { + return err + } + stagedPath := staged.Name() + if err := staged.Close(); err != nil { + _ = os.Remove(stagedPath) + return err + } + defer os.Remove(stagedPath) + if err := os.Remove(stagedPath); err != nil { + return err + } + if err := symlink(target, stagedPath); err != nil { + return err + } + return rename(stagedPath, path) +} diff --git a/cniconf/files_test.go b/cniconf/files_test.go new file mode 100644 index 0000000..5517256 --- /dev/null +++ b/cniconf/files_test.go @@ -0,0 +1,468 @@ +package cniconf + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/PastureStack/network-plugin-manager/internal/metadata" + "github.com/containernetworking/cni/libcni" + "github.com/moby/moby/client" +) + +func migrationNetwork(legacy bool) metadata.Network { + file, name, driver, ipam := currentConfigName, "pasturestack-cni-network", "pasture-bridge", "metadata-cni-ipam" + if legacy { + file, name, driver, ipam = legacyConfigName, "rancher-cni-network", "rancher-bridge", "rancher-cni-ipam" + } + return metadata.Network{ + Name: "ipsec", UUID: "network-ipsec", + Metadata: map[string]interface{}{"cniConfig": map[string]interface{}{ + file: map[string]interface{}{ + "cniVersion": "0.3.1", "name": name, "type": driver, + "bridge": "docker0", "bridgeSubnet": "10.42.0.0/16", "hostNat": true, + "ipam": map[string]interface{}{"type": ipam}, + }, + }}, + } +} + +func migrationFixture(t *testing.T) (*watcher, string, []byte) { + t.Helper() + previousDir := cniDir + cniDir = filepath.Join(t.TempDir(), "%s.d") + t.Cleanup(func() { cniDir = previousDir }) + directory := filepath.Join(filepath.Dir(cniDir), "ipsec.d") + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + legacy := migrationNetwork(true).Metadata["cniConfig"].(map[string]interface{})[legacyConfigName] + content, err := json.MarshalIndent(legacy, "", " ") + if err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(directory, legacyConfigName), content) + return &watcher{applied: map[string]metadata.Network{}}, directory, content +} + +func writeFixture(t *testing.T, path string, content []byte) { + t.Helper() + if err := os.WriteFile(path, content, 0600); err != nil { + t.Fatal(err) + } +} + +func assertFileContent(t *testing.T, path string, expected []byte) { + t.Helper() + content, err := os.ReadFile(path) + if err != nil || !bytes.Equal(content, expected) { + t.Fatalf("%s content = %q, error = %v; want %q", path, content, err, expected) + } +} + +func assertNoFile(t *testing.T, path string) { + t.Helper() + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("expected no %s, got %v", path, err) + } +} + +func TestPlatformConfigMigrationLeavesOneActiveConfigAndIsIdempotent(t *testing.T) { + w, directory, legacy := migrationFixture(t) + for i := 0; i < 2; i++ { + if err := w.apply(migrationNetwork(false)); err != nil { + t.Fatal(err) + } + files, err := libcni.ConfFiles(directory, []string{".conf", ".json"}) + if err != nil || len(files) != 1 || filepath.Base(files[0]) != currentConfigName { + t.Fatalf("active configs = %v, error = %v", files, err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName+retiredSuffix), legacy) + assertNoFile(t, filepath.Join(directory, legacyConfigName)) + } + if _, ok := w.applied["ipsec"]; !ok || w.lastApplied.IsZero() { + t.Fatal("successful migration was not recorded") + } + temporaries, err := filepath.Glob(filepath.Join(directory, ".*.tmp-*")) + if err != nil || len(temporaries) != 0 { + t.Fatalf("temporary config files remain: %v, %v", temporaries, err) + } +} + +func TestLegacyMetadataKeepsItsActiveConfig(t *testing.T) { + w, directory, legacy := migrationFixture(t) + if err := w.apply(migrationNetwork(true)); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) + assertNoFile(t, filepath.Join(directory, currentConfigName)) +} + +func TestPlatformConfigMigrationConvergesUpgradeRollbackAndUpgrade(t *testing.T) { + w, directory, legacy := migrationFixture(t) + for _, rollback := range []bool{false, true, false, true} { + if err := w.apply(migrationNetwork(rollback)); err != nil { + t.Fatal(err) + } + want := currentConfigName + if rollback { + want = legacyConfigName + } + files, err := libcni.ConfFiles(directory, []string{".conf", ".json"}) + if err != nil || len(files) != 1 || filepath.Base(files[0]) != want { + t.Fatalf("rollback=%v: active configs = %v, error = %v, want only %s", rollback, files, err, want) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName+retiredSuffix), legacy) + } +} + +func TestPlatformConfigMigrationNeverRetiresDesiredFiles(t *testing.T) { + w, directory, legacy := migrationFixture(t) + network := migrationNetwork(false) + network.Metadata["cniConfig"].(map[string]interface{})[legacyConfigName] = + migrationNetwork(true).Metadata["cniConfig"].(map[string]interface{})[legacyConfigName] + if err := w.apply(network); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) + assertNoFile(t, filepath.Join(directory, currentConfigName+retiredSuffix)) +} + +func TestPlatformConfigMigrationPreservesForeignFiles(t *testing.T) { + w, directory, _ := migrationFixture(t) + foreign := []byte(`{"name":"administrator-network","type":"bridge","ipam":{"type":"host-local"}}`) + writeFixture(t, filepath.Join(directory, legacyConfigName), foreign) + admin := []byte(`{"name":"audit-network","type":"loopback"}`) + writeFixture(t, filepath.Join(directory, "90-administrator.json"), admin) + if err := w.apply(migrationNetwork(false)); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), foreign) + assertFileContent(t, filepath.Join(directory, "90-administrator.json"), admin) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) +} + +func TestPlatformConfigMigrationRejectsUnclassifiableLegacyConfig(t *testing.T) { + for _, content := range []string{ + `{`, `null`, + `{"name":"rancher-cni-network","type":"bridge","ipam":{"type":"host-local"}}`, + `{"name":"custom","type":"rancher-bridge","ipam":{"type":"rancher-cni-ipam"}}`, + } { + t.Run(content, func(t *testing.T) { + w, directory, _ := migrationFixture(t) + writeFixture(t, filepath.Join(directory, legacyConfigName), []byte(content)) + if err := w.apply(migrationNetwork(false)); err == nil { + t.Fatal("unclassifiable legacy config was silently migrated") + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), []byte(content)) + assertNoFile(t, filepath.Join(directory, currentConfigName)) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) + }) + } +} + +func TestPlatformConfigMigrationRequiresExactSuccessorContract(t *testing.T) { + for _, field := range []string{"name", "type", "ipam"} { + t.Run(field, func(t *testing.T) { + w, directory, legacy := migrationFixture(t) + network := migrationNetwork(false) + config := network.Metadata["cniConfig"].(map[string]interface{})[currentConfigName].(map[string]interface{}) + if field == "ipam" { + config[field] = map[string]interface{}{"type": "host-local"} + } else { + config[field] = "custom" + } + if err := w.apply(network); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) + }) + } +} + +func TestPlatformConfigMigrationHandlesExistingBackupWithoutOverwriting(t *testing.T) { + for _, identical := range []bool{true, false} { + t.Run(map[bool]string{true: "identical", false: "different"}[identical], func(t *testing.T) { + w, directory, legacy := migrationFixture(t) + backup := legacy + if !identical { + backup = []byte("previous retired configuration") + } + backupPath := filepath.Join(directory, legacyConfigName+retiredSuffix) + writeFixture(t, backupPath, backup) + err := w.apply(migrationNetwork(false)) + if identical && err != nil { + t.Fatal(err) + } + if !identical && err == nil { + t.Fatal("different backup was overwritten") + } + assertFileContent(t, backupPath, backup) + if identical { + assertNoFile(t, filepath.Join(directory, legacyConfigName)) + } else { + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, currentConfigName)) + } + }) + } +} + +func TestPlatformConfigMigrationDoesNotRetireAfterAnyWriteFails(t *testing.T) { + w, directory, legacy := migrationFixture(t) + network := migrationNetwork(false) + network.Metadata["cniConfig"].(map[string]interface{})["90-extra.conf"] = map[string]interface{}{"type": "loopback"} + writes := 0 + failure := errors.New("injected config write failure") + err := w.applyWithWriter(network, func(path string, content []byte) error { + writes++ + if writes == 2 { + return failure + } + return writeConfigAtomic(path, content) + }) + if !errors.Is(err, failure) || writes != 2 { + t.Fatalf("apply error = %v, writes = %d", err, writes) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) + if len(w.applied) != 0 || !w.lastApplied.IsZero() { + t.Fatal("failed apply was recorded as successful") + } + if err := w.apply(network); err != nil { + t.Fatal(err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName+retiredSuffix), legacy) +} + +func TestConfigPreflightValidatesEveryDesiredConfigBeforeWriting(t *testing.T) { + for _, invalid := range []interface{}{make(chan int), map[string]interface{}{"name": "missing-type"}} { + w, directory, legacy := migrationFixture(t) + network := migrationNetwork(false) + network.Metadata["cniConfig"].(map[string]interface{})["99-invalid.conf"] = invalid + if err := w.apply(network); err == nil { + t.Fatal("invalid desired config was accepted") + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, currentConfigName)) + } +} + +func TestConfigMigrationRejectsPathTraversal(t *testing.T) { + for _, unsafe := range []string{"../outside", `..\outside`, "/absolute", "driver:stream", ".."} { + t.Run(unsafe, func(t *testing.T) { + w, directory, legacy := migrationFixture(t) + network := migrationNetwork(false) + network.Name = unsafe + if err := w.apply(network); err == nil { + t.Fatal("unsafe network path was accepted") + } + network = migrationNetwork(false) + unsafeFile := unsafe + if unsafe != ".." { + unsafeFile += ".conf" + } + network.Metadata["cniConfig"].(map[string]interface{})[unsafeFile] = map[string]interface{}{"type": "loopback"} + if err := w.apply(network); err == nil { + t.Fatal("unsafe config path was accepted") + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, currentConfigName)) + }) + } +} + +func symlinkFixture(t *testing.T, target, path string) { + t.Helper() + if err := os.Symlink(target, path); err != nil { + if runtime.GOOS == "windows" { + t.Skipf("symlink creation unavailable: %v", err) + } + t.Fatal(err) + } +} + +func TestConfigMigrationRejectsSymlinkedConfigAndBackup(t *testing.T) { + for _, name := range []string{legacyConfigName, currentConfigName, legacyConfigName + retiredSuffix} { + t.Run(name, func(t *testing.T) { + w, directory, legacy := migrationFixture(t) + victim := filepath.Join(t.TempDir(), "administrator-file") + writeFixture(t, victim, legacy) + path := filepath.Join(directory, name) + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + symlinkFixture(t, victim, path) + if err := w.apply(migrationNetwork(false)); err == nil { + t.Fatal("symlinked CNI file was accepted") + } + assertFileContent(t, victim, legacy) + if name != currentConfigName { + assertNoFile(t, filepath.Join(directory, currentConfigName)) + } + }) + } +} + +func TestConfigMigrationRejectsSymlinkedDirectory(t *testing.T) { + w, directory, legacy := migrationFixture(t) + actual := directory + "-administrator" + if err := os.Rename(directory, actual); err != nil { + t.Fatal(err) + } + symlinkFixture(t, actual, directory) + if err := w.apply(migrationNetwork(false)); err == nil { + t.Fatal("symlinked network directory was accepted") + } + assertFileContent(t, filepath.Join(actual, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(actual, currentConfigName)) +} + +func TestLegacyRetirementRejectsChangesAfterPreflight(t *testing.T) { + _, directory, _ := migrationFixture(t) + files, err := prepareConfigFiles(directory, migrationNetwork(false).Metadata["cniConfig"].(map[string]interface{})) + if err != nil { + t.Fatal(err) + } + retirement, err := prepareLegacyRetirement(directory, files) + if err != nil { + t.Fatal(err) + } + changed := []byte(`{"type":"administrator-plugin"}`) + writeFixture(t, filepath.Join(directory, legacyConfigName), changed) + if err := retirement.retire(); err == nil { + t.Fatal("concurrently changed legacy config was retired") + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), changed) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) +} + +func TestManagedSymlinkFailurePreservesPreviousPointer(t *testing.T) { + for _, stage := range []string{"create", "rename"} { + t.Run(stage, func(t *testing.T) { + directory := t.TempDir() + oldDirectory := filepath.Join(directory, "old.d") + if err := os.Mkdir(oldDirectory, 0700); err != nil { + t.Fatal(err) + } + oldConfig := []byte(`{"type":"administrator-plugin"}`) + writeFixture(t, filepath.Join(oldDirectory, "10-old.conf"), oldConfig) + path := filepath.Join(directory, "managed.d") + symlinkFixture(t, "old.d", path) + failure := errors.New("injected symlink failure") + symlink, rename := os.Symlink, os.Rename + if stage == "create" { + symlink = func(string, string) error { return failure } + } else { + rename = func(string, string) error { return failure } + } + if err := replaceManagedSymlinkWithOps(path, "new.d", symlink, rename); !errors.Is(err, failure) { + t.Fatalf("replace error = %v", err) + } + assertFileContent(t, filepath.Join(path, "10-old.conf"), oldConfig) + if target, err := os.Readlink(path); err != nil || target != "old.d" { + t.Fatalf("old pointer changed: %q, %v", target, err) + } + staged, err := filepath.Glob(filepath.Join(directory, ".managed.d.tmp-*")) + if err != nil || len(staged) != 0 { + t.Fatalf("staged pointers remain: %v, %v", staged, err) + } + }) + } +} + +func TestManagedSymlinkRejectsAdministratorDirectory(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "managed.d") + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + admin := []byte(`{"type":"administrator-plugin"}`) + writeFixture(t, filepath.Join(path, "90-admin.conf"), admin) + if err := replaceManagedSymlink(path, "new.d"); err == nil { + t.Fatal("administrator directory was replaced") + } + assertFileContent(t, filepath.Join(path, "90-admin.conf"), admin) +} + +func TestManagedSymlinkReplacementAndUnchangedDefaultPointer(t *testing.T) { + w, directory, legacy := migrationFixture(t) + path := filepath.Join(filepath.Dir(directory), "managed.d") + symlinkFixture(t, "previous.d", path) + network := migrationNetwork(false) + network.Default = true + var previous os.FileInfo + for i := 0; i < 2; i++ { + if err := w.apply(network); err != nil { + t.Fatal(err) + } + if target, err := os.Readlink(path); err != nil || target != "ipsec.d" { + t.Fatalf("managed pointer = %q, %v", target, err) + } + assertFileContent(t, filepath.Join(directory, legacyConfigName+retiredSuffix), legacy) + pointer, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + if previous != nil && !os.SameFile(previous, pointer) { + t.Fatal("already correct managed pointer was unnecessarily replaced") + } + previous = pointer + } +} + +func TestConfigMigrationPreservesOldConfigIfDefaultPointerFails(t *testing.T) { + w, directory, legacy := migrationFixture(t) + if err := os.Mkdir(filepath.Join(filepath.Dir(directory), "managed.d"), 0700); err != nil { + t.Fatal(err) + } + network := migrationNetwork(false) + network.Default = true + if err := w.apply(network); err == nil { + t.Fatal("invalid managed pointer was accepted") + } + assertFileContent(t, filepath.Join(directory, legacyConfigName), legacy) + assertNoFile(t, filepath.Join(directory, legacyConfigName+retiredSuffix)) +} + +type configMetadata struct { + metadata.Client + network metadata.Network +} + +func (m configMetadata) GetNetworks() ([]metadata.Network, error) { + return []metadata.Network{m.network}, nil +} +func (m configMetadata) GetSelfHost() (metadata.Host, error) { + return metadata.Host{UUID: "local-host"}, nil +} +func (m configMetadata) GetServices() ([]metadata.Service, error) { + return []metadata.Service{{Kind: "networkDriverService", Containers: []metadata.Container{{HostUUID: "local-host"}}}}, nil +} + +func TestOnChangeReturnsConfigMigrationFailure(t *testing.T) { + w, directory, _ := migrationFixture(t) + writeFixture(t, filepath.Join(directory, legacyConfigName), []byte("invalid JSON")) + server := httptest.NewServer(http.NotFoundHandler()) + defer server.Close() + docker, err := client.New(client.WithHost("tcp://"+server.Listener.Addr().String()), client.WithAPIVersion("1.55")) + if err != nil { + t.Fatal(err) + } + defer docker.Close() + w.c = configMetadata{network: migrationNetwork(false)} + w.dc = docker + if err := w.onChange(""); err == nil || !strings.Contains(err.Error(), "cannot classify legacy CNI config") { + t.Fatalf("migration error was swallowed: %v", err) + } +} diff --git a/cniconf/watcher.go b/cniconf/watcher.go index 67a42a2..1529c24 100644 --- a/cniconf/watcher.go +++ b/cniconf/watcher.go @@ -1,11 +1,9 @@ package cniconf import ( - "bytes" - "encoding/json" + "errors" "fmt" "os" - "path/filepath" "reflect" "time" @@ -68,6 +66,7 @@ func (w *watcher) onChange(version string) error { logrus.Debugf("localNetworks: %v", localNetworks) forceApply := time.Now().Sub(w.lastApplied) > reapplyEvery + var applyErrors []error for _, network := range networks { if _, local := localNetworks[network.UUID]; !local { @@ -81,12 +80,12 @@ func (w *watcher) onChange(version string) error { if forceApply || !reflect.DeepEqual(w.applied[network.Name], network) { if err := w.apply(network); err != nil { - logrus.Errorf("Failed to apply cni conf: %v", err) + applyErrors = append(applyErrors, fmt.Errorf("network %q: %w", network.Name, err)) } } } - return nil + return errors.Join(applyErrors...) } // localCNINetworks returns the CNI-managed networks that must be configured on @@ -124,30 +123,30 @@ func localCNINetworks(networks []metadata.Network, services []metadata.Service, } func (w *watcher) apply(network metadata.Network) error { + return w.applyWithWriter(network, writeConfigAtomic) +} + +func (w *watcher) applyWithWriter(network metadata.Network, writeConfig func(string, []byte) error) error { cniConf, _ := network.Metadata["cniConfig"].(map[string]interface{}) + if !validConfigName(network.Name) { + return fmt.Errorf("invalid CNI network name %q", network.Name) + } confDir := fmt.Sprintf(cniDir, network.Name) - if err := os.MkdirAll(confDir, 0700); err != nil { + files, err := prepareConfigFiles(confDir, cniConf) + if err != nil { return err } - - var lastErr error - for file, config := range cniConf { - p := filepath.Join(confDir, file) - content, err := json.Marshal(config) - if err != nil { - lastErr = err - continue - } - - out := &bytes.Buffer{} - if err := json.Indent(out, content, "", " "); err != nil { - lastErr = err - continue - } - - logrus.Debugf("Writing %s: %s", p, out) - if err := os.WriteFile(p, out.Bytes(), 0600); err != nil { - lastErr = err + retirement, err := prepareLegacyRetirement(confDir, files) + if err != nil { + return err + } + if err := ensureConfigDirectory(confDir); err != nil { + return err + } + for _, file := range files { + logrus.Debugf("Writing CNI config %s", file.path) + if err := writeConfig(file.path, file.content); err != nil { + return err } } @@ -156,17 +155,20 @@ func (w *watcher) apply(network metadata.Network) error { managedDirTest, err := os.Stat(managedDir) configDirTest, err1 := os.Stat(confDir) if !(err == nil && err1 == nil && os.SameFile(managedDirTest, configDirTest)) { - os.Remove(managedDir) - if err := os.Symlink(network.Name+".d", managedDir); err != nil { - lastErr = err + if err := replaceManagedSymlink(managedDir, network.Name+".d"); err != nil { + return err } } } - if lastErr == nil { - w.applied[network.Name] = network - w.lastApplied = time.Now() + if err := retirement.retire(); err != nil { + return err } + if retirement != nil { + logrus.Infof("Retired superseded platform CNI config %s; recoverable copy: %s", retirement.path, retirement.backup) + } + w.applied[network.Name] = network + w.lastApplied = time.Now() - return lastErr + return nil } From 09cd6fc1dc3bd187044ef086d45c571fc2996166 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Wed, 7 Oct 2026 10:39:42 +0800 Subject: [PATCH 2/3] build: refresh available Alpine TLS and CA package pins --- alpine-apk.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alpine-apk.lock b/alpine-apk.lock index e06aa78..64e835a 100644 --- a/alpine-apk.lock +++ b/alpine-apk.lock @@ -4,7 +4,7 @@ ALPINE_APK_BRANCH='3.23' ALPINE_APK_BASH_VERSION='5.3.3-r1' -ALPINE_APK_CA_CERTIFICATES_VERSION='20260611-r0' +ALPINE_APK_CA_CERTIFICATES_VERSION='20260909-r0' ALPINE_APK_CONNTRACK_TOOLS_VERSION='1.4.8-r0' ALPINE_APK_CURL_VERSION='8.22.0-r0' ALPINE_APK_GAWK_VERSION='5.3.2-r2' @@ -12,8 +12,8 @@ ALPINE_APK_IPROUTE2_VERSION='6.17.0-r0' ALPINE_APK_IPTABLES_VERSION='1.8.11-r1' ALPINE_APK_IPTABLES_LEGACY_VERSION='1.8.11-r1' ALPINE_APK_JQ_VERSION='1.8.2-r0' -ALPINE_APK_LIBCRYPTO3_VERSION='3.5.8-r0' -ALPINE_APK_LIBSSL3_VERSION='3.5.8-r0' +ALPINE_APK_LIBCRYPTO3_VERSION='3.5.9-r0' +ALPINE_APK_LIBSSL3_VERSION='3.5.9-r0' ALPINE_APK_NET_TOOLS_VERSION='2.10-r3' ALPINE_APK_NFTABLES_VERSION='1.1.5-r2' ALPINE_APK_PROCPS_NG_VERSION='4.0.5-r0' From 5d01a1ffa29e5dd885db7de32b9e646beb0afca9 Mon Sep 17 00:00:00 2001 From: chen21019 Date: Wed, 7 Oct 2026 10:41:37 +0800 Subject: [PATCH 3/3] docs: describe current network manager without obsolete release receipts --- README.md | 71 +++++++------------------------------------------------ 1 file changed, 8 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index c58d4e4..bfd1b25 100644 --- a/README.md +++ b/README.md @@ -8,67 +8,12 @@ PastureStack is an independent community effort to preserve, audit, and moderniz ## Runtime image -The `v0.8.14` image was published with GHCR manifest digest -`sha256:59b4bb31df28503337e9f3b8f08c18aa0dbe9749692c721fe8bdfc4cc921f263`. -Its annotated tag resolves to signed source commit -`98ffacd24436d42e33db721ab7026739d0edee41`. The release workflow passed -tests, a reproducible build, Trivy source/binary/image scans, CycloneDX source -and image SBOM checks, and asset/image provenance attestations. Image -publication is separate from Catalog integration and the complete -control-plane host lifecycle gate. - -On an isolated Ubuntu 26.04.1 / Docker 29.8 VM, a source-equivalent release -candidate passed backend detection against Docker's native nftables, -iptables-nft, and iptables-legacy modes, rejected mismatched explicit choices -without changing rules, and passed a Docker restart check and a legacy-mode -host reboot check. This does not establish multi-host rollout or existing-stack -upgrade safety. - -The `v0.8.15` image was published with GHCR manifest digest -`sha256:622cfb38a58f204d23152205e6d850d204d1cb9d3c50392a935afee49d780e3e`. -Its annotated tag resolves to signed source commit -`26eee48df2e3bac96fc97fcd596a16deccc9f4ad`. The release workflow passed -its build, security, checksum, SBOM, and provenance gates. This release moves -same-subnet NAT exclusion into the manager's xtables rules, matching native -nftables ownership. The isolated VM applied, reapplied, inspected, and removed -candidate host NAT and host-port rules under Docker's iptables-nft and -iptables-legacy frontends. Image publication and isolated-VM tests do not by -themselves establish a managed-service or multi-host rollout. - -The `v0.8.16` image was published with GHCR manifest digest -`sha256:a042c582689561b43349fa83ed92269e849038be3b7a2342e8a9ef0149460f92`. -The `v0.8.17` image was published with GHCR manifest digest -`sha256:f13654b27b71f3fbddbcf33272c10b342d513dd402a255bdda1f341cfbe908f8`. -Its signed tag resolves to verified source commit -`e29dd5cefa373140d76e3a21da9bd95a3bec97e3`; the release workflow passed -tests, image scanning, checksums, SBOM, and provenance gates. This version adds -bounded cross-host exceptions for the per-host-subnet -network: only active hosts with distinct, valid subnet labels are peers. Their -traffic retains its container source IP and is marked before Docker's native -nft bridge filter. An active host with a missing or overlapping label fails -closed; an inactive registration does not block live peers. Network Plugin -Manager owns these NAT and forwarding rules, not the CNI driver or an ad-hoc -host firewall script. - -On two isolated Ubuntu 26.04.1 / Docker 29.8 QA hosts, a source-equivalent -`v0.8.17` candidate passed bidirectional container ping and TCP 42, service -DNS, public HTTPS egress, and host port 32792 after Docker restarts and host -reboots. The second host was also explicitly switched to `iptables-nft`, then -`iptables-legacy`, with the same cross-host checks passing in each mode. It was -restored to native nft afterward. The official `v0.8.17` image was then used on -both native-nft hosts with the official IPsec/VXLAN `v0.14.34` image; manager -health, bidirectional TCP 42, Metadata HTTP 200, public HTTPS, and published -host ports all passed. The manager follows the Docker-selected backend; it -does not change the host's firewall preference. This bounded test -does not establish every existing iptables or IPsec deployment's migration safety. - -The `v0.8.18` image was published with GHCR manifest digest -`sha256:1f5d44de03648a771ec9e7bc448e456ef6b21a5fcd4cc51f59f99df96a804822`. -It restores target-scoped authorization for every packet in an owned DNAT -flow, including later UDP datagrams, without accepting unrelated Docker -traffic. - -The current release is `v0.8.21`. Managed bridge subnets can initiate +The current release is `v0.8.22`. Release assets include source and image +SBOMs, SHA256 checksums, and build provenance. Obtain the immutable image +identity from the checksum-covered `published.txt` in the release. Catalog +integration and real-host lifecycle checks are separate from image publication. + +Managed bridge subnets can initiate outbound traffic and receive established or related replies. Shared overlay subnets used by IPsec and VXLAN can also receive new connections from the same validated subnet through the exact managed bridge. Existing templates @@ -93,7 +38,7 @@ image identity from the release's checksum-covered [`published.txt`](https://github.com/PastureStack/network-plugin-manager/releases/latest/download/published.txt) rather than copying an older release digest. -`v0.8.21` also closes two control-plane convergence gaps without moving +The manager also closes two control-plane convergence gaps without moving responsibility between plugins. If Metadata temporarily omits the primary IP of a running container that publishes a host port, the manager reads that exact container's network namespace and accepts an address only when exactly @@ -227,7 +172,7 @@ The Alpine 3.23 base image is digest-pinned. Direct runtime packages are exact-v make test make validate bash scripts/check-build-downloads -VERSION_OVERRIDE=v0.8.21 IMAGE_NAMESPACE=local/pasturestack make package +VERSION_OVERRIDE=v0.8.22 IMAGE_NAMESPACE=local/pasturestack make package ``` Pull requests and `main` run one non-publishing gate: tests, vet/format checks, govulncheck, a reproducible binary build, one runtime image build, and Trivy scans plus CycloneDX SBOMs for the source, binary, and image. All reported vulnerabilities and secrets fail the gate. Publishing remains a separate, explicitly authorized operation.