From 2d3009344186f4d76eb036291ef41b7c4e0a048d Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 21 Oct 2026 19:01:46 +0000 Subject: [PATCH 1/6] DAOS-19809 control: Enable multiprovider functionality This patch enables user access to multiprovider functionality. Multi-provider access may be configured in the server and agent configuration files. Features: control Signed-off-by: Kris Jacque --- src/control/cmd/daos_agent/attachinfo.go | 3 +- src/control/cmd/daos_agent/config.go | 3 +- src/control/cmd/daos_agent/config_test.go | 3 ++ src/control/cmd/dmg/auto_test.go | 4 +- src/control/cmd/dmg/pretty/system.go | 25 +++++++++ src/control/cmd/dmg/pretty/system_test.go | 24 +++++++++ src/control/lib/control/auto.go | 2 +- src/control/server/config/server_test.go | 7 ++- src/control/server/engine/config.go | 21 +++++--- src/control/server/engine/config_test.go | 62 ++++++++++++++++------- utils/config/daos_agent.yml | 20 ++++++-- utils/config/daos_server.yml | 11 ++++ 12 files changed, 150 insertions(+), 35 deletions(-) diff --git a/src/control/cmd/daos_agent/attachinfo.go b/src/control/cmd/daos_agent/attachinfo.go index decd9469df4..963f9bc1ef9 100644 --- a/src/control/cmd/daos_agent/attachinfo.go +++ b/src/control/cmd/daos_agent/attachinfo.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -37,7 +38,7 @@ func (cmd *attachInfoCmd) getAttachInfo(ctx context.Context) (*control.GetAttach type dumpAttachInfoCmd struct { attachInfoCmd Output string `short:"o" long:"output" default:"stdout" description:"Dump output to this location"` - ProviderIdx *uint // TODO SRS-31: Enable with multiprovider functionality: `short:"n" long:"provider_idx" description:"Index of provider to fetch (if multiple)"` + ProviderIdx *uint `short:"n" long:"provider_idx" description:"Index of provider to fetch (if multiple)"` } func (cmd *dumpAttachInfoCmd) Execute(_ []string) error { diff --git a/src/control/cmd/daos_agent/config.go b/src/control/cmd/daos_agent/config.go index 418ffadfefe..8121392da2b 100644 --- a/src/control/cmd/daos_agent/config.go +++ b/src/control/cmd/daos_agent/config.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent @@ -135,7 +136,7 @@ type Config struct { ExcludeFabricIfaces common.StringSet `yaml:"exclude_fabric_ifaces,omitempty"` IncludeFabricIfaces common.StringSet `yaml:"include_fabric_ifaces,omitempty"` FabricInterfaces []*NUMAFabricConfig `yaml:"fabric_ifaces,omitempty"` - ProviderIdx uint // TODO SRS-31: Enable with multiprovider functionality + ProviderIdx uint `yaml:"provider_idx"` Telemetry TelemetryConfig `yaml:",inline"` } diff --git a/src/control/cmd/daos_agent/config_test.go b/src/control/cmd/daos_agent/config_test.go index 69f50152605..e2852161c98 100644 --- a/src/control/cmd/daos_agent/config_test.go +++ b/src/control/cmd/daos_agent/config_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2024 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent @@ -78,6 +79,7 @@ fabric_ifaces: - iface: ib3 domain: mlx5_3 +provider_idx: 1 `) badLogMaskCfg := test.CreateTestFile(t, dir, ` @@ -202,6 +204,7 @@ exclude_fabric_ifaces: ["ib3"] }, }, }, + ProviderIdx: 1, }, }, } { diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index d1c79c07d20..780971b8625 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -614,7 +614,7 @@ engines: - "0000:03:00.0" provider: ofi+verbs fabric_iface: ib0 - fabric_iface_port: 31416 + fabric_iface_port: "31416" pinned_numa_node: 0 - targets: 6 nr_xs_helpers: 0 @@ -631,7 +631,7 @@ engines: - "0000:06:00.0" provider: ofi+verbs fabric_iface: ib1 - fabric_iface_port: 32416 + fabric_iface_port: "32416" pinned_numa_node: 1 disable_vfio: false disable_vmd: false diff --git a/src/control/cmd/dmg/pretty/system.go b/src/control/cmd/dmg/pretty/system.go index 26bb8dd8799..fb38d25d9f3 100644 --- a/src/control/cmd/dmg/pretty/system.go +++ b/src/control/cmd/dmg/pretty/system.go @@ -143,6 +143,8 @@ func PrintSystemQueryResponse(out, outErr io.Writer, resp *control.SystemQueryRe printAbsentHosts(outErr, &resp.AbsentHosts) printSysSelfHealUnsetFlags(out, resp.SysSelfHealPolicy) + printSystemProviders(out, resp.Providers) + return nil } @@ -150,6 +152,8 @@ func PrintSystemQueryResponse(out, outErr io.Writer, resp *control.SystemQueryRe printAbsentRanks(outErr, &resp.AbsentRanks) printSysSelfHealUnsetFlags(out, resp.SysSelfHealPolicy) + printSystemProviders(out, resp.Providers) + return nil } @@ -187,6 +191,27 @@ func printSystemResults(out, outErr io.Writer, results system.MemberResults, abs return nil } +func printSystemProviders(out io.Writer, providers []string) { + if len(providers) == 0 { + return + } + + idxKey := "Idx" + provKey := "Provider" + formatter := txtfmt.NewTableFormatter(idxKey, provKey) + + var table []txtfmt.TableRow + for i, prov := range providers { + table = append(table, txtfmt.TableRow{ + idxKey: fmt.Sprintf("%d", i), + provKey: prov, + }) + } + + fmt.Fprint(out, "\n") + fmt.Fprint(out, formatter.Format(table)) +} + // PrintSystemStartResponse generates a human-readable representation of the // supplied SystemStartResp struct and writes it to the supplied io.Writer. func PrintSystemStartResponse(out, outErr io.Writer, resp *control.SystemStartResp) error { diff --git a/src/control/cmd/dmg/pretty/system_test.go b/src/control/cmd/dmg/pretty/system_test.go index 225cf7aceef..7048955d641 100644 --- a/src/control/cmd/dmg/pretty/system_test.go +++ b/src/control/cmd/dmg/pretty/system_test.go @@ -424,6 +424,30 @@ Rank UUID Control Address Fault Domain State Unknown 3 hosts: foo[7-9] Unknown 3 ranks: 7-9 +`, + }, + "providers": { + resp: &control.SystemQueryResp{ + Members: Members{ + MockMember(t, 0, MemberStateJoined), + MockMember(t, 1, MemberStateJoined), + MockMember(t, 2, MemberStateStopped), + }, + Providers: []string{"ofi+verbs", "ofi+tcp", "ofi+tcp"}, + }, + expPrintStr: ` +Rank State +---- ----- +[0-1] Joined +2 Stopped + +System property self_heal flags disabled: exclude, pool_exclude, pool_rebuild + +Idx Provider +--- -------- +0 ofi+verbs +1 ofi+tcp +2 ofi+tcp `, }, } { diff --git a/src/control/lib/control/auto.go b/src/control/lib/control/auto.go index b33896a3aa9..812d10224de 100644 --- a/src/control/lib/control/auto.go +++ b/src/control/lib/control/auto.go @@ -1223,7 +1223,7 @@ func genEngineConfigs(req ConfGenerateReq, newEngineCfg newEngineCfgFn, nodeSet cfg.Fabric = engine.FabricConfig{ Provider: iface.Provider, Interface: iface.Device, - InterfacePort: ifPort, + InterfacePort: fmt.Sprintf("%d", ifPort), } if err := cfg.SetNUMAAffinity(pnn); err != nil { return nil, errors.Wrapf(err, "setting numa %d affinity on engine config", diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 1e124dd76a8..95bdc836fa8 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -257,6 +257,7 @@ func TestServerConfig_Constructed(t *testing.T) { WithSocketDir("./.daos/daos_server"). WithFabricProvider("ofi+verbs;ofi_rxm"). WithCrtTimeout(30). + WithNumSecondaryEndpoints([]int{2}). WithMgmtSvcReplicas("hostname1", "hostname2", "hostname3"). WithFaultCb("./.daos/fd_callback"). WithFaultPath("/vcdu0/rack1/hostname"). @@ -294,6 +295,7 @@ func TestServerConfig_Constructed(t *testing.T) { WithFabricProvider("ofi+verbs;ofi_rxm"). WithFabricAuthKey("foo:bar"). WithCrtTimeout(30). + WithNumSecondaryEndpoints([]int{2}). WithPinnedNumaNode(0). WithBypassHealthChk(&bypass). WithEnvVars("CRT_TIMEOUT=30"). @@ -324,6 +326,7 @@ func TestServerConfig_Constructed(t *testing.T) { WithFabricProvider("ofi+verbs;ofi_rxm"). WithFabricAuthKey("foo:bar"). WithCrtTimeout(30). + WithNumSecondaryEndpoints([]int{2}). WithBypassHealthChk(&bypass). WithEnvVars("CRT_TIMEOUT=100"). WithLogFile("/var/log/daos/daos_engine.1.log"). @@ -365,7 +368,7 @@ func TestServerConfig_updateServerConfig(t *testing.T) { Fabric: engine.FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 1111, + InterfacePort: "1111", NumSecondaryEndpoints: []int{2, 3, 4}, }, }, @@ -379,7 +382,7 @@ func TestServerConfig_updateServerConfig(t *testing.T) { Fabric: engine.FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 1111, + InterfacePort: "1111", NumSecondaryEndpoints: []int{2, 3, 4}, }, }, diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index e49214ca111..7e57e824a86 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -40,7 +40,7 @@ const ( type FabricConfig struct { Provider string `yaml:"provider,omitempty" cmdEnv:"D_PROVIDER"` Interface string `yaml:"fabric_iface,omitempty" cmdEnv:"D_INTERFACE"` - InterfacePort int `yaml:"fabric_iface_port,omitempty" cmdEnv:"D_PORT,nonzero"` + InterfacePort string `yaml:"fabric_iface_port,omitempty" cmdEnv:"D_PORT"` NumaNodeIndex uint `yaml:"-"` BypassHealthChk *bool `yaml:"bypass_health_chk,omitempty" cmdLongFlag:"--bypass_health_chk" cmdShortFlag:"-b"` CrtTimeout uint32 `yaml:"crt_timeout,omitempty" cmdEnv:"CRT_TIMEOUT"` @@ -126,11 +126,20 @@ func (fc *FabricConfig) GetInterfacePorts() ([]int, error) { return nil, errors.New("FabricConfig is nil") } - if fc.InterfacePort == 0 { + portStrs := splitMultiProviderStr(fc.InterfacePort) + if len(portStrs) == 0 { return nil, errors.New("fabric_iface_port not set") } - return []int{fc.InterfacePort}, nil + ports := make([]int, 0) + for _, str := range portStrs { + intPort, err := strconv.Atoi(str) + if err != nil { + return nil, err + } + ports = append(ports, intPort) + } + return ports, nil } // Update fills in any missing fields from the provided FabricConfig. @@ -145,7 +154,7 @@ func (fc *FabricConfig) Update(other FabricConfig) { if fc.Interface == "" { fc.Interface = other.Interface } - if fc.InterfacePort == 0 { + if fc.InterfacePort == "" { fc.InterfacePort = other.InterfacePort } if fc.CrtTimeout == 0 { @@ -196,7 +205,7 @@ func (fc *FabricConfig) Validate() error { } } - if len(interfaces) != numProv { // TODO SRS-31: check num ports when multiprovider fully enabled: || len(ports) != numProv { + if len(interfaces) != numProv || len(ports) != numProv { return errors.Errorf("provider, fabric_iface and fabric_iface_port must include the same number of items delimited by %q", MultiProviderSeparator) } @@ -701,7 +710,7 @@ func (c *Config) WithFabricInterface(iface string) *Config { // WithFabricInterfacePort sets the numeric interface port to be used by this instance. func (c *Config) WithFabricInterfacePort(ifacePort int) *Config { - c.Fabric.InterfacePort = ifacePort + c.Fabric.InterfacePort = fmt.Sprintf("%d", ifacePort) return c } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 1c771cfe8b5..06543e35595 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -560,14 +560,14 @@ func TestConfig_FabricValidation(t *testing.T) { "missing provider": { cfg: FabricConfig{ Interface: "bar", - InterfacePort: 42, + InterfacePort: "42", }, expErr: errors.New("provider"), }, "missing interface": { cfg: FabricConfig{ Provider: "foo", - InterfacePort: 42, + InterfacePort: "42", }, expErr: errors.New("fabric_iface"), }, @@ -582,7 +582,7 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: "foo", Interface: "bar", - InterfacePort: -42, + InterfacePort: "-42", }, expErr: errors.New("fabric_iface_port"), }, @@ -590,21 +590,21 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: "foo", Interface: "bar", - InterfacePort: 42, + InterfacePort: "42", }, }, "multi provider/interface/port ok": { cfg: FabricConfig{ Provider: multiProviderString("foo", "bar"), Interface: multiProviderString("baz", "net"), - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128"), }, }, "mismatched num providers": { cfg: FabricConfig{ Provider: "foo", Interface: multiProviderString("baz", "net"), - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128"), }, expErr: errors.New("same number"), }, @@ -612,7 +612,15 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: multiProviderString("foo", "bar"), Interface: "baz", - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128"), + }, + expErr: errors.New("same number"), + }, + "mismatched num ports": { + cfg: FabricConfig{ + Provider: multiProviderString("foo", "bar"), + Interface: multiProviderString("baz", "net"), + InterfacePort: "42", }, expErr: errors.New("same number"), }, @@ -620,7 +628,7 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: multiProviderString("foo", "bar"), Interface: multiProviderString("baz", "net"), - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128"), NumSecondaryEndpoints: []int{0}, }, expErr: errors.New("must be > 0"), @@ -629,7 +637,7 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: multiProviderString("foo", "bar", "baz"), Interface: multiProviderString("net0", "net1", "net2"), - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128", "256"), NumSecondaryEndpoints: []int{1, 2}, }, }, @@ -637,7 +645,7 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: multiProviderString("foo", "bar", "baz"), Interface: multiProviderString("net0", "net1", "net2"), - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128", "256"), NumSecondaryEndpoints: []int{1, 2, 3}, }, expErr: errors.New("must have one value for each"), @@ -646,7 +654,7 @@ func TestConfig_FabricValidation(t *testing.T) { cfg: FabricConfig{ Provider: multiProviderString("foo", "bar", "baz"), Interface: multiProviderString("net0", "net1", "net2"), - InterfacePort: 42, + InterfacePort: multiProviderString("42", "128", "256"), NumSecondaryEndpoints: []int{1}, }, expErr: errors.New("must have one value for each"), @@ -949,10 +957,28 @@ func TestFabricConfig_GetInterfacePorts(t *testing.T) { }, "single": { cfg: &FabricConfig{ - InterfacePort: 1234, + InterfacePort: "1234", }, expPorts: []int{1234}, }, + "multi": { + cfg: &FabricConfig{ + InterfacePort: multiProviderString("1234", "5678", "9012"), + }, + expPorts: []int{1234, 5678, 9012}, + }, + "excessive whitespace": { + cfg: &FabricConfig{ + InterfacePort: multiProviderString("1234 ", " 5678 ", "", " 9012"), + }, + expPorts: []int{1234, 5678, 9012}, + }, + "non-integer port": { + cfg: &FabricConfig{ + InterfacePort: multiProviderString("1234", "a123"), + }, + expErr: errors.New("strconv.Atoi"), + }, } { t.Run(name, func(t *testing.T) { ports, err := tc.cfg.GetInterfacePorts() @@ -1024,7 +1050,7 @@ func TestFabricConfig_Update(t *testing.T) { new: FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 9999, + InterfacePort: "9999", CrtTimeout: 60, DisableSRX: true, NumSecondaryEndpoints: []int{1}, @@ -1032,7 +1058,7 @@ func TestFabricConfig_Update(t *testing.T) { expResult: &FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 9999, + InterfacePort: "9999", CrtTimeout: 60, DisableSRX: true, NumSecondaryEndpoints: []int{1}, @@ -1042,7 +1068,7 @@ func TestFabricConfig_Update(t *testing.T) { fc: &FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 9999, + InterfacePort: "9999", CrtTimeout: 60, DisableSRX: true, NumSecondaryEndpoints: []int{1}, @@ -1051,7 +1077,7 @@ func TestFabricConfig_Update(t *testing.T) { expResult: &FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 9999, + InterfacePort: "9999", CrtTimeout: 60, DisableSRX: true, NumSecondaryEndpoints: []int{1}, @@ -1064,14 +1090,14 @@ func TestFabricConfig_Update(t *testing.T) { new: FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 9999, + InterfacePort: "9999", CrtTimeout: 120, DisableSRX: true, }, expResult: &FabricConfig{ Provider: "provider", Interface: "iface", - InterfacePort: 9999, + InterfacePort: "9999", CrtTimeout: 60, DisableSRX: true, }, diff --git a/utils/config/daos_agent.yml b/utils/config/daos_agent.yml index 9e94e38ba75..e4565fdd8ee 100644 --- a/utils/config/daos_agent.yml +++ b/utils/config/daos_agent.yml @@ -157,10 +157,10 @@ # #include_fabric_ifaces: ["eth0"] -# Manually define the fabric interfaces and domains to be used by the agent, -# organized by NUMA node. -# If not defined, the agent will automatically detect all fabric interfaces and -# select appropriate ones based on the server preferences. +## Manually define the fabric interfaces and domains to be used by the agent, +## organized by NUMA node. +## If not defined, the agent will automatically detect all fabric interfaces and +## select appropriate ones based on the server preferences. # #fabric_ifaces: #- @@ -181,3 +181,15 @@ # - # iface: ib3 # domain: mlx5_3 + +## Select the fabric provider to be used by all clients, if the server is configured with +## multiple providers. +## The provider index is 0-based and based on the order of fabric providers listed in the +## "provider" field of the corresponding daos_server configuration. An administrator can +## see the list of configured server providers and their indices by using the "dmg system +## query" command. +## Index 0 corresponds to the primary provider. Any index > 0 is a secondary provider. +# +## default: 0 +# +#provider_idx: 1 diff --git a/utils/config/daos_server.yml b/utils/config/daos_server.yml index 8fbc63e70cc..268ade2ec0b 100644 --- a/utils/config/daos_server.yml +++ b/utils/config/daos_server.yml @@ -274,6 +274,17 @@ #system_ram_reserved: 5 # # +## Number of network endpoints per secondary provider +# +## Specifies the number of secondary network endpoints each engine will create +## for each additional fabric provider after the first. This only applies when +## running in multi-provider mode. +# +## default: 1 +# +#secondary_provider_endpoints: [2] +# +# ## Set specific debug mask for daos_server (control plane). ## The mask specifies minimum level of message significance to pass to logger. ## Currently supported values are DISABLED, TRACE, DEBUG, INFO, NOTICE and ERROR. From 4827161bc825f9bcc82cbef544c01bcee34bbdf7 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Fri, 23 Oct 2026 00:03:39 +0000 Subject: [PATCH 2/6] Workaround for support output parsing Features: control Signed-off-by: Kris Jacque --- src/control/lib/support/log.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/control/lib/support/log.go b/src/control/lib/support/log.go index 9dca3b76b24..24c21368958 100644 --- a/src/control/lib/support/log.go +++ b/src/control/lib/support/log.go @@ -415,6 +415,7 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error } domainName := strings.Split(string(dName), "\n") + // TODO DAOS-19818: use the control API instead of parsing text. cmd := strings.Join([]string{"dmg", "system", "query", "-v", "-o", configPath}, " ") out, err := exec.Command("sh", "-c", cmd).Output() if err != nil { @@ -424,7 +425,11 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error if len(temp) > 0 { for _, hn := range temp[2 : len(temp)-2] { - hn = strings.ReplaceAll(strings.Fields(hn)[3][1:], domainName[0], "") + hnFields := strings.Fields(hn) + if len(hnFields) < 4 { + continue + } + hn = strings.ReplaceAll(hnFields[3][1:], domainName[0], "") hn = strings.TrimSuffix(hn, ".") hostNames = append(hostNames, hn) } From 58a4456a714633e7cb3947da901732f05912ff45 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 28 Oct 2026 21:47:41 +0000 Subject: [PATCH 3/6] Address review comments Features: control Signed-off-by: Kris Jacque --- src/control/lib/support/log.go | 2 +- src/control/server/engine/config.go | 41 +++++++---- src/control/server/engine/config_test.go | 88 +++++++++++++++++++----- 3 files changed, 99 insertions(+), 32 deletions(-) diff --git a/src/control/lib/support/log.go b/src/control/lib/support/log.go index 24c21368958..5b881f58366 100644 --- a/src/control/lib/support/log.go +++ b/src/control/lib/support/log.go @@ -427,7 +427,7 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error for _, hn := range temp[2 : len(temp)-2] { hnFields := strings.Fields(hn) if len(hnFields) < 4 { - continue + break // we're outside the rank table } hn = strings.ReplaceAll(hnFields[3][1:], domainName[0], "") hn = strings.TrimSuffix(hn, ".") diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 7e57e824a86..92ffc70de1e 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -50,6 +50,25 @@ type FabricConfig struct { AuthKey string `yaml:"fabric_auth_key,omitempty" cmdEnv:"D_PROVIDER_AUTH_KEY"` } +// Normalize adjusts the format of the fabric configuration strings that are exported to CaRT. +func (fc *FabricConfig) Normalize() { + normalizeMultiprov := func(multiProvStr string) string { + strs := strings.Split(multiProvStr, MultiProviderSeparator) + result := []string{} + for _, s := range strs { + s = strings.TrimSpace(s) + if s != "" { + result = append(result, s) + } + } + return strings.Join(result, MultiProviderSeparator) + } + + fc.Provider = normalizeMultiprov(fc.Provider) + fc.Interface = normalizeMultiprov(fc.Interface) + fc.InterfacePort = normalizeMultiprov(fc.InterfacePort) +} + // GetPrimaryProvider parses the primary provider from the Provider string. func (fc *FabricConfig) GetPrimaryProvider() (string, error) { providers, err := fc.GetProviders() @@ -76,15 +95,10 @@ func (fc *FabricConfig) GetProviders() ([]string, error) { func splitMultiProviderStr(str string) []string { strs := strings.Split(str, MultiProviderSeparator) - result := make([]string, 0) - for _, s := range strs { - trimmed := strings.TrimSpace(s) - if trimmed != "" { - result = append(result, trimmed) - } + if len(strs) == 1 && strs[0] == "" { + return nil } - - return result + return strs } // GetNumProviders gets the number of fabric providers configured. @@ -121,7 +135,7 @@ func (fc *FabricConfig) GetInterfaces() ([]string, error) { } // GetInterfacePorts parses the InterfacePort string to one or more ports. -func (fc *FabricConfig) GetInterfacePorts() ([]int, error) { +func (fc *FabricConfig) GetInterfacePorts() ([]uint, error) { if fc == nil { return nil, errors.New("FabricConfig is nil") } @@ -131,13 +145,13 @@ func (fc *FabricConfig) GetInterfacePorts() ([]int, error) { return nil, errors.New("fabric_iface_port not set") } - ports := make([]int, 0) + ports := make([]uint, 0) for _, str := range portStrs { - intPort, err := strconv.Atoi(str) + uint64Port, err := strconv.ParseUint(str, 10, 16) if err != nil { - return nil, err + return nil, errors.Errorf("fabric_iface_port item %q: must be an integer in the range 1-65535", str) } - ports = append(ports, intPort) + ports = append(ports, uint(uint64Port)) // OK to truncate to 32 bits } return ports, nil } @@ -325,6 +339,7 @@ func (c *Config) Validate() error { return errors.New("target count must be nonzero") } + c.Fabric.Normalize() if err := c.Fabric.Validate(); err != nil { return errors.Wrap(err, "fabric config validation failed") } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 06543e35595..9c4b0f84314 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -586,6 +586,14 @@ func TestConfig_FabricValidation(t *testing.T) { }, expErr: errors.New("fabric_iface_port"), }, + "signed positive port number": { + cfg: FabricConfig{ + Provider: "foo", + Interface: "bar", + InterfacePort: "+42", + }, + expErr: errors.New("fabric_iface_port"), + }, "success": { cfg: FabricConfig{ Provider: "foo", @@ -758,6 +766,56 @@ func TestConfig_ToCmdVals(t *testing.T) { } } +func TestEngine_FabricConfig_Normalize(t *testing.T) { + for name, tc := range map[string]struct { + cfg *FabricConfig + expCfg *FabricConfig + }{ + "unchanged": { + cfg: &FabricConfig{ + Provider: multiProviderString("ofi+verbs", "ofi+tcp"), + Interface: multiProviderString("if0", "if1"), + InterfacePort: multiProviderString("1234", "5678"), + }, + expCfg: &FabricConfig{ + Provider: multiProviderString("ofi+verbs", "ofi+tcp"), + Interface: multiProviderString("if0", "if1"), + InterfacePort: multiProviderString("1234", "5678"), + }, + }, + "trim whitespace": { + cfg: &FabricConfig{ + Provider: multiProviderString("ofi+verbs ", " ofi+tcp "), + Interface: multiProviderString(" if0 ", " if1"), + InterfacePort: multiProviderString("\t1234 ", "\n5678 "), + }, + expCfg: &FabricConfig{ + Provider: multiProviderString("ofi+verbs", "ofi+tcp"), + Interface: multiProviderString("if0", "if1"), + InterfacePort: multiProviderString("1234", "5678"), + }, + }, + "remove empty elements": { + cfg: &FabricConfig{ + Provider: multiProviderString("ofi+verbs", "ofi+tcp", ""), + Interface: multiProviderString("if0", "", "if1"), + InterfacePort: multiProviderString("", "1234", "5678"), + }, + expCfg: &FabricConfig{ + Provider: multiProviderString("ofi+verbs", "ofi+tcp"), + Interface: multiProviderString("if0", "if1"), + InterfacePort: multiProviderString("1234", "5678"), + }, + }, + } { + t.Run(name, func(t *testing.T) { + tc.cfg.Normalize() + + test.CmpAny(t, "updated FabricConfig", tc.expCfg, tc.cfg) + }) + } +} + func TestFabricConfig_GetProviders(t *testing.T) { for name, tc := range map[string]struct { cfg *FabricConfig @@ -783,12 +841,6 @@ func TestFabricConfig_GetProviders(t *testing.T) { }, expProviders: []string{"p1", "p2", "p3"}, }, - "excessive whitespace": { - cfg: &FabricConfig{ - Provider: multiProviderString(" ", " p1 ", " p2 ", "p3"), - }, - expProviders: []string{"p1", "p2", "p3"}, - }, } { t.Run(name, func(t *testing.T) { providers, err := tc.cfg.GetProviders() @@ -889,12 +941,6 @@ func TestFabricConfig_GetInterfaces(t *testing.T) { }, expInterfaces: []string{"net1", "net2", "net3"}, }, - "excessive whitespace": { - cfg: &FabricConfig{ - Interface: multiProviderString(" net1 ", "", " net2", "net3", ""), - }, - expInterfaces: []string{"net1", "net2", "net3"}, - }, } { t.Run(name, func(t *testing.T) { interfaces, err := tc.cfg.GetInterfaces() @@ -945,7 +991,7 @@ func TestFabricConfig_GetPrimaryInterface(t *testing.T) { func TestFabricConfig_GetInterfacePorts(t *testing.T) { for name, tc := range map[string]struct { cfg *FabricConfig - expPorts []int + expPorts []uint expErr error }{ "nil": { @@ -959,25 +1005,31 @@ func TestFabricConfig_GetInterfacePorts(t *testing.T) { cfg: &FabricConfig{ InterfacePort: "1234", }, - expPorts: []int{1234}, + expPorts: []uint{1234}, }, "multi": { cfg: &FabricConfig{ - InterfacePort: multiProviderString("1234", "5678", "9012"), + InterfacePort: multiProviderString("1234", "5678", "9012", "65535"), }, - expPorts: []int{1234, 5678, 9012}, + expPorts: []uint{1234, 5678, 9012, 65535}, }, "excessive whitespace": { cfg: &FabricConfig{ InterfacePort: multiProviderString("1234 ", " 5678 ", "", " 9012"), }, - expPorts: []int{1234, 5678, 9012}, + expErr: errors.New("fabric_iface_port item \"1234 \""), }, "non-integer port": { cfg: &FabricConfig{ InterfacePort: multiProviderString("1234", "a123"), }, - expErr: errors.New("strconv.Atoi"), + expErr: errors.New("fabric_iface_port item \"a123\""), + }, + "over 16 bits": { + cfg: &FabricConfig{ + InterfacePort: multiProviderString("65536", "1234"), + }, + expErr: errors.New("fabric_iface_port item \"65536\""), }, } { t.Run(name, func(t *testing.T) { From 3d8aaa861323101a009cce2891bd5a341ede3a68 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Wed, 28 Oct 2026 21:49:58 +0000 Subject: [PATCH 4/6] Remove defunct check Features: control Signed-off-by: Kris Jacque --- src/control/server/engine/config.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 92ffc70de1e..bf502c4ca2a 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -213,12 +213,6 @@ func (fc *FabricConfig) Validate() error { return err } - for _, p := range ports { - if p < 0 { - return errors.New("fabric_iface_port cannot be negative") - } - } - if len(interfaces) != numProv || len(ports) != numProv { return errors.Errorf("provider, fabric_iface and fabric_iface_port must include the same number of items delimited by %q", MultiProviderSeparator) } From 55b45f3f4fc36859e278ce13d2837f8406b33416 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Thu, 29 Oct 2026 23:05:52 +0000 Subject: [PATCH 5/6] Require nonzero port Features: control Signed-off-by: Kris Jacque --- src/control/server/engine/config.go | 2 +- src/control/server/engine/config_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index bf502c4ca2a..57c9d2f9e88 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -148,7 +148,7 @@ func (fc *FabricConfig) GetInterfacePorts() ([]uint, error) { ports := make([]uint, 0) for _, str := range portStrs { uint64Port, err := strconv.ParseUint(str, 10, 16) - if err != nil { + if err != nil || uint64Port == 0 { return nil, errors.Errorf("fabric_iface_port item %q: must be an integer in the range 1-65535", str) } ports = append(ports, uint(uint64Port)) // OK to truncate to 32 bits diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 9c4b0f84314..1c3323e4228 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1031,6 +1031,12 @@ func TestFabricConfig_GetInterfacePorts(t *testing.T) { }, expErr: errors.New("fabric_iface_port item \"65536\""), }, + "zero": { + cfg: &FabricConfig{ + InterfacePort: multiProviderString("5678", "0", "1234"), + }, + expErr: errors.New("fabric_iface_port item \"0\""), + }, } { t.Run(name, func(t *testing.T) { ports, err := tc.cfg.GetInterfacePorts() From 414c0b8abb5b7019c6785549fc8e87f27ad0fc51 Mon Sep 17 00:00:00 2001 From: Kris Jacque Date: Fri, 30 Oct 2026 18:21:56 +0000 Subject: [PATCH 6/6] Fix oversights - Remove check blocking multiprovider usage. - Move normalization to config load time. Features: control Signed-off-by: Kris Jacque --- src/control/server/config/server.go | 7 ++----- src/control/server/engine/config.go | 1 - src/control/server/server_utils_test.go | 4 ++++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/control/server/config/server.go b/src/control/server/config/server.go index 1ff155f6bcf..e099ce498c4 100644 --- a/src/control/server/config/server.go +++ b/src/control/server/config/server.go @@ -204,6 +204,7 @@ func (cfg *Server) updateServerConfig(cfgPtr **engine.Config) { engineCfg := *cfgPtr engineCfg.Fabric.Update(cfg.Fabric) + engineCfg.Fabric.Normalize() engineCfg.SystemName = cfg.SystemName engineCfg.SocketDir = cfg.SocketDir engineCfg.Modules = cfg.Modules @@ -408,11 +409,7 @@ func (cfg *Server) Load(log logging.Logger) error { return errors.Errorf("invalid system name: %q", cfg.SystemName) } - // TODO multiprovider: Remove when multiprovider is enabled - if cfg.Fabric.GetNumProviders() > 1 { - return errors.Errorf("fabric provider string %q includes more than one provider", cfg.Fabric.Provider) - } - + cfg.Fabric.Normalize() // propagate top-level settings to engine configs for i := range cfg.Engines { cfg.updateServerConfig(&cfg.Engines[i]) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 57c9d2f9e88..ba2fe0f7de5 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -333,7 +333,6 @@ func (c *Config) Validate() error { return errors.New("target count must be nonzero") } - c.Fabric.Normalize() if err := c.Fabric.Validate(); err != nil { return errors.Wrap(err, "fabric config validation failed") } diff --git a/src/control/server/server_utils_test.go b/src/control/server/server_utils_test.go index 75a063ac03a..349a8749ead 100644 --- a/src/control/server/server_utils_test.go +++ b/src/control/server/server_utils_test.go @@ -2068,6 +2068,10 @@ func TestServer_processFabricProvider(t *testing.T) { cfgFabric: "ucx+ud", expFabric: "ucx+ud", }, + "multiprovider": { + cfgFabric: "ofi+verbs,ofi+tcp", + expFabric: "ofi+verbs;ofi_rxm,ofi+tcp", + }, } { t.Run(name, func(t *testing.T) { cfg := &config.Server{