DAOS-19809 control: Enable multiprovider functionality

src/control/cmd/daos_agent/attachinfo.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -37,7 +38,7 @@ func (cmd *attachInfoCmd) getAttachInfo(ctx context.Context) (*control.GetAttach
 type dumpAttachInfoCmd struct {
 	attachInfoCmd
 	Output      string `short:"o" long:"output" default:"stdout" description:"Dump output to this location"`
-	ProviderIdx *uint  // TODO SRS-31: Enable with multiprovider functionality: `short:"n" long:"provider_idx" description:"Index of provider to fetch (if multiple)"`
+	ProviderIdx *uint  `short:"n" long:"provider_idx" description:"Index of provider to fetch (if multiple)"`
 }
 
 func (cmd *dumpAttachInfoCmd) Execute(_ []string) error {

src/control/cmd/daos_agent/config.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2024 Intel Corporation.
+// (C) Copyright 2026 Hewlett Packard Enterprise Development LP
 // (C) Copyright 2025 Google LLC
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
@@ -135,7 +136,7 @@ type Config struct {
 	ExcludeFabricIfaces common.StringSet           `yaml:"exclude_fabric_ifaces,omitempty"`
 	IncludeFabricIfaces common.StringSet           `yaml:"include_fabric_ifaces,omitempty"`
 	FabricInterfaces    []*NUMAFabricConfig        `yaml:"fabric_ifaces,omitempty"`
-	ProviderIdx         uint                       // TODO SRS-31: Enable with multiprovider functionality
+	ProviderIdx         uint                       `yaml:"provider_idx"`
 	Telemetry           TelemetryConfig            `yaml:",inline"`
 }
 

src/control/cmd/daos_agent/config_test.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2021-2024 Intel Corporation.
+// (C) Copyright 2026 Hewlett Packard Enterprise Development LP
 // (C) Copyright 2025 Google LLC
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
@@ -78,6 +79,7 @@ fabric_ifaces:
   -
      iface: ib3
      domain: mlx5_3
+provider_idx: 1
 `)
 
 	badLogMaskCfg := test.CreateTestFile(t, dir, `
@@ -202,6 +204,7 @@ exclude_fabric_ifaces: ["ib3"]
 						},
 					},
 				},
+				ProviderIdx: 1,
 			},
 		},
 	} {

src/control/cmd/dmg/auto_test.go

@@ -614,7 +614,7 @@ engines:
     - "0000:03:00.0"
   provider: ofi+verbs
   fabric_iface: ib0
-  fabric_iface_port: 31416
+  fabric_iface_port: "31416"
   pinned_numa_node: 0
 - targets: 6
   nr_xs_helpers: 0
@@ -631,7 +631,7 @@ engines:
     - "0000:06:00.0"
   provider: ofi+verbs
   fabric_iface: ib1
-  fabric_iface_port: 32416
+  fabric_iface_port: "32416"
   pinned_numa_node: 1
 disable_vfio: false
 disable_vmd: false

src/control/cmd/dmg/pretty/system.go

@@ -143,13 +143,17 @@ func PrintSystemQueryResponse(out, outErr io.Writer, resp *control.SystemQueryRe
 		printAbsentHosts(outErr, &resp.AbsentHosts)
 		printSysSelfHealUnsetFlags(out, resp.SysSelfHealPolicy)
 
+		printSystemProviders(out, resp.Providers)
+
 		return nil
 	}
 
 	printAbsentHosts(outErr, &resp.AbsentHosts)
 	printAbsentRanks(outErr, &resp.AbsentRanks)
 	printSysSelfHealUnsetFlags(out, resp.SysSelfHealPolicy)
 
+	printSystemProviders(out, resp.Providers)
+
 	return nil
 }
 
@@ -187,6 +191,27 @@ func printSystemResults(out, outErr io.Writer, results system.MemberResults, abs
 	return nil
 }
 
+func printSystemProviders(out io.Writer, providers []string) {
+	if len(providers) == 0 {
+		return
+	}
+
+	idxKey := "Idx"
+	provKey := "Provider"
+	formatter := txtfmt.NewTableFormatter(idxKey, provKey)
+
+	var table []txtfmt.TableRow
+	for i, prov := range providers {
+		table = append(table, txtfmt.TableRow{
+			idxKey:  fmt.Sprintf("%d", i),
+			provKey: prov,
+		})
+	}
+
+	fmt.Fprint(out, "\n")
+	fmt.Fprint(out, formatter.Format(table))
+}
+
 // PrintSystemStartResponse generates a human-readable representation of the
 // supplied SystemStartResp struct and writes it to the supplied io.Writer.
 func PrintSystemStartResponse(out, outErr io.Writer, resp *control.SystemStartResp) error {

src/control/cmd/dmg/pretty/system_test.go

@@ -424,6 +424,30 @@ Rank UUID                                 Control Address Fault Domain State
 
 Unknown 3 hosts: foo[7-9]
 Unknown 3 ranks: 7-9
+`,
+		},
+		"providers": {
+			resp: &control.SystemQueryResp{
+				Members: Members{
+					MockMember(t, 0, MemberStateJoined),
+					MockMember(t, 1, MemberStateJoined),
+					MockMember(t, 2, MemberStateStopped),
+				},
+				Providers: []string{"ofi+verbs", "ofi+tcp", "ofi+tcp"},
+			},
+			expPrintStr: `
+Rank  State   
+----  -----   
+[0-1] Joined  
+2     Stopped 
+
+System property self_heal flags disabled: exclude, pool_exclude, pool_rebuild
+
+Idx Provider  
+--- --------  
+0   ofi+verbs 
+1   ofi+tcp   
+2   ofi+tcp   
 `,
 		},
 	} {

src/control/lib/control/auto.go

@@ -1223,7 +1223,7 @@ func genEngineConfigs(req ConfGenerateReq, newEngineCfg newEngineCfgFn, nodeSet
 		cfg.Fabric = engine.FabricConfig{
 			Provider:      iface.Provider,
 			Interface:     iface.Device,
-			InterfacePort: ifPort,
+			InterfacePort: fmt.Sprintf("%d", ifPort),
 		}
 		if err := cfg.SetNUMAAffinity(pnn); err != nil {
 			return nil, errors.Wrapf(err, "setting numa %d affinity on engine config",

src/control/lib/support/log.go

@@ -415,6 +415,7 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error
 	}
 	domainName := strings.Split(string(dName), "\n")
 
+	// TODO DAOS-19818: use the control API instead of parsing text.
 	cmd := strings.Join([]string{"dmg", "system", "query", "-v", "-o", configPath}, " ")
 	out, err := exec.Command("sh", "-c", cmd).Output()
 	if err != nil {
@@ -424,7 +425,11 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error
 
 	if len(temp) > 0 {
 		for _, hn := range temp[2 : len(temp)-2] {
-			hn = strings.ReplaceAll(strings.Fields(hn)[3][1:], domainName[0], "")
+			hnFields := strings.Fields(hn)
+			if len(hnFields) < 4 {
+				break // we're outside the rank table
+			}
+			hn = strings.ReplaceAll(hnFields[3][1:], domainName[0], "")
 			hn = strings.TrimSuffix(hn, ".")
 			hostNames = append(hostNames, hn)
 		}

src/control/server/config/server.go

@@ -204,6 +204,7 @@ func (cfg *Server) updateServerConfig(cfgPtr **engine.Config) {
 
 	engineCfg := *cfgPtr
 	engineCfg.Fabric.Update(cfg.Fabric)
+	engineCfg.Fabric.Normalize()
 	engineCfg.SystemName = cfg.SystemName
 	engineCfg.SocketDir = cfg.SocketDir
 	engineCfg.Modules = cfg.Modules
@@ -408,11 +409,7 @@ func (cfg *Server) Load(log logging.Logger) error {
 		return errors.Errorf("invalid system name: %q", cfg.SystemName)
 	}
 
-	// TODO multiprovider: Remove when multiprovider is enabled
-	if cfg.Fabric.GetNumProviders() > 1 {
-		return errors.Errorf("fabric provider string %q includes more than one provider", cfg.Fabric.Provider)
-	}
-
+	cfg.Fabric.Normalize()
 	// propagate top-level settings to engine configs
 	for i := range cfg.Engines {
 		cfg.updateServerConfig(&cfg.Engines[i])

src/control/server/config/server_test.go

@@ -257,6 +257,7 @@ func TestServerConfig_Constructed(t *testing.T) {
 		WithSocketDir("./.daos/daos_server").
 		WithFabricProvider("ofi+verbs;ofi_rxm").
 		WithCrtTimeout(30).
+		WithNumSecondaryEndpoints([]int{2}).
 		WithMgmtSvcReplicas("hostname1", "hostname2", "hostname3").
 		WithFaultCb("./.daos/fd_callback").
 		WithFaultPath("/vcdu0/rack1/hostname").
@@ -294,6 +295,7 @@ func TestServerConfig_Constructed(t *testing.T) {
 			WithFabricProvider("ofi+verbs;ofi_rxm").
 			WithFabricAuthKey("foo:bar").
 			WithCrtTimeout(30).
+			WithNumSecondaryEndpoints([]int{2}).
 			WithPinnedNumaNode(0).
 			WithBypassHealthChk(&bypass).
 			WithEnvVars("CRT_TIMEOUT=30").
@@ -324,6 +326,7 @@ func TestServerConfig_Constructed(t *testing.T) {
 			WithFabricProvider("ofi+verbs;ofi_rxm").
 			WithFabricAuthKey("foo:bar").
 			WithCrtTimeout(30).
+			WithNumSecondaryEndpoints([]int{2}).
 			WithBypassHealthChk(&bypass).
 			WithEnvVars("CRT_TIMEOUT=100").
 			WithLogFile("/var/log/daos/daos_engine.1.log").
@@ -365,7 +368,7 @@ func TestServerConfig_updateServerConfig(t *testing.T) {
 				Fabric: engine.FabricConfig{
 					Provider:              "provider",
 					Interface:             "iface",
-					InterfacePort:         1111,
+					InterfacePort:         "1111",
 					NumSecondaryEndpoints: []int{2, 3, 4},
 				},
 			},
@@ -379,7 +382,7 @@ func TestServerConfig_updateServerConfig(t *testing.T) {
 				Fabric: engine.FabricConfig{
 					Provider:              "provider",
 					Interface:             "iface",
-					InterfacePort:         1111,
+					InterfacePort:         "1111",
 					NumSecondaryEndpoints: []int{2, 3, 4},
 				},
 			},

src/control/server/engine/config.go

@@ -40,7 +40,7 @@ const (
 type FabricConfig struct {
 	Provider        string `yaml:"provider,omitempty" cmdEnv:"D_PROVIDER"`
 	Interface       string `yaml:"fabric_iface,omitempty" cmdEnv:"D_INTERFACE"`
-	InterfacePort   int    `yaml:"fabric_iface_port,omitempty" cmdEnv:"D_PORT,nonzero"`
+	InterfacePort   string `yaml:"fabric_iface_port,omitempty" cmdEnv:"D_PORT"`
 	NumaNodeIndex   uint   `yaml:"-"`
 	BypassHealthChk *bool  `yaml:"bypass_health_chk,omitempty" cmdLongFlag:"--bypass_health_chk" cmdShortFlag:"-b"`
 	CrtTimeout      uint32 `yaml:"crt_timeout,omitempty" cmdEnv:"CRT_TIMEOUT"`
@@ -50,6 +50,25 @@ type FabricConfig struct {
 	AuthKey               string `yaml:"fabric_auth_key,omitempty" cmdEnv:"D_PROVIDER_AUTH_KEY"`
 }
 
+// Normalize adjusts the format of the fabric configuration strings that are exported to CaRT.
+func (fc *FabricConfig) Normalize() {
+	normalizeMultiprov := func(multiProvStr string) string {
+		strs := strings.Split(multiProvStr, MultiProviderSeparator)
+		result := []string{}
+		for _, s := range strs {
+			s = strings.TrimSpace(s)
+			if s != "" {
+				result = append(result, s)
+			}
+		}
+		return strings.Join(result, MultiProviderSeparator)
+	}
+
+	fc.Provider = normalizeMultiprov(fc.Provider)
+	fc.Interface = normalizeMultiprov(fc.Interface)
+	fc.InterfacePort = normalizeMultiprov(fc.InterfacePort)
+}
+
 // GetPrimaryProvider parses the primary provider from the Provider string.
 func (fc *FabricConfig) GetPrimaryProvider() (string, error) {
 	providers, err := fc.GetProviders()
@@ -76,15 +95,10 @@ func (fc *FabricConfig) GetProviders() ([]string, error) {
 
 func splitMultiProviderStr(str string) []string {
 	strs := strings.Split(str, MultiProviderSeparator)
-	result := make([]string, 0)
-	for _, s := range strs {
-		trimmed := strings.TrimSpace(s)
-		if trimmed != "" {
-			result = append(result, trimmed)
-		}
+	if len(strs) == 1 && strs[0] == "" {
+		return nil
 	}
-
-	return result
+	return strs
 }
 
 // GetNumProviders gets the number of fabric providers configured.
@@ -121,16 +135,25 @@ func (fc *FabricConfig) GetInterfaces() ([]string, error) {
 }
 
 // GetInterfacePorts parses the InterfacePort string to one or more ports.
-func (fc *FabricConfig) GetInterfacePorts() ([]int, error) {
+func (fc *FabricConfig) GetInterfacePorts() ([]uint, error) {
 	if fc == nil {
 		return nil, errors.New("FabricConfig is nil")
 	}
 
-	if fc.InterfacePort == 0 {
+	portStrs := splitMultiProviderStr(fc.InterfacePort)
+	if len(portStrs) == 0 {
 		return nil, errors.New("fabric_iface_port not set")
 	}
 
-	return []int{fc.InterfacePort}, nil
+	ports := make([]uint, 0)
+	for _, str := range portStrs {
+		uint64Port, err := strconv.ParseUint(str, 10, 16)
+		if err != nil || uint64Port == 0 {
+			return nil, errors.Errorf("fabric_iface_port item %q: must be an integer in the range 1-65535", str)
+		}
+		ports = append(ports, uint(uint64Port)) // OK to truncate to 32 bits
+	}
+	return ports, nil
 }
 
 // Update fills in any missing fields from the provided FabricConfig.
@@ -145,7 +168,7 @@ func (fc *FabricConfig) Update(other FabricConfig) {
 	if fc.Interface == "" {
 		fc.Interface = other.Interface
 	}
-	if fc.InterfacePort == 0 {
+	if fc.InterfacePort == "" {
 		fc.InterfacePort = other.InterfacePort
 	}
 	if fc.CrtTimeout == 0 {
@@ -190,13 +213,7 @@ func (fc *FabricConfig) Validate() error {
 		return err
 	}
 
-	for _, p := range ports {
-		if p < 0 {
-			return errors.New("fabric_iface_port cannot be negative")
-		}
-	}
-
-	if len(interfaces) != numProv { // TODO SRS-31: check num ports when multiprovider fully enabled: || len(ports) != numProv {
+	if len(interfaces) != numProv || len(ports) != numProv {
 		return errors.Errorf("provider, fabric_iface and fabric_iface_port must include the same number of items delimited by %q", MultiProviderSeparator)
 	}
 
@@ -701,7 +718,7 @@ func (c *Config) WithFabricInterface(iface string) *Config {
 
 // WithFabricInterfacePort sets the numeric interface port to be used by this instance.
 func (c *Config) WithFabricInterfacePort(ifacePort int) *Config {
-	c.Fabric.InterfacePort = ifacePort
+	c.Fabric.InterfacePort = fmt.Sprintf("%d", ifacePort)
 	return c
 }