Skip to content

DAOS-19803 control: Fix rebuild timeout caused by self_heal lookup #19395

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mjmac wants to merge 1 commit into
base: master
Choose a base branch
from
+664 −428
Draft

DAOS-19803 control: Fix rebuild timeout caused by self_heal lookup #19395

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/control/cmd/dmg/pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -763,22 +763,23 @@ type poolQueryCmd struct {
poolCmd
HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"`
ShowEnabledRanks bool `short:"e" long:"show-enabled" description:"Show engine unique identifiers (ranks) which are enabled"`
NoSelfHealCheck bool `long:"no-self-heal-check" description:"Disable self_heal pool property check"`
}

// Execute is run when PoolQueryCmd subcommand is activated
func (cmd *poolQueryCmd) Execute(args []string) error {
selfHeal, err := daos.PoolProperties().GetProperty("self_heal")
if err != nil {
return err
}
req := &control.PoolQueryReq{
ID: cmd.PoolID().String(),
QueryMask: daos.DefaultPoolQueryMask,
ID: cmd.PoolID().String(),
QueryMask: daos.DefaultPoolQueryMask,
Properties: []*daos.PoolProperty{selfHeal},
}

if cmd.HealthOnly {
req.QueryMask = daos.HealthOnlyPoolQueryMask
}
if !cmd.NoSelfHealCheck {
req.QueryMask.SetOptions(daos.PoolQueryOptionSelfHealPolicy)
}
if cmd.ShowEnabledRanks {
req.QueryMask.SetOptions(daos.PoolQueryOptionEnabledEngines)
}
Expand All @@ -790,7 +791,6 @@ func (cmd *poolQueryCmd) Execute(args []string) error {
if resp != nil {
poolInfo = &resp.PoolInfo
}
// NOTE: Self heal policy info not returned in JSON response.
return cmd.OutputJSON(poolInfo, err)
}

Expand Down
53 changes: 5 additions & 48 deletions src/control/cmd/dmg/pool_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1242,14 +1242,8 @@ func TestDmg_PoolCommands(t *testing.T) {
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
qm.SetOptions(daos.PoolQueryOptionSelfHealPolicy)
}),
}),
printRequest(t, &control.PoolGetPropReq{
ID: test.MockUUID(0),
Properties: []*daos.PoolProperty{
propWithVal("self_heal", ""),
},
Properties: []*daos.PoolProperty{propWithVal("self_heal", "")},
}),
}, " "),
nil,
Expand All @@ -1262,14 +1256,8 @@ func TestDmg_PoolCommands(t *testing.T) {
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
qm.SetOptions(daos.PoolQueryOptionEnabledEngines)
qm.SetOptions(daos.PoolQueryOptionSelfHealPolicy)
}),
}),
printRequest(t, &control.PoolGetPropReq{
ID: test.MockUUID(0),
Properties: []*daos.PoolProperty{
propWithVal("self_heal", ""),
},
Properties: []*daos.PoolProperty{propWithVal("self_heal", "")},
}),
}, " "),
nil,
Expand All @@ -1282,47 +1270,22 @@ func TestDmg_PoolCommands(t *testing.T) {
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
qm.SetOptions(daos.PoolQueryOptionEnabledEngines)
qm.SetOptions(daos.PoolQueryOptionSelfHealPolicy)
}),
}),
printRequest(t, &control.PoolGetPropReq{
ID: test.MockUUID(0),
Properties: []*daos.PoolProperty{
propWithVal("self_heal", ""),
},
Properties: []*daos.PoolProperty{propWithVal("self_heal", "")},
}),
}, " "),
nil,
},
{
"Query pool for health only",
"pool query --health-only 12345678-1234-1234-1234-1234567890ab",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
*qm = daos.HealthOnlyPoolQueryMask
qm.SetOptions(daos.PoolQueryOptionSelfHealPolicy)
}),
}),
printRequest(t, &control.PoolGetPropReq{
ID: test.MockUUID(0),
Properties: []*daos.PoolProperty{
propWithVal("self_heal", ""),
},
}),
}, " "),
nil,
},
{
"Query pool for health only; skip self_heal fetch",
"pool query --health-only 12345678-1234-1234-1234-1234567890ab --no-self-heal-check",
strings.Join([]string{
printRequest(t, &control.PoolQueryReq{
ID: "12345678-1234-1234-1234-1234567890ab",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
*qm = daos.HealthOnlyPoolQueryMask
}),
Properties: []*daos.PoolProperty{propWithVal("self_heal", "")},
}),
}, " "),
nil,
Expand All @@ -1334,14 +1297,8 @@ func TestDmg_PoolCommands(t *testing.T) {
printRequest(t, &control.PoolQueryReq{
ID: "test_label",
QueryMask: setQueryMask(func(qm *daos.PoolQueryMask) {
qm.SetOptions(daos.PoolQueryOptionSelfHealPolicy)
}),
}),
printRequest(t, &control.PoolGetPropReq{
ID: test.MockUUID(0),
Properties: []*daos.PoolProperty{
propWithVal("self_heal", ""),
},
Properties: []*daos.PoolProperty{propWithVal("self_heal", "")},
}),
}, " "),
nil,
Expand Down
6 changes: 4 additions & 2 deletions src/control/cmd/dmg/pretty/pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//
// (C) Copyright 2020-2024 Intel Corporation.
// (C) Copyright 2025 Hewlett Packard Enterprise Development LP
// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -30,7 +30,9 @@ func PrintPoolQueryResponse(pqr *control.PoolQueryResp, out io.Writer, opts ...P
if err := pretty.PrintPoolInfo(&pqr.PoolInfo, out); err != nil {
return err
}
pretty.PrintPoolSelfHealDisable(pqr.PoolInfo.SelfHealPolicy, pqr.SysSelfHealPolicy, out)
if pqr.SelfHealPolicy != "" {
pretty.PrintPoolSelfHealDisable(pqr.SelfHealPolicy, pqr.SysSelfHealPolicy, out)
}

return nil
}
Expand Down
20 changes: 20 additions & 0 deletions src/control/common/proto/error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//
// (C) Copyright 2020-2022 Intel Corporation.
// (C) Copyright 2026 Hewlett Packard Enterprise Development LP
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -34,6 +35,9 @@ const (
// AnnotatedSystemErrNotReplica defines an identifier for ErrNotReplica errors
// serialized as gRPC status metadata.
AnnotatedSystemErrNotReplica = "proto.system.ErrNotReplica"
// AnnotatedSystemErrNoEngine defines an identifier for ErrNoEngine
// errors serialized as gRPC status metadata.
AnnotatedSystemErrNoEngine = "proto.system.ErrNoEngine"
// AnnotatedSystemErrPoolNotFound defines an identifier for ErrPoolNotFound errors
// serialized as gRPC status metadata.
AnnotatedSystemErrPoolNotFound = "proto.system.ErrPoolNotFound"
Expand All @@ -51,6 +55,8 @@ func ErrFromMeta(meta map[string]string, errType error) error {
err = json.Unmarshal(jm, et)
case *system.ErrNotReplica:
err = json.Unmarshal([]byte(meta["Replicas"]), &et.Replicas)
case *system.ErrNoEngine:
err = json.Unmarshal([]byte(meta["Replicas"]), &et.Replicas)
case *system.ErrNotLeader:
et.LeaderHint = meta["LeaderHint"]
err = json.Unmarshal([]byte(meta["Replicas"]), &et.Replicas)
Expand Down Expand Up @@ -114,6 +120,18 @@ func AnnotateError(in error) error {
"Replicas": string(data),
},
}
case *system.ErrNoEngine:
data, err := json.Marshal(et.Replicas)
if err != nil {
break
}
details = &errdetails.ErrorInfo{
Reason: AnnotatedSystemErrNoEngine,
Domain: "DAOS",
Metadata: map[string]string{
"Replicas": string(data),
},
}
case *system.ErrNotLeader:
data, err := json.Marshal(et.Replicas)
if err != nil {
Expand Down Expand Up @@ -173,6 +191,8 @@ func UnwrapError(st *status.Status) error {
return daos.Status(i)
case AnnotatedSystemErrNotReplica:
return ErrFromMeta(t.Metadata, new(system.ErrNotReplica))
case AnnotatedSystemErrNoEngine:
return ErrFromMeta(t.Metadata, new(system.ErrNoEngine))
case AnnotatedSystemErrNotLeader:
return ErrFromMeta(t.Metadata, new(system.ErrNotLeader))
case AnnotatedSystemErrPoolNotFound:
Expand Down
8 changes: 8 additions & 0 deletions src/control/common/proto/error_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//
// (C) Copyright 2020-2022 Intel Corporation.
// (C) Copyright 2026 Hewlett Packard Enterprise Development LP
//
// SPDX-License-Identifier: BSD-2-Clause-Patent
//
Expand Down Expand Up @@ -78,6 +79,9 @@ func TestProto_AnnotateError(t *testing.T) {
LeaderHint: "foo.bar.baz",
Replicas: []string{"a", "b", "c"},
}
testEngineUnavailable := &system.ErrNoEngine{
Replicas: []string{"b", "c"},
}
testPoolNotFound := system.ErrPoolLabelNotFound("foo")

for name, tc := range map[string]struct {
Expand Down Expand Up @@ -111,6 +115,10 @@ func TestProto_AnnotateError(t *testing.T) {
err: testNotLeader,
expErr: testNotLeader,
},
"wrap/unwrap ErrNoEngine": {
err: testEngineUnavailable,
expErr: testEngineUnavailable,
},
"wrap/unwrap ErrPoolNotFound": {
err: testPoolNotFound,
expErr: testPoolNotFound,
Expand Down
71 changes: 43 additions & 28 deletions src/control/common/proto/mgmt/pool.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading