From a3a8764b12069b46c594eb0ea97f7e94031f8c01 Mon Sep 17 00:00:00 2001 From: spencercjh Date: Tue, 25 Aug 2026 07:07:34 +0000 Subject: [PATCH 1/2] feat: add Kunlun vXPU mock resources Signed-off-by: spencercjh --- README.md | 49 +++- internal/pkg/api/device/kunlun/device.go | 45 +++- internal/pkg/api/device/kunlun/device_test.go | 246 ++++++++++++++++++ internal/pkg/config/config.go | 11 +- internal/pkg/config/kunlun_test.go | 74 ++++++ 5 files changed, 400 insertions(+), 25 deletions(-) create mode 100644 internal/pkg/api/device/kunlun/device_test.go create mode 100644 internal/pkg/config/kunlun_test.go diff --git a/README.md b/README.md index b024359..0359b3a 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,20 @@ The mock plugin **does not detect hardware**. It does the following every ~30s: ```text node annotation: hami.io/node--register = [ {devmem, devcore, ...}, ... ] (1) - node capacity: (e.g. nvidia.com/gpu) > 0 (2) <- health gate + node capacity: (e.g. nvidia.com/gpu) > 0 (2) <- health gate for supplemental resources | mock reads v registers into allocatable: /...mem , /...cores , ... ``` -On a **real** cluster, (1) and (2) are produced by the real device plugin. In a **mock-only** (no hardware) environment you provide them yourself: +On a **real** cluster, each family's required inputs are produced by the real device plugin. In a **mock-only** (no hardware) environment you provide them yourself: - **(1) the `node--register` annotation** describing the fake cards -- `kubectl annotate`. -- **(2) the count extended resource** (e.g. `nvidia.com/gpu`) -- patched onto the node `status`. +- **(2) the count extended resource** (e.g. `nvidia.com/gpu`) -- patched onto the node `status` for NVIDIA, Hygon and Ascend. -> There is **no auto-labeller** in this repo, so (1) and (2) are manual today. Forgetting them is the usual cause of `device xxx is unhealthy` / `no allocation` -- see issues #14 / #16. +Kunlun vXPU uses only its `hami.io/node-register-xpu` annotation. The mock derives both `kunlunxin.com/vxpu` and `kunlunxin.com/vxpu-memory` from healthy annotation entries, so it does not need a separate count-resource status patch. + +> There is **no auto-labeller** in this repo, so the required inputs are manual today. Forgetting them is the usual cause of `device xxx is unhealthy` / `no allocation` -- see issues #14 / #16. ## Prerequisites @@ -47,7 +49,8 @@ The mock **derives the registered resources from the annotation, not from the co - **Number of fake cards = the number of entries in the annotation array** (not the count value). - Registered `...-memory` = **sum of `devmem`** over all entries. - Registered `...-cores` / `...-core` = **sum of `devcore`** over all entries. -- The **count extended resource is only a health gate**: its value just needs to be `> 0`. It does **not** affect the registered memory/cores. By convention it is set to `cards x splits-per-card` (e.g. Ascend `2 x VDeviceCount(4) = 8`), but `1` would work equally well for the memory/cores to appear. +- For NVIDIA, Hygon and Ascend, the **count extended resource is only a health gate**: its value just needs to be `> 0`. It does **not** affect the registered memory/cores. By convention it is set to `cards x splits-per-card` (e.g. Ascend `2 x VDeviceCount(4) = 8`), but `1` would work equally well for the memory/cores to appear. +- Kunlun vXPU is fully annotation-derived: `vxpu = sum(devcore)` and `vxpu-memory = sum(devmem)` across entries whose `health` is `true`. Annotation entry fields: @@ -55,10 +58,10 @@ Annotation entry fields: | :-- | :-- | | `id` | unique device UUID (any string) | | `devmem` | per-card memory in MB -- **summed** into `...-memory` | -| `devcore` | per-card cores. **NVIDIA/Hygon:** summed into `...-cores` (NVIDIA: percentage, 100 = a whole card). **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. | +| `devcore` | per-card cores. **NVIDIA/Hygon:** summed into `...-cores` (NVIDIA: percentage, 100 = a whole card). **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. **Kunlun:** summed into `vxpu`. | | `count` | per-card split count (informational for the mock) | | `type` | device model string | -| `health` | must be `true` to be counted | +| `health` | scheduler-visible health; only healthy Kunlun entries contribute to its mock resources | | `index` | card index `0,1,2,...` (`0` may be omitted) | | `numa`, `mode` | optional | @@ -66,7 +69,7 @@ Annotation entry fields: ## Usage by vendor -To mock one card you always need the **three pieces**: the vendor **config block** (in the `hami-scheduler-device` ConfigMap, gives the resource names), the **count extended resource** (passes the health gate), and the **node-register annotation** (describes the fake cards). Replace `` below with your target node. Resource names follow the ConfigMap (HAMi defaults shown). See [Understanding the values](#understanding-the-values) for how the numbers are derived. +NVIDIA, Hygon and Ascend need the vendor **config block**, the **count extended resource** health gate and the **node-register annotation**. Kunlun vXPU needs its config block and node-register annotation, but not a separate status patch. Replace `` below with your target node. Resource names follow the ConfigMap (HAMi defaults shown). See [Understanding the values](#understanding-the-values) for how the numbers are derived. ### NVIDIA (e.g. A100-80GB) @@ -127,6 +130,33 @@ kubectl get node -o json | jq '.status.allocatable|with_entries(select(.k # expect: hygon.com/dcumem=32768, hygon.com/dcucores=200 (2 cards x devcore 100) ``` +### Kunlun vXPU + +- config block: `kunlun:` | annotation: `hami.io/node-register-xpu` (JSON) +- mock registers: `kunlunxin.com/vxpu`, `kunlunxin.com/vxpu-memory` +- no separate count-resource status patch is required + +The default HAMi config is: + +```yaml +kunlun: + resourceCountName: kunlunxin.com/xpu + resourceVCountName: kunlunxin.com/vxpu + resourceVMemoryName: kunlunxin.com/vxpu-memory +``` + +To advertise one healthy 24 GiB vXPU: + +```bash +kubectl annotate node \ + 'hami.io/node-register-xpu=[{"id":"XPU-MOCK-0","count":1,"devmem":24576,"devcore":1,"type":"XPU","health":true}]' +# verify (~30s later) +kubectl get node -o json | jq '.status.allocatable|with_entries(select(.key|test("kunlunxin.com")))' +# expect: kunlunxin.com/vxpu=1, kunlunxin.com/vxpu-memory=24576 +``` + +Each healthy entry contributes its `devcore` value to `vxpu` and its `devmem` value in MiB to `vxpu-memory`. Unhealthy entries remain visible to the HAMi scheduler through the annotation but do not contribute kubelet capacity, matching HAMi's `FitVXPU` health check. + ## Ascend config compatibility (new vs legacy) The Ascend `vnpus` config has two layouts and the plugin accepts **both**: @@ -155,8 +185,9 @@ The new nested format is tried first; if that fails it falls back to the legacy | Nvidia GPU | `nvidia.com/gpumem`, `nvidia.com/gpumem-percentage`, `nvidia.com/gpucores` | | Hygon DCU | `hygon.com/dcumem`, `hygon.com/dcucores` (when `resourceCoreName` is set) | | Ascend | `huawei.com/Ascend{chip}-memory`, `huawei.com/Ascend{chip}-core` (when `resourceCoreName` is set **and** the node is in `hami-vnpu-core` mode) | +| Kunlun vXPU | `kunlunxin.com/vxpu`, `kunlunxin.com/vxpu-memory` | -**Note:** If the counted memory is too large (e.g. > 120GB) it may display as 0. Set `memoryFactor` in the `hami-scheduler-device` ConfigMap (default 1). +**Note:** For device configs that expose `memoryFactor`, set it in the `hami-scheduler-device` ConfigMap if counted memory is too large (e.g. > 120GB) and displays as 0. Kunlun vXPU does not use `memoryFactor`; its `vxpu-memory` value is the direct sum of healthy entries' `devmem` values. ## Build diff --git a/internal/pkg/api/device/kunlun/device.go b/internal/pkg/api/device/kunlun/device.go index a6dcdaa..c2aaf07 100644 --- a/internal/pkg/api/device/kunlun/device.go +++ b/internal/pkg/api/device/kunlun/device.go @@ -19,6 +19,7 @@ package kunlun import ( "errors" "fmt" + "strings" "github.com/HAMi/mock-device-plugin/internal/pkg/api/device" "github.com/HAMi/mock-device-plugin/internal/pkg/mock" @@ -26,6 +27,7 @@ import ( //"github.com/kubevirt/device-plugin-manager/pkg/dpm" corev1 "k8s.io/api/core/v1" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/klog/v2" ) @@ -36,11 +38,6 @@ const ( HandshakeAnnos = "hami.io/node-handshake-xpu" ) -var ( - KunlunResourceVCount string - KunlunResourceVMemory string -) - type KunlunConfig struct { ResourceCountName string `yaml:"resourceCountName"` ResourceVCountName string `yaml:"resourceVCountName"` @@ -48,12 +45,23 @@ type KunlunConfig struct { } type KunlunVDevices struct { + config KunlunConfig } func InitKunlunVDevice(config KunlunConfig) *KunlunVDevices { - KunlunResourceVCount = config.ResourceVCountName - KunlunResourceVMemory = config.ResourceVMemoryName - return &KunlunVDevices{} + vCountVendor, vCountResource, vCountQualified := strings.Cut(config.ResourceVCountName, "/") + vMemoryVendor, vMemoryResource, vMemoryQualified := strings.Cut(config.ResourceVMemoryName, "/") + if config.ResourceVCountName == "" || config.ResourceVMemoryName == "" || + config.ResourceVCountName == config.ResourceVMemoryName || + !vCountQualified || !vMemoryQualified || + vCountVendor == "" || vMemoryVendor == "" || + vCountResource == "" || vMemoryResource == "" || + len(utilvalidation.IsQualifiedName(config.ResourceVCountName)) > 0 || + len(utilvalidation.IsQualifiedName(config.ResourceVMemoryName)) > 0 || + vCountVendor != vMemoryVendor { + return nil + } + return &KunlunVDevices{config: config} } func (dev *KunlunVDevices) CommonWord() string { @@ -70,8 +78,14 @@ func (dev *KunlunVDevices) GetNodeDevices(n *corev1.Node) ([]*device.DeviceInfo, klog.ErrorS(err, "failed to unmarshal node devices", "node", n.Name, "device annotation", anno) return []*device.DeviceInfo{}, err } - for idx := range nodeDevices { - nodeDevices[idx].DeviceVendor = dev.CommonWord() + for idx, nodeDevice := range nodeDevices { + if nodeDevice == nil { + return []*device.DeviceInfo{}, fmt.Errorf("device %d in %s is null", idx, RegisterAnnos) + } + if nodeDevice.Count < 0 || nodeDevice.Devmem < 0 || nodeDevice.Devcore < 0 { + return []*device.DeviceInfo{}, fmt.Errorf("device %d in %s has negative capacity", idx, RegisterAnnos) + } + nodeDevice.DeviceVendor = dev.CommonWord() } if len(nodeDevices) == 0 { klog.InfoS("no gpu device found", "node", n.Name, "device annotation", anno) @@ -81,8 +95,8 @@ func (dev *KunlunVDevices) GetNodeDevices(n *corev1.Node) ([]*device.DeviceInfo, } func (dev *KunlunVDevices) GetResource(n *corev1.Node) map[string]int { - memoryResourceName := device.GetResourceName(KunlunResourceVMemory) - vCountResourceName := device.GetResourceName(KunlunResourceVCount) + memoryResourceName := device.GetResourceName(dev.config.ResourceVMemoryName) + vCountResourceName := device.GetResourceName(dev.config.ResourceVCountName) resourceMap := map[string]int{ memoryResourceName: 0, vCountResourceName: 0, @@ -93,6 +107,9 @@ func (dev *KunlunVDevices) GetResource(n *corev1.Node) map[string]int { return resourceMap } for _, val := range devInfos { + if !val.Health { + continue + } resourceMap[vCountResourceName] += int(val.Devcore) resourceMap[memoryResourceName] += int(val.Devmem) } @@ -101,8 +118,8 @@ func (dev *KunlunVDevices) GetResource(n *corev1.Node) map[string]int { } func (dev *KunlunVDevices) RunManager() { - lmock := mock.NewMockLister(device.GetVendorName(KunlunResourceVCount)) - device.Register(lmock, dev) + lmock := mock.NewMockLister(device.GetVendorName(dev.config.ResourceVCountName)) + go device.Register(lmock, dev) mockmanager := dpm.NewManager(lmock) klog.Infof("Running mocking dp: %s", dev.CommonWord()) mockmanager.Run() diff --git a/internal/pkg/api/device/kunlun/device_test.go b/internal/pkg/api/device/kunlun/device_test.go new file mode 100644 index 0000000..bf5efda --- /dev/null +++ b/internal/pkg/api/device/kunlun/device_test.go @@ -0,0 +1,246 @@ +/* +Copyright 2026 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kunlun + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func testConfig() KunlunConfig { + return KunlunConfig{ + ResourceCountName: "kunlunxin.com/xpu", + ResourceVCountName: "kunlunxin.com/vxpu", + ResourceVMemoryName: "kunlunxin.com/vxpu-memory", + } +} + +func TestInitKunlunVDevice(t *testing.T) { + tests := []struct { + name string + config KunlunConfig + valid bool + }{ + {name: "complete", config: testConfig(), valid: true}, + { + name: "physical count is optional", + config: KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu", + ResourceVMemoryName: "kunlunxin.com/vxpu-memory", + }, + valid: true, + }, + { + name: "missing virtual count", + config: KunlunConfig{ + ResourceVMemoryName: "kunlunxin.com/vxpu-memory", + }, + }, + { + name: "missing virtual memory", + config: KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu", + }, + }, + { + name: "unqualified resources", + config: KunlunConfig{ + ResourceVCountName: "vxpu", + ResourceVMemoryName: "vxpu-memory", + }, + }, + { + name: "invalid uppercase vendor", + config: KunlunConfig{ + ResourceVCountName: "KUNLUNXIN.COM/vxpu", + ResourceVMemoryName: "KUNLUNXIN.COM/vxpu-memory", + }, + }, + { + name: "multiple resource separators", + config: KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu/bad", + ResourceVMemoryName: "kunlunxin.com/vxpu-memory", + }, + }, + { + name: "different resource vendors", + config: KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu", + ResourceVMemoryName: "example.com/vxpu-memory", + }, + }, + { + name: "same resource twice", + config: KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu", + ResourceVMemoryName: "kunlunxin.com/vxpu", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dev := InitKunlunVDevice(test.config) + if test.valid { + if dev == nil { + t.Fatal("expected device to initialize") + } + if dev.config != test.config { + t.Fatalf("stored config = %#v, want %#v", dev.config, test.config) + } + return + } + if dev != nil { + t.Fatalf("expected invalid config to be ignored, got %#v", dev) + } + }) + } +} + +func TestKunlunVDevicesConfigurationIsPerInstance(t *testing.T) { + first := InitKunlunVDevice(testConfig()) + second := InitKunlunVDevice(KunlunConfig{ + ResourceVCountName: "example.com/second-count", + ResourceVMemoryName: "example.com/second-memory", + }) + node := nodeWithAnnotation(`[{"id":"XPU-0","devmem":24576,"devcore":1,"type":"XPU","health":true}]`) + + firstResources := first.GetResource(node) + if got := firstResources["vxpu"]; got != 1 { + t.Fatalf("first instance count = %d, want 1", got) + } + if got := firstResources["vxpu-memory"]; got != 24576 { + t.Fatalf("first instance memory = %d, want 24576", got) + } + if _, exists := firstResources["second-count"]; exists { + t.Fatalf("first instance used second instance config: %#v", firstResources) + } + + secondResources := second.GetResource(node) + if got := secondResources["second-count"]; got != 1 { + t.Fatalf("second instance count = %d, want 1", got) + } + if got := secondResources["second-memory"]; got != 24576 { + t.Fatalf("second instance memory = %d, want 24576", got) + } +} + +func TestKunlunVDevicesGetNodeDevices(t *testing.T) { + dev := InitKunlunVDevice(testConfig()) + tests := []struct { + name string + annotation *string + wantErr bool + }{ + {name: "missing", wantErr: true}, + {name: "malformed", annotation: stringPointer("not-json"), wantErr: true}, + {name: "empty", annotation: stringPointer("[]"), wantErr: true}, + {name: "null entry", annotation: stringPointer("[null]"), wantErr: true}, + { + name: "negative count", + annotation: stringPointer(`[{"id":"XPU-0","count":-1,"devmem":24576,"devcore":1,"health":true}]`), + wantErr: true, + }, + { + name: "negative memory", + annotation: stringPointer(`[{"id":"XPU-0","count":1,"devmem":-1,"devcore":1,"health":true}]`), + wantErr: true, + }, + { + name: "negative core", + annotation: stringPointer(`[{"id":"XPU-0","count":1,"devmem":24576,"devcore":-1,"health":true}]`), + wantErr: true, + }, + { + name: "valid", + annotation: stringPointer(`[{"id":"XPU-0","devmem":98304,"devcore":1,"type":"XPU","health":true}]`), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "kunlun-node"}} + if test.annotation != nil { + node.Annotations = map[string]string{RegisterAnnos: *test.annotation} + } + + devices, err := dev.GetNodeDevices(node) + if test.wantErr { + if err == nil { + t.Fatalf("expected error, got devices %#v", devices) + } + if len(devices) != 0 { + t.Fatalf("devices = %#v, want empty", devices) + } + return + } + if err != nil { + t.Fatalf("GetNodeDevices returned error: %v", err) + } + if len(devices) != 1 { + t.Fatalf("device count = %d, want 1", len(devices)) + } + if devices[0].DeviceVendor != XPUCommonWord { + t.Fatalf("device vendor = %q, want %q", devices[0].DeviceVendor, XPUCommonWord) + } + }) + } +} + +func TestKunlunVDevicesGetResourceUsesHealthyAnnotationUnits(t *testing.T) { + dev := InitKunlunVDevice(testConfig()) + node := nodeWithAnnotation(`[ + {"id":"XPU-0","devmem":24576,"devcore":1,"type":"XPU","health":true}, + {"id":"XPU-1","index":1,"devmem":49152,"devcore":2,"type":"XPU","health":true}, + {"id":"XPU-2","index":2,"devmem":98304,"devcore":4,"type":"XPU","health":false} + ]`) + + resources := dev.GetResource(node) + if got, want := resources["vxpu"], 3; got != want { + t.Fatalf("vxpu capacity = %d, want %d", got, want) + } + if got, want := resources["vxpu-memory"], 73728; got != want { + t.Fatalf("vxpu-memory capacity = %d, want %d", got, want) + } +} + +func TestKunlunVDevicesGetResourceInvalidAnnotationReturnsZeroes(t *testing.T) { + dev := InitKunlunVDevice(testConfig()) + for _, annotation := range []string{"not-json", "[]", "[null]"} { + resources := dev.GetResource(nodeWithAnnotation(annotation)) + if got := resources["vxpu"]; got != 0 { + t.Fatalf("annotation %q: vxpu capacity = %d, want 0", annotation, got) + } + if got := resources["vxpu-memory"]; got != 0 { + t.Fatalf("annotation %q: vxpu-memory capacity = %d, want 0", annotation, got) + } + } +} + +func nodeWithAnnotation(annotation string) *corev1.Node { + return &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "kunlun-node", + Annotations: map[string]string{RegisterAnnos: annotation}, + }} +} + +func stringPointer(value string) *string { + return &value +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 9cbd2a4..90acb69 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -18,6 +18,7 @@ package config import ( "flag" + "fmt" "os" "gopkg.in/yaml.v2" @@ -93,10 +94,16 @@ func InitDevicesWithConfig(config *Config) error { if enflameDevice != nil { device.DevicesMap[enflameDevice.CommonWord()] = enflameDevice } + */ kunlunDevice := kunlun.InitKunlunVDevice(config.KunlunConfig) if kunlunDevice != nil { - device.DevicesMap[kunlunDevice.CommonWord()] = kunlunDevice - }*/ + commonWord := kunlunDevice.CommonWord() + if _, exists := device.DevicesMap[commonWord]; exists { + return fmt.Errorf("device common word %q is configured more than once", commonWord) + } + device.DevicesMap[commonWord] = kunlunDevice + klog.Infof("Kunlun device %s initialized", commonWord) + } hygonDevice := hygon.InitDCUDevice(config.HygonConfig) if hygonDevice != nil { device.DevicesMap[hygonDevice.CommonWord()] = hygonDevice diff --git a/internal/pkg/config/kunlun_test.go b/internal/pkg/config/kunlun_test.go new file mode 100644 index 0000000..f129749 --- /dev/null +++ b/internal/pkg/config/kunlun_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2026 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "testing" + + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device" + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device/ascend" + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device/kunlun" +) + +func TestInitDevicesWithConfigAddsKunlun(t *testing.T) { + t.Cleanup(func() { device.DevicesMap = nil }) + config := &Config{KunlunConfig: kunlun.KunlunConfig{ + ResourceCountName: "kunlunxin.com/xpu", + ResourceVCountName: "kunlunxin.com/vxpu", + ResourceVMemoryName: "kunlunxin.com/vxpu-memory", + }} + + if err := InitDevicesWithConfig(config); err != nil { + t.Fatalf("InitDevicesWithConfig returned error: %v", err) + } + if _, exists := device.DevicesMap[kunlun.XPUCommonWord]; !exists { + t.Fatalf("Kunlun device %q was not initialized", kunlun.XPUCommonWord) + } +} + +func TestInitDevicesWithConfigSkipsIncompleteKunlun(t *testing.T) { + t.Cleanup(func() { device.DevicesMap = nil }) + config := &Config{KunlunConfig: kunlun.KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu", + }} + + if err := InitDevicesWithConfig(config); err != nil { + t.Fatalf("InitDevicesWithConfig returned error: %v", err) + } + if _, exists := device.DevicesMap[kunlun.XPUCommonWord]; exists { + t.Fatalf("incomplete Kunlun config initialized device %q", kunlun.XPUCommonWord) + } +} + +func TestInitDevicesWithConfigRejectsKunlunCommonWordCollision(t *testing.T) { + t.Cleanup(func() { device.DevicesMap = nil }) + config := &Config{ + VNPUs: ascend.VNPUs{Configs: []ascend.VNPUConfig{{ + CommonWord: kunlun.XPUCommonWord, + ResourceName: "example.com/xpu", + ResourceMemoryName: "example.com/xpu-memory", + }}}, + KunlunConfig: kunlun.KunlunConfig{ + ResourceVCountName: "kunlunxin.com/vxpu", + ResourceVMemoryName: "kunlunxin.com/vxpu-memory", + }, + } + + if err := InitDevicesWithConfig(config); err == nil { + t.Fatal("expected duplicate common word to be rejected") + } +} From a130831ac256385b811c1928a7d95f6e3479e37b Mon Sep 17 00:00:00 2001 From: spencercjh Date: Wed, 26 Aug 2026 03:34:08 +0000 Subject: [PATCH 2/2] fix(kunlun): honor vxpu registration handshake Signed-off-by: spencercjh --- README.md | 14 +++--- internal/pkg/api/device/device.go | 42 +++++++++++++++++- internal/pkg/api/device/device_test.go | 44 ++++++++++++++++++- internal/pkg/api/device/kunlun/device.go | 6 ++- internal/pkg/api/device/kunlun/device_test.go | 8 ++-- 5 files changed, 100 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0359b3a..70b6773 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The mock **derives the registered resources from the annotation, not from the co - Registered `...-memory` = **sum of `devmem`** over all entries. - Registered `...-cores` / `...-core` = **sum of `devcore`** over all entries. - For NVIDIA, Hygon and Ascend, the **count extended resource is only a health gate**: its value just needs to be `> 0`. It does **not** affect the registered memory/cores. By convention it is set to `cards x splits-per-card` (e.g. Ascend `2 x VDeviceCount(4) = 8`), but `1` would work equally well for the memory/cores to appear. -- Kunlun vXPU is fully annotation-derived: `vxpu = sum(devcore)` and `vxpu-memory = sum(devmem)` across entries whose `health` is `true`. +- Kunlun vXPU is fully annotation-derived: `vxpu = sum(count)` and `vxpu-memory = sum(devmem)` across entries whose `health` is `true`. Annotation entry fields: @@ -58,8 +58,8 @@ Annotation entry fields: | :-- | :-- | | `id` | unique device UUID (any string) | | `devmem` | per-card memory in MB -- **summed** into `...-memory` | -| `devcore` | per-card cores. **NVIDIA/Hygon:** summed into `...-cores` (NVIDIA: percentage, 100 = a whole card). **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. **Kunlun:** summed into `vxpu`. | -| `count` | per-card split count (informational for the mock) | +| `devcore` | per-card cores. **NVIDIA/Hygon:** summed into `...-cores` (NVIDIA: percentage, 100 = a whole card). **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. | +| `count` | per-card split count. **Kunlun:** summed into `vxpu`; informational for the other mocks. | | `type` | device model string | | `health` | scheduler-visible health; only healthy Kunlun entries contribute to its mock resources | | `index` | card index `0,1,2,...` (`0` may be omitted) | @@ -145,17 +145,17 @@ kunlun: resourceVMemoryName: kunlunxin.com/vxpu-memory ``` -To advertise one healthy 24 GiB vXPU: +To model one healthy P800-OAM card with four 24 GiB vXPU shares: ```bash kubectl annotate node \ - 'hami.io/node-register-xpu=[{"id":"XPU-MOCK-0","count":1,"devmem":24576,"devcore":1,"type":"XPU","health":true}]' + 'hami.io/node-register-xpu=[{"id":"XPU-MOCK-0","count":4,"devmem":98304,"devcore":100,"type":"XPU","health":true}]' # verify (~30s later) kubectl get node -o json | jq '.status.allocatable|with_entries(select(.key|test("kunlunxin.com")))' -# expect: kunlunxin.com/vxpu=1, kunlunxin.com/vxpu-memory=24576 +# expect: kunlunxin.com/vxpu=4, kunlunxin.com/vxpu-memory=98304 ``` -Each healthy entry contributes its `devcore` value to `vxpu` and its `devmem` value in MiB to `vxpu-memory`. Unhealthy entries remain visible to the HAMi scheduler through the annotation but do not contribute kubelet capacity, matching HAMi's `FitVXPU` health check. +Each healthy entry contributes its `count` value to `vxpu` and its `devmem` value in MiB to `vxpu-memory`. The mock also acknowledges HAMi Scheduler's `hami.io/node-handshake-xpu` probe, matching the real vXPU device-plugin startup protocol. Unhealthy entries remain visible to the HAMi scheduler through the annotation but do not contribute kubelet capacity, matching HAMi's `FitVXPU` health check. ## Ascend config compatibility (new vs legacy) diff --git a/internal/pkg/api/device/device.go b/internal/pkg/api/device/device.go index 60a61a6..73ccc03 100644 --- a/internal/pkg/api/device/device.go +++ b/internal/pkg/api/device/device.go @@ -31,6 +31,8 @@ import ( "github.com/ccoveille/go-safecast" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" ) @@ -77,6 +79,12 @@ type Devices interface { RunManager() } +// HandshakeReporter is implemented by device families whose real device +// plugin acknowledges scheduler health probes through a Node annotation. +type HandshakeReporter interface { + HandshakeAnnotation() string +} + type ResourceNames struct { ResourceCountName string ResourceMemoryName string @@ -114,18 +122,50 @@ func RunManagers() error { func Register(l *mock.MockLister, dev Devices) { nodeName := os.Getenv("NODE_NAME") + kubeClient := client.GetClient() for { - node, err := client.GetClient().CoreV1().Nodes().Get(context.Background(), nodeName, v1.GetOptions{}) + node, err := kubeClient.CoreV1().Nodes().Get(context.Background(), nodeName, v1.GetOptions{}) if err != nil { klog.Error("Get node error", err.Error()) } else { resourceMap := dev.GetResource(node) l.SetResource(resourceMap) + if reporter, ok := dev.(HandshakeReporter); ok { + if err := reportHandshake(context.Background(), kubeClient, node, reporter.HandshakeAnnotation()); err != nil { + klog.ErrorS(err, "Failed to report device handshake", "node", nodeName, "device", dev.CommonWord()) + } + } } time.Sleep(time.Second * 30) } } +func reportHandshake(ctx context.Context, kubeClient kubernetes.Interface, node *corev1.Node, annotation string) error { + patch, ok, err := handshakePatch(node, annotation) + if err != nil || !ok { + return err + } + _, err = kubeClient.CoreV1().Nodes().Patch(ctx, node.Name, types.JSONPatchType, patch, v1.PatchOptions{}) + return err +} + +func handshakePatch(node *corev1.Node, annotation string) ([]byte, bool, error) { + if node == nil || annotation == "" || node.Annotations == nil { + return nil, false, nil + } + request, ok := node.Annotations[annotation] + if !ok || !strings.HasPrefix(request, "Requesting_") { + return nil, false, nil + } + + annotationPath := strings.NewReplacer("~", "~0", "/", "~1").Replace(annotation) + patch, err := json.Marshal([]map[string]string{ + {"op": "test", "path": "/metadata/annotations/" + annotationPath, "value": request}, + {"op": "replace", "path": "/metadata/annotations/" + annotationPath, "value": "Reported_" + strings.TrimPrefix(request, "Requesting_")}, + }) + return patch, true, err +} + func GetResourceName(name string) string { if _, after, found := strings.Cut(name, "/"); found { return after diff --git a/internal/pkg/api/device/device_test.go b/internal/pkg/api/device/device_test.go index 33baf2a..525b7ce 100644 --- a/internal/pkg/api/device/device_test.go +++ b/internal/pkg/api/device/device_test.go @@ -17,10 +17,13 @@ limitations under the License. package device import ( + "encoding/json" "errors" "testing" "gotest.tools/v3/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_DecodeNodeDevices(t *testing.T) { @@ -100,4 +103,43 @@ func Test_DecodeNodeDevices(t *testing.T) { } }) } -} \ No newline at end of file +} + +func TestHandshakePatch(t *testing.T) { + const annotation = "hami.io/node-handshake-xpu" + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "mock-node", + Annotations: map[string]string{annotation: "Requesting_2026-08-26 03:00:00"}, + }} + + patch, ok, err := handshakePatch(node, annotation) + assert.NilError(t, err) + assert.Assert(t, ok) + var operations []map[string]string + err = json.Unmarshal(patch, &operations) + assert.NilError(t, err) + assert.Equal(t, len(operations), 2) + assert.DeepEqual(t, operations[0], map[string]string{ + "op": "test", + "path": "/metadata/annotations/hami.io~1node-handshake-xpu", + "value": "Requesting_2026-08-26 03:00:00", + }) + assert.DeepEqual(t, operations[1], map[string]string{ + "op": "replace", + "path": "/metadata/annotations/hami.io~1node-handshake-xpu", + "value": "Reported_2026-08-26 03:00:00", + }) +} + +func TestHandshakePatchIgnoresNonRequest(t *testing.T) { + const annotation = "hami.io/node-handshake-xpu" + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "mock-node", + Annotations: map[string]string{annotation: "Reported_2026-08-26 03:00:00"}, + }} + + patch, ok, err := handshakePatch(node, annotation) + assert.NilError(t, err) + assert.Assert(t, !ok) + assert.Assert(t, patch == nil) +} diff --git a/internal/pkg/api/device/kunlun/device.go b/internal/pkg/api/device/kunlun/device.go index c2aaf07..cee808f 100644 --- a/internal/pkg/api/device/kunlun/device.go +++ b/internal/pkg/api/device/kunlun/device.go @@ -68,6 +68,10 @@ func (dev *KunlunVDevices) CommonWord() string { return XPUDevice } +func (dev *KunlunVDevices) HandshakeAnnotation() string { + return HandshakeAnnos +} + func (dev *KunlunVDevices) GetNodeDevices(n *corev1.Node) ([]*device.DeviceInfo, error) { anno, ok := n.Annotations[RegisterAnnos] if !ok { @@ -110,7 +114,7 @@ func (dev *KunlunVDevices) GetResource(n *corev1.Node) map[string]int { if !val.Health { continue } - resourceMap[vCountResourceName] += int(val.Devcore) + resourceMap[vCountResourceName] += int(val.Count) resourceMap[memoryResourceName] += int(val.Devmem) } klog.InfoS("Add resource", vCountResourceName, resourceMap[vCountResourceName], memoryResourceName, resourceMap[memoryResourceName]) diff --git a/internal/pkg/api/device/kunlun/device_test.go b/internal/pkg/api/device/kunlun/device_test.go index bf5efda..c42391d 100644 --- a/internal/pkg/api/device/kunlun/device_test.go +++ b/internal/pkg/api/device/kunlun/device_test.go @@ -120,7 +120,7 @@ func TestKunlunVDevicesConfigurationIsPerInstance(t *testing.T) { ResourceVCountName: "example.com/second-count", ResourceVMemoryName: "example.com/second-memory", }) - node := nodeWithAnnotation(`[{"id":"XPU-0","devmem":24576,"devcore":1,"type":"XPU","health":true}]`) + node := nodeWithAnnotation(`[{"id":"XPU-0","count":1,"devmem":24576,"devcore":100,"type":"XPU","health":true}]`) firstResources := first.GetResource(node) if got := firstResources["vxpu"]; got != 1 { @@ -207,9 +207,9 @@ func TestKunlunVDevicesGetNodeDevices(t *testing.T) { func TestKunlunVDevicesGetResourceUsesHealthyAnnotationUnits(t *testing.T) { dev := InitKunlunVDevice(testConfig()) node := nodeWithAnnotation(`[ - {"id":"XPU-0","devmem":24576,"devcore":1,"type":"XPU","health":true}, - {"id":"XPU-1","index":1,"devmem":49152,"devcore":2,"type":"XPU","health":true}, - {"id":"XPU-2","index":2,"devmem":98304,"devcore":4,"type":"XPU","health":false} + {"id":"XPU-0","count":1,"devmem":24576,"devcore":100,"type":"XPU","health":true}, + {"id":"XPU-1","index":1,"count":2,"devmem":49152,"devcore":100,"type":"XPU","health":true}, + {"id":"XPU-2","index":2,"count":4,"devmem":98304,"devcore":100,"type":"XPU","health":false} ]`) resources := dev.GetResource(node)