diff --git a/.gitignore b/.gitignore index cfc599e..af68487 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,7 @@ helm/**/charts helm/**/Chart.lock helm/**/requirements.lock -cmd/k8s-device-plugin/k8s-device-plugin \ No newline at end of file +cmd/k8s-device-plugin/k8s-device-plugin + +# Build output (Makefile) +bin/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7df190c..295ac4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,9 @@ FROM golang:1.21-bullseye AS gobuild +# Go module proxy. Defaults to the public proxy; override for faster builds +# behind a slow network, e.g.: +# docker build --build-arg GOPROXY=https://goproxy.cn,direct . +ARG GOPROXY=https://proxy.golang.org,direct +ENV GOPROXY=${GOPROXY} ADD . /device-plugin RUN cd /device-plugin && go build -o ./k8s-device-plugin cmd/k8s-device-plugin/main.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..43e2576 --- /dev/null +++ b/Makefile @@ -0,0 +1,110 @@ +# Makefile for HAMi mock-device-plugin +# +# Run `make help` to see the available targets. + +# ---- Variables (override on the command line, e.g. `make docker-build IMG=myrepo/mock:dev`) ---- +REGISTRY ?= projecthami +IMAGE_NAME ?= mock-device-plugin +TAG ?= latest +IMG ?= $(REGISTRY)/$(IMAGE_NAME):$(TAG) + +BINARY ?= k8s-device-plugin +MAIN_PKG ?= ./cmd/k8s-device-plugin +OUTPUT_DIR ?= bin + +GIT_DESCRIBE ?= $(shell git describe --always --long --dirty 2>/dev/null || echo "unknown") +LDFLAGS ?= -X main.gitDescribe=$(GIT_DESCRIBE) + +GO ?= go +DOCKER ?= docker +# Use `make ... DOCKER=nerdctl` on containerd-only hosts. + +# Optional Go module proxy passed into the image build. Set it for faster builds +# behind a slow network, e.g. `make docker-build GOPROXY=https://goproxy.cn,direct`. +GOPROXY ?= +DOCKER_BUILD_ARGS := +ifneq ($(GOPROXY),) +DOCKER_BUILD_ARGS += --build-arg GOPROXY=$(GOPROXY) +endif + +# Detect golangci-lint availability for the lint target. +GOLANGCI_LINT := $(shell command -v golangci-lint 2>/dev/null) + +.DEFAULT_GOAL := help + +.PHONY: all +all: build ## Alias for `build` (compatibility with standard Makefile tooling). + +##@ General + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: fmt +fmt: ## Run go fmt against code. + $(GO) fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + $(GO) vet ./... + +.PHONY: tidy +tidy: ## Tidy go.mod / go.sum. + $(GO) mod tidy + +.PHONY: lint +lint: ## Run golangci-lint (if installed). +ifeq ($(GOLANGCI_LINT),) + @echo "golangci-lint not found; install: https://golangci-lint.run/usage/install/ -- skipping." +else + golangci-lint run ./... +endif + +.PHONY: test +test: ## Run unit tests. + $(GO) test ./... + +.PHONY: test-verbose +test-verbose: ## Run unit tests with verbose output. + $(GO) test -v ./... + +.PHONY: cover +cover: ## Run unit tests and report coverage. + $(GO) test -coverprofile=coverage.out ./... + $(GO) tool cover -func=coverage.out + +##@ Build + +.PHONY: build +build: vet ## Build the device-plugin binary into bin/. (run `make fmt` separately to format) + mkdir -p $(OUTPUT_DIR) + CGO_ENABLED=0 $(GO) build -ldflags "$(LDFLAGS)" -o $(OUTPUT_DIR)/$(BINARY) $(MAIN_PKG) + +.PHONY: clean +clean: ## Remove build artifacts. + rm -rf $(OUTPUT_DIR) coverage.out + +##@ Container + +.PHONY: docker-build +docker-build: ## Build the container image ($(IMG)). Set GOPROXY=... for a faster module proxy. + $(DOCKER) build $(DOCKER_BUILD_ARGS) -t $(IMG) . + +.PHONY: docker-push +docker-push: ## Push the container image ($(IMG)). + $(DOCKER) push $(IMG) + +##@ Deployment (requires kubectl + a valid KUBECONFIG) + +.PHONY: deploy +deploy: ## Deploy RBAC + DaemonSet to the current cluster. + kubectl apply -f k8s-mock-rbac.yaml + kubectl apply -f k8s-mock-plugin.yaml + +.PHONY: undeploy +undeploy: ## Remove the DaemonSet + RBAC from the current cluster. + kubectl delete -f k8s-mock-plugin.yaml --ignore-not-found + kubectl delete -f k8s-mock-rbac.yaml --ignore-not-found diff --git a/README.md b/README.md index 7602aa9..b024359 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,175 @@ # Mock device plugin for HAMi [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FProject-HAMi%2Fmock-device-plugin.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2FProject-HAMi%2Fmock-device-plugin?ref=badge_shield) - ## Introduction -This is a Kubernetes device plugin implementation that enables the registration of virtual-devices which would normally be ignored by scheduler (i.e gpu-memory, gpu-cores, etc..)on each node. After deployment, these resources will be available on `node.status.allocatable` and `node.status.capacity` +This is a Kubernetes device plugin that registers **virtual** device resources (e.g. `gpu-memory`, `gpu-cores`) -- the resources HAMi tracks but the kubelet would normally ignore -- onto a node, **without requiring real hardware**. It lets you test the HAMi scheduler (scheduling policies, quotas, webhooks) on machines that have no GPU/NPU. + +After deployment these resources show up under `node.status.allocatable` and `node.status.capacity`. + +## How it works (read this first) + +The mock plugin **does not detect hardware**. It does the following every ~30s: + +```text + node annotation: hami.io/node--register = [ {devmem, devcore, ...}, ... ] (1) + node capacity: (e.g. nvidia.com/gpu) > 0 (2) <- health gate + | mock reads + v + registers into allocatable: /...mem , /...cores , ... +``` + +On a **real** cluster, (1) and (2) are produced by the real device plugin. In a **mock-only** (no hardware) environment you provide them yourself: + +- **(1) the `node--register` annotation** describing the fake cards -- `kubectl annotate`. +- **(2) the count extended resource** (e.g. `nvidia.com/gpu`) -- patched onto the node `status`. + +> There is **no auto-labeller** in this repo, so (1) and (2) are manual today. Forgetting them is the usual cause of `device xxx is unhealthy` / `no allocation` -- see issues #14 / #16. ## Prerequisites -- Kubernetes version >= v1.18 +- Kubernetes >= v1.18 +- The `hami-scheduler-device` ConfigMap (the device config). If HAMi is installed it already exists; otherwise create it from [device-configmap.yaml](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml). ## Deployment -If the `hami-scheduler-device` ConfigMap is not deployed, it needs to be deployed first. refer [device-configmap.yaml](https://github.com/Project-HAMi/HAMi/blob/master/charts/hami/templates/scheduler/device-configmap.yaml) - +```bash +make deploy # = kubectl apply -f k8s-mock-rbac.yaml && kubectl apply -f k8s-mock-plugin.yaml +# or manually: +kubectl apply -f k8s-mock-rbac.yaml +kubectl apply -f k8s-mock-plugin.yaml ``` -$ kubectl apply -f k8s-mock-rbac.yaml -$ kubectl apply -f k8s-mock-plugin.yaml + +## Understanding the values + +The mock **derives the registered resources from the annotation, not from the count resource.** This trips people up, so to be explicit: + +- **Number of fake cards = the number of entries in the annotation array** (not the count value). +- Registered `...-memory` = **sum of `devmem`** over all entries. +- Registered `...-cores` / `...-core` = **sum of `devcore`** over all entries. +- The **count extended resource is only a health gate**: its value just needs to be `> 0`. It does **not** affect the registered memory/cores. By convention it is set to `cards x splits-per-card` (e.g. Ascend `2 x VDeviceCount(4) = 8`), but `1` would work equally well for the memory/cores to appear. + +Annotation entry fields: + +| field | meaning | +| :-- | :-- | +| `id` | unique device UUID (any string) | +| `devmem` | per-card memory in MB -- **summed** into `...-memory` | +| `devcore` | per-card cores. **NVIDIA/Hygon:** summed into `...-cores` (NVIDIA: percentage, 100 = a whole card). **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. | +| `count` | per-card split count (informational for the mock) | +| `type` | device model string | +| `health` | must be `true` to be counted | +| `index` | card index `0,1,2,...` (`0` may be omitted) | +| `numa`, `mode` | optional | + +> **Worked example (Ascend, below):** the annotation has **2 entries**, each `devmem=32768`. So `...-memory = 2x32768 = 65536` and `...-core = 2x100 = 200` (Ascend `...-core` is **percentage-based**: a whole card is always **100**). The count resource `=8` is a separate health-gate value and is unrelated to these numbers. + +## Usage by vendor + +To mock one card you always need the **three pieces**: the vendor **config block** (in the `hami-scheduler-device` ConfigMap, gives the resource names), the **count extended resource** (passes the health gate), and the **node-register annotation** (describes the fake cards). Replace `` below with your target node. Resource names follow the ConfigMap (HAMi defaults shown). See [Understanding the values](#understanding-the-values) for how the numbers are derived. + +### NVIDIA (e.g. A100-80GB) + +- config block: `nvidia:` | annotation: `hami.io/node-nvidia-register` (JSON) | count: `nvidia.com/gpu` +- mock registers: `nvidia.com/gpumem`, `nvidia.com/gpucores`, `nvidia.com/gpumem-percentage` + +```bash +# (2) count resource: splitCount=10 -> one card advertises 10 +kubectl patch node --subresource=status --type=json \ + -p '[{"op":"add","path":"/status/capacity/nvidia.com~1gpu","value":"10"}]' +# (1) annotation: 1 x A100-80G (devmem=81920 MB, devcore=100) +kubectl annotate node \ + 'hami.io/node-nvidia-register=[{"id":"GPU-MOCK-0","count":10,"devmem":81920,"devcore":100,"type":"NVIDIA-A100-SXM4-80GB","health":true,"numa":0,"mode":"hami-core"}]' +# verify (~30s later) +kubectl get node -o json | jq '.status.allocatable|with_entries(select(.key|test("nvidia.com")))' +# expect: nvidia.com/gpu=10, nvidia.com/gpumem=81920, nvidia.com/gpucores=100, nvidia.com/gpumem-percentage=100 ``` -## Build +### Ascend NPU (e.g. 910B4) + +- config block: `vnpus.configs` (entry for `910B4`) | annotation: `hami.io/node-register-Ascend910B4` (JSON) | count: `huawei.com/Ascend910B4` +- mock registers: `huawei.com/Ascend910B4-memory`, and -- **only when the node runs in `hami-vnpu-core` soft mode** (see below) -- `huawei.com/Ascend910B4-core` + +```bash +# (2) count resource: 2 cards x VDeviceCount(4) = 8 +kubectl patch node --subresource=status --type=json \ + -p '[{"op":"add","path":"/status/capacity/huawei.com~1Ascend910B4","value":"8"}]' +# (1) annotation: 2 x 910B4 (devmem=32768 MB, devcore=20), matching the real ascend-device-plugin report +kubectl annotate node \ + 'hami.io/node-register-Ascend910B4=[{"id":"MOCK-0","count":4,"devmem":32768,"devcore":20,"type":"Ascend910B4","health":true},{"id":"MOCK-1","index":1,"count":4,"devmem":32768,"devcore":20,"type":"Ascend910B4","health":true}]' +# (1b) soft mode: mark the node as hami-vnpu-core so the -core resource is reported. +# Either set vnpus.hamiVnpuCore: true globally in the ConfigMap, or annotate this node +# (the node annotation overrides the global setting, same precedence as the real plugin): +kubectl annotate node hami-vnpu-core=true +# verify +kubectl get node -o json | jq '.status.allocatable|with_entries(select(.key|test("Ascend910B4")))' +# expect: huawei.com/Ascend910B4-memory=65536, huawei.com/Ascend910B4-core=200 (2 cards x 100) ``` -docker build . + +The Ascend `-core` resource is **percentage-based**: each physical card contributes **100** (a whole card), independent of the annotation's `devcore`. HAMi caps a core request at 100 and, in `hami-vnpu-core` soft mode, treats a card's total core as 100. + +**When is `-core` reported?** Only on a node that runs in `hami-vnpu-core` (soft-partition) mode, mirroring the real ascend-device-plugin and the HAMi scheduler. Mode is decided by: the node's `hami-vnpu-core` annotation if present, otherwise the global `vnpus.hamiVnpuCore` in the ConfigMap (`true`/`false`, default `false`). On a hard/template node (`hami-vnpu-core` off) the HAMi scheduler filters out any pod that requests `-core` (`ModeNotFit`), so the mock does not register `-core` there. There is **no ascend-device-plugin** in a mock environment, so nothing writes the `hami-vnpu-core` node annotation automatically -- set it yourself (like the `node-register` annotation and the count resource). + +### Hygon DCU + +- config block: `hygon:` | annotation: `hami.io/node-dcu-register` (**CSV**, not JSON) | count: `hygon.com/dcunum` +- mock registers: `hygon.com/dcumem`, and -- when the config sets `resourceCoreName` -- `hygon.com/dcucores` (sum of per-card `devcore`) + +```bash +# (2) count resource: 2 DCUs +kubectl patch node --subresource=status --type=json \ + -p '[{"op":"add","path":"/status/capacity/hygon.com~1dcunum","value":"2"}]' +# (1) annotation: CSV form is "id,count,devmem,devcore,type,numa,health,index,mode:" per card +kubectl annotate node \ + 'hami.io/node-dcu-register=DCU-MOCK-0,2,16384,100,Z100L,0,true,0,hami-core:DCU-MOCK-1,2,16384,100,Z100L,1,true,1,hami-core:' +# verify +kubectl get node -o json | jq '.status.allocatable|with_entries(select(.key|test("hygon.com")))' +# expect: hygon.com/dcumem=32768, hygon.com/dcucores=200 (2 cards x devcore 100) ``` -## Examples +## Ascend config compatibility (new vs legacy) + +The Ascend `vnpus` config has two layouts and the plugin accepts **both**: +```yaml +# New (HAMi >= v2.9.0): nested object +vnpus: + hamiVnpuCore: true # node runs in soft mode -> the -core resource is reported + configs: + - chipName: 910B4 + resourceCoreName: huawei.com/Ascend910B4-core # name of the -core resource + ... ``` -Allocatable: - ... - memory: 769189866507 - nvidia.com/gpu: 20 - nvidia.com/gpucores: 200 - nvidia.com/gpumem: 65536 - nvidia.com/gpumem-percentage: 200 - pods: 110 - ... +```yaml +# Legacy (older / downstream): a bare list +vnpus: + - chipName: 910B4 + ... ``` +The new nested format is tried first; if that fails it falls back to the legacy flat list; if neither matches an error is returned. This lets clusters that have not upgraded their ConfigMap keep working. ## ManagedResources -| Devices | Mocking Resources | -| :--- | :----: | -| Nvidia GPU | nvidia.com/gpumem, nvidia.com/gpumem-percentage, nvidia.com/gpucores | -| Hygon DCU | hygon.com/dcumem | -| Ascend | huawei.com/Ascend{chip-name}-memory | +| Devices | Mocking Resources | +| :--- | :---- | +| Nvidia GPU | `nvidia.com/gpumem`, `nvidia.com/gpumem-percentage`, `nvidia.com/gpucores` | +| Hygon DCU | `hygon.com/dcumem`, `hygon.com/dcucores` (when `resourceCoreName` is set) | +| Ascend | `huawei.com/Ascend{chip}-memory`, `huawei.com/Ascend{chip}-core` (when `resourceCoreName` is set **and** the node is in `hami-vnpu-core` mode) | -**Note:** If the counted memory is too large, for example exceeding 120GB, it will display as 0. In this case, you can set the `memoryFactor` in `hami-scheduler-device` ConfigMap. The default value of `memoryFactor` is 1. +**Note:** If the counted memory is too large (e.g. > 120GB) it may display as 0. Set `memoryFactor` in the `hami-scheduler-device` ConfigMap (default 1). + +## Build + +```bash +make build # build the binary into bin/ +make test # run unit tests +make docker-build # build the image (override: IMG=myrepo/mock:tag) +make help # list all targets +``` ## Maintainer limengxuan@4paradigm.com ## License -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FProject-HAMi%2Fmock-device-plugin.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2FProject-HAMi%2Fmock-device-plugin?ref=badge_large) \ No newline at end of file +[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FProject-HAMi%2Fmock-device-plugin.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2FProject-HAMi%2Fmock-device-plugin?ref=badge_large) diff --git a/internal/pkg/api/device/ascend/device.go b/internal/pkg/api/device/ascend/device.go index dc3edf2..b173a39 100644 --- a/internal/pkg/api/device/ascend/device.go +++ b/internal/pkg/api/device/ascend/device.go @@ -29,6 +29,19 @@ import ( "k8s.io/klog/v2" ) +// ascendCardCorePercentage is the per-physical-card core capacity reported for +// the percentage-based huawei.com/-core resource. A whole card is 100: +// HAMi caps a core request at 100 and, in hami-vnpu-core soft mode, treats a +// card's total core as 100 regardless of the physical AI-core count. +const ascendCardCorePercentage = 100 + +// vnpuNodeSelectorAnnotation is the node annotation the real ascend-device-plugin +// writes to advertise whether a node runs in hami-vnpu-core (soft-partition) mode. +// The HAMi scheduler reads it as the authoritative per-node signal (it takes +// precedence over the global vnpus.hamiVnpuCore config). See ascend-device-plugin +// register.go and HAMi pkg/device/ascend/device.go (VNPUNodeSelectorAnnotation). +const vnpuNodeSelectorAnnotation = "hami-vnpu-core" + type Template struct { Name string `yaml:"name"` Memory int64 `yaml:"memory"` @@ -41,6 +54,7 @@ type VNPUConfig struct { ChipName string `yaml:"chipName"` ResourceName string `yaml:"resourceName"` ResourceMemoryName string `yaml:"resourceMemoryName"` + ResourceCoreName string `yaml:"resourceCoreName"` MemoryAllocatable int64 `yaml:"memoryAllocatable"` MemoryCapacity int64 `yaml:"memoryCapacity"` MemoryFactor int32 `yaml:"memoryFactor"` @@ -50,16 +64,20 @@ type VNPUConfig struct { } type Devices struct { - config VNPUConfig + config VNPUConfig + // hamiVnpuCore is the global vnpus.hamiVnpuCore default for this chip. It is + // overridden per node by the "hami-vnpu-core" node annotation (see GetResource). + hamiVnpuCore bool nodeRegisterAnno string } -func InitDevices(config []VNPUConfig) []*Devices { +func InitDevices(vnpus VNPUs) []*Devices { var devs []*Devices - for _, vnpu := range config { + for _, vnpu := range vnpus.Configs { commonWord := vnpu.CommonWord dev := &Devices{ config: vnpu, + hamiVnpuCore: vnpus.HamiVnpuCore, nodeRegisterAnno: fmt.Sprintf("hami.io/node-register-%s", commonWord), } sort.Slice(dev.config.Templates, func(i, j int) bool { @@ -100,6 +118,25 @@ func (dev *Devices) GetResource(n *corev1.Node) map[string]int { resourceMap := map[string]int{ resourceName: 0, } + // huawei.com/-core is the soft-partition (hami-vnpu-core) core resource: + // a percentage where a whole card is 100. Only report it when the node runs in + // hami-vnpu-core mode, mirroring the real ascend-device-plugin (which reports + // the 100-per-card core only when IsHamiVnpuCore) and the HAMi scheduler, which + // derives nodeSupportHamiCore as: the node "hami-vnpu-core" annotation if present, + // otherwise the global vnpus.hamiVnpuCore config. On a non-soft node the scheduler + // filters out any pod requesting -core (ModeNotFit), so registering it there would + // be a dead resource. core and memory must share resourceMap from the first call + // so that MockLister registers both plugins in a single shot. + nodeSupportHamiCore := dev.hamiVnpuCore + if v, ok := n.Annotations[vnpuNodeSelectorAnnotation]; ok { + nodeSupportHamiCore = v == "true" + } + hasCore := dev.config.ResourceCoreName != "" && nodeSupportHamiCore + var coreResourceName string + if hasCore { + coreResourceName = device.GetResourceName(dev.config.ResourceCoreName) + resourceMap[coreResourceName] = 0 + } if !device.CheckHealthy(n, dev.config.ResourceName) { klog.Infof("device %s is unhealthy on this node", dev.CommonWord()) return resourceMap @@ -111,6 +148,14 @@ func (dev *Devices) GetResource(n *corev1.Node) map[string]int { } for _, val := range devInfos { resourceMap[resourceName] += int(val.Devmem) + if hasCore { + // huawei.com/-core is a percentage-based resource: a whole + // physical card is 100. HAMi caps a core request at 100 and, in + // hami-vnpu-core soft mode, treats the card's total core as 100 + // (it ignores the physical AI-core count). So register 100 per + // physical card, independent of the annotation's devcore. + resourceMap[coreResourceName] += ascendCardCorePercentage + } } if dev.config.MemoryFactor > 1 { rawMemory := resourceMap[resourceName] @@ -118,6 +163,9 @@ func (dev *Devices) GetResource(n *corev1.Node) map[string]int { klog.InfoS("Update memory", "raw", rawMemory, "after", resourceMap[resourceName], "factor", dev.config.MemoryFactor) } klog.InfoS("Add resource", resourceName, resourceMap[resourceName]) + if hasCore { + klog.InfoS("Add resource", coreResourceName, resourceMap[coreResourceName]) + } return resourceMap } diff --git a/internal/pkg/api/device/ascend/device_core_test.go b/internal/pkg/api/device/ascend/device_core_test.go new file mode 100644 index 0000000..83c9082 --- /dev/null +++ b/internal/pkg/api/device/ascend/device_core_test.go @@ -0,0 +1,263 @@ +/* +Copyright 2026 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ascend + +import ( + "fmt" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// newAscendNode builds a node carrying the hami.io/node-register- annotation +// and a count-resource capacity used by CheckHealthy. +func newAscendNode(commonWord, resourceName, annoJSON, capacity string) *corev1.Node { + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: map[string]string{ + "hami.io/node-register-" + commonWord: annoJSON, + }, + }, + Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceName(resourceName): resource.MustParse(capacity), + }, + }, + } +} + +// cards builds a JSON device list with n identical cards (devmem/devcore per card). +func cards(n int, devmem, devcore int) string { + items := make([]string, 0, n) + for i := 0; i < n; i++ { + items = append(items, fmt.Sprintf(`{"id":"MOCK-%d","index":%d,"devmem":%d,"devcore":%d,"type":"Ascend910B4","health":true}`, i, i, devmem, devcore)) + } + return "[" + strings.Join(items, ",") + "]" +} + +func newDev(coreName string, aiCore, memFactor int32, hamiVnpuCore bool) *Devices { + return &Devices{ + config: VNPUConfig{ + CommonWord: "Ascend910B4", + ResourceName: "huawei.com/Ascend910B4", + ResourceMemoryName: "huawei.com/Ascend910B4-memory", + ResourceCoreName: coreName, + AICore: aiCore, + MemoryFactor: memFactor, + }, + hamiVnpuCore: hamiVnpuCore, + nodeRegisterAnno: "hami.io/node-register-Ascend910B4", + } +} + +// TestGetResource_Core covers core-resource registration. +func TestGetResource_Core(t *testing.T) { + const ( + memName = "Ascend910B4-memory" + coreName = "Ascend910B4-core" + coreFull = "huawei.com/Ascend910B4-core" + ) + tests := []struct { + name string + dev *Devices + anno string + // nodeHamiCore, when non-empty, sets the node "hami-vnpu-core" annotation + // ("true"/"false") to exercise the per-node soft-mode override. + nodeHamiCore string + capacity string + wantMem int + wantCore int + wantCoreKey bool + }{ + { + name: "1 two cards -> 100 core per card (global hami-vnpu-core on)", + dev: newDev(coreFull, 20, 0, true), + anno: cards(2, 32768, 20), + capacity: "8", + wantMem: 65536, + wantCore: 200, // 2 cards * 100 (percentage-based, devcore ignored) + wantCoreKey: true, + }, + { + name: "2 real scale (8 cards)", + dev: newDev(coreFull, 20, 0, true), + anno: cards(8, 32768, 20), + capacity: "32", + wantMem: 262144, + wantCore: 800, // 8 cards * 100 + wantCoreKey: true, + }, + { + name: "3 annotation devcore omitted -> still 100 per card", + dev: newDev(coreFull, 20, 0, true), + anno: cards(2, 21527, 0), + capacity: "8", + wantMem: 43054, + wantCore: 200, // 2 cards * 100, independent of devcore + wantCoreKey: true, + }, + { + name: "4 no resourceCoreName -> memory only (backward compat)", + dev: newDev("", 20, 0, true), + anno: cards(2, 32768, 20), + capacity: "8", + wantMem: 65536, + wantCore: 0, + wantCoreKey: false, + }, + { + name: "5 MemoryFactor does not affect core", + dev: newDev(coreFull, 20, 2, true), + anno: cards(2, 16384, 20), + capacity: "8", + wantMem: 16384, // (16384*2)/2 + wantCore: 200, // 2 cards * 100 + wantCoreKey: true, + }, + { + name: "6 unhealthy node (capacity 0)", + dev: newDev(coreFull, 20, 0, true), + anno: cards(2, 32768, 20), + capacity: "0", + wantMem: 0, + wantCore: 0, + wantCoreKey: true, // key present but zero + }, + { + name: "7 varying devcore in annotation is ignored for core", + dev: newDev(coreFull, 20, 0, true), + anno: `[{"id":"A","devmem":32768,"devcore":20,"type":"Ascend910B4","health":true},{"id":"B","index":1,"devmem":32768,"devcore":0,"type":"Ascend910B4","health":true}]`, + capacity: "8", + wantMem: 65536, + wantCore: 200, // 2 cards * 100 regardless of per-card devcore + wantCoreKey: true, + }, + { + name: "8 no register annotation", + dev: newDev(coreFull, 20, 0, true), + anno: "", // overwritten below to remove annotation + capacity: "8", + wantMem: 0, + wantCore: 0, + wantCoreKey: true, + }, + { + name: "9 hami-vnpu-core off globally, no node annotation -> memory only", + dev: newDev(coreFull, 20, 0, false), + anno: cards(2, 32768, 20), + capacity: "8", + wantMem: 65536, + wantCore: 0, + wantCoreKey: false, // hard/template node: -core not reported + }, + { + name: "10 node annotation hami-vnpu-core=true overrides global off -> core reported", + dev: newDev(coreFull, 20, 0, false), + anno: cards(2, 32768, 20), + nodeHamiCore: "true", + capacity: "8", + wantMem: 65536, + wantCore: 200, // 2 cards * 100 + wantCoreKey: true, + }, + { + name: "11 node annotation hami-vnpu-core=false overrides global on -> memory only", + dev: newDev(coreFull, 20, 0, true), + anno: cards(2, 32768, 20), + nodeHamiCore: "false", + capacity: "8", + wantMem: 65536, + wantCore: 0, + wantCoreKey: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := newAscendNode("Ascend910B4", "huawei.com/Ascend910B4", tt.anno, tt.capacity) + if tt.anno == "" { + // case 8: healthy node but no register annotation at all + delete(node.Annotations, "hami.io/node-register-Ascend910B4") + } + if tt.nodeHamiCore != "" { + node.Annotations[vnpuNodeSelectorAnnotation] = tt.nodeHamiCore + } + res := node // alias + got := tt.dev.GetResource(res) + + if got[memName] != tt.wantMem { + t.Errorf("memory = %d, want %d", got[memName], tt.wantMem) + } + _, hasCore := got[coreName] + if hasCore != tt.wantCoreKey { + t.Errorf("core key present = %v, want %v (map=%v)", hasCore, tt.wantCoreKey, got) + } + if tt.wantCoreKey && got[coreName] != tt.wantCore { + t.Errorf("core = %d, want %d", got[coreName], tt.wantCore) + } + }) + } +} + +// TestGetNodeDevices_RealAnnotation feeds a real 183 node-register annotation slice +// (2 cards, including index omitempty / custominfo / devicepairscore) to ensure the +// mock DeviceInfo can losslessly decode what the real ascend-device-plugin reports +// A tag drift that drops devcore would make the core resource 0. +func TestGetNodeDevices_RealAnnotation(t *testing.T) { + const realAnno = `[{"id":"7D16F664-806034F3-5BC13B72-D0D00485-104301E3","count":4,"devmem":32768,"devcore":20,"type":"Ascend910B4","health":true,"custominfo":{"NetworkID":0},"devicepairscore":{}},{"id":"16DFF664-806050DD-5CE21592-87D28485-104301E3","index":1,"count":4,"devmem":32768,"devcore":20,"type":"Ascend910B4","health":true,"custominfo":{"NetworkID":0},"devicepairscore":{}}]` + + dev := newDev("huawei.com/Ascend910B4-core", 20, 0, true) + node := newAscendNode("Ascend910B4", "huawei.com/Ascend910B4", realAnno, "32") + + devs, err := dev.GetNodeDevices(node) + if err != nil { + t.Fatalf("GetNodeDevices error: %v", err) + } + if len(devs) != 2 { + t.Fatalf("got %d devices, want 2", len(devs)) + } + for i, d := range devs { + if d.Devmem != 32768 { + t.Errorf("dev[%d].Devmem = %d, want 32768", i, d.Devmem) + } + if d.Devcore != 20 { + t.Errorf("dev[%d].Devcore = %d, want 20", i, d.Devcore) + } + if d.Count != 4 { + t.Errorf("dev[%d].Count = %d, want 4", i, d.Count) + } + if d.Type != "Ascend910B4" { + t.Errorf("dev[%d].Type = %q, want Ascend910B4", i, d.Type) + } + if !d.Health { + t.Errorf("dev[%d].Health = false, want true", i) + } + if d.DeviceVendor != "Ascend910B4" { + t.Errorf("dev[%d].DeviceVendor = %q, want Ascend910B4", i, d.DeviceVendor) + } + } + if devs[0].Index != 0 { + t.Errorf("dev[0].Index = %d, want 0 (omitempty)", devs[0].Index) + } + if devs[1].Index != 1 { + t.Errorf("dev[1].Index = %d, want 1", devs[1].Index) + } +} diff --git a/internal/pkg/api/device/ascend/device_test.go b/internal/pkg/api/device/ascend/device_test.go index 373eff8..7f9808e 100644 --- a/internal/pkg/api/device/ascend/device_test.go +++ b/internal/pkg/api/device/ascend/device_test.go @@ -26,12 +26,14 @@ import ( func TestInitDevices(t *testing.T) { tests := []struct { name string + hamiVnpuCore bool configs []VNPUConfig expectedCount int expectedCommon []string }{ { - name: "single device config", + name: "single device config", + hamiVnpuCore: true, configs: []VNPUConfig{ { CommonWord: "Ascend310P", @@ -53,7 +55,8 @@ func TestInitDevices(t *testing.T) { expectedCommon: []string{"Ascend310P"}, }, { - name: "multiple device configs", + name: "multiple device configs", + hamiVnpuCore: false, configs: []VNPUConfig{ { CommonWord: "Ascend310P", @@ -93,13 +96,16 @@ func TestInitDevices(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - devices := InitDevices(tt.configs) + devices := InitDevices(VNPUs{HamiVnpuCore: tt.hamiVnpuCore, Configs: tt.configs}) if len(devices) != tt.expectedCount { t.Errorf("expected %d devices, got %d", tt.expectedCount, len(devices)) } for i, dev := range devices { + if dev.hamiVnpuCore != tt.hamiVnpuCore { + t.Errorf("device %d: hamiVnpuCore = %v, want %v", i, dev.hamiVnpuCore, tt.hamiVnpuCore) + } if tt.expectedCommon[i] != dev.CommonWord() { t.Errorf("device %d: expected CommonWord %s, got %s", i, tt.expectedCommon[i], dev.CommonWord()) diff --git a/internal/pkg/api/device/ascend/vnpu.go b/internal/pkg/api/device/ascend/vnpu.go new file mode 100644 index 0000000..37e8a54 --- /dev/null +++ b/internal/pkg/api/device/ascend/vnpu.go @@ -0,0 +1,61 @@ +/* +Copyright 2026 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ascend + +import "k8s.io/klog/v2" + +// VNPUs holds the global Ascend vNPU configuration. It mirrors the upstream HAMi +// structure (pkg/device/ascend/vnpu.go): a global hami-vnpu-core switch plus the +// per-chip config list under `configs`. +// +// Upstream HAMi switched the `vnpus` config from a bare list ([]VNPUConfig) to this +// nested object (since v2.9.0). To avoid breaking users who have not upgraded their +// device-config ConfigMap (e.g. downstream/commercial deployments still on the old +// flat-list format), UnmarshalYAML accepts BOTH formats. +type VNPUs struct { + HamiVnpuCore bool `yaml:"hamiVnpuCore"` + Configs []VNPUConfig `yaml:"configs"` +} + +// UnmarshalYAML implements dual-format parsing for the `vnpus` field: +// 1. Prefer the new nested object: {hamiVnpuCore: bool, configs: [...]}. +// 2. Fall back to the legacy flat list: [VNPUConfig, ...]. +// 3. If neither matches, return the error from the preferred (new) format so the +// failure surfaces with the original semantics. +func (v *VNPUs) UnmarshalYAML(unmarshal func(interface{}) error) error { + // 1) Preferred: new nested structure. Use an alias type to avoid recursing + // into this very UnmarshalYAML method. + type vnpusAlias VNPUs + var nested vnpusAlias + errNew := unmarshal(&nested) + if errNew == nil { + *v = VNPUs(nested) + return nil + } + + // 2) Fallback: legacy flat list of per-chip configs. + var list []VNPUConfig + if errOld := unmarshal(&list); errOld == nil { + v.HamiVnpuCore = false + v.Configs = list + klog.Warning("vnpus parsed as legacy flat-list format; consider upgrading the device-config to {hamiVnpuCore, configs: [...]}") + return nil + } + + // 3) Neither format matched: report the preferred-format error. + return errNew +} diff --git a/internal/pkg/api/device/ascend/vnpu_test.go b/internal/pkg/api/device/ascend/vnpu_test.go new file mode 100644 index 0000000..229e10d --- /dev/null +++ b/internal/pkg/api/device/ascend/vnpu_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2026 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ascend + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +// wrapper mirrors how config.Config embeds VNPUs under the top-level `vnpus` key, +// so the test exercises the same code path as the real config parsing. +type wrapper struct { + VNPUs VNPUs `yaml:"vnpus"` +} + +// TestVNPUs_UnmarshalYAML_DualFormat verifies the backward-compatible parsing: +// new nested format is preferred, the legacy flat-list format is accepted as a +// fallback, and anything else surfaces an error. +func TestVNPUs_UnmarshalYAML_DualFormat(t *testing.T) { + tests := []struct { + name string + yamlStr string + wantErr bool + wantHamiVnpuCore bool + wantConfigs int + wantFirstCommon string + wantFirstCore string + }{ + { + name: "new nested format", + yamlStr: ` +vnpus: + hamiVnpuCore: true + configs: + - chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + resourceCoreName: huawei.com/Ascend910B4-core + aiCore: 20 + templates: + - name: vir05_1c_8g + memory: 8192 + aiCore: 5 +`, + wantErr: false, + wantHamiVnpuCore: true, + wantConfigs: 1, + wantFirstCommon: "Ascend910B4", + wantFirstCore: "huawei.com/Ascend910B4-core", + }, + { + name: "legacy flat-list format", + yamlStr: ` +vnpus: +- chipName: 910B4 + commonWord: Ascend910B4 + resourceName: huawei.com/Ascend910B4 + resourceMemoryName: huawei.com/Ascend910B4-memory + aiCore: 20 +`, + wantErr: false, + wantHamiVnpuCore: false, + wantConfigs: 1, + wantFirstCommon: "Ascend910B4", + wantFirstCore: "", + }, + { + name: "invalid scalar", + yamlStr: "vnpus: \"not-a-list-or-map\"", + wantErr: true, + wantConfigs: 0, + }, + { + name: "empty/null vnpus", + yamlStr: "vnpus:", + wantErr: false, + wantConfigs: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var w wrapper + err := yaml.Unmarshal([]byte(tt.yamlStr), &w) + if tt.wantErr != (err != nil) { + t.Fatalf("unmarshal err = %v, wantErr = %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if w.VNPUs.HamiVnpuCore != tt.wantHamiVnpuCore { + t.Errorf("HamiVnpuCore = %v, want %v", w.VNPUs.HamiVnpuCore, tt.wantHamiVnpuCore) + } + if len(w.VNPUs.Configs) != tt.wantConfigs { + t.Fatalf("Configs len = %d, want %d", len(w.VNPUs.Configs), tt.wantConfigs) + } + if tt.wantConfigs > 0 { + if w.VNPUs.Configs[0].CommonWord != tt.wantFirstCommon { + t.Errorf("Configs[0].CommonWord = %q, want %q", w.VNPUs.Configs[0].CommonWord, tt.wantFirstCommon) + } + if w.VNPUs.Configs[0].ResourceCoreName != tt.wantFirstCore { + t.Errorf("Configs[0].ResourceCoreName = %q, want %q", w.VNPUs.Configs[0].ResourceCoreName, tt.wantFirstCore) + } + } + }) + } +} diff --git a/internal/pkg/api/device/hygon/device.go b/internal/pkg/api/device/hygon/device.go index 4a2a26a..e9f7f46 100644 --- a/internal/pkg/api/device/hygon/device.go +++ b/internal/pkg/api/device/hygon/device.go @@ -89,6 +89,16 @@ func (dev *DCUDevices) GetResource(n *corev1.Node) map[string]int { resourceMap := map[string]int{ memoryResourceName: 0, } + // HAMi consumes the DCU core resource (e.g. hygon.com/dcucores) for compute + // partitioning, so register it as allocatable too when the config declares it. + // core and memory must share resourceMap from the first call so that + // MockLister registers both plugins in a single shot. + hasCore := HygonResourceCores != "" + var coreResourceName string + if hasCore { + coreResourceName = device.GetResourceName(HygonResourceCores) + resourceMap[coreResourceName] = 0 + } if !device.CheckHealthy(n, HygonResourceCount) { klog.Infof("device %s is unhealthy on this node", dev.CommonWord()) return resourceMap @@ -100,6 +110,9 @@ func (dev *DCUDevices) GetResource(n *corev1.Node) map[string]int { } for _, val := range devs { resourceMap[memoryResourceName] += int(val.Devmem) + if hasCore { + resourceMap[coreResourceName] += int(val.Devcore) + } } if MemoryFactor > 1 { rawMemory := resourceMap[memoryResourceName] @@ -107,12 +120,15 @@ func (dev *DCUDevices) GetResource(n *corev1.Node) map[string]int { klog.InfoS("Update memory", "raw", rawMemory, "after", resourceMap[memoryResourceName], "factor", MemoryFactor) } klog.InfoS("Add resources", memoryResourceName, resourceMap[memoryResourceName]) + if hasCore { + klog.InfoS("Add resources", coreResourceName, resourceMap[coreResourceName]) + } return resourceMap } func (dev *DCUDevices) RunManager() { lmock := mock.NewMockLister(device.GetVendorName(HygonResourceMemory)) - device.Register(lmock, dev) + go device.Register(lmock, dev) mockmanager := dpm.NewManager(lmock) klog.Infof("Running mocking dp: %s", dev.CommonWord()) mockmanager.Run() diff --git a/internal/pkg/api/device/hygon/device_test.go b/internal/pkg/api/device/hygon/device_test.go index 6389d5b..db2a3c2 100644 --- a/internal/pkg/api/device/hygon/device_test.go +++ b/internal/pkg/api/device/hygon/device_test.go @@ -60,5 +60,40 @@ func TestDCUDevices_GetResource(t *testing.T) { t.Errorf("Expected total memory %d, got %d", expectedTotalMemory, result[resourceName]) } + // cores are summed from per-card devcore (6 cards x 100) + coreName := device.GetResourceName(config.ResourceCoreName) + expectedTotalCores := 100 * 6 + if result[coreName] != expectedTotalCores { + t.Errorf("Expected total cores %d, got %d", expectedTotalCores, result[coreName]) + } + }) + + t.Run("WithoutResourceCoreName", func(t *testing.T) { + // When resourceCoreName is not configured, only memory is registered. + cfg := HygonConfig{ + ResourceCountName: "hygon.com/dcunum", + ResourceMemoryName: "hygon.com/dcumem", + } + dev := InitDCUDevice(cfg) + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node-2", + Annotations: map[string]string{ + RegisterAnnos: "DCU-A,4,65520,100,DCU-K100_AI,0,true,0,hami:", + }, + }, + Status: corev1.NodeStatus{ + Capacity: corev1.ResourceList{ + corev1.ResourceName(cfg.ResourceCountName): resource.MustParse("4"), + }, + }, + } + result := dev.GetResource(&node) + if _, ok := result["dcucores"]; ok { + t.Errorf("did not expect a core resource key, got map %v", result) + } + if result[device.GetResourceName(cfg.ResourceMemoryName)] != 65520 { + t.Errorf("expected memory 65520, got %d", result[device.GetResourceName(cfg.ResourceMemoryName)]) + } }) } diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index d7a7dfc..9cbd2a4 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -48,7 +48,7 @@ type Config struct { KunlunConfig kunlun.KunlunConfig `yaml:"kunlun"` AWSNeuronConfig awsneuron.AWSNeuronConfig `yaml:"awsneuron"` AMDGPUConfig amd.AMDConfig `yaml:"amd"` - VNPUs []ascend.VNPUConfig `yaml:"vnpus"` + VNPUs ascend.VNPUs `yaml:"vnpus"` } var (