From e9c16154e0a4972e4e4a8d6d2f3e36a918070585 Mon Sep 17 00:00:00 2001 From: Jifei Wang Date: Sun, 21 Sep 2025 22:24:41 +0800 Subject: [PATCH 1/7] add resourcequota Signed-off-by: Jifei Wang Signed-off-by: Jifei Wang Signed-off-by: Jifei Wang Signed-off-by: Jifei Wang Signed-off-by: Jifei Wang --- pkg/device/ascend/device.go | 8 + pkg/device/awsneuron/device.go | 8 + pkg/device/cambricon/device.go | 8 + pkg/device/common/common.go | 1 + pkg/device/devices.go | 1 + pkg/device/enflame/device.go | 8 + pkg/device/enflame/gcu.go | 8 + pkg/device/hygon/device.go | 8 + pkg/device/iluvatar/device.go | 8 + pkg/device/kunlun/device.go | 8 + pkg/device/kunlun/vdevice.go | 8 + pkg/device/metax/device.go | 8 + pkg/device/metax/sdevice.go | 8 + pkg/device/mthreads/device.go | 8 + pkg/device/nvidia/device.go | 24 +++ pkg/device/quota.go | 241 ++++++++++++++++++++++++++ pkg/scheduler/pods.go | 4 +- pkg/scheduler/scheduler.go | 57 +++++- pkg/scheduler/scheduler_test.go | 295 ++++++++++++++++++++++++++++++++ 19 files changed, 710 insertions(+), 9 deletions(-) create mode 100644 pkg/device/quota.go diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 4512b1523..54d23269a 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -289,6 +289,14 @@ func (dev *Devices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceUsage, ctr return nil } +func (dev *Devices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: dev.config.ResourceName, + ResourceMemoryName: dev.config.ResourceMemoryName, + ResourceCoreName: "", + } +} + func (npu *Devices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) (bool, map[string]device.ContainerDevices, string) { k := request originReq := k.Nums diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index b897d9f56..73dfd1ede 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -211,6 +211,14 @@ func (dev *AWSNeuronDevices) CheckHealth(devType string, n *corev1.Node) (bool, return true, true } +func (dev *AWSNeuronDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: dev.resourceCountName, + ResourceMemoryName: "", + ResourceCoreName: dev.resourceCoreName, + } +} + func (dev *AWSNeuronDevices) GenerateResourceRequests(ctr *corev1.Container) device.ContainerDeviceRequest { klog.Info("Start to count awsNeuron devices for container ", ctr.Name) awsResourceCount := corev1.ResourceName(dev.resourceCountName) diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index bbfbb3ce9..001b025fb 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -421,3 +421,11 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C } return false, tmpDevs, common.GenReason(reason, len(devices)) } + +func (dev *CambriconDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: MLUResourceCount, + ResourceMemoryName: MLUResourceMemory, + ResourceCoreName: MLUResourceCores, + } +} diff --git a/pkg/device/common/common.go b/pkg/device/common/common.go index c64d2ae0a..750cbe95d 100644 --- a/pkg/device/common/common.go +++ b/pkg/device/common/common.go @@ -36,6 +36,7 @@ const ( AllocatedCardsInsufficientRequest = "AllocatedCardsInsufficientRequest" NodeUnfitPod = "NodeUnfitPod" NodeFitPod = "NodeFitPod" + ResourceQuotaNotFit = "ResourceQuotaNotFit" ) func GenReason(reasons map[string]int, cards int) string { diff --git a/pkg/device/devices.go b/pkg/device/devices.go index dd8fc8066..3994702a5 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -35,6 +35,7 @@ type Devices interface { MutateAdmission(ctr *corev1.Container, pod *corev1.Pod) (bool, error) CheckHealth(devType string, n *corev1.Node) (bool, bool) NodeCleanUp(nn string) error + GetResourceNames() ResoureNames GetNodeDevices(n corev1.Node) ([]*DeviceInfo, error) LockNode(n *corev1.Node, p *corev1.Pod) error ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index 3e84e96fc..9b50dc7d7 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -327,3 +327,11 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con } return false, tmpDevs, common.GenReason(reason, len(devices)) } + +func (dev *EnflameDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: EnflameResourceNameVGCU, + ResourceMemoryName: EnflameResourceNameVGCUPercentage, + ResourceCoreName: "", + } +} diff --git a/pkg/device/enflame/gcu.go b/pkg/device/enflame/gcu.go index bac63224b..2e411bacb 100644 --- a/pkg/device/enflame/gcu.go +++ b/pkg/device/enflame/gcu.go @@ -182,6 +182,14 @@ func (gcuDev *GCUDevices) Fit(devices []*device.DeviceUsage, request device.Cont return false, tmpDevs, common.GenReason(reason, len(devices)) } +func (dev *GCUDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: EnflameResourceNameGCU, + ResourceMemoryName: "", + ResourceCoreName: "", + } +} + func (dev *GCUDevices) checkType(n device.ContainerDeviceRequest) bool { return n.Type == EnflameGCUDevice } diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index 135dd3f2f..1432bdd9b 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -373,3 +373,11 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain } return false, tmpDevs, common.GenReason(reason, len(devices)) } + +func (dev *DCUDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: HygonResourceCount, + ResourceMemoryName: HygonResourceMemory, + ResourceCoreName: HygonResourceCores, + } +} diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index 729db20e8..df620699a 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -332,3 +332,11 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co } return false, tmpDevs, common.GenReason(reason, len(devices)) } + +func (dev *IluvatarDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: IluvatarResourceCount, + ResourceMemoryName: IluvatarResourceMemory, + ResourceCoreName: IluvatarResourceCores, + } +} diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go index 163535da8..b6411edce 100644 --- a/pkg/device/kunlun/device.go +++ b/pkg/device/kunlun/device.go @@ -224,6 +224,14 @@ func (kl *KunlunDevices) Fit(devices []*device.DeviceUsage, request device.Conta return true, tmpDevs, "" } +func (dev *KunlunDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: KunlunResourceCount, + ResourceMemoryName: "", + ResourceCoreName: "", + } +} + func FitXPU(device *device.DeviceUsage, request device.ContainerDeviceRequest) bool { return device.Used == 0 } diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index 80f7f2684..6ba0a0905 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -232,6 +232,14 @@ func (dev *KunlunVDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceUsa return nil } +func (dev *KunlunVDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: KunlunResourceVCount, + ResourceMemoryName: KunlunResourceVMemory, + ResourceCoreName: "", + } +} + func (dev *KunlunVDevices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) (bool, map[string]device.ContainerDevices, string) { klog.InfoS("Allocating device for container request", "pod", klog.KObj(pod), "card request", request) tmpDevs := make(map[string]device.ContainerDevices) diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 184d0e0d0..1b0eacf23 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -317,3 +317,11 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta } return false, tmpDevs, common.GenReason(reason, len(devices)) } + +func (dev *MetaxDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: MetaxResourceCount, + ResourceMemoryName: "", + ResourceCoreName: "", + } +} diff --git a/pkg/device/metax/sdevice.go b/pkg/device/metax/sdevice.go index 8e809eb5f..0b4070700 100644 --- a/pkg/device/metax/sdevice.go +++ b/pkg/device/metax/sdevice.go @@ -414,6 +414,14 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con return true, map[string]device.ContainerDevices{request.Type: bestDevices}, "" } +func (dev *MetaxSDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: MetaxResourceNameVCount, + ResourceMemoryName: MetaxResourceNameVMemory, + ResourceCoreName: MetaxResourceNameVCore, + } +} + func (sdev *MetaxSDevices) getMetaxSDevices(n corev1.Node) ([]*MetaxSDeviceInfo, error) { anno, ok := n.Annotations[MetaxSDeviceAnno] if !ok { diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 4c9eb864e..365efa3d0 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -387,3 +387,11 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co } return false, tmpDevs, common.GenReason(reason, len(devices)) } + +func (dev *MthreadsDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: MthreadsResourceCount, + ResourceMemoryName: MthreadsResourceMemory, + ResourceCoreName: MthreadsResourceCores, + } +} diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 9f46be60e..482a4b6a2 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -603,6 +603,17 @@ func (dev *NvidiaGPUDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceU return nil } +func fitQuota(tmpDevs map[string]device.ContainerDevices, ns string, memreq int64, coresreq int64) bool { + mem := memreq + core := coresreq + for _, val := range tmpDevs[NvidiaGPUDevice] { + mem += int64(val.Usedmem) + core += int64(val.Usedcores) + } + klog.V(4).Infoln("Allocating...", mem, "cores", core) + return device.GetLocalCache().FitQuota(ns, mem, core, NvidiaGPUDevice) +} + func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *device.NodeInfo, allocated *device.PodDevices) (bool, map[string]device.ContainerDevices, string) { k := request originReq := k.Nums @@ -659,6 +670,11 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co //This incurs an issue memreq = dev.Totalmem * k.MemPercentagereq / 100 } + if !fitQuota(tmpDevs, pod.Namespace, int64(memreq), int64(k.Coresreq)) { + reason[common.ResourceQuotaNotFit]++ + klog.V(3).InfoS(common.ResourceQuotaNotFit, "pod", pod.Name, "memreq", memreq, "coresreq", k.Coresreq) + continue + } if dev.Totalmem-dev.Usedmem < memreq { reason[common.CardInsufficientMemory]++ klog.V(5).InfoS(common.CardInsufficientMemory, "pod", klog.KObj(pod), "device", dev.ID, "device index", i, "device total memory", dev.Totalmem, "device used memory", dev.Usedmem, "request memory", memreq) @@ -736,6 +752,14 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co return false, tmpDevs, common.GenReason(reason, len(devices)) } +func (dev *NvidiaGPUDevices) GetResourceNames() device.ResoureNames { + return device.ResoureNames{ + ResourceCountName: dev.config.ResourceCountName, + ResourceMemoryName: dev.config.ResourceMemoryName, + ResourceCoreName: dev.config.ResourceCoreName, + } +} + func generateCombinations(request device.ContainerDeviceRequest, tmpDevs map[string]device.ContainerDevices) []device.ContainerDevices { k := request num := int(k.Nums) diff --git a/pkg/device/quota.go b/pkg/device/quota.go new file mode 100644 index 000000000..99a923c56 --- /dev/null +++ b/pkg/device/quota.go @@ -0,0 +1,241 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package device + +import ( + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) + +type Quota struct { + Used int64 + Limit int64 +} + +type DeviceQuota map[string]*Quota + +type QuotaManager struct { + Quotas map[string]*DeviceQuota + mutex sync.RWMutex +} + +var localCache QuotaManager + +func GetLocalCache() *QuotaManager { + return &localCache +} + +var once sync.Once + +func NewQuotaManager() *QuotaManager { + once.Do(func() { + localCache = QuotaManager{ + Quotas: make(map[string]*DeviceQuota), + } + }) + return &localCache +} + +func (q *QuotaManager) FitQuota(ns string, memreq int64, coresreq int64, deviceName string) bool { + q.mutex.RLock() + defer q.mutex.RUnlock() + dq := q.Quotas[ns] + if dq == nil { + return true + } + devs, ok := GetDevices()[deviceName] + if !ok { + return true + } + resourceNames := devs.GetResourceNames() + memResourceName := resourceNames.ResourceMemoryName + coreResourceName := resourceNames.ResourceCoreName + _, ok = (*dq)[memResourceName] + if ok { + klog.V(4).InfoS("resourceMem quota judging", "limit", (*dq)[memResourceName].Limit, "used", (*dq)[memResourceName].Used, "alloc", memreq) + if (*dq)[memResourceName].Limit != 0 && (*dq)[memResourceName].Used+memreq > (*dq)[memResourceName].Limit { + klog.V(4).InfoS("resourceMem quota not fitted", "limit", (*dq)[memResourceName].Limit, "used", (*dq)[memResourceName].Used, "alloc", memreq) + return false + } + } + _, ok = (*dq)[coreResourceName] + if ok && (*dq)[coreResourceName].Limit != 0 && (*dq)[coreResourceName].Used+coresreq > (*dq)[coreResourceName].Limit { + klog.V(4).InfoS("resourceCores quota not fitted", "limit", (*dq)[coreResourceName].Limit, "used", (*dq)[coreResourceName].Used, "alloc", memreq) + return false + } + return true +} + +func countPodDevices(podDev PodDevices) map[string]int64 { + res := make(map[string]int64) + for deviceName, podSingle := range podDev { + if !strings.Contains(deviceName, "NVIDIA") { + continue + } + devs, ok := GetDevices()[deviceName] + if !ok { + continue + } + resourceNames := devs.GetResourceNames() + for _, ctrdevices := range podSingle { + for _, ctrdevice := range ctrdevices { + if len(resourceNames.ResourceMemoryName) > 0 { + res[resourceNames.ResourceMemoryName] += int64(ctrdevice.Usedmem) + } + if len(resourceNames.ResourceCoreName) > 0 { + res[resourceNames.ResourceCoreName] += int64(ctrdevice.Usedcores) + } + } + } + } + return res +} + +func (q *QuotaManager) AddUsage(pod *corev1.Pod, podDev PodDevices) { + q.mutex.Lock() + defer q.mutex.Unlock() + usage := countPodDevices(podDev) + if len(usage) == 0 { + return + } + if q.Quotas[pod.Namespace] == nil { + q.Quotas[pod.Namespace] = &DeviceQuota{} + } + dp, ok := q.Quotas[pod.Namespace] + if !ok { + return + } + for idx, val := range usage { + _, ok := (*dp)[idx] + if !ok { + (*dp)[idx] = &Quota{ + Used: 0, + Limit: 0, + } + } + (*dp)[idx].Used += val + } + for _, val := range q.Quotas { + for idx, val1 := range *val { + klog.V(4).Infoln("add usage val=", idx, ":", val1) + } + } +} + +func (q *QuotaManager) RmUsage(pod *corev1.Pod, podDev PodDevices) { + q.mutex.Lock() + defer q.mutex.Unlock() + usage := countPodDevices(podDev) + if len(usage) == 0 { + return + } + dp, ok := q.Quotas[pod.Namespace] + if !ok { + return + } + for idx, val := range usage { + _, ok = (*dp)[idx] + if ok { + (*dp)[idx].Used -= val + } + } + for _, val := range q.Quotas { + for idx, val1 := range *val { + klog.V(4).Infoln("after val=", idx, ":", val1) + } + } +} + +func IsManagedQuota(quotaName string) bool { + for _, val := range GetDevices() { + names := val.GetResourceNames() + if len(names.ResourceMemoryName) > 0 && names.ResourceMemoryName == quotaName { + return true + } + if len(names.ResourceCoreName) > 0 && names.ResourceCoreName == quotaName { + return true + } + } + return false +} + +func (q *QuotaManager) AddQuota(quota *corev1.ResourceQuota) { + q.mutex.Lock() + defer q.mutex.Unlock() + for idx, val := range quota.Spec.Hard { + value, ok := val.AsInt64() + if ok { + if !strings.HasPrefix(idx.String(), "limits.") { + continue + } + dn := strings.TrimPrefix(idx.String(), "limits.") + if !IsManagedQuota(dn) { + continue + } + if q.Quotas[quota.Namespace] == nil { + q.Quotas[quota.Namespace] = &DeviceQuota{} + } + dp := q.Quotas[quota.Namespace] + _, ok := (*dp)[dn] + if !ok { + (*dp)[dn] = &Quota{ + Used: 0, + Limit: value, + } + } + (*dp)[dn].Limit = value + klog.V(4).InfoS("quota set:", "idx=", idx, "val", value) + } + } + for _, val := range q.Quotas { + for idx, val1 := range *val { + klog.V(4).Infoln("after val=", idx, ":", val1) + } + } +} + +func (q *QuotaManager) DelQuota(quota *corev1.ResourceQuota) { + q.mutex.Lock() + defer q.mutex.Unlock() + for idx, val := range quota.Spec.Hard { + value, ok := val.AsInt64() + if ok { + if len(idx.String()) <= len("limits.") { + continue + } + dn := idx.String()[len("limits."):] + if !IsManagedQuota(dn) { + continue + } + klog.V(4).InfoS("quota remove:", "idx=", idx, "val", value) + if dq, ok := q.Quotas[quota.Namespace]; ok { + if quotaInfo, ok := (*dq)[dn]; ok { + quotaInfo.Limit = 0 + } + } + } + } + for _, val := range q.Quotas { + for idx, val1 := range *val { + klog.V(4).Infoln("after val=", idx, ":", val1) + } + } + +} diff --git a/pkg/scheduler/pods.go b/pkg/scheduler/pods.go index 8cb86b59a..8e44587ba 100644 --- a/pkg/scheduler/pods.go +++ b/pkg/scheduler/pods.go @@ -55,7 +55,7 @@ func newPodManager() *podManager { return pm } -func (m *podManager) addPod(pod *corev1.Pod, nodeID string, devices device.PodDevices) { +func (m *podManager) addPod(pod *corev1.Pod, nodeID string, devices device.PodDevices) bool { m.mutex.Lock() defer m.mutex.Unlock() @@ -81,6 +81,8 @@ func (m *podManager) addPod(pod *corev1.Pod, nodeID string, devices device.PodDe "devices", devices, ) } + + return !exists } func (m *podManager) delPod(pod *corev1.Pod) { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 7e08ae935..7da8112c5 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -47,17 +47,18 @@ type Scheduler struct { *nodeManager *podManager - stopCh chan struct{} - kubeClient kubernetes.Interface - podLister listerscorev1.PodLister - nodeLister listerscorev1.NodeLister + stopCh chan struct{} + kubeClient kubernetes.Interface + podLister listerscorev1.PodLister + nodeLister listerscorev1.NodeLister + quotaLister listerscorev1.ResourceQuotaLister //Node status returned by filter cachedstatus map[string]*NodeUsage nodeNotify chan struct{} //Node Overview overviewstatus map[string]*NodeUsage - - eventRecorder record.EventRecorder + eventRecorder record.EventRecorder + quotaManager *device.QuotaManager } func NewScheduler() *Scheduler { @@ -69,10 +70,15 @@ func NewScheduler() *Scheduler { } s.nodeManager = newNodeManager() s.podManager = newPodManager() + s.quotaManager = device.NewQuotaManager() klog.V(2).InfoS("Scheduler initialized successfully") return s } +func (s *Scheduler) GetQuotas() *device.QuotaManager { + return s.quotaManager +} + func (s *Scheduler) doNodeNotify() { select { case s.nodeNotify <- struct{}{}: @@ -96,7 +102,9 @@ func (s *Scheduler) onAddPod(obj any) { return } podDev, _ := device.DecodePodDevices(device.SupportDevices, pod.Annotations) - s.addPod(pod, nodeID, podDev) + if s.addPod(pod, nodeID, podDev) { + s.quotaManager.AddUsage(pod, podDev) + } } func (s *Scheduler) onUpdatePod(_, newObj any) { @@ -113,7 +121,34 @@ func (s *Scheduler) onDelPod(obj any) { if !ok { return } - s.delPod(pod) + pi, ok := s.pods[pod.UID] + if ok { + s.quotaManager.RmUsage(pod, pi.Devices) + s.delPod(pod) + } +} + +func (s *Scheduler) onAddQuota(obj interface{}) { + quota, ok := obj.(*corev1.ResourceQuota) + if !ok { + klog.Errorf("unknown add object type") + return + } + s.quotaManager.AddQuota(quota) +} + +func (s *Scheduler) onUpdateQuota(_, newObj interface{}) { + s.onDelQuota(newObj) + s.onAddQuota(newObj) +} + +func (s *Scheduler) onDelQuota(obj interface{}) { + quota, ok := obj.(*corev1.ResourceQuota) + if !ok { + klog.Errorf("unknown add object type") + return + } + s.quotaManager.DelQuota(quota) } func (s *Scheduler) Start() { @@ -122,6 +157,7 @@ func (s *Scheduler) Start() { informerFactory := informers.NewSharedInformerFactoryWithOptions(s.kubeClient, time.Hour*1) s.podLister = informerFactory.Core().V1().Pods().Lister() s.nodeLister = informerFactory.Core().V1().Nodes().Lister() + s.quotaLister = informerFactory.Core().V1().ResourceQuotas().Lister() informerFactory.Core().V1().Pods().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: s.onAddPod, @@ -133,6 +169,11 @@ func (s *Scheduler) Start() { UpdateFunc: func(_, _ any) { s.doNodeNotify() }, DeleteFunc: func(_ any) { s.doNodeNotify() }, }) + informerFactory.Core().V1().ResourceQuotas().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: s.onAddQuota, + UpdateFunc: s.onUpdateQuota, + DeleteFunc: s.onDelQuota, + }) informerFactory.Start(s.stopCh) informerFactory.WaitForCacheSync(s.stopCh) s.addAllEventHandlers() diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index d71fd7238..5d4f87be8 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -835,3 +835,298 @@ func Test_RegisterFromNodeAnnotations_NIL(t *testing.T) { }) } } + +func Test_ResourceQuota(t *testing.T) { + s := NewScheduler() + client.KubeClient = fake.NewSimpleClientset() + s.kubeClient = client.KubeClient + informerFactory := informers.NewSharedInformerFactoryWithOptions(client.KubeClient, time.Hour*1) + s.podLister = informerFactory.Core().V1().Pods().Lister() + informer := informerFactory.Core().V1().Pods().Informer() + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: s.onAddPod, + UpdateFunc: s.onUpdatePod, + DeleteFunc: s.onDelPod, + }) + informerFactory.Start(s.stopCh) + informerFactory.WaitForCacheSync(s.stopCh) + s.addAllEventHandlers() + sConfig := &config.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", + ResourceMemoryName: "hami.io/gpumem", + ResourceMemoryPercentageName: "hami.io/gpumem-percentage", + ResourceCoreName: "hami.io/gpucores", + DefaultMemory: 0, + DefaultCores: 0, + DefaultGPUNum: 1, + }, + } + + if err := config.InitDevicesWithConfig(sConfig); err != nil { + klog.Fatalf("Failed to initialize devices with config: %v", err) + } + + initNode := func() { + nodes, _ := s.ListNodes() + for index := range nodes { + s.rmNodeDevices(index, nvidia.NvidiaGPUDevice) + } + pods, _ := s.ListPodsUID() + for index := range pods { + s.delPod(pods[index]) + } + + s.addNode("node1", &device.NodeInfo{ + ID: "node1", + Node: &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}}, + Devices: []device.DeviceInfo{ + { + ID: "device1", + Index: 0, + Count: 10, + Devmem: 2000, + Devcore: 100, + Numa: 0, + Mode: "hami", + Type: nvidia.NvidiaGPUDevice, + Health: true, + DeviceVendor: nvidia.NvidiaGPUDevice, + }, + { + ID: "device2", + Index: 1, + Count: 10, + Devmem: 8000, + Devcore: 100, + Numa: 0, + Mode: "hami", + Type: nvidia.NvidiaGPUDevice, + Health: true, + DeviceVendor: nvidia.NvidiaGPUDevice, + }, + { + ID: "device3", + Index: 0, + Count: 10, + Devmem: 4000, + Devcore: 100, + Numa: 0, + Mode: "hami", + Type: nvidia.NvidiaGPUDevice, + Health: true, + }, + { + ID: "device4", + Index: 1, + Count: 10, + Devmem: 6000, + Devcore: 100, + Numa: 0, + Type: nvidia.NvidiaGPUDevice, + Health: true, + }, + }, + }) + } + + tests := []struct { + name string + args extenderv1.ExtenderArgs + quota corev1.ResourceQuota + want *extenderv1.ExtenderFilterResult + wantErr error + }{ + { + name: "multi device Resourcequota pass", + args: extenderv1.ExtenderArgs{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test1", + UID: "test1-uid1", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "gpu-burn", + Image: "chrstnhntschl/gpu_burn", + Args: []string{"6000"}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "hami.io/gpu": *resource.NewQuantity(2, resource.BinarySI), + "hami.io/gpucores": *resource.NewQuantity(100, resource.BinarySI), + "hami.io/gpumem": *resource.NewQuantity(2000, resource.BinarySI), + }, + }, + }, + }, + }, + }, + NodeNames: &[]string{"node1"}, + }, + quota: corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-quota", + Namespace: "default", + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + "limits.hami.io/gpucores": *resource.NewQuantity(200, resource.BinarySI), + "limits.hami.io/gpumem": *resource.NewQuantity(4000, resource.BinarySI), + }, + }, + }, + wantErr: nil, + want: &extenderv1.ExtenderFilterResult{ + NodeNames: &[]string{"node1"}, + }, + }, + { + name: "multi device Resourcequota deny", + args: extenderv1.ExtenderArgs{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test2", + UID: "test2-uid2", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "gpu-burn", + Image: "chrstnhntschl/gpu_burn", + Args: []string{"6000"}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "hami.io/gpu": *resource.NewQuantity(2, resource.BinarySI), + "hami.io/gpucores": *resource.NewQuantity(60, resource.BinarySI), + "hami.io/gpumem": *resource.NewQuantity(3000, resource.BinarySI), + }, + }, + }, + }, + }, + }, + NodeNames: &[]string{"node1"}, + }, + quota: corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-quota", + Namespace: "default", + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + "limits.hami.io/gpucores": *resource.NewQuantity(200, resource.BinarySI), + "limits.hami.io/gpumem": *resource.NewQuantity(4000, resource.BinarySI), + }, + }, + }, + wantErr: nil, + want: &extenderv1.ExtenderFilterResult{ + FailedNodes: map[string]string{ + "node1": "NodeUnfitPod", + }, + }, + }, + { + name: "unspecified device Resourcequota pass", + args: extenderv1.ExtenderArgs{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test3", + UID: "test3-uid3", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "gpu-burn", + Image: "chrstnhntschl/gpu_burn", + Args: []string{"6000"}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI), + }, + }, + }, + }, + }, + }, + NodeNames: &[]string{"node1"}, + }, + quota: corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-quota", + Namespace: "default", + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + "limits.hami.io/gpucores": *resource.NewQuantity(100, resource.BinarySI), + "limits.hami.io/gpumem": *resource.NewQuantity(2000, resource.BinarySI), + }, + }, + }, + wantErr: nil, + want: &extenderv1.ExtenderFilterResult{ + NodeNames: &[]string{"node1"}, + }, + }, + { + name: "unspecified device Resourcequota deny", + args: extenderv1.ExtenderArgs{ + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test4", + UID: "test4-uid4", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "gpu-burn", + Image: "chrstnhntschl/gpu_burn", + Args: []string{"6000"}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "hami.io/gpu": *resource.NewQuantity(1, resource.BinarySI), + }, + }, + }, + }, + }, + }, + NodeNames: &[]string{"node1"}, + }, + quota: corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-quota", + Namespace: "default", + }, + Spec: corev1.ResourceQuotaSpec{ + Hard: corev1.ResourceList{ + "limits.hami.io/gpucores": *resource.NewQuantity(100, resource.BinarySI), + "limits.hami.io/gpumem": *resource.NewQuantity(1500, resource.BinarySI), + }, + }, + }, + wantErr: nil, + want: &extenderv1.ExtenderFilterResult{ + FailedNodes: map[string]string{ + "node1": "NodeUnfitPod", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s.onAddQuota(&test.quota) + initNode() + client.KubeClient.CoreV1().Pods(test.args.Pod.Namespace).Create(context.Background(), test.args.Pod, metav1.CreateOptions{}) + got, gotErr := s.Filter(test.args) + assert.DeepEqual(t, test.wantErr, gotErr) + assert.DeepEqual(t, test.want, got) + }) + } +} From 6aca355a8342047253c860cab36abcb355f96664 Mon Sep 17 00:00:00 2001 From: Jifei Wang Date: Wed, 24 Sep 2025 15:40:36 +0800 Subject: [PATCH 2/7] add quota_test Signed-off-by: Jifei Wang --- .../hami/templates/scheduler/clusterrole.yaml | 3 + pkg/device/ascend/device.go | 4 +- pkg/device/awsneuron/device.go | 4 +- pkg/device/cambricon/device.go | 4 +- pkg/device/devices.go | 4 +- pkg/device/enflame/device.go | 4 +- pkg/device/enflame/gcu.go | 4 +- pkg/device/hygon/device.go | 4 +- pkg/device/iluvatar/device.go | 4 +- pkg/device/kunlun/device.go | 4 +- pkg/device/kunlun/vdevice.go | 4 +- pkg/device/metax/device.go | 4 +- pkg/device/metax/sdevice.go | 4 +- pkg/device/mthreads/device.go | 4 +- pkg/device/nvidia/device.go | 4 +- pkg/device/quota_test.go | 237 ++++++++++++++++++ 16 files changed, 268 insertions(+), 28 deletions(-) create mode 100644 pkg/device/quota_test.go diff --git a/charts/hami/templates/scheduler/clusterrole.yaml b/charts/hami/templates/scheduler/clusterrole.yaml index 6a3d41295..16d8b9c7a 100644 --- a/charts/hami/templates/scheduler/clusterrole.yaml +++ b/charts/hami/templates/scheduler/clusterrole.yaml @@ -18,4 +18,7 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["create", "get", "list"] + - apiGroups: [""] + resources: ["resourcequotas"] + verbs: ["get", "list", "watch"] diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 54d23269a..3088c0215 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -289,8 +289,8 @@ func (dev *Devices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceUsage, ctr return nil } -func (dev *Devices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *Devices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: dev.config.ResourceName, ResourceMemoryName: dev.config.ResourceMemoryName, ResourceCoreName: "", diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index 73dfd1ede..60250cf97 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -211,8 +211,8 @@ func (dev *AWSNeuronDevices) CheckHealth(devType string, n *corev1.Node) (bool, return true, true } -func (dev *AWSNeuronDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *AWSNeuronDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: dev.resourceCountName, ResourceMemoryName: "", ResourceCoreName: dev.resourceCoreName, diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 001b025fb..2cd809175 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -422,8 +422,8 @@ func (cam *CambriconDevices) Fit(devices []*device.DeviceUsage, request device.C return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *CambriconDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *CambriconDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: MLUResourceCount, ResourceMemoryName: MLUResourceMemory, ResourceCoreName: MLUResourceCores, diff --git a/pkg/device/devices.go b/pkg/device/devices.go index 3994702a5..cb3f0ecc3 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -35,7 +35,7 @@ type Devices interface { MutateAdmission(ctr *corev1.Container, pod *corev1.Pod) (bool, error) CheckHealth(devType string, n *corev1.Node) (bool, bool) NodeCleanUp(nn string) error - GetResourceNames() ResoureNames + GetResourceNames() ResourceNames GetNodeDevices(n corev1.Node) ([]*DeviceInfo, error) LockNode(n *corev1.Node, p *corev1.Pod) error ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error @@ -118,7 +118,7 @@ type NodeInfo struct { Devices []DeviceInfo } -type ResoureNames struct { +type ResourceNames struct { ResourceCountName string ResourceMemoryName string ResourceCoreName string diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index 9b50dc7d7..943cf5639 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -328,8 +328,8 @@ func (enf *EnflameDevices) Fit(devices []*device.DeviceUsage, request device.Con return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *EnflameDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *EnflameDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: EnflameResourceNameVGCU, ResourceMemoryName: EnflameResourceNameVGCUPercentage, ResourceCoreName: "", diff --git a/pkg/device/enflame/gcu.go b/pkg/device/enflame/gcu.go index 2e411bacb..eecbcd22a 100644 --- a/pkg/device/enflame/gcu.go +++ b/pkg/device/enflame/gcu.go @@ -182,8 +182,8 @@ func (gcuDev *GCUDevices) Fit(devices []*device.DeviceUsage, request device.Cont return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *GCUDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *GCUDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: EnflameResourceNameGCU, ResourceMemoryName: "", ResourceCoreName: "", diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index 1432bdd9b..94d8bbf86 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -374,8 +374,8 @@ func (dcu *DCUDevices) Fit(devices []*device.DeviceUsage, request device.Contain return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *DCUDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *DCUDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: HygonResourceCount, ResourceMemoryName: HygonResourceMemory, ResourceCoreName: HygonResourceCores, diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index df620699a..fe00a9936 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -333,8 +333,8 @@ func (ilu *IluvatarDevices) Fit(devices []*device.DeviceUsage, request device.Co return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *IluvatarDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *IluvatarDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: IluvatarResourceCount, ResourceMemoryName: IluvatarResourceMemory, ResourceCoreName: IluvatarResourceCores, diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go index b6411edce..66eb08e92 100644 --- a/pkg/device/kunlun/device.go +++ b/pkg/device/kunlun/device.go @@ -224,8 +224,8 @@ func (kl *KunlunDevices) Fit(devices []*device.DeviceUsage, request device.Conta return true, tmpDevs, "" } -func (dev *KunlunDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *KunlunDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: KunlunResourceCount, ResourceMemoryName: "", ResourceCoreName: "", diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index 6ba0a0905..0ff7f2356 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -232,8 +232,8 @@ func (dev *KunlunVDevices) AddResourceUsage(pod *corev1.Pod, n *device.DeviceUsa return nil } -func (dev *KunlunVDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *KunlunVDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: KunlunResourceVCount, ResourceMemoryName: KunlunResourceVMemory, ResourceCoreName: "", diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 1b0eacf23..1ad45198e 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -318,8 +318,8 @@ func (mat *MetaxDevices) Fit(devices []*device.DeviceUsage, request device.Conta return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *MetaxDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *MetaxDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: MetaxResourceCount, ResourceMemoryName: "", ResourceCoreName: "", diff --git a/pkg/device/metax/sdevice.go b/pkg/device/metax/sdevice.go index 0b4070700..94ffb25e7 100644 --- a/pkg/device/metax/sdevice.go +++ b/pkg/device/metax/sdevice.go @@ -414,8 +414,8 @@ func (mats *MetaxSDevices) Fit(devices []*device.DeviceUsage, request device.Con return true, map[string]device.ContainerDevices{request.Type: bestDevices}, "" } -func (dev *MetaxSDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *MetaxSDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: MetaxResourceNameVCount, ResourceMemoryName: MetaxResourceNameVMemory, ResourceCoreName: MetaxResourceNameVCore, diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 365efa3d0..390665766 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -388,8 +388,8 @@ func (mth *MthreadsDevices) Fit(devices []*device.DeviceUsage, request device.Co return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *MthreadsDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *MthreadsDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: MthreadsResourceCount, ResourceMemoryName: MthreadsResourceMemory, ResourceCoreName: MthreadsResourceCores, diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 482a4b6a2..f50e48b6d 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -752,8 +752,8 @@ func (nv *NvidiaGPUDevices) Fit(devices []*device.DeviceUsage, request device.Co return false, tmpDevs, common.GenReason(reason, len(devices)) } -func (dev *NvidiaGPUDevices) GetResourceNames() device.ResoureNames { - return device.ResoureNames{ +func (dev *NvidiaGPUDevices) GetResourceNames() device.ResourceNames { + return device.ResourceNames{ ResourceCountName: dev.config.ResourceCountName, ResourceMemoryName: dev.config.ResourceMemoryName, ResourceCoreName: dev.config.ResourceCoreName, diff --git a/pkg/device/quota_test.go b/pkg/device/quota_test.go new file mode 100644 index 000000000..a359d4473 --- /dev/null +++ b/pkg/device/quota_test.go @@ -0,0 +1,237 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package device + +import ( + "sync" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type MockDevices struct { + resourceNames ResourceNames +} + +func (m *MockDevices) CommonWord() string { + return "mock" +} + +func (m *MockDevices) MutateAdmission(ctr *corev1.Container, pod *corev1.Pod) (bool, error) { + return true, nil +} + +func (m *MockDevices) CheckHealth(devType string, n *corev1.Node) (bool, bool) { + return true, true +} + +func (m *MockDevices) NodeCleanUp(nn string) error { + return nil +} + +func (m *MockDevices) GetResourceNames() ResourceNames { + return m.resourceNames +} + +func (m *MockDevices) GetNodeDevices(n corev1.Node) ([]*DeviceInfo, error) { + return []*DeviceInfo{}, nil +} + +func (m *MockDevices) LockNode(n *corev1.Node, p *corev1.Pod) error { + return nil +} + +func (m *MockDevices) ReleaseNodeLock(n *corev1.Node, p *corev1.Pod) error { + return nil +} + +func (m *MockDevices) GenerateResourceRequests(ctr *corev1.Container) ContainerDeviceRequest { + return ContainerDeviceRequest{} +} + +func (m *MockDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd PodDevices) map[string]string { + return map[string]string{} +} + +func (m *MockDevices) ScoreNode(node *corev1.Node, podDevices PodSingleDevice, previous []*DeviceUsage, policy string) float32 { + return 1.0 +} + +func (m *MockDevices) AddResourceUsage(pod *corev1.Pod, n *DeviceUsage, ctr *ContainerDevice) error { + return nil +} + +func (m *MockDevices) Fit(devices []*DeviceUsage, request ContainerDeviceRequest, pod *corev1.Pod, nodeInfo *NodeInfo, allocated *PodDevices) (bool, map[string]ContainerDevices, string) { + return true, nil, "" +} + +type PodDeviceInfo struct { + Usedmem int + Usedcores int +} + +type TestPodDevices map[string]map[string][]PodDeviceInfo + +func initTest() { + DevicesMap = make(map[string]Devices) + DevicesMap["NVIDIA"] = &MockDevices{ + resourceNames: ResourceNames{ + ResourceMemoryName: "nvidia.com/gpumem", + ResourceCoreName: "nvidia.com/gpucore", + }, + } +} + +func TestNewQuotaManagerSingleton(t *testing.T) { + var wg sync.WaitGroup + var managers [2]*QuotaManager + + wg.Add(2) + go func() { + managers[0] = NewQuotaManager() + wg.Done() + }() + go func() { + managers[1] = NewQuotaManager() + wg.Done() + }() + wg.Wait() + + if managers[0] != managers[1] { + t.Error("NewQuotaManager should return the same instance (singleton)") + } +} + +func TestFitQuota(t *testing.T) { + initTest() + qm := NewQuotaManager() + ns := "testns" + deviceName := "NVIDIA" + memName := "nvidia.com/gpumem" + coreName := "nvidia.com/gpucore" + + qm.Quotas[ns] = &DeviceQuota{ + memName: &Quota{Used: 1000, Limit: 2000}, + coreName: &Quota{Used: 200, Limit: 400}, + } + + // Should fit + if !qm.FitQuota(ns, 500, 100, deviceName) { + t.Error("FitQuota should return true when within limits") + } + // Should not fit memory + if qm.FitQuota(ns, 1500, 100, deviceName) { + t.Error("FitQuota should return false when memory exceeds limit") + } + // Should not fit core + if qm.FitQuota(ns, 500, 300, deviceName) { + t.Error("FitQuota should return false when core exceeds limit") + } + // Should fit if namespace not present + if !qm.FitQuota("otherns", 1500, 100, deviceName) { + t.Error("FitQuota should return true if namespace not present") + } + // Should fit if device not present + if !qm.FitQuota(ns, 1000, 100, "unknown-device") { + t.Error("FitQuota should return true if device not present") + } +} + +func TestAddUsageAndRmUsage(t *testing.T) { + initTest() + qm := NewQuotaManager() + ns := "testns" + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: ns}} + podDev := PodDevices{ + "NVIDIA": PodSingleDevice{ + []ContainerDevice{ + { + Idx: 0, + UUID: "GPU0", + Usedmem: 1000, + Usedcores: 100, + }, + }, + }, + } + + qm.Quotas[ns] = &DeviceQuota{} + qm.AddUsage(pod, podDev) + + memName := "nvidia.com/gpumem" + coreName := "nvidia.com/gpucore" + + if (*qm.Quotas[ns])[memName].Used != 1000 { + t.Errorf("AddUsage: expected Used memory 1000, got %d", (*qm.Quotas[ns])[memName].Used) + } + if (*qm.Quotas[ns])[coreName].Used != 100 { + t.Errorf("AddUsage: expected Used core 100, got %d", (*qm.Quotas[ns])[coreName].Used) + } + + qm.RmUsage(pod, podDev) + if (*qm.Quotas[ns])[memName].Used != 0 { + t.Errorf("RmUsage: expected Used memory 0, got %d", (*qm.Quotas[ns])[memName].Used) + } + if (*qm.Quotas[ns])[coreName].Used != 0 { + t.Errorf("RmUsage: expected Used core 0, got %d", (*qm.Quotas[ns])[coreName].Used) + } +} + +func TestIsManagedQuota(t *testing.T) { + initTest() + if !IsManagedQuota("nvidia.com/gpumem") { + t.Error("IsManagedQuota should return true for managed memory quota") + } + if !IsManagedQuota("nvidia.com/gpucore") { + t.Error("IsManagedQuota should return true for managed core quota") + } + if IsManagedQuota("other-resource") { + t.Error("IsManagedQuota should return false for unmanaged quota") + } +} + +func TestAddQuotaAndDelQuota(t *testing.T) { + initTest() + qm := NewQuotaManager() + ns := "testns" + memName := "nvidia.com/gpumem" + coreName := "nvidia.com/gpucore" + + rq := &corev1.ResourceQuota{} + rq.Namespace = ns + rq.Spec.Hard = corev1.ResourceList{ + corev1.ResourceName("limits." + memName): *resource.NewQuantity(100, resource.DecimalSI), + corev1.ResourceName("limits." + coreName): *resource.NewQuantity(10, resource.DecimalSI), + } + + qm.AddQuota(rq) + if (*qm.Quotas[ns])[memName].Limit != 100 { + t.Errorf("AddQuota: expected memory limit 100, got %d", (*qm.Quotas[ns])[memName].Limit) + } + if (*qm.Quotas[ns])[coreName].Limit != 10 { + t.Errorf("AddQuota: expected core limit 10, got %d", (*qm.Quotas[ns])[coreName].Limit) + } + + qm.DelQuota(rq) + if (*qm.Quotas[ns])[memName].Limit != 0 { + t.Errorf("DelQuota: expected memory limit 0, got %d", (*qm.Quotas[ns])[memName].Limit) + } + if (*qm.Quotas[ns])[coreName].Limit != 0 { + t.Errorf("DelQuota: expected core limit 0, got %d", (*qm.Quotas[ns])[coreName].Limit) + } +} From 3f013a96e83a90456116be820d7b7ad46b3b1366 Mon Sep 17 00:00:00 2001 From: limengxuan Date: Thu, 25 Sep 2025 12:10:44 +0800 Subject: [PATCH 3/7] update Signed-off-by: limengxuan --- pkg/device/ascend/device.go | 2 +- pkg/device/devices.go | 2 +- pkg/device/kunlun/vdevice.go | 2 +- pkg/device/nvidia/device.go | 2 +- pkg/scheduler/pods.go | 2 +- pkg/scheduler/scheduler.go | 5 ++--- pkg/util/util.go | 4 ++-- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 3088c0215..3c10243fd 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -237,7 +237,6 @@ func (dev *Devices) CheckHealth(devType string, n *corev1.Node) (bool, bool) { } func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.ContainerDeviceRequest { - klog.Infof("Counting %s devices", dev.config.CommonWord) ascendResourceCount := corev1.ResourceName(dev.config.ResourceName) ascendResourceMem := corev1.ResourceName(dev.config.ResourceMemoryName) v, ok := ctr.Resources.Limits[ascendResourceCount] @@ -245,6 +244,7 @@ func (dev *Devices) GenerateResourceRequests(ctr *corev1.Container) device.Conta v, ok = ctr.Resources.Requests[ascendResourceCount] } if ok { + klog.V(3).Infof("Counting %s devices", dev.config.CommonWord) if n, ok := v.AsInt64(); ok { klog.Info("Found AscendDevices devices") memnum := 0 diff --git a/pkg/device/devices.go b/pkg/device/devices.go index cb3f0ecc3..d31359e30 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -375,7 +375,7 @@ func DecodePodDevices(checklist map[string]string, annos map[string]string) (Pod pd[devID] = append(pd[devID], cd) } } - klog.InfoS("Decoded pod annos", "poddevices", pd) + klog.V(5).InfoS("Decoded pod annos", "poddevices", pd) return pd, nil } diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index 0ff7f2356..fe0973af3 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -181,7 +181,6 @@ func (dev *KunlunVDevices) CheckUUID(annos map[string]string, d device.DeviceUsa } func (dev *KunlunVDevices) GenerateResourceRequests(ctr *corev1.Container) device.ContainerDeviceRequest { - klog.Infof("Counting %s devices", dev.CommonWord()) xpuResourceCount := corev1.ResourceName(KunlunResourceVCount) xpuResourceMem := corev1.ResourceName(KunlunResourceVMemory) v, ok := ctr.Resources.Limits[xpuResourceCount] @@ -189,6 +188,7 @@ func (dev *KunlunVDevices) GenerateResourceRequests(ctr *corev1.Container) devic v, ok = ctr.Resources.Requests[xpuResourceCount] } if ok { + klog.V(3).Infof("Counting %s devices", dev.CommonWord()) if n, ok := v.AsInt64(); ok { memnum := 0 mem, ok := ctr.Resources.Limits[xpuResourceMem] diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index f50e48b6d..89d680a9a 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -275,7 +275,7 @@ func (dev *NvidiaGPUDevices) GetNodeDevices(n corev1.Node) ([]*device.DeviceInfo pairScores, ok := n.Annotations[RegisterGPUPairScore] if !ok { - klog.Warning("no topology score found", "node", n.Name) + klog.V(5).InfoS("no topology score found", "node", n.Name) } else { devicePairScores, err := device.DecodePairScores(pairScores) if err != nil { diff --git a/pkg/scheduler/pods.go b/pkg/scheduler/pods.go index 8e44587ba..5fc72959e 100644 --- a/pkg/scheduler/pods.go +++ b/pkg/scheduler/pods.go @@ -76,7 +76,7 @@ func (m *podManager) addPod(pod *corev1.Pod, nodeID string, devices device.PodDe ) } else { m.pods[pod.UID].Devices = devices - klog.InfoS("Pod devices updated", + klog.V(5).InfoS("Pod devices updated", "pod", klog.KRef(pod.Namespace, pod.Name), "devices", devices, ) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 7da8112c5..a2f1b718f 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -166,7 +166,6 @@ func (s *Scheduler) Start() { }) informerFactory.Core().V1().Nodes().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(_ any) { s.doNodeNotify() }, - UpdateFunc: func(_, _ any) { s.doNodeNotify() }, DeleteFunc: func(_ any) { s.doNodeNotify() }, }) informerFactory.Core().V1().ResourceQuotas().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -198,7 +197,7 @@ func (s *Scheduler) RegisterFromNodeAnnotations() { case <-s.nodeNotify: klog.V(5).InfoS("Received node notification") case <-ticker.C: - klog.InfoS("Ticker triggered") + klog.V(5).InfoS("Ticker triggered") case <-s.stopCh: klog.InfoS("Received stop signal, exiting RegisterFromNodeAnnotations") return @@ -244,7 +243,7 @@ func (s *Scheduler) RegisterFromNodeAnnotations() { if ok { tmppat := make(map[string]string) tmppat[util.HandshakeAnnos[devhandsk]] = "Requesting_" + time.Now().Format(time.DateTime) - klog.InfoS("New timestamp for annotation", "nodeName", val.Name, "annotationKey", util.HandshakeAnnos[devhandsk], "annotationValue", tmppat[util.HandshakeAnnos[devhandsk]]) + klog.V(5).InfoS("New timestamp for annotation", "nodeName", val.Name, "annotationKey", util.HandshakeAnnos[devhandsk], "annotationValue", tmppat[util.HandshakeAnnos[devhandsk]]) n, err := util.GetNode(val.Name) if err != nil { klog.ErrorS(err, "Failed to get node", "nodeName", val.Name) diff --git a/pkg/util/util.go b/pkg/util/util.go index be174e19b..83e6a2eee 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -48,7 +48,7 @@ func GetNode(nodename string) (*corev1.Node, error) { return nil, fmt.Errorf("nodename is empty") } - klog.InfoS("Fetching node", "nodeName", nodename) + klog.V(5).InfoS("Fetching node", "nodeName", nodename) n, err := client.GetClient().CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{}) if err != nil { switch { @@ -64,7 +64,7 @@ func GetNode(nodename string) (*corev1.Node, error) { } } - klog.InfoS("Successfully fetched node", "nodeName", nodename) + klog.V(5).InfoS("Successfully fetched node", "nodeName", nodename) return n, nil } From 23af77a23cd9761951c7fc1242292c8658acb472 Mon Sep 17 00:00:00 2001 From: limengxuan Date: Thu, 25 Sep 2025 15:04:47 +0800 Subject: [PATCH 4/7] update Signed-off-by: limengxuan --- pkg/device/ascend/device.go | 9 ++-- pkg/device/awsneuron/device.go | 5 +- pkg/device/cambricon/device.go | 7 ++- pkg/device/devices.go | 2 +- pkg/device/enflame/device.go | 21 ++++---- pkg/device/enflame/device_test.go | 74 ++++++++++++++--------------- pkg/device/enflame/gcu.go | 7 ++- pkg/device/hygon/device.go | 9 ++-- pkg/device/iluvatar/device.go | 7 ++- pkg/device/kunlun/device.go | 5 +- pkg/device/kunlun/vdevice.go | 9 ++-- pkg/device/metax/device.go | 7 ++- pkg/device/metax/sdevice.go | 7 ++- pkg/device/mthreads/device.go | 7 ++- pkg/device/nvidia/device.go | 9 ++-- pkg/scheduler/config/config.go | 4 +- pkg/scheduler/config/config_test.go | 2 +- 17 files changed, 115 insertions(+), 76 deletions(-) diff --git a/pkg/device/ascend/device.go b/pkg/device/ascend/device.go index 3c10243fd..882073c5e 100644 --- a/pkg/device/ascend/device.go +++ b/pkg/device/ascend/device.go @@ -88,9 +88,12 @@ func InitDevices(config []VNPUConfig) []*Devices { sort.Slice(dev.config.Templates, func(i, j int) bool { return dev.config.Templates[i].Memory < dev.config.Templates[j].Memory }) - device.InRequestDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-to-allocate", commonWord) - device.SupportDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-allocated", commonWord) - util.HandshakeAnnos[commonWord] = dev.handshakeAnno + _, ok := device.InRequestDevices[commonWord] + if !ok { + device.InRequestDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-to-allocate", commonWord) + device.SupportDevices[commonWord] = fmt.Sprintf("hami.io/%s-devices-allocated", commonWord) + util.HandshakeAnnos[commonWord] = dev.handshakeAnno + } devs = append(devs, dev) klog.Infof("load ascend vnpu config %s: %v", commonWord, dev.config) } diff --git a/pkg/device/awsneuron/device.go b/pkg/device/awsneuron/device.go index 60250cf97..bec641a28 100644 --- a/pkg/device/awsneuron/device.go +++ b/pkg/device/awsneuron/device.go @@ -61,7 +61,10 @@ type AWSNeuronConfig struct { } func InitAWSNeuronDevice(config AWSNeuronConfig) *AWSNeuronDevices { - device.SupportDevices[AWSNeuronDevice] = "hami.io/aws-neuron-devices-allocated" + _, ok := device.SupportDevices[AWSNeuronDevice] + if !ok { + device.SupportDevices[AWSNeuronDevice] = "hami.io/aws-neuron-devices-allocated" + } return &AWSNeuronDevices{ resourceCountName: config.ResourceCountName, resourceCoreName: config.ResourceCoreName, diff --git a/pkg/device/cambricon/device.go b/pkg/device/cambricon/device.go index 2cd809175..9f384f330 100644 --- a/pkg/device/cambricon/device.go +++ b/pkg/device/cambricon/device.go @@ -80,8 +80,11 @@ func InitMLUDevice(config CambriconConfig) *CambriconDevices { MLUResourceCount = config.ResourceCountName MLUResourceMemory = config.ResourceMemoryName MLUResourceCores = config.ResourceCoreName - device.InRequestDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-to-allocate" - device.SupportDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-allocated" + _, ok := device.InRequestDevices[CambriconMLUDevice] + if !ok { + device.InRequestDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-to-allocate" + device.SupportDevices[CambriconMLUDevice] = "hami.io/cambricon-mlu-devices-allocated" + } return &CambriconDevices{} } diff --git a/pkg/device/devices.go b/pkg/device/devices.go index d31359e30..4ac75deda 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -353,7 +353,7 @@ func DecodeContainerDevices(str string) (ContainerDevices, error) { } func DecodePodDevices(checklist map[string]string, annos map[string]string) (PodDevices, error) { - klog.V(5).Infof("checklist is [%+v], annos is [%+v]", checklist, annos) + klog.Infof("=-=-=-=-=---=-=-=-=checklist is [%+v], annos is [%+v]", checklist, annos) if len(annos) == 0 { return PodDevices{}, nil } diff --git a/pkg/device/enflame/device.go b/pkg/device/enflame/device.go index 943cf5639..43e20de87 100644 --- a/pkg/device/enflame/device.go +++ b/pkg/device/enflame/device.go @@ -35,8 +35,8 @@ type EnflameDevices struct { } const ( - EnflameGPUDevice = "Enflame" - EnflameGPUCommonWord = "Enflame" + EnflameVGCUDevice = "Enflame" + EnflameVGCUCommonWord = "Enflame" // IluvatarUseUUID is user can use specify Iluvatar device for set Iluvatar UUID. EnflameUseUUID = "enflame.com/use-gpuuuid" // IluvatarNoUseUUID is user can not use specify Iluvatar device for set Iluvatar UUID. @@ -54,14 +54,17 @@ const ( func InitEnflameDevice(config EnflameConfig) *EnflameDevices { EnflameResourceNameVGCU = config.ResourceNameVGCU EnflameResourceNameVGCUPercentage = config.ResourceNameVGCUPercentage - device.SupportDevices[EnflameGPUDevice] = "hami.io/enflame-vgpu-devices-allocated" + _, ok := device.SupportDevices[EnflameVGCUDevice] + if !ok { + device.SupportDevices[EnflameVGCUDevice] = "hami.io/enflame-vgpu-devices-allocated" + } return &EnflameDevices{ factor: 0, } } func (dev *EnflameDevices) CommonWord() string { - return EnflameGPUCommonWord + return EnflameVGCUCommonWord } func (dev *EnflameDevices) MutateAdmission(ctr *corev1.Container, p *corev1.Pod) (bool, error) { @@ -111,7 +114,7 @@ func (dev *EnflameDevices) GetNodeDevices(n corev1.Node) ([]*device.DeviceInfo, Count: 100, Devmem: 100, Devcore: 100, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Numa: 0, Health: true, }) @@ -121,9 +124,9 @@ func (dev *EnflameDevices) GetNodeDevices(n corev1.Node) ([]*device.DeviceInfo, } func (dev *EnflameDevices) PatchAnnotations(pod *corev1.Pod, annoinput *map[string]string, pd device.PodDevices) map[string]string { - devlist, ok := pd[EnflameGPUDevice] + devlist, ok := pd[EnflameVGCUDevice] if ok && len(devlist) > 0 { - (*annoinput)[device.SupportDevices[EnflameGPUDevice]] = device.EncodePodSingleDevice(devlist) + (*annoinput)[device.SupportDevices[EnflameVGCUDevice]] = device.EncodePodSingleDevice(devlist) (*annoinput)[PodHasAssignedGCU] = "false" (*annoinput)[PodAssignedGCUTime] = strconv.FormatInt(time.Now().UnixNano(), 10) annoKey := PodAssignedGCUID @@ -151,7 +154,7 @@ func (dev *EnflameDevices) NodeCleanUp(nn string) error { } func (dev *EnflameDevices) checkType(annos map[string]string, d device.DeviceUsage, n device.ContainerDeviceRequest) (bool, bool, bool) { - if strings.Compare(n.Type, EnflameGPUDevice) == 0 { + if strings.Compare(n.Type, EnflameVGCUDevice) == 0 { return true, true, false } return false, false, false @@ -209,7 +212,7 @@ func (dev *EnflameDevices) GenerateResourceRequests(ctr *corev1.Container) devic } return device.ContainerDeviceRequest{ Nums: int32(n), - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Memreq: int32(memnum), MemPercentagereq: 0, Coresreq: 0, diff --git a/pkg/device/enflame/device_test.go b/pkg/device/enflame/device_test.go index 66265207e..1d9ec57f3 100644 --- a/pkg/device/enflame/device_test.go +++ b/pkg/device/enflame/device_test.go @@ -58,7 +58,7 @@ func TestGetNodeDevices(t *testing.T) { Count: 100, Devmem: 100, Devcore: 100, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Numa: 0, Health: true, }, @@ -118,7 +118,7 @@ func TestPatchAnnotations(t *testing.T) { name: "With devices", annoInput: map[string]string{}, podDevices: device.PodDevices{ - EnflameGPUDevice: device.PodSingleDevice{ + EnflameVGCUDevice: device.PodSingleDevice{ []device.ContainerDevice{ { Idx: 0, @@ -129,10 +129,10 @@ func TestPatchAnnotations(t *testing.T) { }, }, expected: map[string]string{ - device.SupportDevices[EnflameGPUDevice]: "k8s-gpu-enflame-0,Enflame,0,0:;", - PodHasAssignedGCU: "false", - PodAssignedGCUTime: strconv.FormatInt(time.Now().UnixNano(), 10), - PodAssignedGCUID: "0", + device.SupportDevices[EnflameVGCUDevice]: "k8s-gpu-enflame-0,Enflame,0,0:;", + PodHasAssignedGCU: "false", + PodAssignedGCUTime: strconv.FormatInt(time.Now().UnixNano(), 10), + PodAssignedGCUID: "0", }, }, } @@ -239,7 +239,7 @@ func Test_checkType(t *testing.T) { annos: map[string]string{}, d: device.DeviceUsage{}, n: device.ContainerDeviceRequest{ - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, }, want1: true, @@ -389,7 +389,7 @@ func Test_GenerateResourceRequests(t *testing.T) { }, want: device.ContainerDeviceRequest{ Nums: int32(1), - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Memreq: int32(15), MemPercentagereq: int32(0), Coresreq: int32(0), @@ -419,7 +419,7 @@ func Test_GenerateResourceRequests(t *testing.T) { }, want: device.ContainerDeviceRequest{ Nums: int32(1), - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Memreq: int32(100), MemPercentagereq: int32(0), Coresreq: int32(0), @@ -467,7 +467,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }, { @@ -480,7 +480,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }, }, @@ -489,7 +489,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 64, MemPercentagereq: 0, Coresreq: 50, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: true, @@ -509,7 +509,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -517,7 +517,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 50, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: false, @@ -537,7 +537,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 100, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -545,7 +545,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 50, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: false, @@ -566,7 +566,7 @@ func TestDevices_Fit(t *testing.T) { Usedcores: 0, Numa: 0, Health: true, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }}, request: device.ContainerDeviceRequest{ Nums: 1, @@ -593,7 +593,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -601,7 +601,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 50, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{EnflameUseUUID: "dev-0"}, wantFit: false, @@ -621,7 +621,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -629,7 +629,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 50, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{EnflameNoUseUUID: "dev-0"}, wantFit: false, @@ -649,7 +649,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -657,7 +657,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 50, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: false, @@ -677,7 +677,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -685,7 +685,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 120, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: true, @@ -705,7 +705,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 0, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -713,7 +713,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 100, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: false, @@ -733,7 +733,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 100, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -741,7 +741,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: false, @@ -761,7 +761,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 10, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -769,7 +769,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 512, MemPercentagereq: 0, Coresreq: 20, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: false, @@ -789,7 +789,7 @@ func TestDevices_Fit(t *testing.T) { Totalcore: 100, Usedcores: 10, Numa: 0, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, Health: true, }}, request: device.ContainerDeviceRequest{ @@ -797,7 +797,7 @@ func TestDevices_Fit(t *testing.T) { Memreq: 0, MemPercentagereq: 10, Coresreq: 20, - Type: EnflameGPUDevice, + Type: EnflameVGCUDevice, }, annos: map[string]string{}, wantFit: true, @@ -820,12 +820,12 @@ func TestDevices_Fit(t *testing.T) { t.Errorf("Fit: got %v, want %v", fit, test.wantFit) } if test.wantFit { - if len(result[EnflameGPUDevice]) != test.wantLen { - t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameGPUDevice])) + if len(result[EnflameVGCUDevice]) != test.wantLen { + t.Errorf("expected len: %d, got len %d", test.wantLen, len(result[EnflameVGCUDevice])) } for idx, id := range test.wantDevIDs { - if id != result[EnflameGPUDevice][idx].UUID { - t.Errorf("expected device id: %s, got device id %s", id, result[EnflameGPUDevice][idx].UUID) + if id != result[EnflameVGCUDevice][idx].UUID { + t.Errorf("expected device id: %s, got device id %s", id, result[EnflameVGCUDevice][idx].UUID) } } } diff --git a/pkg/device/enflame/gcu.go b/pkg/device/enflame/gcu.go index eecbcd22a..5d8b6ede4 100644 --- a/pkg/device/enflame/gcu.go +++ b/pkg/device/enflame/gcu.go @@ -37,8 +37,11 @@ type GCUDevices struct { func InitGCUDevice(config EnflameConfig) *GCUDevices { EnflameResourceNameGCU = config.ResourceNameGCU - device.InRequestDevices[EnflameGCUDevice] = "hami.io/enflame-gcu-devices-to-allocate" - device.SupportDevices[EnflameGCUDevice] = "hami.io/enflame-gcu-devices-allocated" + _, ok := device.InRequestDevices[EnflameGCUDevice] + if !ok { + device.InRequestDevices[EnflameGCUDevice] = "hami.io/enflame-gcu-devices-to-allocate" + device.SupportDevices[EnflameGCUDevice] = "hami.io/enflame-gcu-devices-allocated" + } return &GCUDevices{} } diff --git a/pkg/device/hygon/device.go b/pkg/device/hygon/device.go index 94d8bbf86..09bcb3c94 100644 --- a/pkg/device/hygon/device.go +++ b/pkg/device/hygon/device.go @@ -68,9 +68,12 @@ func InitDCUDevice(config HygonConfig) *DCUDevices { HygonResourceCount = config.ResourceCountName HygonResourceMemory = config.ResourceMemoryName HygonResourceCores = config.ResourceCoreName - device.InRequestDevices[HygonDCUDevice] = "hami.io/dcu-devices-to-allocate" - device.SupportDevices[HygonDCUDevice] = "hami.io/dcu-devices-allocated" - util.HandshakeAnnos[HygonDCUDevice] = HandshakeAnnos + _, ok := device.InRequestDevices[HygonDCUDevice] + if !ok { + device.InRequestDevices[HygonDCUDevice] = "hami.io/dcu-devices-to-allocate" + device.SupportDevices[HygonDCUDevice] = "hami.io/dcu-devices-allocated" + util.HandshakeAnnos[HygonDCUDevice] = HandshakeAnnos + } return &DCUDevices{} } diff --git a/pkg/device/iluvatar/device.go b/pkg/device/iluvatar/device.go index fe00a9936..8cd4252f1 100644 --- a/pkg/device/iluvatar/device.go +++ b/pkg/device/iluvatar/device.go @@ -61,8 +61,11 @@ func InitIluvatarDevice(config IluvatarConfig) *IluvatarDevices { IluvatarResourceCount = config.ResourceCountName IluvatarResourceMemory = config.ResourceMemoryName IluvatarResourceCores = config.ResourceCoreName - device.InRequestDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-to-allocate" - device.SupportDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-allocated" + _, ok := device.InRequestDevices[IluvatarGPUDevice] + if !ok { + device.InRequestDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-to-allocate" + device.SupportDevices[IluvatarGPUDevice] = "hami.io/iluvatar-vgpu-devices-allocated" + } return &IluvatarDevices{} } diff --git a/pkg/device/kunlun/device.go b/pkg/device/kunlun/device.go index 66eb08e92..1144ec803 100644 --- a/pkg/device/kunlun/device.go +++ b/pkg/device/kunlun/device.go @@ -45,7 +45,10 @@ type KunlunDevices struct { func InitKunlunDevice(config KunlunConfig) *KunlunDevices { KunlunResourceCount = config.ResourceCountName - device.SupportDevices[KunlunGPUDevice] = "hami.io/kunlun-allocated" + _, ok := device.SupportDevices[KunlunGPUDevice] + if !ok { + device.SupportDevices[KunlunGPUDevice] = "hami.io/kunlun-allocated" + } return &KunlunDevices{} } diff --git a/pkg/device/kunlun/vdevice.go b/pkg/device/kunlun/vdevice.go index fe0973af3..0ecd25977 100644 --- a/pkg/device/kunlun/vdevice.go +++ b/pkg/device/kunlun/vdevice.go @@ -53,9 +53,12 @@ type KunlunVDevices struct { func InitKunlunVDevice(config KunlunConfig) *KunlunVDevices { KunlunResourceVCount = config.ResourceVCountName KunlunResourceVMemory = config.ResourceVMemoryName - device.InRequestDevices[XPUDevice] = "hami.io/xpu-devices-to-allocate" - device.SupportDevices[XPUDevice] = "hami.io/xpu-devices-allocated" - util.HandshakeAnnos[XPUDevice] = HandshakeAnnos + _, ok := device.InRequestDevices[XPUDevice] + if !ok { + device.InRequestDevices[XPUDevice] = "hami.io/xpu-devices-to-allocate" + device.SupportDevices[XPUDevice] = "hami.io/xpu-devices-allocated" + util.HandshakeAnnos[XPUDevice] = HandshakeAnnos + } return &KunlunVDevices{} } diff --git a/pkg/device/metax/device.go b/pkg/device/metax/device.go index 1ad45198e..bdf8a4071 100644 --- a/pkg/device/metax/device.go +++ b/pkg/device/metax/device.go @@ -46,8 +46,11 @@ var ( func InitMetaxDevice(config MetaxConfig) *MetaxDevices { MetaxResourceCount = config.ResourceCountName - device.InRequestDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-to-allocate" - device.SupportDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-allocated" + _, ok := device.InRequestDevices[MetaxGPUDevice] + if !ok { + device.InRequestDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-to-allocate" + device.SupportDevices[MetaxGPUDevice] = "hami.io/metax-gpu-devices-allocated" + } return &MetaxDevices{} } diff --git a/pkg/device/metax/sdevice.go b/pkg/device/metax/sdevice.go index 94ffb25e7..4cdbc65d7 100644 --- a/pkg/device/metax/sdevice.go +++ b/pkg/device/metax/sdevice.go @@ -60,8 +60,11 @@ func InitMetaxSDevice(config MetaxConfig) *MetaxSDevices { MetaxResourceNameVMemory = config.ResourceVMemoryName MetaxTopologyAware = config.TopologyAware - device.InRequestDevices[MetaxSGPUDevice] = "hami.io/metax-sgpu-devices-to-allocate" - device.SupportDevices[MetaxSGPUDevice] = "hami.io/metax-sgpu-devices-allocated" + _, ok := device.InRequestDevices[MetaxSGPUDevice] + if !ok { + device.InRequestDevices[MetaxSGPUDevice] = "hami.io/metax-sgpu-devices-to-allocate" + device.SupportDevices[MetaxSGPUDevice] = "hami.io/metax-sgpu-devices-allocated" + } return &MetaxSDevices{ jqCache: NewJitteryQosCache(), diff --git a/pkg/device/mthreads/device.go b/pkg/device/mthreads/device.go index 390665766..4adb3ae46 100644 --- a/pkg/device/mthreads/device.go +++ b/pkg/device/mthreads/device.go @@ -69,8 +69,11 @@ func InitMthreadsDevice(config MthreadsConfig) *MthreadsDevices { MthreadsResourceCount = config.ResourceCountName MthreadsResourceCores = config.ResourceCoreName MthreadsResourceMemory = config.ResourceMemoryName - device.InRequestDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-to-allocate" - device.SupportDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-allocated" + _, ok := device.InRequestDevices[MthreadsGPUDevice] + if !ok { + device.InRequestDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-to-allocate" + device.SupportDevices[MthreadsGPUDevice] = "hami.io/mthreads-vgpu-devices-allocated" + } return &MthreadsDevices{} } diff --git a/pkg/device/nvidia/device.go b/pkg/device/nvidia/device.go index 89d680a9a..5df4d0b91 100644 --- a/pkg/device/nvidia/device.go +++ b/pkg/device/nvidia/device.go @@ -160,9 +160,12 @@ type NvidiaGPUDevices struct { func InitNvidiaDevice(nvconfig NvidiaConfig) *NvidiaGPUDevices { klog.InfoS("initializing nvidia device", "resourceName", nvconfig.ResourceCountName, "resourceMem", nvconfig.ResourceMemoryName, "DefaultGPUNum", nvconfig.DefaultGPUNum) - device.InRequestDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-to-allocate" - device.SupportDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-allocated" - util.HandshakeAnnos[NvidiaGPUDevice] = HandshakeAnnos + _, ok := device.InRequestDevices[NvidiaGPUDevice] + if !ok { + device.InRequestDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-to-allocate" + device.SupportDevices[NvidiaGPUDevice] = "hami.io/vgpu-devices-allocated" + util.HandshakeAnnos[NvidiaGPUDevice] = HandshakeAnnos + } return &NvidiaGPUDevices{ config: nvconfig, } diff --git a/pkg/scheduler/config/config.go b/pkg/scheduler/config/config.go index 3249db8aa..ce14c3a64 100644 --- a/pkg/scheduler/config/config.go +++ b/pkg/scheduler/config/config.go @@ -153,10 +153,10 @@ func InitDevicesWithConfig(config *Config) error { } return enflame.InitGCUDevice(enflameConfig), nil }, config.EnflameConfig}, - {enflame.EnflameGPUDevice, enflame.EnflameGPUCommonWord, func(cfg any) (device.Devices, error) { + {enflame.EnflameVGCUDevice, enflame.EnflameVGCUCommonWord, func(cfg any) (device.Devices, error) { enflameConfig, ok := cfg.(enflame.EnflameConfig) if !ok { - return nil, fmt.Errorf("invalid configuration for %s", enflame.EnflameGPUCommonWord) + return nil, fmt.Errorf("invalid configuration for %s", enflame.EnflameVGCUCommonWord) } return enflame.InitEnflameDevice(enflameConfig), nil }, config.EnflameConfig}, diff --git a/pkg/scheduler/config/config_test.go b/pkg/scheduler/config/config_test.go index 7e094071b..5ec4e9d33 100644 --- a/pkg/scheduler/config/config_test.go +++ b/pkg/scheduler/config/config_test.go @@ -370,7 +370,7 @@ func setupTest(t *testing.T) (map[string]string, map[string]device.Devices) { metax.MetaxGPUDevice: metax.MetaxGPUCommonWord, metax.MetaxSGPUDevice: metax.MetaxSGPUCommonWord, enflame.EnflameGCUDevice: enflame.EnflameGCUCommonWord, - enflame.EnflameGPUDevice: enflame.EnflameGPUCommonWord, + enflame.EnflameVGCUDevice: enflame.EnflameVGCUCommonWord, kunlun.KunlunGPUDevice: kunlun.KunlunGPUCommonWord, kunlun.XPUDevice: kunlun.XPUCommonWord, awsneuron.AWSNeuronDevice: awsneuron.AWSNeuronCommonWord, From d2957d4a9ced71ab09f03b70296f9cb6d09cc923 Mon Sep 17 00:00:00 2001 From: Jifei Wang Date: Thu, 25 Sep 2025 15:40:34 +0800 Subject: [PATCH 5/7] add quota metrics Signed-off-by: Jifei Wang --- cmd/scheduler/metrics.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index 83b7f396a..14878f850 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -194,6 +194,22 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) { "vGPU core allocated from a container", []string{"podnamespace", "nodename", "podname", "containeridx", "deviceuuid"}, nil, ) + quotaUsedDesc := prometheus.NewDesc( + "QuotaUsed", + "resourcequota usage for a certain device", + []string{"quotanamespace", "quotaName", "limit"}, nil, + ) + quotas := sher.GetQuotas() + for ns, val := range quotas.Quotas { + for quotaname, q := range *val { + ch <- prometheus.MustNewConstMetric( + quotaUsedDesc, + prometheus.GaugeValue, + float64(q.Used), + ns, quotaname, fmt.Sprint(q.Limit), + ) + } + } schedpods, _ := sher.GetScheduledPods() for _, val := range schedpods { for _, podSingleDevice := range val.Devices { From 895b9322466355843506730a02afdcf6dbee7eb6 Mon Sep 17 00:00:00 2001 From: Jifei Wang Date: Thu, 25 Sep 2025 19:03:38 +0800 Subject: [PATCH 6/7] fix fail update quota Signed-off-by: Jifei Wang --- pkg/scheduler/scheduler.go | 4 +++- pkg/scheduler/scheduler_test.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index a2f1b718f..3792b6b51 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -537,7 +537,9 @@ func (s *Scheduler) Filter(args extenderv1.ExtenderArgs) (*extenderv1.ExtenderFi val.PatchAnnotations(args.Pod, &annotations, m.Devices) } - s.addPod(args.Pod, m.NodeID, m.Devices) + if s.addPod(args.Pod, m.NodeID, m.Devices) { + s.quotaManager.AddUsage(args.Pod, m.Devices) + } err = util.PatchPodAnnotations(args.Pod, annotations) if err != nil { s.recordScheduleFilterResultEvent(args.Pod, EventReasonFilteringFailed, "", err) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 5d4f87be8..6334c2804 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1125,6 +1125,8 @@ func Test_ResourceQuota(t *testing.T) { initNode() client.KubeClient.CoreV1().Pods(test.args.Pod.Namespace).Create(context.Background(), test.args.Pod, metav1.CreateOptions{}) got, gotErr := s.Filter(test.args) + client.KubeClient.CoreV1().Pods(test.args.Pod.Namespace).Delete(context.Background(), test.args.Pod.Name, metav1.DeleteOptions{}) + s.onDelQuota(&test.quota) assert.DeepEqual(t, test.wantErr, gotErr) assert.DeepEqual(t, test.want, got) }) From 16f3b411463808d91dea4ad8e1c20c4eb121d9fc Mon Sep 17 00:00:00 2001 From: Jifei Wang Date: Fri, 26 Sep 2025 10:43:25 +0800 Subject: [PATCH 7/7] resolve comment Signed-off-by: Jifei Wang --- pkg/device/devices.go | 2 +- pkg/device/quota.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/device/devices.go b/pkg/device/devices.go index 4ac75deda..d31359e30 100644 --- a/pkg/device/devices.go +++ b/pkg/device/devices.go @@ -353,7 +353,7 @@ func DecodeContainerDevices(str string) (ContainerDevices, error) { } func DecodePodDevices(checklist map[string]string, annos map[string]string) (PodDevices, error) { - klog.Infof("=-=-=-=-=---=-=-=-=checklist is [%+v], annos is [%+v]", checklist, annos) + klog.V(5).Infof("checklist is [%+v], annos is [%+v]", checklist, annos) if len(annos) == 0 { return PodDevices{}, nil } diff --git a/pkg/device/quota.go b/pkg/device/quota.go index 99a923c56..722a12e17 100644 --- a/pkg/device/quota.go +++ b/pkg/device/quota.go @@ -86,9 +86,6 @@ func (q *QuotaManager) FitQuota(ns string, memreq int64, coresreq int64, deviceN func countPodDevices(podDev PodDevices) map[string]int64 { res := make(map[string]int64) for deviceName, podSingle := range podDev { - if !strings.Contains(deviceName, "NVIDIA") { - continue - } devs, ok := GetDevices()[deviceName] if !ok { continue