diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 0b40366a8665..c751f81f9273 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -72,7 +72,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findSystemVMTemplate(long zoneId); - VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); + VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch); List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, String preferredArch); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 12c00a3209ae..b6796cf8f9d1 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -578,11 +579,19 @@ public List listAllReadySystemVMTemplates(Long zoneId) { } @Override - public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) { + public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch) { List templates = listAllReadySystemVMTemplates(zoneId); if (CollectionUtils.isEmpty(templates)) { return null; } + if (StringUtils.isNotBlank(preferredArch)) { + // Sort the templates by preferred architecture first + templates = templates.stream() + .sorted(Comparator.comparing( + x -> !x.getArch().getType().equalsIgnoreCase(preferredArch) + )) + .collect(Collectors.toList()); + } if (hypervisorType == HypervisorType.Any) { return templates.get(0); } diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java index df0b36ebdbf6..e97a887cc471 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -186,4 +187,24 @@ public void testFindLatestTemplateByTypeAndHypervisorAndArch_NotFound() { VMTemplateVO result = templateDao.findLatestTemplateByTypeAndHypervisorAndArch(hypervisorType, arch, type); assertNull(result); } + + @Test + public void testFindSystemVMReadyTemplate() { + Long zoneId = 1L; + VMTemplateVO systemVmTemplate1 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate1.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate2 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate2.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate3 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate3.getArch()).thenReturn(CPU.CPUArch.arm64); + Mockito.when(systemVmTemplate3.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + List templates = Arrays.asList(systemVmTemplate1, systemVmTemplate2, systemVmTemplate3); + Mockito.when(hostDao.listDistinctHypervisorTypes(zoneId)).thenReturn(Arrays.asList(Hypervisor.HypervisorType.KVM)); + SearchBuilder sb = mock(SearchBuilder.class); + templateDao.readySystemTemplateSearch = sb; + when(sb.create()).thenReturn(mock(SearchCriteria.class)); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + VMTemplateVO readyTemplate = templateDao.findSystemVMReadyTemplate(zoneId, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType()); + Assert.assertEquals(CPU.CPUArch.arm64, readyTemplate.getArch()); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 411e6af883e3..d1babb547f85 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -433,8 +433,14 @@ private IpAddress getSourceNatIp(Network network) { return null; } - public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) { - VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType); + public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType, + KubernetesSupportedVersion clusterKubernetesVersion) { + String systemVMPreferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(dataCenter.getId()); + VMTemplateVO cksIso = clusterKubernetesVersion != null ? + templateDao.findById(clusterKubernetesVersion.getIsoId()) : + null; + String preferredArchitecture = getCksClusterPreferredArch(systemVMPreferredArchitecture, cksIso); + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType, preferredArchitecture); if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) { logger.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter)); template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId())); @@ -445,6 +451,14 @@ public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervis return template; } + protected String getCksClusterPreferredArch(String systemVMPreferredArchitecture, VMTemplateVO cksIso) { + if (cksIso == null) { + return systemVMPreferredArchitecture; + } + String cksIsoArchName = cksIso.getArch().name(); + return cksIsoArchName.equals(systemVMPreferredArchitecture) ? systemVMPreferredArchitecture : cksIsoArchName; + } + protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) { List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose); for (FirewallRuleVO rule : rules) { @@ -1302,7 +1316,10 @@ public KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterC } final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType()); + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType(), clusterKubernetesVersion); + + compareKubernetesIsoArchToSelectedTemplateArch(clusterKubernetesVersion, finalTemplate); + final long cores = serviceOffering.getCpu() * (controlNodeCount + clusterSize); final long memory = serviceOffering.getRamSize() * (controlNodeCount + clusterSize); @@ -1331,6 +1348,21 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { return cluster; } + private void compareKubernetesIsoArchToSelectedTemplateArch(KubernetesSupportedVersion clusterKubernetesVersion, VMTemplateVO finalTemplate) { + VMTemplateVO cksIso = templateDao.findById(clusterKubernetesVersion.getIsoId()); + if (cksIso == null) { + String err = String.format("Cannot find Kubernetes ISO associated to the Kubernetes version %s (id=%s)", + clusterKubernetesVersion.getName(), clusterKubernetesVersion.getUuid()); + throw new CloudRuntimeException(err); + } + if (!cksIso.getArch().equals(finalTemplate.getArch())) { + String err = String.format("The selected Kubernetes ISO %s arch (%s) doesn't match the template %s arch (%s) " + + "to deploy the Kubernetes cluster", + clusterKubernetesVersion.getName(), cksIso.getArch(), finalTemplate.getName(), finalTemplate.getArch()); + throw new CloudRuntimeException(err); + } + } + private SecurityGroup getOrCreateSecurityGroupForAccount(Account owner) { String securityGroupName = String.format("%s-%s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME, owner.getUuid()); String securityGroupDesc = String.format("%s and account %s", KubernetesClusterActionWorker.CKS_SECURITY_GROUP_DESCRIPTION, owner.getName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 29ecde477a44..99c4d4c9c96c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -29,6 +29,7 @@ import javax.inject.Inject; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -194,7 +195,8 @@ protected void init() { DataCenterVO dataCenterVO = dataCenterDao.findById(zoneId); VMTemplateVO template = templateDao.findById(templateId); Hypervisor.HypervisorType type = template.getHypervisorType(); - this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type); + KubernetesSupportedVersionVO kubernetesSupportedVersion = kubernetesSupportedVersionDao.findById(this.kubernetesCluster.getKubernetesVersionId()); + this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type, kubernetesSupportedVersion); this.sshKeyFile = getManagementServerSshPublicKeyFile(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 8c983149d02d..04c63bb54838 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -248,7 +248,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); HostVO h = hp.first(); - if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType())) { + if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType()) || !h.getArch().equals(clusterTemplate.getArch())) { continue; } hostDao.loadHostTags(h); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index a6d46ffc9aa1..287e045a18e0 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -21,6 +21,7 @@ import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenter; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -292,4 +293,22 @@ public void removeVmsFromCluster() { Mockito.when(kubernetesClusterDao.findById(Mockito.anyLong())).thenReturn(cluster); Assert.assertTrue(kubernetesClusterManager.removeVmsFromCluster(cmd).size() > 0); } + + @Test + public void testGetCksClusterPreferredArchDifferentArchsPreferCKSIsoArch() { + String systemVMArch = "x86_64"; + VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class); + Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.arm64); + String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); + Assert.assertEquals(CPU.CPUArch.arm64.name(), cksClusterPreferredArch); + } + + @Test + public void testGetCksClusterPreferredArchSameArch() { + String systemVMArch = "x86_64"; + VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class); + Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.amd64); + String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); + Assert.assertEquals(CPU.CPUArch.amd64.name(), cksClusterPreferredArch); + } } diff --git a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java index 31159e7d3d95..3f018c1c9b3c 100644 --- a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java +++ b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java @@ -174,10 +174,11 @@ private UserVm deploySharedFSVM(Long zoneId, Account owner, List networkId customParameterMap.put("maxIopsDo", maxIops.toString()); } List keypairs = new ArrayList(); + String preferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(zoneId); for (final Iterator iter = hypervisors.iterator(); iter.hasNext();) { final Hypervisor.HypervisorType hypervisor = iter.next(); - VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor); + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor, preferredArchitecture); if (template == null && !iter.hasNext()) { throw new CloudRuntimeException(String.format("Unable to find the systemvm template for %s or it was not downloaded in %s.", hypervisor.toString(), zone.toString())); } diff --git a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java index 4393b0565f89..1e5c8774b261 100644 --- a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java +++ b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java @@ -236,7 +236,7 @@ private SharedFS prepareDeploySharedFS() throws ResourceUnavailableException, In when(serviceOfferingDao.findById(s_serviceOfferingId)).thenReturn(serviceOffering); VMTemplateVO template = mock(VMTemplateVO.class); - when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(template); + when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM, ResourceManager.SystemVmPreferredArchitecture.defaultValue())).thenReturn(template); when(template.getId()).thenReturn(s_templateId); return sharedFS; @@ -298,7 +298,6 @@ public void testDeploySharedFSTemplateNotFound() throws ResourceUnavailableExcep when(dataCenterDao.findById(s_zoneId)).thenReturn(zone); when(resourceMgr.getSupportedHypervisorTypes(s_zoneId, false, null)).thenReturn(List.of(Hypervisor.HypervisorType.KVM)); - when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(null); lifeCycle.deploySharedFS(sharedFS, s_networkId, s_diskOfferingId, s_size, s_minIops, s_maxIops); }