Skip to content

Find system VM templates for CKS clusters and SharedFS honouring the preferred architecture #10946

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Jul 31, 2025
Merged

Find system VM templates for CKS clusters and SharedFS honouring the preferred architecture #10946

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
81c6f5a
Find system VM templates for CKS cluster honouring the preferred arch…
nvazquez Jun 2, 2025
778ccc1
Fix unit tests
nvazquez Jun 2, 2025
02de12f
Fix checkstyle
nvazquez Jun 2, 2025
70f4e26
Sort instead of filtering by preferred arch
nvazquez Jun 2, 2025
efa3bdc
Remove unnecesary stubs
nvazquez Jun 2, 2025
dabd833
Restore java version
nvazquez Jun 2, 2025
a44dde0
Address review comments
nvazquez Jun 17, 2025
337b0ee
Fail and display error message in case the CKS ISO arch doesnt match …
nvazquez Jun 18, 2025
c3b83a4
Prefer CKS ISO arch instead of the system VM setting
nvazquez Jul 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long>, StateDao<

VMTemplateVO findSystemVMTemplate(long zoneId);

VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType);
VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch);

List<VMTemplateVO> findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, String preferredArch);

Expand Down
11 changes: 10 additions & 1 deletion engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.naming.ConfigurationException;
Expand Down Expand Up @@ -578,11 +579,19 @@ public List<VMTemplateVO> listAllReadySystemVMTemplates(Long zoneId) {
}

@Override
public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) {
public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch) {
List<VMTemplateVO> templates = listAllReadySystemVMTemplates(zoneId);
if (CollectionUtils.isEmpty(templates)) {
return null;
}
if (StringUtils.isNotBlank(preferredArch)) {
// Sort the templates by preferred architecture first
templates = templates.stream()
.sorted(Comparator.comparing(
x -> !x.getArch().getType().equalsIgnoreCase(preferredArch)
))
.collect(Collectors.toList());
}
if (hypervisorType == HypervisorType.Any) {
return templates.get(0);
}
Expand Down
21 changes: 21 additions & 0 deletions engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.util.List;
import java.util.Map;

import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
Expand Down Expand Up @@ -186,4 +187,24 @@ public void testFindLatestTemplateByTypeAndHypervisorAndArch_NotFound() {
VMTemplateVO result = templateDao.findLatestTemplateByTypeAndHypervisorAndArch(hypervisorType, arch, type);
assertNull(result);
}

@Test
public void testFindSystemVMReadyTemplate() {
Long zoneId = 1L;
VMTemplateVO systemVmTemplate1 = mock(VMTemplateVO.class);
Mockito.when(systemVmTemplate1.getArch()).thenReturn(CPU.CPUArch.x86);
VMTemplateVO systemVmTemplate2 = mock(VMTemplateVO.class);
Mockito.when(systemVmTemplate2.getArch()).thenReturn(CPU.CPUArch.x86);
VMTemplateVO systemVmTemplate3 = mock(VMTemplateVO.class);
Mockito.when(systemVmTemplate3.getArch()).thenReturn(CPU.CPUArch.arm64);
Mockito.when(systemVmTemplate3.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
List<VMTemplateVO> templates = Arrays.asList(systemVmTemplate1, systemVmTemplate2, systemVmTemplate3);
Mockito.when(hostDao.listDistinctHypervisorTypes(zoneId)).thenReturn(Arrays.asList(Hypervisor.HypervisorType.KVM));
SearchBuilder<VMTemplateVO> sb = mock(SearchBuilder.class);
templateDao.readySystemTemplateSearch = sb;
when(sb.create()).thenReturn(mock(SearchCriteria.class));
doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class));
VMTemplateVO readyTemplate = templateDao.findSystemVMReadyTemplate(zoneId, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType());
Assert.assertEquals(CPU.CPUArch.arm64, readyTemplate.getArch());
}
}
38 changes: 35 additions & 3 deletions ...etes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,8 +433,14 @@ private IpAddress getSourceNatIp(Network network) {
return null;
}

public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) {
VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType);
public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType,
KubernetesSupportedVersion clusterKubernetesVersion) {
String systemVMPreferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(dataCenter.getId());
VMTemplateVO cksIso = clusterKubernetesVersion != null ?
templateDao.findById(clusterKubernetesVersion.getIsoId()) :
null;
String preferredArchitecture = getCksClusterPreferredArch(systemVMPreferredArchitecture, cksIso);
VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType, preferredArchitecture);
if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) {
logger.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter));
template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId()));
Expand All @@ -445,6 +451,14 @@ public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervis
return template;
}

protected String getCksClusterPreferredArch(String systemVMPreferredArchitecture, VMTemplateVO cksIso) {
if (cksIso == null) {
return systemVMPreferredArchitecture;
}
String cksIsoArchName = cksIso.getArch().name();
return cksIsoArchName.equals(systemVMPreferredArchitecture) ? systemVMPreferredArchitecture : cksIsoArchName;
}

protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) {
List<FirewallRuleVO> rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose);
for (FirewallRuleVO rule : rules) {
Expand Down Expand Up @@ -1302,7 +1316,10 @@ public KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterC
}

final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId());
final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType());
final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType(), clusterKubernetesVersion);

compareKubernetesIsoArchToSelectedTemplateArch(clusterKubernetesVersion, finalTemplate);

final long cores = serviceOffering.getCpu() * (controlNodeCount + clusterSize);
final long memory = serviceOffering.getRamSize() * (controlNodeCount + clusterSize);

Expand Down Expand Up @@ -1331,6 +1348,21 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) {
return cluster;
}

private void compareKubernetesIsoArchToSelectedTemplateArch(KubernetesSupportedVersion clusterKubernetesVersion, VMTemplateVO finalTemplate) {
VMTemplateVO cksIso = templateDao.findById(clusterKubernetesVersion.getIsoId());
if (cksIso == null) {
String err = String.format("Cannot find Kubernetes ISO associated to the Kubernetes version %s (id=%s)",
clusterKubernetesVersion.getName(), clusterKubernetesVersion.getUuid());
throw new CloudRuntimeException(err);
}
if (!cksIso.getArch().equals(finalTemplate.getArch())) {
String err = String.format("The selected Kubernetes ISO %s arch (%s) doesn't match the template %s arch (%s) " +
"to deploy the Kubernetes cluster",
clusterKubernetesVersion.getName(), cksIso.getArch(), finalTemplate.getName(), finalTemplate.getArch());
throw new CloudRuntimeException(err);
Copy link
Member

@weizhouapache weizhouapache Jul 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we consider the arch of cks iso before preferred arch of zone ?

Copy link
Contributor Author

@nvazquez nvazquez Jul 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it could also make sense, what do you think @harikrishna-patnala @sureshanaparti @shwstppr ?

Copy link
Member

@rohityadavcloud rohityadavcloud Jul 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes that makes sense - pl address that @nvazquez - then it's ready for merging.

}
}

private SecurityGroup getOrCreateSecurityGroupForAccount(Account owner) {
String securityGroupName = String.format("%s-%s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME, owner.getUuid());
String securityGroupDesc = String.format("%s and account %s", KubernetesClusterActionWorker.CKS_SECURITY_GROUP_DESCRIPTION, owner.getName());
Expand Down
4 changes: 3 additions & 1 deletion ...c/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

import javax.inject.Inject;

import com.cloud.kubernetes.version.KubernetesSupportedVersionVO;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -194,7 +195,8 @@ protected void init() {
DataCenterVO dataCenterVO = dataCenterDao.findById(zoneId);
VMTemplateVO template = templateDao.findById(templateId);
Hypervisor.HypervisorType type = template.getHypervisorType();
this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type);
KubernetesSupportedVersionVO kubernetesSupportedVersion = kubernetesSupportedVersionDao.findById(this.kubernetesCluster.getKubernetesVersionId());
this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type, kubernetesSupportedVersion);
this.sshKeyFile = getManagementServerSshPublicKeyFile();
}

Expand Down
2 changes: 1 addition & 1 deletion ...cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ protected DeployDestination plan(final long nodesCount, final DataCenter zone, f
for (Map.Entry<String, Pair<HostVO, Integer>> hostEntry : hosts_with_resevered_capacity.entrySet()) {
Pair<HostVO, Integer> hp = hostEntry.getValue();
HostVO h = hp.first();
if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType())) {
if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType()) || !h.getArch().equals(clusterTemplate.getArch())) {
continue;
}
hostDao.loadHostTags(h);
Expand Down
19 changes: 19 additions & 0 deletions ...-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import com.cloud.api.query.dao.TemplateJoinDao;
import com.cloud.api.query.vo.TemplateJoinVO;
import com.cloud.cpu.CPU;
import com.cloud.dc.DataCenter;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.PermissionDeniedException;
Expand Down Expand Up @@ -292,4 +293,22 @@ public void removeVmsFromCluster() {
Mockito.when(kubernetesClusterDao.findById(Mockito.anyLong())).thenReturn(cluster);
Assert.assertTrue(kubernetesClusterManager.removeVmsFromCluster(cmd).size() > 0);
}

@Test
public void testGetCksClusterPreferredArchDifferentArchsPreferCKSIsoArch() {
String systemVMArch = "x86_64";
VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class);
Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.arm64);
String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso);
Assert.assertEquals(CPU.CPUArch.arm64.name(), cksClusterPreferredArch);
}

@Test
public void testGetCksClusterPreferredArchSameArch() {
String systemVMArch = "x86_64";
VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class);
Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.amd64);
String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso);
Assert.assertEquals(CPU.CPUArch.amd64.name(), cksClusterPreferredArch);
}
}
3 changes: 2 additions & 1 deletion ...ain/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,11 @@ private UserVm deploySharedFSVM(Long zoneId, Account owner, List<Long> networkId
customParameterMap.put("maxIopsDo", maxIops.toString());
}
List<String> keypairs = new ArrayList<String>();
String preferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(zoneId);

for (final Iterator<Hypervisor.HypervisorType> iter = hypervisors.iterator(); iter.hasNext();) {
final Hypervisor.HypervisorType hypervisor = iter.next();
VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor);
VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor, preferredArchitecture);
if (template == null && !iter.hasNext()) {
throw new CloudRuntimeException(String.format("Unable to find the systemvm template for %s or it was not downloaded in %s.", hypervisor.toString(), zone.toString()));
}
Expand Down
3 changes: 1 addition & 2 deletions ...java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ private SharedFS prepareDeploySharedFS() throws ResourceUnavailableException, In
when(serviceOfferingDao.findById(s_serviceOfferingId)).thenReturn(serviceOffering);

VMTemplateVO template = mock(VMTemplateVO.class);
when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(template);
when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM, ResourceManager.SystemVmPreferredArchitecture.defaultValue())).thenReturn(template);
when(template.getId()).thenReturn(s_templateId);

return sharedFS;
Expand Down Expand Up @@ -298,7 +298,6 @@ public void testDeploySharedFSTemplateNotFound() throws ResourceUnavailableExcep
when(dataCenterDao.findById(s_zoneId)).thenReturn(zone);
when(resourceMgr.getSupportedHypervisorTypes(s_zoneId, false, null)).thenReturn(List.of(Hypervisor.HypervisorType.KVM));

when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(null);
lifeCycle.deploySharedFS(sharedFS, s_networkId, s_diskOfferingId, s_size, s_minIops, s_maxIops);
}

Expand Down
Loading