Merge release branch 4.20 to main

* 4.20:
  log name change after merge forward
  check tags while fetching storage pool for importing vm (#9764)
  UI: Add cluster arch type to the zone creation wizard (#10080)

Author: DaanHoogland

server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java

@@ -538,7 +538,7 @@ private Map<String, Network.IpAddresses> getNicIpAddresses(final List<UnmanagedI
         return nicIpAddresses;
     }
 
-    private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final DataCenter zone, final Cluster cluster) {
+    private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final DataCenter zone, final Cluster cluster, String diskOfferingTags) {
         StoragePool storagePool = null;
         final String dsHost = disk.getDatastoreHost();
         final String dsPath = disk.getDatastorePath();
@@ -548,7 +548,8 @@ private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final Da
             List<StoragePoolVO> pools = primaryDataStoreDao.listPoolByHostPath(dsHost, dsPath);
             for (StoragePool pool : pools) {
                 if (pool.getDataCenterId() == zone.getId() &&
-                        (pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId()))) {
+                        (pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId())) &&
+                        volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOfferingTags)) {
                     storagePool = pool;
                     break;
                 }
@@ -560,7 +561,8 @@ private StoragePool getStoragePool(final UnmanagedInstanceTO.Disk disk, final Da
             pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId()));
             for (StoragePool pool : pools) {
                 String searchPoolParam = StringUtils.isNotBlank(dsPath) ? dsPath : dsName;
-                if (StringUtils.contains(pool.getPath(), searchPoolParam)) {
+                if (StringUtils.contains(pool.getPath(), searchPoolParam) &&
+                        volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOfferingTags)) {
                     storagePool = pool;
                     break;
                 }
@@ -623,7 +625,8 @@ private void checkUnmanagedDiskAndOfferingForImport(String instanceName, Unmanag
         if (diskOffering != null && !diskOffering.isCustomized() && diskOffering.getDiskSize() < disk.getCapacity()) {
             throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Size of disk offering(ID: %s) %dGB is found less than the size of disk(ID: %s) %dGB during VM import", diskOffering.getUuid(), (diskOffering.getDiskSize() / Resource.ResourceType.bytesToGiB), disk.getDiskId(), (disk.getCapacity() / (Resource.ResourceType.bytesToGiB))));
         }
-        StoragePool storagePool = getStoragePool(disk, zone, cluster);
+        diskOffering = diskOffering != null ? diskOffering : diskOfferingDao.findById(serviceOffering.getDiskOfferingId());
+        StoragePool storagePool = getStoragePool(disk, zone, cluster, diskOffering != null ? diskOffering.getTags() : null);
         if (diskOffering != null && !migrateAllowed && !storagePoolSupportsDiskOffering(storagePool, diskOffering)) {
             throw new InvalidParameterValueException(String.format("Disk offering: %s is not compatible with storage pool: %s of unmanaged disk: %s", diskOffering.getUuid(), storagePool.getUuid(), disk.getDiskId()));
         }
@@ -860,7 +863,7 @@ private Pair<DiskProfile, StoragePool> importDisk(UnmanagedInstanceTO.Disk disk,
             diskInfo.setDiskChain(new String[]{disk.getImagePath()});
             chainInfo = gson.toJson(diskInfo);
         }
-        StoragePool storagePool = getStoragePool(disk, zone, cluster);
+        StoragePool storagePool = getStoragePool(disk, zone, cluster, diskOffering != null ? diskOffering.getTags() : null);
         DiskProfile profile = volumeManager.importVolume(type, name, diskOffering, diskSize,
                 minIops, maxIops, vm.getDataCenterId(), vm.getHypervisorType(), vm, template, owner, deviceId, storagePool.getId(), path, chainInfo);
 
@@ -1645,7 +1648,7 @@ protected UserVm importUnmanagedInstanceFromVmwareToKvm(DataCenter zone, Cluster
 
             temporaryConvertLocation = selectInstanceConversionTemporaryLocation(
                     destinationCluster, convertHost, convertStoragePoolId);
-            List<StoragePoolVO> convertStoragePools = findInstanceConversionStoragePoolsInCluster(destinationCluster);
+            List<StoragePoolVO> convertStoragePools = findInstanceConversionStoragePoolsInCluster(destinationCluster, serviceOffering, dataDiskOfferingMap);
             long importStartTime = System.currentTimeMillis();
             Pair<UnmanagedInstanceTO, Boolean> sourceInstanceDetails = getSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password, clusterName, sourceHostName, sourceVMName);
             sourceVMwareInstance = sourceInstanceDetails.first();
@@ -1661,15 +1664,18 @@ protected UserVm importUnmanagedInstanceFromVmwareToKvm(DataCenter zone, Cluster
             if (cmd.getForceMsToImportVmFiles() || !conversionSupportAnswer.isOvfExportSupported()) {
                 // Uses MS for OVF export to temporary conversion location
                 int noOfThreads = UnmanagedVMsManager.ThreadsOnMSToImportVMwareVMFiles.value();
-                ovfTemplateOnConvertLocation = createOvfTemplateOfSourceVmwareUnmanagedInstance(vcenter, datacenterName, username, password,
-                        clusterName, sourceHostName, sourceVMwareInstance.getName(), temporaryConvertLocation, noOfThreads);
+                ovfTemplateOnConvertLocation = createOvfTemplateOfSourceVmwareUnmanagedInstance(
+                        vcenter, datacenterName, username, password, clusterName, sourceHostName,
+                        sourceVMwareInstance.getName(), temporaryConvertLocation, noOfThreads);
                 convertedInstance = convertVmwareInstanceToKVMWithOVFOnConvertLocation(sourceVMName,
-                                sourceVMwareInstance, convertHost, importHost, convertStoragePools,
-                        temporaryConvertLocation, ovfTemplateOnConvertLocation);
+                        sourceVMwareInstance, convertHost, importHost, convertStoragePools,
+                        serviceOffering, dataDiskOfferingMap, temporaryConvertLocation,
+                        ovfTemplateOnConvertLocation);
             } else {
                 // Uses KVM Host for OVF export to temporary conversion location, through ovftool
                 convertedInstance = convertVmwareInstanceToKVMAfterExportingOVFToConvertLocation(
-                        sourceVMName, sourceVMwareInstance, convertHost, importHost, convertStoragePools,
+                        sourceVMName, sourceVMwareInstance, convertHost, importHost,
+                        convertStoragePools, serviceOffering, dataDiskOfferingMap,
                         temporaryConvertLocation, vcenter, username, password, datacenterName);
             }
 
@@ -1759,9 +1765,9 @@ private void sanitizeConvertedInstance(UnmanagedInstanceTO convertedInstance, Un
         convertedInstance.setPowerState(UnmanagedInstanceTO.PowerState.PowerOff);
         List<UnmanagedInstanceTO.Disk> convertedInstanceDisks = convertedInstance.getDisks();
         List<UnmanagedInstanceTO.Disk> sourceVMwareInstanceDisks = sourceVMwareInstance.getDisks();
-        for (int i = 0; i < convertedInstanceDisks.size(); i++) {
-            UnmanagedInstanceTO.Disk disk = convertedInstanceDisks.get(i);
-            disk.setDiskId(sourceVMwareInstanceDisks.get(i).getDiskId());
+        for (UnmanagedInstanceTO.Disk sourceVMwareInstanceDisk : sourceVMwareInstanceDisks) {
+            UnmanagedInstanceTO.Disk convertedDisk = convertedInstanceDisks.get(sourceVMwareInstanceDisk.getPosition());
+            convertedDisk.setDiskId(sourceVMwareInstanceDisk.getDiskId());
         }
         List<UnmanagedInstanceTO.Nic> convertedInstanceNics = convertedInstance.getNics();
         List<UnmanagedInstanceTO.Nic> sourceVMwareInstanceNics = sourceVMwareInstance.getNics();
@@ -1945,16 +1951,16 @@ private CheckConvertInstanceAnswer checkConversionSupportOnHost(HostVO convertHo
     }
 
     private UnmanagedInstanceTO convertVmwareInstanceToKVMWithOVFOnConvertLocation(
-            String sourceVM, UnmanagedInstanceTO sourceVMwareInstance,
-            HostVO convertHost, HostVO importHost,
-            List<StoragePoolVO> convertStoragePools, DataStoreTO temporaryConvertLocation,
-            String ovfTemplateDirConvertLocation
+            String sourceVM, UnmanagedInstanceTO sourceVMwareInstance, HostVO convertHost,
+            HostVO importHost, List<StoragePoolVO> convertStoragePools,
+            ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap,
+            DataStoreTO temporaryConvertLocation, String ovfTemplateDirConvertLocation
     ) {
             logger.debug(String.format("Delegating the conversion of instance %s from VMware to KVM to the host %s (%s) using OVF %s on conversion datastore",
                 sourceVM, convertHost.getId(), convertHost.getName(), ovfTemplateDirConvertLocation));
 
         RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(sourceVM);
-        List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks());
+        List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks(), serviceOffering, dataDiskOfferingMap);
         ConvertInstanceCommand cmd = new ConvertInstanceCommand(remoteInstanceTO,
                 Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation, ovfTemplateDirConvertLocation, false, false);
         int timeoutSeconds = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
@@ -1980,16 +1986,17 @@ private UnmanagedInstanceTO convertVmwareInstanceToKVMWithOVFOnConvertLocation(
     }
 
     private UnmanagedInstanceTO convertVmwareInstanceToKVMAfterExportingOVFToConvertLocation(
-            String sourceVM, UnmanagedInstanceTO sourceVMwareInstance,
-            HostVO convertHost, HostVO importHost, List<StoragePoolVO> convertStoragePools,
-            DataStoreTO temporaryConvertLocation, String vcenterHost,
-            String vcenterUsername, String vcenterPassword, String datacenterName
+            String sourceVM, UnmanagedInstanceTO sourceVMwareInstance, HostVO convertHost,
+            HostVO importHost, List<StoragePoolVO> convertStoragePools,
+            ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap,
+            DataStoreTO temporaryConvertLocation, String vcenterHost, String vcenterUsername,
+            String vcenterPassword, String datacenterName
     ) {
         logger.debug(String.format("Delegating the conversion of instance %s from VMware to KVM to the host %s (%s) after OVF export through ovftool",
                 sourceVM, convertHost.getId(), convertHost.getName()));
 
         RemoteInstanceTO remoteInstanceTO = new RemoteInstanceTO(sourceVMwareInstance.getName(), vcenterHost, vcenterUsername, vcenterPassword, datacenterName);
-        List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks());
+        List<String> destinationStoragePools = selectInstanceConversionStoragePools(convertStoragePools, sourceVMwareInstance.getDisks(), serviceOffering, dataDiskOfferingMap);
         ConvertInstanceCommand cmd = new ConvertInstanceCommand(remoteInstanceTO,
                 Hypervisor.HypervisorType.KVM, destinationStoragePools, temporaryConvertLocation, null, false, true);
         int timeoutSeconds = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout.value() * 60 * 60;
@@ -2052,12 +2059,31 @@ private UnmanagedInstanceTO convertAndImportToKVM(ConvertInstanceCommand convert
         return ((ImportConvertedInstanceAnswer) importAnswer).getConvertedInstance();
     }
 
-    private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(Cluster destinationCluster) {
+    private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(
+            Cluster destinationCluster, ServiceOfferingVO serviceOffering,
+            Map<String, Long> dataDiskOfferingMap
+    ) {
         List<StoragePoolVO> pools = new ArrayList<>();
-        List<StoragePoolVO> clusterPools = primaryDataStoreDao.findClusterWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem);
-        pools.addAll(clusterPools);
-        List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getDataCenterId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem);
-        pools.addAll(zonePools);
+        pools.addAll(primaryDataStoreDao.findClusterWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem));
+        pools.addAll(primaryDataStoreDao.findZoneWideStoragePoolsByHypervisorAndPoolType(destinationCluster.getDataCenterId(), Hypervisor.HypervisorType.KVM, Storage.StoragePoolType.NetworkFilesystem));
+        List<String> diskOfferingTags = new ArrayList<>();
+        for (Long diskOfferingId : dataDiskOfferingMap.values()) {
+            DiskOfferingVO diskOffering = diskOfferingDao.findById(diskOfferingId);
+            if (diskOffering == null) {
+                String msg = String.format("Cannot find disk offering with ID %s", diskOfferingId);
+                logger.error(msg);
+                throw new CloudRuntimeException(msg);
+            }
+            diskOfferingTags.add(diskOffering.getTags());
+        }
+        if (serviceOffering.getDiskOfferingId() != null) {
+            DiskOfferingVO diskOffering = diskOfferingDao.findById(serviceOffering.getDiskOfferingId());
+            if (diskOffering != null) {
+                diskOfferingTags.add(diskOffering.getTags());
+            }
+        }
+
+        pools = getPoolsWithMatchingTags(pools, diskOfferingTags);
         if (pools.isEmpty()) {
             String msg = String.format("Cannot find suitable storage pools in cluster %s for the conversion", destinationCluster.getName());
             logger.error(msg);
@@ -2066,12 +2092,54 @@ private List<StoragePoolVO> findInstanceConversionStoragePoolsInCluster(Cluster
         return pools;
     }
 
-    private List<String> selectInstanceConversionStoragePools(List<StoragePoolVO> pools, List<UnmanagedInstanceTO.Disk> disks) {
+    private List<StoragePoolVO> getPoolsWithMatchingTags(List<StoragePoolVO> pools, List<String> diskOfferingTags) {
+        if (diskOfferingTags.isEmpty()) {
+            return pools;
+        }
+        List<StoragePoolVO> poolsSupportingTags = new ArrayList<>(pools);
+        for (String tags : diskOfferingTags) {
+            boolean tagsMatched = false;
+            for (StoragePoolVO pool : pools) {
+                if (volumeApiService.doesTargetStorageSupportDiskOffering(pool, tags)) {
+                    poolsSupportingTags.add(pool);
+                    tagsMatched = true;
+                }
+            }
+            if (!tagsMatched) {
+                String msg = String.format("Cannot find suitable storage pools for the conversion with disk offering tags %s", tags);
+                logger.error(msg);
+                throw new CloudRuntimeException(msg);
+            }
+        }
+        return poolsSupportingTags;
+    }
+
+    private List<String> selectInstanceConversionStoragePools(
+            List<StoragePoolVO> pools, List<UnmanagedInstanceTO.Disk> disks,
+            ServiceOfferingVO serviceOffering, Map<String, Long> dataDiskOfferingMap
+    ) {
         List<String> storagePools = new ArrayList<>(disks.size());
-        //TODO: Choose pools by capacity
+        for (int i = 0; i < disks.size(); i++) {
+            storagePools.add(null);
+        }
+        Set<String> dataDiskIds = dataDiskOfferingMap.keySet();
         for (UnmanagedInstanceTO.Disk disk : disks) {
-            Long capacity = disk.getCapacity();
-            storagePools.add(pools.get(0).getUuid());
+            Long diskOfferingId = dataDiskOfferingMap.get(disk.getDiskId());
+            if (diskOfferingId == null && !dataDiskIds.contains(disk.getDiskId())) {
+                diskOfferingId = serviceOffering.getDiskOfferingId();
+            }
+            //TODO: Choose pools by capacity
+            if (diskOfferingId == null) {
+                storagePools.set(disk.getPosition(), pools.get(0).getUuid());
+            } else {
+                DiskOfferingVO diskOffering = diskOfferingDao.findById(diskOfferingId);
+                for (StoragePoolVO pool : pools) {
+                    if (volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOffering.getTags())) {
+                        storagePools.set(disk.getPosition(), pool.getUuid());
+                        break;
+                    }
+                }
+            }
         }
         return storagePools;
     }

server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java

@@ -279,6 +279,7 @@ public void setUp() throws Exception {
         List<UnmanagedInstanceTO.Disk> instanceDisks = new ArrayList<>();
         UnmanagedInstanceTO.Disk instanceDisk = new UnmanagedInstanceTO.Disk();
         instanceDisk.setDiskId("1000-1");
+        instanceDisk.setPosition(0);
         instanceDisk.setLabel("DiskLabel");
         instanceDisk.setController("scsi");
         instanceDisk.setImagePath("[b6ccf44a1fa13e29b3667b4954fa10ee] TestInstance/ROOT-1.vmdk");
@@ -424,6 +425,7 @@ public void importUnmanagedInstanceTest() {
         ImportUnmanagedInstanceCmd importUnmanageInstanceCmd = Mockito.mock(ImportUnmanagedInstanceCmd.class);
         when(importUnmanageInstanceCmd.getName()).thenReturn("TestInstance");
         when(importUnmanageInstanceCmd.getDomainId()).thenReturn(null);
+        when(volumeApiService.doesTargetStorageSupportDiskOffering(any(StoragePool.class), any())).thenReturn(true);
         try (MockedStatic<UsageEventUtils> ignored = Mockito.mockStatic(UsageEventUtils.class)) {
             unmanagedVMsManager.importUnmanagedInstance(importUnmanageInstanceCmd);
         }
@@ -703,6 +705,8 @@ private void baseTestImportVmFromVmwareToKvm(VcenterParameter vcenterParameter,
             when(agentManager.send(Mockito.eq(convertHostId), Mockito.any(CheckConvertInstanceCommand.class))).thenReturn(checkConvertInstanceAnswer);
         }
 
+        when(volumeApiService.doesTargetStorageSupportDiskOffering(any(StoragePool.class), any())).thenReturn(true);
+
         ConvertInstanceAnswer convertInstanceAnswer = mock(ConvertInstanceAnswer.class);
         ImportConvertedInstanceAnswer convertImportedInstanceAnswer = mock(ImportConvertedInstanceAnswer.class);
         when(convertInstanceAnswer.getConvertedInstance()).thenReturn(instance);

ui/src/views/infra/zone/ZoneWizardAddResources.vue

@@ -192,6 +192,13 @@ export default {
           placeHolder: 'message.error.cluster.name',
           required: true
         },
+        {
+          title: 'label.arch',
+          key: 'arch',
+          required: false,
+          select: true,
+          options: this.architectureTypes
+        },
         {
           title: 'label.vcenter.host',
           key: 'vCenterHost',
@@ -846,6 +853,13 @@ export default {
       primaryStorageScopes: [],
       primaryStorageProtocols: [],
       primaryStorageProviders: [],
+      architectureTypes: [{
+        id: 'x86_64',
+        description: 'AMD 64 bits (x86_64)'
+      }, {
+        id: 'aarch64',
+        description: 'ARM 64 bits (aarch64)'
+      }],
       storageProviders: [],
       currentStep: null,
       options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider']

ui/src/views/infra/zone/ZoneWizardLaunchZone.vue

@@ -1283,6 +1283,7 @@ export default {
       if (this.isEdgeZone) {
         clusterName = 'Cluster-' + this.stepData.zoneReturned.name
       }
+      params.arch = this.prefillContent?.arch || null
 
       if (hypervisor === 'VMware') {
         params.username = this.prefillContent?.vCenterUsername || null