diff --git a/api/src/main/java/com/cloud/agent/api/to/DiskTO.java b/api/src/main/java/com/cloud/agent/api/to/DiskTO.java index d22df2df172e..9755ed1f8f2a 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DiskTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DiskTO.java @@ -29,6 +29,8 @@ public class DiskTO { public static final String CHAP_TARGET_SECRET = "chapTargetSecret"; public static final String SCSI_NAA_DEVICE_ID = "scsiNaaDeviceId"; public static final String MANAGED = "managed"; + public static final String SCOPE = "scope"; + public static final String INTER_CLUSTER_MIGRATION = "interClusterMigration"; public static final String IQN = "iqn"; public static final String STORAGE_HOST = "storageHost"; public static final String STORAGE_PORT = "storagePort"; diff --git a/core/src/main/java/com/cloud/agent/api/UnmountDatastoresCommand.java b/core/src/main/java/com/cloud/agent/api/UnmountDatastoresCommand.java new file mode 100644 index 000000000000..073dcbc49ac9 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UnmountDatastoresCommand.java @@ -0,0 +1,42 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +import java.util.List; + +public class UnmountDatastoresCommand extends Command { + private List datastorePools; + + public UnmountDatastoresCommand() { + + } + + public UnmountDatastoresCommand(List datastorePools) { + this.datastorePools = datastorePools; + } + + public List getDatastorePools() { + return datastorePools; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index c3525466ce19..a4462a36ff5a 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -124,7 +124,7 @@ DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Lon boolean storageMigration(VirtualMachineProfile vm, Map volumeToPool) throws StorageUnavailableException; - void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest); + void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest, Long srcHostId); void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException, StorageAccessException; @@ -178,4 +178,6 @@ DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachin * Unmanage VM volumes */ void unmanageVolumes(long vmId); + + List postMigrationReleaseDatastoresOnOriginHost(VirtualMachineProfile profile, long vmId); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java index 0e70c7b528dd..012ce43aed54 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java @@ -183,4 +183,15 @@ default boolean volumesRequireGrantAccessWhenUsed() { default boolean zoneWideVolumesAvailableWithoutClusterMotion() { return false; } + + /** + * Disabled by default. Set to true if the data store driver needs to unmount/revoke volumes datastore on the + * origin host in case of: + * - zone wide storage + * - inter-cluster VM migration (without storage motion) + * - the hypervisor has restrictions on the number of mounted datastores per host/cluster + */ + default boolean zoneWideVolumesDatastoreCleanupOnOriginHostAfterInterClusterMigration() { + return false; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 41fd631d88fd..bb27d37f909b 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -47,6 +47,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; +import com.cloud.agent.api.UnmountDatastoresCommand; import com.cloud.event.ActionEventUtils; import com.google.gson.Gson; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -2754,7 +2755,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy profile.setHost(dest.getHost()); _networkMgr.prepareNicForMigration(profile, dest); - volumeMgr.prepareForMigration(profile, dest); + volumeMgr.prepareForMigration(profile, dest, srcHostId); profile.setConfigDriveLabel(VmConfigDriveLabel.value()); updateOverCommitRatioForVmProfile(profile, dest.getHost().getClusterId()); @@ -2883,8 +2884,8 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy } else { _networkMgr.commitNicForMigration(vmSrc, profile); volumeMgr.release(vm.getId(), srcHostId); + postMigrationRelease(profile, vm.getId(), srcHostId); _networkMgr.setHypervisorHostname(profile, dest, true); - updateVmPod(vm, dstHostId); } @@ -2893,6 +2894,20 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy } } + private void postMigrationRelease(VirtualMachineProfile profile, long vmId, long srcHostId) { + List poolsToReleaseOnOrigin = volumeMgr.postMigrationReleaseDatastoresOnOriginHost(profile, vmId); + if (CollectionUtils.isEmpty(poolsToReleaseOnOrigin)) { + return; + } + // Unmount the datastores from this host only (useful for SolidFire 1:1 plugin zone-wide storage migrations) + UnmountDatastoresCommand cmd = new UnmountDatastoresCommand(poolsToReleaseOnOrigin); + try { + _agentMgr.send(srcHostId, cmd); + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new RuntimeException(e); + } + } + /** * Create and set parameters for the {@link MigrateCommand} used in the migration and scaling of VMs. */ @@ -3243,7 +3258,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo } _networkMgr.prepareNicForMigration(profile, destination); - volumeMgr.prepareForMigration(profile, destination); + volumeMgr.prepareForMigration(profile, destination, srcHostId); final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); final VirtualMachineTO to = hvGuru.implement(profile); @@ -4410,7 +4425,7 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); _networkMgr.prepareNicForMigration(profile, dest); - volumeMgr.prepareForMigration(profile, dest); + volumeMgr.prepareForMigration(profile, dest, srcHostId); final VirtualMachineTO to = toVmTO(profile); final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index a08e74fc13cb..0b8296bcf72d 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -84,6 +84,7 @@ import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -1496,7 +1497,7 @@ public boolean storageMigration(VirtualMachineProfile vm, Map vols = _volsDao.findUsableVolumesForInstance(vm.getId()); if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("Preparing to migrate [%s] volumes for VM [%s].", vols.size(), vm.getVirtualMachine())); @@ -1514,7 +1515,7 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest // make sure this is done AFTER grantAccess, as grantAccess may change the volume's state DataTO volTO = volumeInfo.getTO(); DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId()); - disk.setDetails(getDetails(volumeInfo, dataStore)); + disk.setDetails(getDetails(volumeInfo, dataStore, vm, srcHostId)); vm.addDisk(disk); } @@ -1527,7 +1528,7 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest } } - private Map getDetails(VolumeInfo volumeInfo, DataStore dataStore) { + private Map getDetails(VolumeInfo volumeInfo, DataStore dataStore, VirtualMachineProfile vmProfile, Long srcHostId) { Map details = new HashMap(); StoragePoolVO storagePool = _storagePoolDao.findById(dataStore.getId()); @@ -1560,9 +1561,33 @@ private Map getDetails(VolumeInfo volumeInfo, DataStore dataStor details.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret()); } + // Zone wide storage Solidfire inter-cluster VM migrations needs the destination host mount the LUN before migrating + if (storagePool.isManaged() && ScopeType.ZONE.equals(storagePool.getScope()) && + isCleanupNeededOnOriginHostAfterInterClusterMigration(dataStore)) { + details.put(DiskTO.SCOPE, storagePool.getScope().name()); + if (vmProfile.getHostId() != null && srcHostId != null) { + HostVO host = _hostDao.findById(vmProfile.getHostId()); + HostVO lastHost = _hostDao.findById(srcHostId); + boolean interClusterMigration = isVmwareInterClusterMigration(lastHost, host); + details.put(DiskTO.INTER_CLUSTER_MIGRATION, BooleanUtils.toStringTrueFalse(interClusterMigration)); + } + } + return details; } + private boolean isCleanupNeededOnOriginHostAfterInterClusterMigration(DataStore dataStore) { + if (dataStore == null || dataStore.getDriver() == null) { + return false; + } + + DataStoreDriver driver = dataStore.getDriver(); + if (driver instanceof PrimaryDataStoreDriver) { + return ((PrimaryDataStoreDriver)driver).zoneWideVolumesDatastoreCleanupOnOriginHostAfterInterClusterMigration(); + } + return false; + } + private void setIoDriverPolicy(Map details, StoragePoolVO storagePool, VolumeVO volume) { if (volume.getInstanceId() != null) { UserVmDetailVO ioDriverPolicy = userVmDetailsDao.findDetail(volume.getInstanceId(), @@ -1937,7 +1962,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId()); DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary); - disk.setDetails(getDetails(volumeInfo, dataStore)); + disk.setDetails(getDetails(volumeInfo, dataStore, vm, null)); vm.addDisk(disk); @@ -2308,4 +2333,52 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } }); } + + protected boolean isVmwareInterClusterMigration(Host lastHost, Host host) { + if (ObjectUtils.anyNull(lastHost, host)) { + return false; + } + if (host.getHypervisorType() != HypervisorType.VMware) { + return false; + } + Long lastHostClusterId = lastHost.getClusterId(); + Long clusterId = host.getClusterId(); + if (ObjectUtils.anyNull(lastHostClusterId, clusterId)) { + return false; + } + return lastHostClusterId.compareTo(clusterId) != 0; + } + + @Override + public List postMigrationReleaseDatastoresOnOriginHost(VirtualMachineProfile profile, long vmId) { + List pools = new ArrayList<>(); + if (profile.getVirtualMachine() == null) { + s_logger.debug("Cannot release from source as the virtual machine profile is not set"); + return pools; + } + HostVO lastHost = _hostDao.findById(profile.getVirtualMachine().getLastHostId()); + HostVO host = _hostDao.findById(profile.getHostId()); + + // Consider only Vmware inter-cluster migration + if (!isVmwareInterClusterMigration(lastHost, host)) { + return pools; + } + + List volumesForVm = _volsDao.findUsableVolumesForInstance(vmId); + for (VolumeVO volumeForVm : volumesForVm) { + if (volumeForVm.getPoolId() != null) { + DataStore dataStore = dataStoreMgr.getDataStore(volumeForVm.getPoolId(), DataStoreRole.Primary); + PrimaryDataStore primaryDataStore = (PrimaryDataStore) dataStore; + if (primaryDataStore.getScope() != null && primaryDataStore.getScope().getScopeType() == ScopeType.ZONE) { + // Consider zone-wide storage + PrimaryDataStoreDriver driver = (PrimaryDataStoreDriver) primaryDataStore.getDriver(); + if (driver.zoneWideVolumesDatastoreCleanupOnOriginHostAfterInterClusterMigration()) { + // Currently enabled for SolidFire only on inter-cluster migrations on zone-wide pools. + pools.add(volumeForVm.get_iScsiName()); + } + } + } + } + return pools; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 9e105749da98..f26dca574e35 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -49,8 +49,10 @@ import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; +import com.cloud.agent.api.UnmountDatastoresCommand; import com.cloud.capacity.CapacityManager; import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; +import com.cloud.storage.ScopeType; import com.vmware.vim25.FileInfo; import com.vmware.vim25.FileQueryFlags; import com.vmware.vim25.FolderFileInfo; @@ -74,6 +76,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -619,6 +622,8 @@ public Answer executeRequest(Command cmd) { answer = execute((ListDataStoreObjectsCommand) cmd); } else if (clz == PrepareForBackupRestorationCommand.class) { answer = execute((PrepareForBackupRestorationCommand) cmd); + } else if (clz == UnmountDatastoresCommand.class) { + answer = execute((UnmountDatastoresCommand) cmd); } else { answer = Answer.createUnsupportedCommandAnswer(cmd); } @@ -4624,6 +4629,11 @@ protected Answer execute(PrepareForMigrationCommand cmd) { prepareNetworkFromNicInfo(new HostMO(getServiceContext(), _morHyperHost), nic, false, cmd.getVirtualMachine().getType()); } + DiskTO[] disks = vm.getDisks(); + for (DiskTO disk : disks) { + prepareDatastoreForZoneWideManagedStorageInterClusterMigration(disk, hyperHost); + } + List> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId)); for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { String secStoreUrl = secStoreUrlAndId.first(); @@ -4645,6 +4655,29 @@ protected Answer execute(PrepareForMigrationCommand cmd) { } } + private void prepareDatastoreForZoneWideManagedStorageInterClusterMigration(DiskTO disk, VmwareHypervisorHost hyperHost) throws Exception { + Map details = disk.getDetails(); + if (MapUtils.isEmpty(details) || !details.containsKey(DiskTO.SCOPE) || + !details.containsKey(DiskTO.MANAGED) && !details.containsKey(DiskTO.INTER_CLUSTER_MIGRATION)) { + return; + } + + VmwareContext context = hyperHost.getContext(); + boolean isManaged = details.containsKey(DiskTO.MANAGED) && BooleanUtils.toBoolean(details.get(DiskTO.MANAGED)); + boolean isZoneWideStorage = details.containsKey(DiskTO.SCOPE) && details.get(DiskTO.SCOPE).equalsIgnoreCase(ScopeType.ZONE.name()); + boolean isInterClusterMigration = details.containsKey(DiskTO.INTER_CLUSTER_MIGRATION) && BooleanUtils.toBoolean(details.get(DiskTO.INTER_CLUSTER_MIGRATION)); + + if (isManaged && isZoneWideStorage && isInterClusterMigration) { + s_logger.debug(String.format("Preparing storage on destination cluster for host %s", hyperHost.getHyperHostName())); + String iScsiName = details.get(DiskTO.IQN); // details should not be null for managed storage (it may or may not be null for non-managed storage) + String datastoreName = VmwareResource.getDatastoreName(iScsiName); + s_logger.debug(String.format("Ensuring datastore %s is mounted on destination cluster", datastoreName)); + _storageProcessor.prepareManagedDatastore(context, hyperHost, datastoreName, + details.get(DiskTO.IQN), details.get(DiskTO.STORAGE_HOST), + Integer.parseInt(details.get(DiskTO.STORAGE_PORT))); + } + } + protected Answer execute(MigrateVmToPoolCommand cmd) { final String vmName = cmd.getVmName(); @@ -5339,6 +5372,34 @@ private void handleTargets(boolean add, ModifyTargetsCommand.TargetTypeToRemove } } + private Answer execute(UnmountDatastoresCommand cmd) { + VmwareContext context = getServiceContext(cmd); + VmwareHypervisorHost hyperHost = getHyperHost(context, cmd); + if (hyperHost == null) { + throw new CloudRuntimeException("No hypervisor host found to unmount datastore"); + } + try { + List datastorePools = cmd.getDatastorePools(); + if (CollectionUtils.isNotEmpty(datastorePools)) { + ManagedObjectReference clusterMor = hyperHost.getHyperHostCluster(); + if (clusterMor == null) { + return new Answer(cmd, false, "Cannot unmount datastore pools as the cluster is not found"); + } + ClusterMO clusterMO = new ClusterMO(context, clusterMor); + List> clusterHosts = clusterMO.getClusterHosts(); + for (String datastorePool : datastorePools) { + String datastoreName = VmwareResource.getDatastoreName(datastorePool); + s_logger.debug(String.format("Unmounting datastore %s from cluster %s hosts", datastoreName, clusterMO.getName())); + _storageProcessor.unmountVmfsDatastore(context, hyperHost, datastoreName, clusterHosts); + } + } + } catch (Exception e) { + s_logger.error("Error unmounting datastores", e); + return new Answer(cmd, e); + } + return new Answer(cmd, true, "success"); + } + protected Answer execute(DeleteStoragePoolCommand cmd) { try { if (cmd.getRemoveDatastore()) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 40ebc4cb02ad..05beeb978299 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -3038,7 +3038,7 @@ private void mountVmfsDatastore2(DatastoreMO dsMO, List hosts) throws Ex } } - private void unmountVmfsDatastore(VmwareContext context, VmwareHypervisorHost hyperHost, String datastoreName, + public void unmountVmfsDatastore(VmwareContext context, VmwareHypervisorHost hyperHost, String datastoreName, List> hosts) throws Exception { for (Pair host : hosts) { HostMO hostMO = new HostMO(context, host.first()); diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index a5d1a39a7344..c71a805d036f 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -25,6 +25,7 @@ import javax.inject.Inject; +import com.cloud.agent.AgentManager; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -118,6 +119,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { @Inject private VolumeDao volumeDao; @Inject private VolumeDetailsDao volumeDetailsDao; @Inject private VolumeDataFactory volumeFactory; + @Inject private AgentManager agentManager; @Override public Map getCapabilities() { @@ -1677,4 +1679,19 @@ public void detachVolumeFromAllStorageNodes(Volume volume) { public boolean volumesRequireGrantAccessWhenUsed() { return true; } + + @Override + public boolean zoneWideVolumesAvailableWithoutClusterMotion() { + return true; + } + + @Override + public boolean requiresAccessForMigration(DataObject dataObject) { + return true; + } + + @Override + public boolean zoneWideVolumesDatastoreCleanupOnOriginHostAfterInterClusterMigration() { + return true; + } }