Skip to content

Commit 8b0b163

Browse files
nvazqueznvazquez
committed
[Solidfire] Fix inter-cluster VM migrations on zone wide storage
1 parent b4c39ad commit 8b0b163

File tree

8 files changed

+216
-9
lines changed

8 files changed

+216
-9
lines changed

‎api/src/main/java/com/cloud/agent/api/to/DiskTO.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ public class DiskTO {
2929
public static final String CHAP_TARGET_SECRET = "chapTargetSecret";
3030
public static final String SCSI_NAA_DEVICE_ID = "scsiNaaDeviceId";
3131
public static final String MANAGED = "managed";
32+
public static final String SCOPE = "scope";
33+
public static final String INTER_CLUSTER_MIGRATION = "interClusterMigration";
3234
public static final String IQN = "iqn";
3335
public static final String STORAGE_HOST = "storageHost";
3436
public static final String STORAGE_PORT = "storagePort";

‎core/src/main/java/com/cloud/agent/api/UnmountDatastoresCommand.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
package com.cloud.agent.api;
20+
21+
import java.util.List;
22+
23+
public class UnmountDatastoresCommand extends Command {
24+
private List<String> datastorePools;
25+
26+
public UnmountDatastoresCommand() {
27+
28+
}
29+
30+
public UnmountDatastoresCommand(List<String> datastorePools) {
31+
this.datastorePools = datastorePools;
32+
}
33+
34+
public List<String> getDatastorePools() {
35+
return datastorePools;
36+
}
37+
38+
@Override
39+
public boolean executeInSequence() {
40+
return false;
41+
}
42+
}

‎engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Lon
124124

125125
boolean storageMigration(VirtualMachineProfile vm, Map<Volume, StoragePool> volumeToPool) throws StorageUnavailableException;
126126

127-
void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest);
127+
void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest, Long srcHostId);
128128

129129
void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException, StorageAccessException;
130130

@@ -178,4 +178,6 @@ DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachin
178178
* Unmanage VM volumes
179179
*/
180180
void unmanageVolumes(long vmId);
181+
182+
List<String> postMigrationReleaseDatastoresOnOriginHost(VirtualMachineProfile profile, long vmId);
181183
}

‎engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,4 +183,15 @@ default boolean volumesRequireGrantAccessWhenUsed() {
183183
default boolean zoneWideVolumesAvailableWithoutClusterMotion() {
184184
return false;
185185
}
186+
187+
/**
188+
* Disabled by default. Set to true if the data store driver needs to unmount/revoke volumes datastore on the
189+
* origin host in case of:
190+
* - zone wide storage
191+
* - inter-cluster VM migration (without storage motion)
192+
* - the hypervisor has restrictions on the number of mounted datastores per host/cluster
193+
*/
194+
default boolean zoneWideVolumesDatastoreCleanupOnOriginHostAfterInterClusterMigration() {
195+
return false;
196+
}
186197
}

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import javax.naming.ConfigurationException;
4848
import javax.persistence.EntityExistsException;
4949

50+
import com.cloud.agent.api.UnmountDatastoresCommand;
5051
import com.cloud.event.ActionEventUtils;
5152
import com.google.gson.Gson;
5253
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
@@ -2754,7 +2755,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
27542755
profile.setHost(dest.getHost());
27552756

27562757
_networkMgr.prepareNicForMigration(profile, dest);
2757-
volumeMgr.prepareForMigration(profile, dest);
2758+
volumeMgr.prepareForMigration(profile, dest, srcHostId);
27582759
profile.setConfigDriveLabel(VmConfigDriveLabel.value());
27592760
updateOverCommitRatioForVmProfile(profile, dest.getHost().getClusterId());
27602761

@@ -2883,8 +2884,8 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
28832884
} else {
28842885
_networkMgr.commitNicForMigration(vmSrc, profile);
28852886
volumeMgr.release(vm.getId(), srcHostId);
2887+
postMigrationRelease(profile, vm.getId(), srcHostId);
28862888
_networkMgr.setHypervisorHostname(profile, dest, true);
2887-
28882889
updateVmPod(vm, dstHostId);
28892890
}
28902891

@@ -2893,6 +2894,20 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
28932894
}
28942895
}
28952896

2897+
private void postMigrationRelease(VirtualMachineProfile profile, long vmId, long srcHostId) {
2898+
List<String> poolsToReleaseOnOrigin = volumeMgr.postMigrationReleaseDatastoresOnOriginHost(profile, vmId);
2899+
if (CollectionUtils.isEmpty(poolsToReleaseOnOrigin)) {
2900+
return;
2901+
}
2902+
// Unmount the datastores from this host only (useful for SolidFire 1:1 plugin zone-wide storage migrations)
2903+
UnmountDatastoresCommand cmd = new UnmountDatastoresCommand(poolsToReleaseOnOrigin);
2904+
try {
2905+
_agentMgr.send(srcHostId, cmd);
2906+
} catch (AgentUnavailableException | OperationTimedoutException e) {
2907+
throw new RuntimeException(e);
2908+
}
2909+
}
2910+
28962911
/**
28972912
* Create and set parameters for the {@link MigrateCommand} used in the migration and scaling of VMs.
28982913
*/
@@ -3243,7 +3258,7 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
32433258
}
32443259

32453260
_networkMgr.prepareNicForMigration(profile, destination);
3246-
volumeMgr.prepareForMigration(profile, destination);
3261+
volumeMgr.prepareForMigration(profile, destination, srcHostId);
32473262
final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType());
32483263
final VirtualMachineTO to = hvGuru.implement(profile);
32493264

@@ -4410,7 +4425,7 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI
44104425
final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
44114426
_networkMgr.prepareNicForMigration(profile, dest);
44124427

4413-
volumeMgr.prepareForMigration(profile, dest);
4428+
volumeMgr.prepareForMigration(profile, dest, srcHostId);
44144429

44154430
final VirtualMachineTO to = toVmTO(profile);
44164431
final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to);

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 64 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
8585
import org.apache.commons.collections.CollectionUtils;
8686
import org.apache.commons.collections.MapUtils;
87+
import org.apache.commons.lang.BooleanUtils;
8788
import org.apache.commons.lang3.ObjectUtils;
8889
import org.apache.commons.lang3.StringUtils;
8990
import org.apache.log4j.Logger;
@@ -1496,7 +1497,7 @@ public boolean storageMigration(VirtualMachineProfile vm, Map<Volume, StoragePoo
14961497
}
14971498

14981499
@Override
1499-
public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest) {
1500+
public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest, Long srcHostId) {
15001501
List<VolumeVO> vols = _volsDao.findUsableVolumesForInstance(vm.getId());
15011502
if (s_logger.isDebugEnabled()) {
15021503
s_logger.debug(String.format("Preparing to migrate [%s] volumes for VM [%s].", vols.size(), vm.getVirtualMachine()));
@@ -1514,7 +1515,7 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest
15141515
// make sure this is done AFTER grantAccess, as grantAccess may change the volume's state
15151516
DataTO volTO = volumeInfo.getTO();
15161517
DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId());
1517-
disk.setDetails(getDetails(volumeInfo, dataStore));
1518+
disk.setDetails(getDetails(volumeInfo, dataStore, vm, srcHostId));
15181519
vm.addDisk(disk);
15191520
}
15201521

@@ -1527,7 +1528,7 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest
15271528
}
15281529
}
15291530

1530-
private Map<String, String> getDetails(VolumeInfo volumeInfo, DataStore dataStore) {
1531+
private Map<String, String> getDetails(VolumeInfo volumeInfo, DataStore dataStore, VirtualMachineProfile vmProfile, Long srcHostId) {
15311532
Map<String, String> details = new HashMap<String, String>();
15321533

15331534
StoragePoolVO storagePool = _storagePoolDao.findById(dataStore.getId());
@@ -1560,6 +1561,17 @@ private Map<String, String> getDetails(VolumeInfo volumeInfo, DataStore dataStor
15601561
details.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret());
15611562
}
15621563

1564+
// Zone wide storage Solidfire inter-cluster VM migrations needs the destination host mount the LUN before migrating
1565+
if (storagePool.isManaged() && storagePool.getScope() != null && ScopeType.ZONE == storagePool.getScope()) {
1566+
details.put(DiskTO.SCOPE, storagePool.getScope().name());
1567+
if (vmProfile.getHostId() != null && srcHostId != null) {
1568+
HostVO host = _hostDao.findById(vmProfile.getHostId());
1569+
HostVO lastHost = _hostDao.findById(srcHostId);
1570+
boolean interClusterMigration = isVmwareInterClusterMigration(lastHost, host);
1571+
details.put(DiskTO.INTER_CLUSTER_MIGRATION, BooleanUtils.toStringTrueFalse(interClusterMigration));
1572+
}
1573+
}
1574+
15631575
return details;
15641576
}
15651577

@@ -1937,7 +1949,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto
19371949
DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(), vm.getServiceOfferingId(), vol.getDiskOfferingId());
19381950
DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary);
19391951

1940-
disk.setDetails(getDetails(volumeInfo, dataStore));
1952+
disk.setDetails(getDetails(volumeInfo, dataStore, vm, null));
19411953

19421954
vm.addDisk(disk);
19431955

@@ -2308,4 +2320,52 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
23082320
}
23092321
});
23102322
}
2323+
2324+
protected boolean isVmwareInterClusterMigration(Host lastHost, Host host) {
2325+
if (ObjectUtils.anyNull(lastHost, host)) {
2326+
return false;
2327+
}
2328+
if (host.getHypervisorType() != HypervisorType.VMware) {
2329+
return false;
2330+
}
2331+
Long lastHostClusterId = lastHost.getClusterId();
2332+
Long clusterId = host.getClusterId();
2333+
if (ObjectUtils.anyNull(lastHostClusterId, clusterId)) {
2334+
return false;
2335+
}
2336+
return lastHostClusterId.compareTo(clusterId) != 0;
2337+
}
2338+
2339+
@Override
2340+
public List<String> postMigrationReleaseDatastoresOnOriginHost(VirtualMachineProfile profile, long vmId) {
2341+
List<String> pools = new ArrayList<>();
2342+
if (profile.getVirtualMachine() == null) {
2343+
s_logger.debug("Cannot release from source as the virtual machine profile is not set");
2344+
return pools;
2345+
}
2346+
HostVO lastHost = _hostDao.findById(profile.getVirtualMachine().getLastHostId());
2347+
HostVO host = _hostDao.findById(profile.getHostId());
2348+
2349+
// Consider only Vmware inter-cluster migration
2350+
if (!isVmwareInterClusterMigration(lastHost, host)) {
2351+
return pools;
2352+
}
2353+
2354+
List<VolumeVO> volumesForVm = _volsDao.findUsableVolumesForInstance(vmId);
2355+
for (VolumeVO volumeForVm : volumesForVm) {
2356+
if (volumeForVm.getPoolId() != null) {
2357+
DataStore dataStore = dataStoreMgr.getDataStore(volumeForVm.getPoolId(), DataStoreRole.Primary);
2358+
PrimaryDataStore primaryDataStore = (PrimaryDataStore) dataStore;
2359+
if (primaryDataStore.getScope() != null && primaryDataStore.getScope().getScopeType() == ScopeType.ZONE) {
2360+
// Consider zone-wide storage
2361+
PrimaryDataStoreDriver driver = (PrimaryDataStoreDriver) primaryDataStore.getDriver();
2362+
if (driver.zoneWideVolumesDatastoreCleanupOnOriginHostAfterInterClusterMigration()) {
2363+
// Currently enabled for SolidFire only on inter-cluster migrations on zone-wide pools.
2364+
pools.add(volumeForVm.get_iScsiName());
2365+
}
2366+
}
2367+
}
2368+
}
2369+
return pools;
2370+
}
23112371
}

‎plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@
4949
import javax.naming.ConfigurationException;
5050
import javax.xml.datatype.XMLGregorianCalendar;
5151

52+
import com.cloud.agent.api.UnmountDatastoresCommand;
5253
import com.cloud.capacity.CapacityManager;
5354
import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO;
55+
import com.cloud.storage.ScopeType;
5456
import com.vmware.vim25.FileInfo;
5557
import com.vmware.vim25.FileQueryFlags;
5658
import com.vmware.vim25.FolderFileInfo;
@@ -74,6 +76,7 @@
7476
import org.apache.commons.collections.CollectionUtils;
7577
import org.apache.commons.collections.MapUtils;
7678
import org.apache.commons.lang.ArrayUtils;
79+
import org.apache.commons.lang.BooleanUtils;
7780
import org.apache.commons.lang.math.NumberUtils;
7881
import org.apache.commons.lang3.StringUtils;
7982
import org.apache.log4j.Logger;
@@ -619,6 +622,8 @@ public Answer executeRequest(Command cmd) {
619622
answer = execute((ListDataStoreObjectsCommand) cmd);
620623
} else if (clz == PrepareForBackupRestorationCommand.class) {
621624
answer = execute((PrepareForBackupRestorationCommand) cmd);
625+
} else if (clz == UnmountDatastoresCommand.class) {
626+
answer = execute((UnmountDatastoresCommand) cmd);
622627
} else {
623628
answer = Answer.createUnsupportedCommandAnswer(cmd);
624629
}
@@ -4624,6 +4629,11 @@ protected Answer execute(PrepareForMigrationCommand cmd) {
46244629
prepareNetworkFromNicInfo(new HostMO(getServiceContext(), _morHyperHost), nic, false, cmd.getVirtualMachine().getType());
46254630
}
46264631

4632+
DiskTO[] disks = vm.getDisks();
4633+
for (DiskTO disk : disks) {
4634+
prepareDatastoreForZoneWideManagedStorageInterClusterMigration(disk, hyperHost);
4635+
}
4636+
46274637
List<Pair<String, Long>> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId));
46284638
for (Pair<String, Long> secStoreUrlAndId : secStoreUrlAndIdList) {
46294639
String secStoreUrl = secStoreUrlAndId.first();
@@ -4645,6 +4655,36 @@ protected Answer execute(PrepareForMigrationCommand cmd) {
46454655
}
46464656
}
46474657

4658+
private void prepareDatastoreForZoneWideManagedStorageInterClusterMigration(DiskTO disk, VmwareHypervisorHost hyperHost) throws Exception {
4659+
Map<String, String> details = disk.getDetails();
4660+
if (MapUtils.isEmpty(details) || !details.containsKey(DiskTO.SCOPE) ||
4661+
!details.containsKey(DiskTO.MANAGED) && !details.containsKey(DiskTO.INTER_CLUSTER_MIGRATION)) {
4662+
return;
4663+
}
4664+
4665+
VmwareContext context = hyperHost.getContext();
4666+
boolean isManaged = details.containsKey(DiskTO.MANAGED) && BooleanUtils.toBoolean(details.get(DiskTO.MANAGED));
4667+
boolean isZoneWideStorage = details.containsKey(DiskTO.SCOPE) && details.get(DiskTO.SCOPE).equalsIgnoreCase(ScopeType.ZONE.name());
4668+
boolean isInterClusterMigration = details.containsKey(DiskTO.INTER_CLUSTER_MIGRATION) && BooleanUtils.toBoolean(details.get(DiskTO.INTER_CLUSTER_MIGRATION));
4669+
4670+
if (isManaged && isZoneWideStorage && isInterClusterMigration) {
4671+
s_logger.debug("Preparing storage on destination host " + hyperHost.getHyperHostName());
4672+
String iScsiName = details.get(DiskTO.IQN); // details should not be null for managed storage (it may or may not be null for non-managed storage)
4673+
String datastoreName = VmwareResource.getDatastoreName(iScsiName);
4674+
s_logger.debug("Checking for datastore " + datastoreName);
4675+
ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, datastoreName);
4676+
4677+
// if the datastore is not present, we need to discover the iSCSI device that will support it,
4678+
// create the datastore
4679+
if (morDatastore == null) {
4680+
s_logger.debug("The datastore " + datastoreName + " is not mounted, mounting");
4681+
_storageProcessor.prepareManagedDatastore(context, getHyperHost(context), datastoreName,
4682+
details.get(DiskTO.IQN), details.get(DiskTO.STORAGE_HOST),
4683+
Integer.parseInt(details.get(DiskTO.STORAGE_PORT)));
4684+
}
4685+
}
4686+
}
4687+
46484688
protected Answer execute(MigrateVmToPoolCommand cmd) {
46494689
final String vmName = cmd.getVmName();
46504690

@@ -5339,6 +5379,29 @@ private void handleTargets(boolean add, ModifyTargetsCommand.TargetTypeToRemove
53395379
}
53405380
}
53415381

5382+
private Answer execute(UnmountDatastoresCommand cmd) {
5383+
VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
5384+
if (hyperHost == null) {
5385+
throw new CloudRuntimeException("No hypervisor host found to unmount datastore");
5386+
}
5387+
List<String> datastorePools = cmd.getDatastorePools();
5388+
if (CollectionUtils.isNotEmpty(datastorePools)) {
5389+
try {
5390+
for (String datastorePool : datastorePools) {
5391+
String datastoreName = VmwareResource.getDatastoreName(datastorePool);
5392+
s_logger.debug("Checking for datastore " + datastoreName);
5393+
ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, datastoreName);
5394+
if (morDatastore != null) {
5395+
hyperHost.unmountDatastore(datastoreName);
5396+
}
5397+
}
5398+
} catch (Exception e) {
5399+
throw new RuntimeException(e);
5400+
}
5401+
}
5402+
return new Answer(cmd, true, "success");
5403+
}
5404+
53425405
protected Answer execute(DeleteStoragePoolCommand cmd) {
53435406
try {
53445407
if (cmd.getRemoveDatastore()) {

0 commit comments

Comments
 (0)