Skip to content

Commit e574259

Browse files
sureshanapartisureshanaparti
committed
Migrate volume improvements, to bypass secondary storage when copy volume between pools is allowed directly
1 parent 70a4503 commit e574259

File tree

28 files changed

+116
-23
lines changed

28 files changed

+116
-23
lines changed

‎engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.Set;
2424

2525
import com.cloud.exception.ResourceAllocationException;
26+
import com.cloud.storage.Storage;
2627
import com.cloud.utils.Pair;
2728
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
2829
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -182,10 +183,10 @@ List<DiskProfile> allocateTemplatedVolumes(Type type, String name, DiskOffering
182183
*/
183184
DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops,
184185
Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template,
185-
Account owner, Long deviceId, Long poolId, String path, String chainInfo);
186+
Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo);
186187

187188
DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template,
188-
Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile);
189+
Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile);
189190

190191
/**
191192
* Unmanage VM volumes

‎engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ClusterScope.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.cloudstack.engine.subsystem.api.storage;
2020

2121
import com.cloud.storage.ScopeType;
22+
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
2223

2324
public class ClusterScope extends AbstractScope {
2425
private ScopeType type = ScopeType.CLUSTER;
@@ -51,4 +52,9 @@ public Long getZoneId() {
5152
return this.zoneId;
5253
}
5354

55+
@Override
56+
public String toString() {
57+
return String.format("ClusterScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
58+
this, "zoneId", "clusterId", "podId"));
59+
}
5460
}

‎engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.cloudstack.engine.subsystem.api.storage;
2020

2121
import com.cloud.storage.ScopeType;
22+
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
2223

2324
public class HostScope extends AbstractScope {
2425
private Long hostId;
@@ -49,4 +50,10 @@ public Long getClusterId() {
4950
public Long getZoneId() {
5051
return zoneId;
5152
}
53+
54+
@Override
55+
public String toString() {
56+
return String.format("HostScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
57+
this, "zoneId", "clusterId", "hostId"));
58+
}
5259
}

‎engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ZoneScope.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.cloudstack.engine.subsystem.api.storage;
2020

2121
import com.cloud.storage.ScopeType;
22+
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
2223

2324
public class ZoneScope extends AbstractScope {
2425
private ScopeType type = ScopeType.ZONE;
@@ -39,4 +40,9 @@ public Long getScopeId() {
3940
return this.zoneId;
4041
}
4142

43+
@Override
44+
public String toString() {
45+
return String.format("ZoneScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
46+
this, "zoneId"));
47+
}
4248
}

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2792,6 +2792,7 @@ private void markVolumesInPool(VMInstanceVO vm, Answer[] hypervisorMigrationResu
27922792
}
27932793
volume.setPath(result.getPath());
27942794
volume.setPoolId(pool.getId());
2795+
volume.setPoolType(pool.getPoolType());
27952796
if (result.getChainInfo() != null) {
27962797
volume.setChainInfo(result.getChainInfo());
27972798
}

‎engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,7 +1422,7 @@ public Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageU
14221422
String volumeToString = getVolumeIdentificationInfos(volume);
14231423

14241424
VolumeInfo vol = volFactory.getVolume(volume.getId());
1425-
if (vol == null){
1425+
if (vol == null) {
14261426
throw new CloudRuntimeException(String.format("Volume migration failed because volume [%s] is null.", volumeToString));
14271427
}
14281428
if (destPool == null) {
@@ -2303,6 +2303,7 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo,
23032303
StoragePoolVO pool = _storagePoolDao.findByUuid(updatedDataStoreUUID);
23042304
if (pool != null) {
23052305
vol.setPoolId(pool.getId());
2306+
vol.setPoolType(pool.getPoolType());
23062307
}
23072308
}
23082309
_volsDao.update(volumeId, vol);
@@ -2312,7 +2313,7 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo,
23122313
@Override
23132314
public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops,
23142315
Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner,
2315-
Long deviceId, Long poolId, String path, String chainInfo) {
2316+
Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo) {
23162317
if (sizeInBytes == null) {
23172318
sizeInBytes = offering.getDiskSize();
23182319
}
@@ -2353,6 +2354,7 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L
23532354

23542355
vol.setFormat(getSupportedImageFormatForCluster(hypervisorType));
23552356
vol.setPoolId(poolId);
2357+
vol.setPoolType(poolType);
23562358
vol.setPath(path);
23572359
vol.setChainInfo(chainInfo);
23582360
vol.setState(Volume.State.Ready);
@@ -2362,7 +2364,7 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L
23622364

23632365
@Override
23642366
public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template,
2365-
Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile) {
2367+
Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile) {
23662368

23672369
VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId());
23682370
if (vm != null) {
@@ -2396,6 +2398,7 @@ public DiskProfile updateImportedVolume(Type type, DiskOffering offering, Virtua
23962398

23972399
vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType()));
23982400
vol.setPoolId(poolId);
2401+
vol.setPoolType(poolType);
23992402
vol.setPath(path);
24002403
vol.setChainInfo(chainInfo);
24012404
vol.setSize(diskProfile.getSize());

‎engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ public void testImportVolume() {
241241

242242
volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null,
243243
zoneId, hypervisorType, null, null, owner,
244-
deviceId, poolId, path, chainInfo);
244+
deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo);
245245

246246
VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0);
247247
Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong());

‎engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,7 @@ public void updateAndRemoveVolume(VolumeVO volume) {
822822
if (volume.getState() != Volume.State.Destroy) {
823823
volume.setState(Volume.State.Destroy);
824824
volume.setPoolId(null);
825+
volume.setPoolType(null);
825826
volume.setInstanceId(null);
826827
update(volume.getId(), volume);
827828
remove(volume.getId());

‎engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@
1818
*/
1919
package org.apache.cloudstack.storage.motion;
2020

21+
import java.util.Arrays;
2122
import java.util.HashMap;
2223
import java.util.List;
2324
import java.util.Map;
25+
import java.util.Objects;
2426

2527
import javax.inject.Inject;
2628

@@ -67,6 +69,7 @@
6769
import com.cloud.host.Host;
6870
import com.cloud.hypervisor.Hypervisor;
6971
import com.cloud.storage.DataStoreRole;
72+
import com.cloud.storage.ScopeType;
7073
import com.cloud.storage.Snapshot.Type;
7174
import com.cloud.storage.SnapshotVO;
7275
import com.cloud.storage.StorageManager;
@@ -85,6 +88,10 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
8588
protected Logger logger = LogManager.getLogger(getClass());
8689
private static final String NO_REMOTE_ENDPOINT_SSVM = "No remote endpoint to send command, check if host or ssvm is down?";
8790
private static final String NO_REMOTE_ENDPOINT_WITH_ENCRYPTION = "No remote endpoint to send command, unable to find a valid endpoint. Requires encryption support: %s";
91+
private static final List<StoragePoolType> SUPPORTED_POOL_TYPES_TO_BYPASS_SECONDARY_STORE = Arrays.asList(
92+
StoragePoolType.NetworkFilesystem,
93+
StoragePoolType.Filesystem
94+
);
8895

8996
@Inject
9097
EndPointSelector selector;
@@ -240,7 +247,6 @@ protected DataTO addFullCloneAndDiskprovisiongStrictnessFlagOnVMwareDest(DataTO
240247
return dataTO;
241248
}
242249

243-
244250
protected Answer copyObject(DataObject srcData, DataObject destData) {
245251
return copyObject(srcData, destData, null);
246252
}
@@ -352,10 +358,7 @@ protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData)
352358

353359
Scope destScope = getZoneScope(destData.getDataStore().getScope());
354360
DataStore cacheStore = cacheMgr.getCacheStorage(destScope);
355-
boolean bypassSecondaryStorage = false;
356-
if (srcData instanceof VolumeInfo && ((VolumeInfo)srcData).isDirectDownload()) {
357-
bypassSecondaryStorage = true;
358-
}
361+
boolean bypassSecondaryStorage = canBypassSecondaryStorage(srcData, destData);
359362
boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData);
360363

361364
if (cacheStore == null) {
@@ -448,7 +451,45 @@ protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData)
448451
}
449452
return answer;
450453
}
454+
}
455+
456+
private boolean canBypassSecondaryStorage(DataObject srcData, DataObject destData) {
457+
if (srcData instanceof VolumeInfo) {
458+
if (((VolumeInfo)srcData).isDirectDownload()) {
459+
return true;
460+
}
461+
462+
if (destData instanceof VolumeInfo) {
463+
Scope srcDataStoreScope = srcData.getDataStore().getScope();
464+
Scope destDataStoreScope = destData.getDataStore().getScope();
465+
logger.info("srcDataStoreScope: {}, destDataStoreScope: {}", srcDataStoreScope, destDataStoreScope);
466+
logger.info("srcData - pool type: {}, scope: {}; destData - pool type: {}, scope: {}",
467+
((VolumeInfo)srcData).getStoragePoolType(), srcDataStoreScope != null ? srcDataStoreScope.getScopeType() : null, ((VolumeInfo)destData).getStoragePoolType(), destDataStoreScope != null ? destDataStoreScope.getScopeType() : null);
468+
469+
if (srcDataStoreScope != null && destDataStoreScope != null &&
470+
SUPPORTED_POOL_TYPES_TO_BYPASS_SECONDARY_STORE.contains(((VolumeInfo)srcData).getStoragePoolType()) &&
471+
SUPPORTED_POOL_TYPES_TO_BYPASS_SECONDARY_STORE.contains(((VolumeInfo)destData).getStoragePoolType())) {
472+
473+
if (srcDataStoreScope.isSameScope(destDataStoreScope)) {
474+
return true;
475+
}
476+
477+
if (srcDataStoreScope.getScopeType() == ScopeType.CLUSTER &&
478+
destDataStoreScope.getScopeType() == ScopeType.HOST &&
479+
(Objects.equals(((ClusterScope) srcDataStoreScope).getScopeId(), ((HostScope) destDataStoreScope).getClusterId()))) {
480+
return true;
481+
}
482+
483+
if (srcDataStoreScope.getScopeType() == ScopeType.HOST &&
484+
destDataStoreScope.getScopeType() == ScopeType.CLUSTER &&
485+
(Objects.equals(((HostScope) srcDataStoreScope).getClusterId(), ((ClusterScope) destDataStoreScope).getScopeId()))) {
486+
return true;
487+
}
488+
}
489+
}
490+
}
451491

492+
return false;
452493
}
453494

454495
protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) {
@@ -492,6 +533,7 @@ protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) {
492533
}
493534
volumeVo.setPodId(destPool.getPodId());
494535
volumeVo.setPoolId(destPool.getId());
536+
volumeVo.setPoolType(destPool.getPoolType());
495537
volumeVo.setLastPoolId(oldPoolId);
496538
// For SMB, pool credentials are also stored in the uri query string. We trim the query string
497539
// part here to make sure the credentials do not get stored in the db unencrypted.

‎engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -796,6 +796,7 @@ private void handleSuccessfulVolumeMigration(VolumeInfo srcVolumeInfo, StoragePo
796796

797797
volumeVO.setPodId(destPool.getPodId());
798798
volumeVO.setPoolId(destPool.getId());
799+
volumeVO.setPoolType(destPool.getPoolType());
799800
volumeVO.setLastPoolId(srcVolumeInfo.getPoolId());
800801

801802
_volumeDao.update(srcVolumeInfo.getId(), volumeVO);
@@ -2398,13 +2399,13 @@ private VolumeVO duplicateVolumeOnAnotherStorage(Volume volume, StoragePoolVO st
23982399
Long lastPoolId = volume.getPoolId();
23992400

24002401
VolumeVO newVol = new VolumeVO(volume);
2401-
24022402
newVol.setInstanceId(null);
24032403
newVol.setChainInfo(null);
24042404
newVol.setPath(null);
24052405
newVol.setFolder(null);
24062406
newVol.setPodId(storagePoolVO.getPodId());
24072407
newVol.setPoolId(storagePoolVO.getId());
2408+
newVol.setPoolType(storagePoolVO.getPoolType());
24082409
newVol.setLastPoolId(lastPoolId);
24092410
newVol.setLastId(volume.getId());
24102411

0 commit comments

Comments
 (0)