diff --git a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java index 94fbb7a80af8..1c88c7df124c 100644 --- a/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java +++ b/engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java @@ -94,6 +94,14 @@ public static void publishUsageEvent(String usageType, long accountId, long zone } + public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, + Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) { + if (displayResource) { + saveUsageEvent(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName); + } + publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID); + } + public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId, Long size, Long virtualSize, String entityType, String entityUUID, Map details) { saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, virtualSize, details); @@ -202,6 +210,10 @@ public static void saveUsageEvent(String usageType, long accountId, long zoneId, s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, vmId, securityGroupId)); } + public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) { + s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName)); + } + private static void publishUsageEvent(String usageEventType, Long accountId, Long zoneId, String resourceType, String resourceUUID) { String configKey = "publish.usage.events"; String value = s_configDao.getValue(configKey); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 7af9b6b84923..4e83cf16163f 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -903,7 +903,7 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri // Save usage event and update resource count for user vm volumes if (vm.getType() == VirtualMachine.Type.User) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size, - Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); + Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); } DiskProfile diskProfile = toDiskProfile(vol, offering); @@ -981,7 +981,7 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering } UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size, - Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); + Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); } diff --git a/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java b/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java index 3fc9fda94873..41ecec0c7fb8 100644 --- a/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java +++ b/engine/schema/src/main/java/com/cloud/event/UsageEventVO.java @@ -75,6 +75,9 @@ public enum DynamicParameters { @Column(name = "virtual_size") private Long virtualSize; + @Column(name = "vm_id") + private Long vmId; + public UsageEventVO() { } @@ -143,6 +146,18 @@ public UsageEventVO(String usageType, long accountId, long zoneId, long vmId, lo this.offeringId = securityGroupId; } + public UsageEventVO(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) { + this.type = usageType; + this.accountId = accountId; + this.zoneId = zoneId; + this.resourceId = resourceId; + this.offeringId = offeringId; + this.templateId = templateId; + this.size = size; + this.vmId = vmId; + this.resourceName = resourceName; + } + @Override public long getId() { return id; @@ -248,4 +263,11 @@ public void setVirtualSize(Long virtualSize) { this.virtualSize = virtualSize; } + public Long getVmId() { + return vmId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } } diff --git a/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java index fdef509da5bd..bce9c474e2d2 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java @@ -45,11 +45,11 @@ public class UsageEventDaoImpl extends GenericDaoBase implem private final SearchBuilder latestEventsSearch; private final SearchBuilder IpeventsSearch; private static final String COPY_EVENTS = - "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) " - + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? "; + "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) " + + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? "; private static final String COPY_ALL_EVENTS = - "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) " - + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?"; + "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) " + + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id <= ?"; private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) " + "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? "; private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) " diff --git a/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java b/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java index 96abd2d69c08..6d5315e33464 100644 --- a/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java +++ b/engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java @@ -59,6 +59,9 @@ public class UsageVolumeVO implements InternalIdentity { @Column(name = "size") private long size; + @Column(name = "vm_id") + private Long vmId; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created = null; @@ -70,13 +73,14 @@ public class UsageVolumeVO implements InternalIdentity { protected UsageVolumeVO() { } - public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, long size, Date created, Date deleted) { + public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, Long vmId, long size, Date created, Date deleted) { this.volumeId = id; this.zoneId = zoneId; this.accountId = accountId; this.domainId = domainId; this.diskOfferingId = diskOfferingId; this.templateId = templateId; + this.vmId = vmId; this.size = size; this.created = created; this.deleted = deleted; @@ -126,4 +130,12 @@ public void setDeleted(Date deleted) { public long getVolumeId() { return volumeId; } + + public Long getVmId() { + return vmId; + } + + public void setVmId(Long vmId) { + this.vmId = vmId; + } } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java index 1da533493997..f863cd1e3a35 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java @@ -57,6 +57,7 @@ public UsageStorageDaoImpl() { IdSearch.and("accountId", IdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); IdSearch.and("id", IdSearch.entity().getEntityId(), SearchCriteria.Op.EQ); IdSearch.and("type", IdSearch.entity().getStorageType(), SearchCriteria.Op.EQ); + IdSearch.and("deleted", IdSearch.entity().getDeleted(), SearchCriteria.Op.NULL); IdSearch.done(); IdZoneSearch = createSearchBuilder(); @@ -74,6 +75,7 @@ public List listById(long accountId, long id, int type) { sc.setParameters("accountId", accountId); sc.setParameters("id", id); sc.setParameters("type", type); + sc.setParameters("deleted", null); return listBy(sc, null); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java index 09590b739930..05287240f254 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java @@ -23,9 +23,7 @@ import com.cloud.utils.db.GenericDao; public interface UsageVolumeDao extends GenericDao { - public void removeBy(long userId, long id); - - public void update(UsageVolumeVO usage); - public List getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate, boolean limit, int page); + + List listByVolumeId(long volumeId, long accountId); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java index 4662a6f26ce8..095070feac1c 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java @@ -18,81 +18,46 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.TimeZone; -import com.cloud.exception.CloudException; +import javax.annotation.PostConstruct; + import org.springframework.stereotype.Component; import com.cloud.usage.UsageVolumeVO; import com.cloud.utils.DateUtil; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; @Component public class UsageVolumeDaoImpl extends GenericDaoBase implements UsageVolumeDao { - protected static final String REMOVE_BY_USERID_VOLID = "DELETE FROM usage_volume WHERE account_id = ? AND volume_id = ?"; - protected static final String UPDATE_DELETED = "UPDATE usage_volume SET deleted = ? WHERE account_id = ? AND volume_id = ? and deleted IS NULL"; - protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted " + protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted " + "FROM usage_volume " + "WHERE account_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))"; - protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted " + protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted " + "FROM usage_volume " + "WHERE domain_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))"; - protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted " + protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted " + "FROM usage_volume " + "WHERE (deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?))"; + private SearchBuilder volumeSearch; public UsageVolumeDaoImpl() { } - @Override - public void removeBy(long accountId, long volId) { - TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); - try { - txn.start(); - try(PreparedStatement pstmt = txn.prepareStatement(REMOVE_BY_USERID_VOLID);) { - if (pstmt != null) { - pstmt.setLong(1, accountId); - pstmt.setLong(2, volId); - pstmt.executeUpdate(); - } - }catch (SQLException e) { - throw new CloudException("Error removing usageVolumeVO:"+e.getMessage(), e); - } - txn.commit(); - } catch (Exception e) { - txn.rollback(); - logger.warn("Error removing usageVolumeVO:"+e.getMessage(), e); - } finally { - txn.close(); - } - } - - @Override - public void update(UsageVolumeVO usage) { - TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); - PreparedStatement pstmt = null; - try { - txn.start(); - if (usage.getDeleted() != null) { - pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED); - pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getDeleted())); - pstmt.setLong(2, usage.getAccountId()); - pstmt.setLong(3, usage.getVolumeId()); - pstmt.executeUpdate(); - } - txn.commit(); - } catch (Exception e) { - txn.rollback(); - logger.warn("Error updating UsageVolumeVO", e); - } finally { - txn.close(); - } + @PostConstruct + protected void init() { + volumeSearch = createSearchBuilder(); + volumeSearch.and("accountId", volumeSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + volumeSearch.and("deleted", volumeSearch.entity().getDeleted(), SearchCriteria.Op.NULL); + volumeSearch.done(); } @Override @@ -150,11 +115,15 @@ public List getUsageRecords(Long accountId, Long domainId, Date s if (tId == 0) { tId = null; } - long size = Long.valueOf(rs.getLong(7)); + Long vmId = Long.valueOf(rs.getLong(7)); + if (vmId == 0) { + vmId = null; + } + long size = Long.valueOf(rs.getLong(8)); Date createdDate = null; Date deletedDate = null; - String createdTS = rs.getString(8); - String deletedTS = rs.getString(9); + String createdTS = rs.getString(9); + String deletedTS = rs.getString(10); if (createdTS != null) { createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS); @@ -163,7 +132,7 @@ public List getUsageRecords(Long accountId, Long domainId, Date s deletedDate = DateUtil.parseDateString(s_gmtTimeZone, deletedTS); } - usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, size, createdDate, deletedDate)); + usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, vmId, size, createdDate, deletedDate)); } } catch (Exception e) { txn.rollback(); @@ -174,4 +143,13 @@ public List getUsageRecords(Long accountId, Long domainId, Date s return usageRecords; } + + @Override + public List listByVolumeId(long volumeId, long accountId) { + SearchCriteria sc = volumeSearch.create(); + sc.setParameters("accountId", accountId); + sc.setParameters("volumeId", volumeId); + sc.setParameters("deleted", null); + return listBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 62ae10b7cc9c..d6046a7a09b8 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -32,5 +32,12 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHA -- Populate uuid for existing rows where uuid is NULL or empty UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid = ''; +-- Add vm_id column to usage_event table for volume usage events +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"'); +CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"'); + +-- Add vm_id column to cloud_usage.usage_volume table +CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"'); + -- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository. CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone'''); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index c44239b4447b..b954dbcaaae0 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -4330,6 +4330,9 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map t // For the Resize Volume Event, this publishes an event with an incorrect disk offering ID, so do nothing for now } else { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), vol.getDiskOfferingId(), null, vol.getSize(), - Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume()); + Volume.class.getName(), vol.getUuid(), instanceId, vol.isDisplayVolume()); } } else if (transition.getToState() == State.Destroy && vol.getVolumeType() != Volume.Type.ROOT) { //Do not Publish Usage Event for ROOT Disk as it would have been published already while destroying a VM UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 854886d233e0..4ae80d3d4b21 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2402,6 +2402,8 @@ protected void recoverRootVolume(VolumeVO volume, Long vmId) { if (Volume.State.Destroy.equals(volume.getState())) { _volumeService.recoverVolume(volume.getId()); _volsDao.attachVolume(volume.getId(), vmId, ROOT_DEVICE_ID); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), + volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), vmId, volume.isDisplay()); } else { _volumeService.publishVolumeCreationUsageEvent(volume); } @@ -8094,7 +8096,7 @@ protected void updateVolumesOwner(final List volumes, Account oldAccou logger.trace("Generating a create volume event for volume [{}].", volume); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), - volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume()); + volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), volume.isDisplayVolume()); } } @@ -8867,6 +8869,8 @@ public Pair doInTransaction(final TransactionStatus status) th handleManagedStorage(vm, root); _volsDao.attachVolume(newVol.getId(), vmId, newVol.getDeviceId()); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), + newVol.getDiskOfferingId(), newVol.getTemplateId(), newVol.getSize(), Volume.class.getName(), newVol.getUuid(), vmId, newVol.isDisplay()); // Detach, destroy and create the usage event for the old root volume. _volsDao.detachVolume(root.getId()); diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 79be3695fbde..0575b430ef10 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -1545,7 +1545,7 @@ public void publishVolumeCreationUsageEventTestNullDiskOfferingId() { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } @@ -1558,7 +1558,7 @@ public void publishVolumeCreationUsageEventTestNullDiskOfferingVo() { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } @@ -1573,7 +1573,7 @@ public void publishVolumeCreationUsageEventTestDiskOfferingVoTypeNotDisk() { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } @@ -1589,7 +1589,7 @@ public void publishVolumeCreationUsageEventTestOfferingIdNotNull() { volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock); usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(), - offeringMockId, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay())); + offeringMockId, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay())); } } diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 66e8783e77b6..37f1fa9e5dde 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -1102,10 +1102,12 @@ public void testResetVMUserDataSuccessResetWithUserdataId() { public void recoverRootVolumeTestDestroyState() { Mockito.doReturn(Volume.State.Destroy).when(volumeVOMock).getState(); - userVmManagerImpl.recoverRootVolume(volumeVOMock, vmId); + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + userVmManagerImpl.recoverRootVolume(volumeVOMock, vmId); - Mockito.verify(volumeApiService).recoverVolume(volumeVOMock.getId()); - Mockito.verify(volumeDaoMock).attachVolume(volumeVOMock.getId(), vmId, UserVmManagerImpl.ROOT_DEVICE_ID); + Mockito.verify(volumeApiService).recoverVolume(volumeVOMock.getId()); + Mockito.verify(volumeDaoMock).attachVolume(volumeVOMock.getId(), vmId, UserVmManagerImpl.ROOT_DEVICE_ID); + } } @Test(expected = InvalidParameterValueException.class) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 49d799997162..864ad35c9229 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1008,7 +1008,12 @@ private boolean isIPEvent(String eventType) { private boolean isVolumeEvent(String eventType) { return eventType != null && - (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) || eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD)); + (eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || + eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || + eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) || + eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD) || + eventType.equals(EventTypes.EVENT_VOLUME_ATTACH) || + eventType.equals(EventTypes.EVENT_VOLUME_DETACH)); } private boolean isTemplateEvent(String eventType) { @@ -1424,92 +1429,112 @@ private void createIPHelperEvent(UsageEventVO event) { } } + private void deleteExistingSecondaryStorageUsageForVolume(long volId, long accountId, Date deletedDate) { + List storageVOs = _usageStorageDao.listById(accountId, volId, StorageTypes.VOLUME); + for (UsageStorageVO storageVO : storageVOs) { + logger.debug("Setting the volume with id: {} to 'deleted' in the usage_storage table for account: {}.", volId, accountId); + storageVO.setDeleted(deletedDate); + _usageStorageDao.update(storageVO); + } + } + + private void deleteExistingInstanceVolumeUsage(long volId, long accountId, Date deletedDate) { + List volumesVOs = _usageVolumeDao.listByVolumeId(volId, accountId); + for (UsageVolumeVO volumesVO : volumesVOs) { + if (volumesVO.getVmId() != null) { + logger.debug("Setting the volume with id: {} for instance id: {} to 'deleted' in the usage_volume table for account {}.", + volumesVO.getVolumeId(), volumesVO.getVmId(), accountId); + volumesVO.setDeleted(deletedDate); + _usageVolumeDao.update(volumesVO.getId(), volumesVO); + } + } + } + + private void deleteExistingVolumeUsage(long volId, long accountId, Date deletedDate) { + List volumesVOs = _usageVolumeDao.listByVolumeId(volId, accountId); + for (UsageVolumeVO volumesVO : volumesVOs) { + logger.debug("Setting the volume with id: {} to 'deleted' in the usage_volume table for account: {}.", volId, accountId); + volumesVO.setDeleted(deletedDate); + _usageVolumeDao.update(volumesVO.getId(), volumesVO); + } + } + private void createVolumeHelperEvent(UsageEventVO event) { long volId = event.getResourceId(); + Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); + List volumesVOs; + UsageVolumeVO volumeVO; - if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType())) { - //For volumes which are 'attached' successfully, set the 'deleted' column in the usage_storage table, + switch (event.getType()) { + case EventTypes.EVENT_VOLUME_CREATE: + //For volumes which are 'attached' successfully from uploaded state, set the 'deleted' column in the usage_storage table, //so that the secondary storage should stop accounting and only primary will be accounted. - SearchCriteria sc = _usageStorageDao.createSearchCriteria(); - sc.addAnd("entityId", SearchCriteria.Op.EQ, volId); - sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME); - List volumesVOs = _usageStorageDao.search(sc, null); - if (volumesVOs != null) { - if (volumesVOs.size() == 1) { - logger.debug("Setting the volume with id: " + volId + " to 'deleted' in the usage_storage table."); - volumesVOs.get(0).setDeleted(event.getCreateDate()); - _usageStorageDao.update(volumesVOs.get(0)); - } - } - } - if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType()) || EventTypes.EVENT_VOLUME_RESIZE.equals(event.getType())) { - SearchCriteria sc = _usageVolumeDao.createSearchCriteria(); - sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId()); - sc.addAnd("volumeId", SearchCriteria.Op.EQ, volId); - sc.addAnd("deleted", SearchCriteria.Op.NULL); - List volumesVOs = _usageVolumeDao.search(sc, null); + deleteExistingSecondaryStorageUsageForVolume(volId, event.getAccountId(), event.getCreateDate()); + + volumesVOs = _usageVolumeDao.listByVolumeId(volId, event.getAccountId()); if (volumesVOs.size() > 0) { //This is a safeguard to avoid double counting of volumes. logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted..."); + deleteExistingVolumeUsage(volId, event.getAccountId(), event.getCreateDate()); } - //an entry exists if it is a resize volume event. marking the existing deleted and creating a new one in the case of resize. - for (UsageVolumeVO volumesVO : volumesVOs) { - if (logger.isDebugEnabled()) { - logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId()); - } - volumesVO.setDeleted(event.getCreateDate()); - _usageVolumeDao.update(volumesVO); - } - if (logger.isDebugEnabled()) { - logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId()); - } - Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); - UsageVolumeVO volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getSize(), event.getCreateDate(), null); + + logger.debug("Creating a new entry in usage_volume for volume with id: {} for account: {}", volId, event.getAccountId()); + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), null, event.getSize(), event.getCreateDate(), null); _usageVolumeDao.persist(volumeVO); - } else if (EventTypes.EVENT_VOLUME_DELETE.equals(event.getType())) { - SearchCriteria sc = _usageVolumeDao.createSearchCriteria(); - sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId()); - sc.addAnd("volumeId", SearchCriteria.Op.EQ, volId); - sc.addAnd("deleted", SearchCriteria.Op.NULL); - List volumesVOs = _usageVolumeDao.search(sc, null); - if (volumesVOs.size() > 1) { - logger.warn("More that one usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking them all as deleted..."); - } - for (UsageVolumeVO volumesVO : volumesVOs) { - if (logger.isDebugEnabled()) { - logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId()); - } - volumesVO.setDeleted(event.getCreateDate()); // there really shouldn't be more than one - _usageVolumeDao.update(volumesVO); - } - } else if (EventTypes.EVENT_VOLUME_UPLOAD.equals(event.getType())) { - //For Upload event add an entry to the usage_storage table. - SearchCriteria sc = _usageStorageDao.createSearchCriteria(); - sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId()); - sc.addAnd("entityId", SearchCriteria.Op.EQ, volId); - sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME); - sc.addAnd("deleted", SearchCriteria.Op.NULL); - List volumesVOs = _usageStorageDao.search(sc, null); - if (volumesVOs.size() > 0) { - //This is a safeguard to avoid double counting of volumes. - logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted..."); + if (event.getVmId() != null) { + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getVmId(), event.getSize(), event.getCreateDate(), null); + _usageVolumeDao.persist(volumeVO); } - for (UsageStorageVO volumesVO : volumesVOs) { - if (logger.isDebugEnabled()) { - logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId()); + break; + + case EventTypes.EVENT_VOLUME_RESIZE: + volumesVOs = _usageVolumeDao.listByVolumeId(volId, event.getAccountId()); + for (UsageVolumeVO volumesVO : volumesVOs) { + String delete_msg = String.format("Setting the volume with id: %s to 'deleted' in the usage_volume table for account: %s.", volId, event.getAccountId()); + String create_msg = String.format("Creating a new entry in usage_volume for volume with id: %s after resize for account: %s", volId, event.getAccountId()); + Long vmId = volumesVO.getVmId(); + if (vmId != null) { + delete_msg = String.format("Setting the volume with id: %s for instance id: %s to 'deleted' in the usage_volume table for account: %s.", + volId, vmId, event.getAccountId()); + create_msg = String.format("Creating a new entry in usage_volume for volume with id: %s and instance id: %s after resize for account: %s", + volId, vmId, event.getAccountId()); } + logger.debug(delete_msg); volumesVO.setDeleted(event.getCreateDate()); - _usageStorageDao.update(volumesVO); - } + _usageVolumeDao.update(volumesVO.getId(), volumesVO); - if (logger.isDebugEnabled()) { - logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId()); + logger.debug(create_msg); + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), vmId, event.getSize(), event.getCreateDate(), null); + _usageVolumeDao.persist(volumeVO); } - Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); - UsageStorageVO volumeVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null); - _usageStorageDao.persist(volumeVO); + break; + + case EventTypes.EVENT_VOLUME_DELETE: + deleteExistingVolumeUsage(volId, event.getAccountId(), event.getCreateDate()); + break; + + case EventTypes.EVENT_VOLUME_ATTACH: + deleteExistingInstanceVolumeUsage(event.getResourceId(), event.getAccountId(), event.getCreateDate()); + + logger.debug("Creating a new entry in usage_volume for volume with id: {}, and instance id: {} for account: {}", + volId, event.getVmId(), event.getAccountId()); + volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getVmId(), event.getSize(), event.getCreateDate(), null); + _usageVolumeDao.persist(volumeVO); + break; + + case EventTypes.EVENT_VOLUME_DETACH: + deleteExistingInstanceVolumeUsage(event.getResourceId(), event.getAccountId(), event.getCreateDate()); + break; + + case EventTypes.EVENT_VOLUME_UPLOAD: + deleteExistingSecondaryStorageUsageForVolume(volId, event.getAccountId(), event.getCreateDate()); + + logger.debug("Creating a new entry in usage_storage for volume with id : {} for account: {}", volId, event.getAccountId()); + UsageStorageVO storageVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null); + _usageStorageDao.persist(storageVO); + break; } } diff --git a/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java index e834b713d420..0210b899e8c9 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VolumeUsageParser.java @@ -73,12 +73,13 @@ protected boolean parse(AccountVO account, Date startDate, Date endDate) { for (UsageVolumeVO usageVol : usageUsageVols) { long volId = usageVol.getVolumeId(); Long doId = usageVol.getDiskOfferingId(); + Long vmId = usageVol.getVmId(); long zoneId = usageVol.getZoneId(); Long templateId = usageVol.getTemplateId(); long size = usageVol.getSize(); - String key = volId + "-" + doId + "-" + size; + String key = volId + "-" + doId + "-" + vmId + "-" + size; - diskOfferingMap.put(key, new VolInfo(volId, zoneId, doId, templateId, size)); + diskOfferingMap.put(key, new VolInfo(volId, zoneId, doId, templateId, size, vmId)); Date volCreateDate = usageVol.getCreated(); Date volDeleteDate = usageVol.getDeleted(); @@ -110,7 +111,7 @@ protected boolean parse(AccountVO account, Date startDate, Date endDate) { if (useTime > 0L) { VolInfo info = diskOfferingMap.get(volIdKey); createUsageRecord(UsageTypes.VOLUME, useTime, startDate, endDate, account, info.getVolumeId(), info.getZoneId(), info.getDiskOfferingId(), - info.getTemplateId(), info.getSize()); + info.getTemplateId(), info.getVmId(), info.getSize()); } } @@ -130,7 +131,7 @@ private void updateVolUsageData(Map> usageDataMap, Stri } private void createUsageRecord(int type, long runningTime, Date startDate, Date endDate, AccountVO account, long volId, long zoneId, Long doId, - Long templateId, long size) { + Long templateId, Long vmId, long size) { // Our smallest increment is hourly for now logger.debug("Total running time {} ms", runningTime); @@ -152,7 +153,11 @@ private void createUsageRecord(int type, long runningTime, Date startDate, Date usageDesc += " (DiskOffering: " + doId + ")"; } - UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), null, null, doId, templateId, volId, + if (vmId != null) { + usageDesc += " (VM: " + vmId + ")"; + } + + UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), vmId, null, doId, templateId, volId, size, startDate, endDate); usageDao.persist(usageRecord); } @@ -163,13 +168,15 @@ private static class VolInfo { private Long diskOfferingId; private Long templateId; private long size; + private Long vmId; - public VolInfo(long volId, long zoneId, Long diskOfferingId, Long templateId, long size) { + public VolInfo(long volId, long zoneId, Long diskOfferingId, Long templateId, long size, Long vmId) { this.volId = volId; this.zoneId = zoneId; this.diskOfferingId = diskOfferingId; this.templateId = templateId; this.size = size; + this.vmId = vmId; } public long getZoneId() { @@ -191,5 +198,9 @@ public Long getTemplateId() { public long getSize() { return size; } + + public Long getVmId() { + return vmId; + } } }