Track volume usage data at a vm granularity as well

engine/components-api/src/main/java/com/cloud/event/UsageEventUtils.java

@@ -94,6 +94,14 @@ public static void publishUsageEvent(String usageType, long accountId, long zone
 
     }
 
+    public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
+                                         Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) {
+        if (displayResource) {
+            saveUsageEvent(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName);
+        }
+        publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID);
+    }
+
     public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
                                          Long size, Long virtualSize, String entityType, String entityUUID, Map<String, String> details) {
         saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, virtualSize, details);
@@ -202,6 +210,10 @@ public static void saveUsageEvent(String usageType, long accountId, long zoneId,
         s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, vmId, securityGroupId));
     }
 
+    public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
+        s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName));
+    }
+
     private static void publishUsageEvent(String usageEventType, Long accountId, Long zoneId, String resourceType, String resourceUUID) {
         String configKey = "publish.usage.events";
         String value = s_configDao.getValue(configKey);

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -903,7 +903,7 @@ public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offeri
         // Save usage event and update resource count for user vm volumes
         if (vm.getType() == VirtualMachine.Type.User) {
             UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size,
-                    Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
+                    Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
             _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
         }
         DiskProfile diskProfile = toDiskProfile(vol, offering);
@@ -981,7 +981,7 @@ private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering
             }
 
             UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size,
-                    Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
+                    Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
 
             _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
         }

engine/schema/src/main/java/com/cloud/event/UsageEventVO.java

@@ -75,6 +75,9 @@ public enum DynamicParameters {
     @Column(name = "virtual_size")
     private Long virtualSize;
 
+    @Column(name = "vm_id")
+    private Long vmId;
+
     public UsageEventVO() {
     }
 
@@ -143,6 +146,18 @@ public UsageEventVO(String usageType, long accountId, long zoneId, long vmId, lo
         this.offeringId = securityGroupId;
     }
 
+    public UsageEventVO(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
+        this.type = usageType;
+        this.accountId = accountId;
+        this.zoneId = zoneId;
+        this.resourceId = resourceId;
+        this.offeringId = offeringId;
+        this.templateId = templateId;
+        this.size = size;
+        this.vmId = vmId;
+        this.resourceName = resourceName;
+    }
+
     @Override
     public long getId() {
         return id;
@@ -248,4 +263,11 @@ public void setVirtualSize(Long virtualSize) {
         this.virtualSize = virtualSize;
     }
 
+    public Long getVmId() {
+        return vmId;
+    }
+
+    public void setVmId(Long vmId) {
+        this.vmId = vmId;
+    }
 }

engine/schema/src/main/java/com/cloud/event/dao/UsageEventDaoImpl.java

@@ -45,11 +45,11 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
     private final SearchBuilder<UsageEventVO> latestEventsSearch;
     private final SearchBuilder<UsageEventVO> IpeventsSearch;
     private static final String COPY_EVENTS =
-        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
-            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
+        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
+            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
     private static final String COPY_ALL_EVENTS =
-        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
-            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
+        "INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
+            + "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
     private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "
             + "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? ";
     private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "

engine/schema/src/main/java/com/cloud/usage/UsageVolumeVO.java

@@ -59,6 +59,9 @@ public class UsageVolumeVO implements InternalIdentity {
     @Column(name = "size")
     private long size;
 
+    @Column(name = "vm_id")
+    private Long vmId;
+
     @Column(name = "created")
     @Temporal(value = TemporalType.TIMESTAMP)
     private Date created = null;
@@ -70,13 +73,14 @@ public class UsageVolumeVO implements InternalIdentity {
     protected UsageVolumeVO() {
     }
 
-    public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, long size, Date created, Date deleted) {
+    public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, Long vmId, long size, Date created, Date deleted) {
         this.volumeId = id;
         this.zoneId = zoneId;
         this.accountId = accountId;
         this.domainId = domainId;
         this.diskOfferingId = diskOfferingId;
         this.templateId = templateId;
+        this.vmId = vmId;
         this.size = size;
         this.created = created;
         this.deleted = deleted;
@@ -126,4 +130,12 @@ public void setDeleted(Date deleted) {
     public long getVolumeId() {
         return volumeId;
     }
+
+    public Long getVmId() {
+        return vmId;
+    }
+
+    public void setVmId(Long vmId) {
+        this.vmId = vmId;
+    }
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageStorageDaoImpl.java

@@ -57,6 +57,7 @@ public UsageStorageDaoImpl() {
         IdSearch.and("accountId", IdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
         IdSearch.and("id", IdSearch.entity().getEntityId(), SearchCriteria.Op.EQ);
         IdSearch.and("type", IdSearch.entity().getStorageType(), SearchCriteria.Op.EQ);
+        IdSearch.and("deleted", IdSearch.entity().getDeleted(), SearchCriteria.Op.NULL);
         IdSearch.done();
 
         IdZoneSearch = createSearchBuilder();
@@ -74,6 +75,7 @@ public List<UsageStorageVO> listById(long accountId, long id, int type) {
         sc.setParameters("accountId", accountId);
         sc.setParameters("id", id);
         sc.setParameters("type", type);
+        sc.setParameters("deleted", null);
         return listBy(sc, null);
     }
 

engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDao.java

@@ -23,9 +23,7 @@
 import com.cloud.utils.db.GenericDao;
 
 public interface UsageVolumeDao extends GenericDao<UsageVolumeVO, Long> {
-    public void removeBy(long userId, long id);
-
-    public void update(UsageVolumeVO usage);
-
     public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate, boolean limit, int page);
+
+    List<UsageVolumeVO> listByVolumeId(long volumeId, long accountId);
 }

engine/schema/src/main/java/com/cloud/usage/dao/UsageVolumeDaoImpl.java

@@ -18,81 +18,46 @@
 
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
-import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.TimeZone;
 
 
-import com.cloud.exception.CloudException;
+import javax.annotation.PostConstruct;
+
 import org.springframework.stereotype.Component;
 
 import com.cloud.usage.UsageVolumeVO;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.TransactionLegacy;
 
 @Component
 public class UsageVolumeDaoImpl extends GenericDaoBase<UsageVolumeVO, Long> implements UsageVolumeDao {
 
-    protected static final String REMOVE_BY_USERID_VOLID = "DELETE FROM usage_volume WHERE account_id = ? AND volume_id = ?";
-    protected static final String UPDATE_DELETED = "UPDATE usage_volume SET deleted = ? WHERE account_id = ? AND volume_id = ? and deleted IS NULL";
-    protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
+    protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
         + "FROM usage_volume " + "WHERE account_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR "
         + "      (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))";
-    protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
+    protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
         + "FROM usage_volume " + "WHERE domain_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR "
         + "      (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))";
-    protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
+    protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
         + "FROM usage_volume " + "WHERE (deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + "      (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?))";
+    private SearchBuilder<UsageVolumeVO> volumeSearch;
 
     public UsageVolumeDaoImpl() {
     }
 
-    @Override
-    public void removeBy(long accountId, long volId) {
-        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
-        try {
-            txn.start();
-            try(PreparedStatement pstmt = txn.prepareStatement(REMOVE_BY_USERID_VOLID);) {
-                if (pstmt != null) {
-                    pstmt.setLong(1, accountId);
-                    pstmt.setLong(2, volId);
-                    pstmt.executeUpdate();
-                }
-            }catch (SQLException e) {
-                throw new CloudException("Error removing usageVolumeVO:"+e.getMessage(), e);
-            }
-            txn.commit();
-        } catch (Exception e) {
-            txn.rollback();
-            logger.warn("Error removing usageVolumeVO:"+e.getMessage(), e);
-        } finally {
-            txn.close();
-        }
-    }
-
-    @Override
-    public void update(UsageVolumeVO usage) {
-        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
-        PreparedStatement pstmt = null;
-        try {
-            txn.start();
-            if (usage.getDeleted() != null) {
-                pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED);
-                pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getDeleted()));
-                pstmt.setLong(2, usage.getAccountId());
-                pstmt.setLong(3, usage.getVolumeId());
-                pstmt.executeUpdate();
-            }
-            txn.commit();
-        } catch (Exception e) {
-            txn.rollback();
-            logger.warn("Error updating UsageVolumeVO", e);
-        } finally {
-            txn.close();
-        }
+    @PostConstruct
+    protected void init() {
+        volumeSearch = createSearchBuilder();
+        volumeSearch.and("accountId", volumeSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
+        volumeSearch.and("deleted", volumeSearch.entity().getDeleted(), SearchCriteria.Op.NULL);
+        volumeSearch.done();
     }
 
     @Override
@@ -150,11 +115,15 @@ public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date s
                 if (tId == 0) {
                     tId = null;
                 }
-                long size = Long.valueOf(rs.getLong(7));
+                Long vmId = Long.valueOf(rs.getLong(7));
+                if (vmId == 0) {
+                    vmId = null;
+                }
+                long size = Long.valueOf(rs.getLong(8));
                 Date createdDate = null;
                 Date deletedDate = null;
-                String createdTS = rs.getString(8);
-                String deletedTS = rs.getString(9);
+                String createdTS = rs.getString(9);
+                String deletedTS = rs.getString(10);
 
                 if (createdTS != null) {
                     createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS);
@@ -163,7 +132,7 @@ public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date s
                     deletedDate = DateUtil.parseDateString(s_gmtTimeZone, deletedTS);
                 }
 
-                usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, size, createdDate, deletedDate));
+                usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, vmId, size, createdDate, deletedDate));
             }
         } catch (Exception e) {
             txn.rollback();
@@ -174,4 +143,13 @@ public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date s
 
         return usageRecords;
     }
+
+    @Override
+    public List<UsageVolumeVO> listByVolumeId(long volumeId, long accountId) {
+        SearchCriteria<UsageVolumeVO> sc = volumeSearch.create();
+        sc.setParameters("accountId", accountId);
+        sc.setParameters("volumeId", volumeId);
+        sc.setParameters("deleted", null);
+        return listBy(sc);
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql

@@ -32,5 +32,12 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHA
 -- Populate uuid for existing rows where uuid is NULL or empty
 UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid = '';
 
+-- Add vm_id column to usage_event table for volume usage events
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"');
+CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"');
+
+-- Add vm_id column to cloud_usage.usage_volume table
+CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"');
+
 -- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository.
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone''');

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -4330,6 +4330,9 @@ public UsageRecordResponse createUsageResponse(Usage usageRecord, Map<String, Se
                 if (volume != null) {
                     builder.append("for ").append(volume.getName()).append(" (").append(volume.getUuid()).append(")");
                 }
+                if (vmInstance != null) {
+                    builder.append(" attached to VM ").append(vmInstance.getHostName()).append(" (").append(vmInstance.getUuid()).append(")");
+                }
                 if (diskOff != null) {
                     builder.append(" with disk offering ").append(diskOff.getName()).append(" (").append(diskOff.getUuid()).append(")");
                 }

server/src/main/java/com/cloud/hypervisor/KVMGuru.java

@@ -21,6 +21,8 @@
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.configuration.ConfigurationManagerImpl;
+import com.cloud.event.EventTypes;
+import com.cloud.event.UsageEventUtils;
 import com.cloud.host.HostVO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
@@ -370,6 +372,8 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId,
                    _volumeDao.update(volume.getId(), volume);
                    _volumeDao.attachVolume(volume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes));
                }
+               UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(),
+                       volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), vm.getId(), volume.isDisplay());
            }
         } catch (Exception e) {
             throw new RuntimeException("Could not restore VM " + vm.getName() + " due to : " + e.getMessage());
@@ -387,6 +391,8 @@ public VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId,
                 _volumeDao.attachVolume(restoredVolume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes));
                 restoredVolume.setState(Volume.State.Ready);
                 _volumeDao.update(restoredVolume.getId(), restoredVolume);
+                UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, restoredVolume.getAccountId(), restoredVolume.getDataCenterId(), restoredVolume.getId(), restoredVolume.getName(),
+                        restoredVolume.getDiskOfferingId(), restoredVolume.getTemplateId(), restoredVolume.getSize(), Volume.class.getName(), restoredVolume.getUuid(), vm.getId(), restoredVolume.isDisplay());
                 return true;
             } catch (Exception e) {
                 restoredVolume.setDisplay(false);