Make libvirt assign BDFs to PCI devices that are in sync with CHV

src/ch/ch_domain.c

@@ -83,6 +83,7 @@ virCHDomainObjPrivateFree(void *data)
 {
     virCHDomainObjPrivate *priv = data;
 
+    g_clear_pointer(&priv->pciAddrSet, virDomainPCIAddressSetFree);
     virChrdevFree(priv->chrdevs);
     g_free(priv->machineName);
     virBitmapFree(priv->autoCpuset);

src/ch/ch_domain.h

@@ -26,6 +26,7 @@
 #include "vircgroup.h"
 #include "virdomainjob.h"
 #include "virthread.h"
+#include "domain_addr.h"
 
 typedef struct _chMigrationDstArgs chMigrationDstArgs;
 
@@ -47,6 +48,7 @@ struct _virCHDomainObjPrivate {
      * events.
      */
     int shutdown_done;
+    virDomainPCIAddressSet *pciAddrSet;
 };
 
 struct _chMigrationDstArgs {

src/ch/ch_driver.c

@@ -27,6 +27,7 @@
 #include "ch_domain.h"
 #include "ch_driver.h"
 #include "ch_monitor.h"
+#include "ch_pci_addr.h"
 #include "ch_process.h"
 #include "domain_capabilities.h"
 #include "domain_cgroup.h"
@@ -45,8 +46,6 @@
 #include "virlog.h"
 #include "virobject.h"
 #include "virfile.h"
-#include "virstring.h"
-#include "virtime.h"
 #include "virtypedparam.h"
 #include "virutil.h"
 #include "viruuid.h"
@@ -3486,6 +3485,10 @@ chDomainAttachDeviceLive(virDomainObj *vm,
             break;
         }
 
+        if (1 == chEnsurePciAddress(vm, dev))
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+    _("Couldn't allocate PCI device slot for device %s!"), dev->data.disk->info.alias);
+
         disks = virJSONValueNewArray();
         if (virCHMonitorBuildDiskJson(disks, dev->data.disk) < 0) {
             DBG("Attach disk failed");
@@ -3502,14 +3505,17 @@ chDomainAttachDeviceLive(virDomainObj *vm,
             break;
         }
 
+        VIR_WARN("Disk : dst: %s drivername: %s alias: %s PCI slot: %d", dev->data.disk->dst, dev->data.disk->driverName, dev->data.disk->info.alias, dev->data.disk->info.addr.pci.slot);
         virDomainDiskInsert(vm->def, dev->data.disk);
         dev->data.disk = NULL;
-
         ret = 0;
         break;
     }
     case VIR_DOMAIN_DEVICE_NET:
     {
+        if (chEnsurePciAddress(vm, dev))
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+            ("Couldn't allocate PCI device slot for Net device: %s!"), dev->data.net->info.alias);
         virDomainNetInsert(vm->def, dev->data.net);
         ret = chProcessAddNetworkDevice(driver, mon, vm->def, dev->data.net);
         dev->data.net = NULL;
@@ -3935,6 +3941,14 @@ chDomainDetachDeviceLive(virDomainObj *vm,
         return -1;
     }
 
+    if (info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+        DBG("Release PCI address for device '%s'", info->alias);
+        chDomainReleaseDeviceAddress(vm, info);
+        DBG("");
+    } else {
+        DBG("Detached non-PCI device '%s'!", info->alias);
+    }
+
     if (match->type == VIR_DOMAIN_DEVICE_DISK) {
         idx = chFindDisk(vm->def, match->data.disk->dst);
         if (idx >= 0) {

src/ch/ch_monitor.c

@@ -26,14 +26,15 @@
 #include <unistd.h>
 #include <curl/curl.h>
 
-#include "datatypes.h"
 #include "ch_conf.h"
 #include "ch_domain.h"
 #include "ch_events.h"
 #include "ch_interface.h"
 #include "ch_monitor.h"
+#include "ch_pci_addr.h"
 #include "ch_socket.h"
 #include "domain_interface.h"
+#include "libvirt/libvirt.h"
 #include "viralloc.h"
 #include "vircommand.h"
 #include "virerror.h"
@@ -196,13 +197,26 @@ virCHMonitorBuildConsoleJson(virJSONValue *content,
 
     if (vmdef->nconsoles &&
         vmdef->consoles[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) {
+        DBG("Create Console with type: %d", vmdef->consoles[0]->info.type);
+        if (vmdef->consoles[0]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+            DBG("Found Console '%s' bound to PCI slot %d", vmdef->consoles[0]->info.alias, vmdef->consoles[0]->info.addr.pci.slot);
+        }
         if (virJSONValueObjectAppendString(console, "mode", "Pty") < 0)
             return -1;
         if (virJSONValueObjectAppend(content, "console", &console) < 0)
             return -1;
     }
 
     if (vmdef->nserials) {
+        if (vmdef->serials[0]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+            DBG("Found serial Console '%s' bound to PCI slot %d", vmdef->serials[0]->info.alias, vmdef->serials[0]->info.addr.pci.slot);
+            if (virJSONValueObjectAppendNumberInt(serial, "bdf_device_id", vmdef->serials[0]->info.addr.pci.slot) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR,
+                           ("Failed to add slot number to JSON for console with alias '%s'"),
+                           vmdef->serials[0]->info.alias);
+                return -1;
+            }
+        }
         if (vmdef->serials[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) {
             if (virJSONValueObjectAppendString(serial, "mode", "Pty") < 0)
                 return -1;
@@ -461,6 +475,11 @@ virCHMonitorBuildDiskJson(virJSONValue *disks, virDomainDiskDef *diskdef)
     if (!diskdef->src)
         return -1;
 
+    if (!disk) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "Failed to allocate memory for disk JSON!");
+        return -1;
+    }
+
     switch (diskdef->src->type) {
     case VIR_STORAGE_TYPE_FILE:
         if (!diskdef->src->path) {
@@ -491,6 +510,16 @@ virCHMonitorBuildDiskJson(virJSONValue *disks, virDomainDiskDef *diskdef)
             if (virJSONValueObjectAppendBoolean(disk, "readonly", true) < 0)
                 return -1;
         }
+
+        if (diskdef->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+            if (virJSONValueObjectAppendNumberInt(disk, "bdf_device_id", diskdef->info.addr.pci.slot) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR,
+                           ("Failed to add slot number to JSON for disk with alias '%s'"),
+                           diskdef->info.alias);
+                return -1;
+            }
+        }
+
         if (virJSONValueArrayAppend(disks, &disk) < 0)
             return -1;
 
@@ -522,8 +551,12 @@ virCHMonitorBuildDisksJson(virJSONValue *content, virDomainDef *vmdef)
         disks = virJSONValueNewArray();
 
         for (i = 0; i < vmdef->ndisks; i++) {
-            if (virCHMonitorBuildDiskJson(disks, vmdef->disks[i]) < 0)
+            if (virCHMonitorBuildDiskJson(disks, vmdef->disks[i]) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR,
+                           _("Failed to attach disk with alias '%s'"),
+                           vmdef->disks[i]->info.alias);
                 return -1;
+            }
         }
         if (virJSONValueObjectAppend(content, "disks", &disks) < 0)
             return -1;
@@ -552,6 +585,16 @@ virCHMonitorBuildRngJson(virJSONValue *content, virDomainDef *vmdef)
         if (virJSONValueObjectAppendString(rng, "src", vmdef->rngs[0]->source.file) < 0)
             return -1;
 
+            // We already know that we have a VIRTIO device from above
+        if (vmdef->rngs[0]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+            if (virJSONValueObjectAppendNumberInt(rng, "bdf_device_id", vmdef->rngs[0]->info.addr.pci.slot) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR,
+                            ("Failed to add slot number to JSON for RNG device with alias '%s'"),
+                            vmdef->rngs[0]->info.alias);
+                return -1;
+            }
+        }
+
         if (virJSONValueObjectAppend(content, "rng", &rng) < 0)
             return -1;
 
@@ -597,19 +640,12 @@ virCHMonitorBuildNetJson(virDomainNetDef *net,
 
     // Populate the <interface type="pci"> XML tag, relevant for OpenStack
     // Currently not needed to have sane values here.
-	net->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
-    net->info.addr.pci.bus = 0;
-    assert(netindex <= 7);
-    net->info.addr.pci.slot = netindex + 1;
-    net->info.addr.pci.function = 0;
-
     if (actualType == VIR_DOMAIN_NET_TYPE_ETHERNET &&
         net->guestIP.nips == 1) {
         const virNetDevIPAddr *ip;
         g_autofree char *addr = NULL;
         virSocketAddr netmask;
         g_autofree char *netmaskStr = NULL;
-
         ip = net->guestIP.ips[0];
 
         if (!(addr = virSocketAddrFormat(&ip->address)))
@@ -671,6 +707,11 @@ virCHMonitorBuildNetJson(virDomainNetDef *net,
             return -1;
     }
 
+    if (net->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+        if (virJSONValueObjectAppendNumberInt(net_json, "bdf_device_id", net->info.addr.pci.slot) < 0)
+            return -1;
+    }
+
     if (!(*jsonstr = virJSONValueToString(net_json, false)))
         return -1;
 
@@ -939,6 +980,20 @@ virCHMonitorReattach(virDomainObj *vm, virCHDriverConfig *cfg, virCHDriver *driv
     mon->eventmonitorfd = event_monitor_fd;
     VIR_DEBUG("%s: Opened the event monitor FIFO(%s)", vm->def->name, mon->eventmonitorpath);
 
+    // Initialize our one and only PCI bus
+    if (chDomainPCIAddressSetCreate(vm)) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                   "CHV driver only supports `ethernet` network types!");
+        return NULL;
+    }
+
+    // Attach all devices from the config to the PCI bus
+    if (chInitPciDevices(vm)) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                    "Failed to assign addresses to PCI devices defined in XML!");
+        return NULL;
+    }
+
     /* now has its own reference */
     mon->vm = virObjectRef(vm);
 
@@ -1863,15 +1918,12 @@ int virCHMonitorMigrationReceive(virCHMonitor *mon,
             net_json = virJSONValueNewObject();
             // TODO switch to chAssignDeviceNetAlias from ch_alias.c
             id = g_strdup_printf("%s_%zu", CH_NET_ID_PREFIX, i);
-			vmdef->nets[i]->info.alias = g_strdup_printf("%s", id);
-
-			// Populate the <interface type="pci"> XML tag, relevant for OpenStack
-			// Currently not needed to have sane values here.
-			vmdef->nets[i]->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
-			vmdef->nets[i]->info.addr.pci.bus = 0;
-			assert(i <= 7);
-			vmdef->nets[i]->info.addr.pci.slot = i + 1;
-			vmdef->nets[i]->info.addr.pci.function = 0;
+            vmdef->nets[i]->info.alias = g_strdup_printf("%s", id);
+
+            if (vmdef->nets[i]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
+                if (virJSONValueObjectAppendNumberInt(net_json, "bdf_device_id", vmdef->nets[i]->info.addr.pci.slot) < 0)
+                    return -1;
+            }
 
             if (virJSONValueObjectAppendString(net_json, "id", id) < 0) {
                 rc = -1;

src/ch/ch_monitor.h

@@ -24,7 +24,6 @@
 
 #include "virobject.h"
 #include "virjson.h"
-#include "domain_conf.h"
 #include "domain_logcontext.h"
 #include "ch_conf.h"