diff --git a/src/ch/ch_domain.c b/src/ch/ch_domain.c index f6c2914960..370d90dd98 100644 --- a/src/ch/ch_domain.c +++ b/src/ch/ch_domain.c @@ -83,6 +83,7 @@ virCHDomainObjPrivateFree(void *data) { virCHDomainObjPrivate *priv = data; + g_clear_pointer(&priv->pciAddrSet, virDomainPCIAddressSetFree); virChrdevFree(priv->chrdevs); g_free(priv->machineName); virBitmapFree(priv->autoCpuset); diff --git a/src/ch/ch_domain.h b/src/ch/ch_domain.h index 771543bfe8..87515ce796 100644 --- a/src/ch/ch_domain.h +++ b/src/ch/ch_domain.h @@ -26,6 +26,7 @@ #include "vircgroup.h" #include "virdomainjob.h" #include "virthread.h" +#include "domain_addr.h" typedef struct _chMigrationDstArgs chMigrationDstArgs; @@ -47,6 +48,7 @@ struct _virCHDomainObjPrivate { * events. */ int shutdown_done; + virDomainPCIAddressSet *pciAddrSet; }; struct _chMigrationDstArgs { diff --git a/src/ch/ch_driver.c b/src/ch/ch_driver.c index bcaa1a5a31..3dfba465b9 100644 --- a/src/ch/ch_driver.c +++ b/src/ch/ch_driver.c @@ -27,6 +27,7 @@ #include "ch_domain.h" #include "ch_driver.h" #include "ch_monitor.h" +#include "ch_pci_addr.h" #include "ch_process.h" #include "domain_capabilities.h" #include "domain_cgroup.h" @@ -45,8 +46,6 @@ #include "virlog.h" #include "virobject.h" #include "virfile.h" -#include "virstring.h" -#include "virtime.h" #include "virtypedparam.h" #include "virutil.h" #include "viruuid.h" @@ -3486,6 +3485,10 @@ chDomainAttachDeviceLive(virDomainObj *vm, break; } + if (1 == chEnsurePciAddress(vm, dev)) + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Couldn't allocate PCI device slot for device %s!"), dev->data.disk->info.alias); + disks = virJSONValueNewArray(); if (virCHMonitorBuildDiskJson(disks, dev->data.disk) < 0) { DBG("Attach disk failed"); @@ -3502,14 +3505,17 @@ chDomainAttachDeviceLive(virDomainObj *vm, break; } + VIR_WARN("Disk : dst: %s drivername: %s alias: %s PCI slot: %d", dev->data.disk->dst, dev->data.disk->driverName, dev->data.disk->info.alias, dev->data.disk->info.addr.pci.slot); virDomainDiskInsert(vm->def, dev->data.disk); dev->data.disk = NULL; - ret = 0; break; } case VIR_DOMAIN_DEVICE_NET: { + if (chEnsurePciAddress(vm, dev)) + virReportError(VIR_ERR_INTERNAL_ERROR, + ("Couldn't allocate PCI device slot for Net device: %s!"), dev->data.net->info.alias); virDomainNetInsert(vm->def, dev->data.net); ret = chProcessAddNetworkDevice(driver, mon, vm->def, dev->data.net); dev->data.net = NULL; @@ -3935,6 +3941,14 @@ chDomainDetachDeviceLive(virDomainObj *vm, return -1; } + if (info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + DBG("Release PCI address for device '%s'", info->alias); + chDomainReleaseDeviceAddress(vm, info); + DBG(""); + } else { + DBG("Detached non-PCI device '%s'!", info->alias); + } + if (match->type == VIR_DOMAIN_DEVICE_DISK) { idx = chFindDisk(vm->def, match->data.disk->dst); if (idx >= 0) { diff --git a/src/ch/ch_monitor.c b/src/ch/ch_monitor.c index 236ad8a4f0..5d0f72eed6 100644 --- a/src/ch/ch_monitor.c +++ b/src/ch/ch_monitor.c @@ -26,14 +26,15 @@ #include #include -#include "datatypes.h" #include "ch_conf.h" #include "ch_domain.h" #include "ch_events.h" #include "ch_interface.h" #include "ch_monitor.h" +#include "ch_pci_addr.h" #include "ch_socket.h" #include "domain_interface.h" +#include "libvirt/libvirt.h" #include "viralloc.h" #include "vircommand.h" #include "virerror.h" @@ -196,6 +197,10 @@ virCHMonitorBuildConsoleJson(virJSONValue *content, if (vmdef->nconsoles && vmdef->consoles[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) { + DBG("Create Console with type: %d", vmdef->consoles[0]->info.type); + if (vmdef->consoles[0]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + DBG("Found Console '%s' bound to PCI slot %d", vmdef->consoles[0]->info.alias, vmdef->consoles[0]->info.addr.pci.slot); + } if (virJSONValueObjectAppendString(console, "mode", "Pty") < 0) return -1; if (virJSONValueObjectAppend(content, "console", &console) < 0) @@ -203,6 +208,15 @@ virCHMonitorBuildConsoleJson(virJSONValue *content, } if (vmdef->nserials) { + if (vmdef->serials[0]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + DBG("Found serial Console '%s' bound to PCI slot %d", vmdef->serials[0]->info.alias, vmdef->serials[0]->info.addr.pci.slot); + if (virJSONValueObjectAppendNumberInt(serial, "bdf_device_id", vmdef->serials[0]->info.addr.pci.slot) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + ("Failed to add slot number to JSON for console with alias '%s'"), + vmdef->serials[0]->info.alias); + return -1; + } + } if (vmdef->serials[0]->source->type == VIR_DOMAIN_CHR_TYPE_PTY) { if (virJSONValueObjectAppendString(serial, "mode", "Pty") < 0) return -1; @@ -461,6 +475,11 @@ virCHMonitorBuildDiskJson(virJSONValue *disks, virDomainDiskDef *diskdef) if (!diskdef->src) return -1; + if (!disk) { + virReportError(VIR_ERR_INTERNAL_ERROR, "Failed to allocate memory for disk JSON!"); + return -1; + } + switch (diskdef->src->type) { case VIR_STORAGE_TYPE_FILE: if (!diskdef->src->path) { @@ -491,6 +510,16 @@ virCHMonitorBuildDiskJson(virJSONValue *disks, virDomainDiskDef *diskdef) if (virJSONValueObjectAppendBoolean(disk, "readonly", true) < 0) return -1; } + + if (diskdef->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + if (virJSONValueObjectAppendNumberInt(disk, "bdf_device_id", diskdef->info.addr.pci.slot) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + ("Failed to add slot number to JSON for disk with alias '%s'"), + diskdef->info.alias); + return -1; + } + } + if (virJSONValueArrayAppend(disks, &disk) < 0) return -1; @@ -522,8 +551,12 @@ virCHMonitorBuildDisksJson(virJSONValue *content, virDomainDef *vmdef) disks = virJSONValueNewArray(); for (i = 0; i < vmdef->ndisks; i++) { - if (virCHMonitorBuildDiskJson(disks, vmdef->disks[i]) < 0) + if (virCHMonitorBuildDiskJson(disks, vmdef->disks[i]) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to attach disk with alias '%s'"), + vmdef->disks[i]->info.alias); return -1; + } } if (virJSONValueObjectAppend(content, "disks", &disks) < 0) return -1; @@ -552,6 +585,16 @@ virCHMonitorBuildRngJson(virJSONValue *content, virDomainDef *vmdef) if (virJSONValueObjectAppendString(rng, "src", vmdef->rngs[0]->source.file) < 0) return -1; + // We already know that we have a VIRTIO device from above + if (vmdef->rngs[0]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + if (virJSONValueObjectAppendNumberInt(rng, "bdf_device_id", vmdef->rngs[0]->info.addr.pci.slot) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + ("Failed to add slot number to JSON for RNG device with alias '%s'"), + vmdef->rngs[0]->info.alias); + return -1; + } + } + if (virJSONValueObjectAppend(content, "rng", &rng) < 0) return -1; @@ -597,19 +640,12 @@ virCHMonitorBuildNetJson(virDomainNetDef *net, // Populate the XML tag, relevant for OpenStack // Currently not needed to have sane values here. - net->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; - net->info.addr.pci.bus = 0; - assert(netindex <= 7); - net->info.addr.pci.slot = netindex + 1; - net->info.addr.pci.function = 0; - if (actualType == VIR_DOMAIN_NET_TYPE_ETHERNET && net->guestIP.nips == 1) { const virNetDevIPAddr *ip; g_autofree char *addr = NULL; virSocketAddr netmask; g_autofree char *netmaskStr = NULL; - ip = net->guestIP.ips[0]; if (!(addr = virSocketAddrFormat(&ip->address))) @@ -671,6 +707,11 @@ virCHMonitorBuildNetJson(virDomainNetDef *net, return -1; } + if (net->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + if (virJSONValueObjectAppendNumberInt(net_json, "bdf_device_id", net->info.addr.pci.slot) < 0) + return -1; + } + if (!(*jsonstr = virJSONValueToString(net_json, false))) return -1; @@ -939,6 +980,20 @@ virCHMonitorReattach(virDomainObj *vm, virCHDriverConfig *cfg, virCHDriver *driv mon->eventmonitorfd = event_monitor_fd; VIR_DEBUG("%s: Opened the event monitor FIFO(%s)", vm->def->name, mon->eventmonitorpath); + // Initialize our one and only PCI bus + if (chDomainPCIAddressSetCreate(vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "CHV driver only supports `ethernet` network types!"); + return NULL; + } + + // Attach all devices from the config to the PCI bus + if (chInitPciDevices(vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Failed to assign addresses to PCI devices defined in XML!"); + return NULL; + } + /* now has its own reference */ mon->vm = virObjectRef(vm); @@ -1863,15 +1918,12 @@ int virCHMonitorMigrationReceive(virCHMonitor *mon, net_json = virJSONValueNewObject(); // TODO switch to chAssignDeviceNetAlias from ch_alias.c id = g_strdup_printf("%s_%zu", CH_NET_ID_PREFIX, i); - vmdef->nets[i]->info.alias = g_strdup_printf("%s", id); - - // Populate the XML tag, relevant for OpenStack - // Currently not needed to have sane values here. - vmdef->nets[i]->info.type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; - vmdef->nets[i]->info.addr.pci.bus = 0; - assert(i <= 7); - vmdef->nets[i]->info.addr.pci.slot = i + 1; - vmdef->nets[i]->info.addr.pci.function = 0; + vmdef->nets[i]->info.alias = g_strdup_printf("%s", id); + + if (vmdef->nets[i]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + if (virJSONValueObjectAppendNumberInt(net_json, "bdf_device_id", vmdef->nets[i]->info.addr.pci.slot) < 0) + return -1; + } if (virJSONValueObjectAppendString(net_json, "id", id) < 0) { rc = -1; diff --git a/src/ch/ch_monitor.h b/src/ch/ch_monitor.h index 5d73381aca..6ce9ab31d1 100644 --- a/src/ch/ch_monitor.h +++ b/src/ch/ch_monitor.h @@ -24,7 +24,6 @@ #include "virobject.h" #include "virjson.h" -#include "domain_conf.h" #include "domain_logcontext.h" #include "ch_conf.h" diff --git a/src/ch/ch_pci_addr.c b/src/ch/ch_pci_addr.c new file mode 100644 index 0000000000..373f13d142 --- /dev/null +++ b/src/ch/ch_pci_addr.c @@ -0,0 +1,496 @@ +/* + * Copyright Cyberus Technology GmbH. 2025 + * + * ch_pci_addr.c: Manage Cloud-Hypervisor PCI device addresses + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * . + */ + +#include + +#include +#include +#include + +#include +#include "device_conf.h" +#include "domain_addr.h" +#include "domain_conf.h" +#include "glib.h" +#include "libvirt/libvirt.h" +#include "libvirt/virterror.h" +#include "virerror.h" +#include "virlog.h" +#include "ch_domain.h" +#include "ch_pci_addr.h" +#include "virerror.h" +#include "virlog.h" +#include "virpci.h" + +#define VIR_FROM_THIS VIR_FROM_CH +VIR_LOG_INIT("ch.ch_pci_addr"); + +const char* DEFAULT_RNG_SOURCE = "/dev/urandom"; +const char* DEFAULT_RNG_ALIAS = "implicit-rng-device"; + +// Helper to collect `virDomainDeviceInfo`s that need to allocate a completely new PCI device ID. +typedef struct _dynamicAddressQueue { + // Current length of the queue + size_t size; + // Index of the next free slot in the queue; Equals the number of items in the queue + size_t nextIdx; + // Array of pointers that hold pointers to the device infos + virDomainDeviceInfo **devInfos; +} dynamicAddressQueue; + +/** + * Initialize a new empty queue + * @queue: Out; Pointer to the newly allocated queue + * + * Returns -1 if memory allocation fails, otherwise 0 to indicate success + */ +static int new_addr_queue(dynamicAddressQueue **queue) { + dynamicAddressQueue *new_queue = g_new0 (dynamicAddressQueue, 1); + if (new_queue) { + new_queue->size = 10; + new_queue->nextIdx = 0; + new_queue->devInfos = g_new0(virDomainDeviceInfo*, new_queue->size); + *queue = new_queue; + return 0; + } + return -1; +} + +/** + * Add a new `virDomainDeviceInfo` to the queue + * @queue: Queue to add the virDomainDeviceInfo to + * @devInfo: virDomainDeviceInfo to add to the queue + * + * Returns 0 on success and -1 if the queue grows and reallocation fails. + */ +static int add_info_to_queue(dynamicAddressQueue *queue, virDomainDeviceInfo *devInfo) { + if (queue->nextIdx >= queue->size) { + queue->size *= 2; + queue->devInfos = g_realloc(queue->devInfos, queue->size); + if (!queue->devInfos) { + return -1; + } + } + queue->devInfos[queue->nextIdx++] = devInfo; + return 0; +} + +/** + * Frees all memory owned by the queue struct and sets queue to NULL on success. + * @queue: Pointer to a pointer of the queue to free. Will be overwritten with 0 + */ +static void free_queue(dynamicAddressQueue **queue) { + g_free((*queue)->devInfos); + g_free(*queue); + *queue = NULL; +} + +/** + * Initializes the Array of virDomainRNGDef with a default device. + * CHV creates a default RNG device if none is present, so we do the same add one to the configuration. + * The default as of writing this code is as follows: + * - random with path /dev/urandom + * - no iommu + * - no bdf, but we assign one as we want libvirt and CHV to be in sync + */ +static int chAddDefaultVirtioRngDevice(virDomainRNGDef ***deviceDefs) { + int ret = -1; + char* rng_device_alias = g_strdup(DEFAULT_RNG_ALIAS); + char* rng_source_file = g_strdup(DEFAULT_RNG_SOURCE); + virDomainDeviceInfo implicit_rng_device_info = { + .alias = NULL, + .addr.pci = {0}, + .type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI, + .pciConnectFlags = VIR_PCI_CONNECT_TYPE_PCI_DEVICE | VIR_PCI_CONNECT_HOTPLUGGABLE, + }; + virDomainRNGDef* implicit_rng_device = g_new0 (virDomainRNGDef, 1); + + if (NULL == implicit_rng_device) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to allocate memory for implicit RNG device!")); + goto cleanup; + } + + // Create the default RNG device + implicit_rng_device->model = VIR_DOMAIN_RNG_MODEL_VIRTIO; + implicit_rng_device->backend = VIR_DOMAIN_RNG_BACKEND_RANDOM; + implicit_rng_device->info = implicit_rng_device_info; + implicit_rng_device->source.file = g_steal_pointer(&rng_source_file); + implicit_rng_device->info.alias = g_steal_pointer(&rng_device_alias); + // All well so far, now we add it to the configuration by allocating the list and adding the device to its head + *deviceDefs = g_malloc0(sizeof(**deviceDefs)); + if (NULL == deviceDefs) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to allocate memory for RNG device list when creating implicit RNG device!")); + goto cleanup; + } + (*deviceDefs)[0] = g_steal_pointer(&implicit_rng_device); + ret = 0; + + cleanup: + if(NULL != rng_source_file) + g_free(rng_source_file); + if(NULL != rng_device_alias) + g_free(rng_device_alias); + if(NULL != rng_device_alias) + g_free(implicit_rng_device); + + return ret; +} + +/** + * Creates a PCIAddressSet and initializes the structure for a single PCI segment. + * @obj: Vir domain for which a bus is to be created and added to + * + * Returns 0 if the bus bus successfully created and added to the domain. Otherwise -1. + */ +int chDomainPCIAddressSetCreate(virDomainObj *obj) { + int ret = -1; + int nbuses = 0; + virDomainPCIAddressSet *addrs = NULL; + virCHDomainObjPrivate *priv = NULL; + virDomainDef *def = obj->def; + if (!obj) { + virReportError(VIR_ERR_INVALID_ARG, "Domain pointer is NULL!"); + goto cleanup; + } + + if (!def) { + virReportError(VIR_ERR_INVALID_ARG, "Domain def pointer is NULL!"); + goto cleanup; + } + + if (0 < def->ncontrollers) { + virReportError(VIR_ERR_INVALID_ARG, + _("No additional PCI controllers supported right now!")); + goto cleanup; + } + + // We currently support only one bus + nbuses = 1; + // Currently we do *not* support more than 1 PCI bus so abort with error in case + if (nbuses > 1) { + virReportError(VIR_ERR_INVALID_ARG, + _("CHV currently does't support more than one PCI bus!")); + goto cleanup; + } + + // We need to allocate some representation of the PCI bus for libvirt to manage devices + if (!(addrs = virDomainPCIAddressSetAlloc(nbuses, VIR_PCI_ADDRESS_EXTENSION_NONE))) + goto cleanup; + // We support exactly one PCI bus currently. We need to set the respective model for usign it + if (virDomainPCIAddressBusSetModel(&addrs->buses[0], VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT, true) < 0) + goto cleanup; + if (obj && obj->privateData) { + priv = obj->privateData; + /* if this is the live domain object, we persist the PCI addresses */ + priv->pciAddrSet = g_steal_pointer(&addrs); + } + ret = 0; + + cleanup: + virDomainPCIAddressSetFree(addrs); + + + return ret; +} + +/** + * Reserve a PCI slot ID for an device, if one is given by the XML or queue it for later assignment. + * + * @addrSet: PciAddressSet to reserve the address from + * @devInfo: Pointer to the DeviceInfo of the device to reserve a PCI slot ID for + * @queue: Queue of DeviceInfos for which a slot ID is reserved later + * + * Returns: 0 on success, -1 in case of error + */ +static int chReserveOrQueueForPciSlotId(virDomainPCIAddressSet *addrSet, virDomainDeviceInfo *devInfo, dynamicAddressQueue* queue) { + // Enforce correct connection flags + devInfo->pciConnectFlags = VIR_PCI_CONNECT_TYPE_PCI_DEVICE; + // If the device is configured per XML as PCI device with a fixed slot ID, reserve this ID + if (1 == virDeviceInfoPCIAddressIsWanted(devInfo)) { + // ... add it to the queue for later slot id assignment to prevent conflicts + if (0 != add_info_to_queue(queue, devInfo)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to add device '%s' to the queue for PCI ID assignment!"), devInfo->alias); + return -1; + } + DBG("Queued device '%s' for later PCI slot assignment", devInfo->alias); + // ... else, if the device is not configured to be of any type... + } else { + if ((devInfo->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE) && (devInfo->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Found non-PCI device '%s'. CHV only supports PCI virtio devices!"), devInfo->alias); + return -1; + } + // We don't need to assign a new slot, but must mark the slot as taken + if (0 != virDomainPCIAddressReserveAddr(addrSet, &devInfo->addr.pci, devInfo->pciConnectFlags, 0)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Couldn't reserve PCI slot %d from config of device with alias '%s'!"), devInfo->addr.pci.slot, devInfo->alias); + return -1; + } + DBG("Assigned Slot %d to device '%s", devInfo->addr.pci.slot, devInfo->alias); + } + return 0; +} + +/** + * Initialize all RNG devices in the domain with an PCI slot ID. Adds a default RNG device if given an empty list of + * devices. + * + * CHV adds a default RNG device if none is provided in the VM config. We mimic this behavior by creating + * an RNG device with CHV defaults if the domain does not contain an RNG device. This function allocates a new + * array in case an empty one is provided. + * Device without a slot ID given in the XML are queued for later ID assignment and therefore not initialized by this + * function. + * + * @addrSet: PciAddressSet to reserve the slot IDs from + * @numDefs: Pointer to the length of the RNG device array + * @deviceDevs: Pointer to the array of RNG devices of a domain + * @queue: Queue of DeviceInfos for which a slot ID is reserved later + * + * Returns: 0 on success, -1 in case of error + */ +static int chInitRngVirtioPciDevices(virDomainPCIAddressSet *addrSet, size_t *numDefs, virDomainRNGDef ***deviceDefs, dynamicAddressQueue* queue) { + size_t idx; + // Cloudhypervisor creates an implicit RNG device if non is found in the config, so we do the same. + if (0 == *numDefs) { + if (0 != chAddDefaultVirtioRngDevice(deviceDefs)) + virReportError(VIR_ERR_INTERNAL_ERROR,_("Couldn't add implicit RNG device to PCI tree!")); + *numDefs = 1; + DBG("Added Implicit RNG device to config"); + } + // Assign an address to all RNG devices + for (idx = 0; idx < *numDefs; ++idx) { + virDomainRNGDef *rng = (*deviceDefs)[idx]; + if (0 != chReserveOrQueueForPciSlotId(addrSet, &rng->info, queue)) + return -1; + } + return 0; +} + +/** + * Initialize all network devices in the domain with an PCI slot ID or places them in a queue for later assignment. + * + * Device without a slot ID given in the XML are queued for later ID assignment and therefore not initialized by this + * function. + * + * @addrSet: PciAddressSet to reserve the slot IDs from + * @numDefs: Length of the network device array + * @deviceDevs: Array of network devices definitions of a domain + * @queue: Queue of DeviceInfos for which a slot ID is reserved later + * + * Returns: 0 on success, -1 in case of error + */ +static int chInitNetworkVirtioPciDevices(virDomainPCIAddressSet *addrSet, size_t numDefs, virDomainNetDef **netDevDefs, dynamicAddressQueue* queue) { + size_t idx; + for (idx = 0; idx < numDefs; ++idx) { + virDomainNetDef *net = netDevDefs[idx]; + if (net->type != VIR_DOMAIN_NET_TYPE_ETHERNET) { + virReportError(VIR_ERR_INVALID_NETWORK, + _("CHV driver only supports `ethernet` network types!")); + return -1; + } + if (!virDomainNetIsVirtioModel(net)) { + VIR_WARN("Found non PCI net device with model type %s", virDomainNetModelTypeToString(net->model)); + continue; + } + if (chReserveOrQueueForPciSlotId(addrSet, &net->info, queue)) + return -1; + } + return 0; +} + +/** + * Initialize all disk devices in the domain with an PCI slot ID or places them in a queue for later assignment. + * + * Device without a slot ID given in the XML are queued for later ID assignment and therefore not initialized by this + * function. + * + * @addrSet: PciAddressSet to reserve the slot IDs from + * @numDefs: Length of the disk device array + * @deviceDevs: Array of disk devices definitions of a domain + * @queue: Queue of DeviceInfos for which a slot ID is reserved later + * + * Returns: 0 on success, -1 in case of error + */ +static int chInitDiskVirtioPciDevices(virDomainPCIAddressSet *addrSet, size_t numDefs, virDomainDiskDef **diskDevDefs, dynamicAddressQueue* queue) { + size_t idx; + for (idx = 0; idx < numDefs; ++idx) { + virDomainDiskDef *disk = diskDevDefs[idx]; + if (disk->bus != VIR_DOMAIN_DISK_BUS_VIRTIO) { + DBG("Found unsupported disk model with type %d", disk->model); + continue; + } + if (chReserveOrQueueForPciSlotId(addrSet, &disk->info, queue)) + return -1; + } + return 0; +} + +/** + * Initialize all serial devices in the domain with an PCI slot ID or places them in a queue for later assignment. + * + * Device without a slot ID given in the XML are queued for later ID assignment and therefore not initialized by this + * function. + * + * @addrSet: PciAddressSet to reserve the slot IDs from + * @numDefs: Length of the serial device array + * @deviceDevs: Array of serial devices definitions of a domain + * @queue: Queue of DeviceInfos for which a slot ID is reserved later + * + * Returns: 0 on success, -1 in case of error + */ +static int chInitSerialVirtioPciDevices(virDomainPCIAddressSet *addrSet, size_t numDefs, virDomainChrDef **charDevDefs, dynamicAddressQueue* queue) { + size_t idx; + for (idx = 0; idx < numDefs; ++idx) { + virDomainChrDef *device = charDevDefs[idx]; + // Device is neither `None` nor VIR_DOMAIN_CHR_SERIAL_TARGET_TYPE_PCI, so incompatible + if (device->targetType & (~VIR_DOMAIN_CHR_SERIAL_TARGET_TYPE_PCI)) { + VIR_WARN("Found unsupported non-PCI target type for serial: %s", virDomainChrSerialTargetTypeToString(device->targetType)); + continue; + } + // Check if device has a compatible mode; expect either 0 or PCI_SERIAL + if (device->targetModel & (~VIR_DOMAIN_CHR_SERIAL_TARGET_MODEL_PCI_SERIAL)) { + VIR_WARN("Found unsupported non-PCI target model of: %d", device->targetModel); + continue; + } + // Set the expected model and type + device->targetType = VIR_DOMAIN_CHR_SERIAL_TARGET_TYPE_PCI; + device->targetModel = VIR_DOMAIN_CHR_SERIAL_TARGET_MODEL_PCI_SERIAL; + if (chReserveOrQueueForPciSlotId(addrSet, &device->info, queue)) + return -1; + } + return 0; +} + +/** + * Assigns static PCI device BDFs to devices with not BDF specified. + * + * @addrSet: PciAddrSet to reserve the PCI BDFs from + * @queue: Queue of devices for which a PCI BDF will be reserved + * + * Returns: 0 on success, -1 in case of error + */ +static int chReserveDynamicPciIds(virDomainPCIAddressSet *addrSet, dynamicAddressQueue *queue) { + for (size_t idx = 0; idx < queue->nextIdx; ++idx) { + queue->devInfos[idx]->type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; + if (virDomainPCIAddressReserveNextAddr(addrSet, queue->devInfos[idx], + queue->devInfos[idx]->pciConnectFlags, -1)) + { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Failed to fix the PCI device id for '%s'!"), queue->devInfos[idx]->alias); + return -1; + } + DBG("Assigned new Slot %d to device '%s' without BDF", queue->devInfos[idx]->addr.pci.slot, queue->devInfos[idx]->alias); + } + return 0; + +} +/* Initializes all PCI devices in the configuration with PCI bus addresses. + * + * First allocates all PCI slot IDs given by the XML. Then allocates slot IDs for all devices without a fixed slot ID. + * + * Succeeds if after successfully assigning an address to each device. + * + * @dom: Domain object for which all PCI devices are initialized with an PCI address. + * + * 0 indicates success, -1 failure + */ +int chInitPciDevices(virDomainObj *dom) { + virCHDomainObjPrivate *priv = dom->privateData; + virDomainDef *def = dom->def; + virDomainPCIAddressSet *addrSet = priv->pciAddrSet; + dynamicAddressQueue* queue = NULL; + int rc = 0; + new_addr_queue(&queue); + + /* Either allocate PCI slot IDs for all devices of all device classes if their XML contains a fixed slot id + * or place them in a queue to later assign fixes slot IDs. This is necessary as there is no order with respect + * to fixed IDs. Moreover, there is no guarantee that even if there was an order in one device class, there would be + * fixed slot ID in a class that is later initialized. So we need a global queue of all devices without a fixed slot + * ID. + */ + if (chInitRngVirtioPciDevices(addrSet, &def->nrngs, &def->rngs, queue)) { + rc = -1; + VIR_ERROR("%s:%d: Failed to attach Virtio RNG devices.", __FILE_NAME__, __LINE__); + goto cleanup; + } + if (chInitNetworkVirtioPciDevices(addrSet,def->nnets, def->nets, queue)) { + rc = -1; + VIR_ERROR("%s:%d: Failed to attach Virtio net devices.", __FILE_NAME__, __LINE__); + goto cleanup; + } + if (chInitDiskVirtioPciDevices(addrSet,def->ndisks, def->disks, queue)) { + rc = -1; + VIR_ERROR("%s:%d: Failed to attach Virtio disk devices.", __FILE_NAME__, __LINE__); + goto cleanup; + } + if (chInitSerialVirtioPciDevices(addrSet,def->nserials, def->serials, queue)) { + rc = -1; + VIR_ERROR("%s:%d: Failed to attach Virtio serial devices", __FILE_NAME__, __LINE__); + goto cleanup; + } + + // Assign a fixed slot ID to all devices in the queue + if (chReserveDynamicPciIds(addrSet, queue)) { + rc = -1; + VIR_ERROR("%s:%d: Could not assigned fixed PCI device IDs", __FILE_NAME__, __LINE__); + } + + cleanup: + free_queue(&queue); + + return rc; +} + +/* Ensures that a hotplugged PCI devices has a associated address. + * + * Performs all necessary configuration for a hotplugable PCI device. + * Succeeds if after successfully assigning an address to the device. + * + * 0 indicates success, -1 failure + */ +int chEnsurePciAddress(virDomainObj *obj, virDomainDeviceDef *dev) { + virCHDomainObjPrivate *priv = obj->privateData; + virDomainDeviceInfo *info = virDomainDeviceGetInfo(dev); + if (!info) + return 0; + + info->type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI; + info->pciConnectFlags = VIR_PCI_CONNECT_TYPE_PCI_DEVICE | VIR_PCI_CONNECT_HOTPLUGGABLE; + + return virDomainPCIAddressEnsureAddr(priv->pciAddrSet, info, + info->pciConnectFlags); +} + +/** + * Releases the PCI bus address reserved for a PCI device. + * + * @vm: VM from which the PCI address is released + * @info: DeviceInfo that holds information about the PCI address to release. + */ +void chDomainReleaseDeviceAddress(virDomainObj *vm, virDomainDeviceInfo *info) +{ + virCHDomainObjPrivate *priv = vm->privateData; + if (1 == virDeviceInfoPCIAddressIsPresent(info)) { + virDomainPCIAddressReleaseAddr(priv->pciAddrSet, &info->addr.pci); + } +} diff --git a/src/ch/ch_pci_addr.h b/src/ch/ch_pci_addr.h new file mode 100644 index 0000000000..b068b0f25e --- /dev/null +++ b/src/ch/ch_pci_addr.h @@ -0,0 +1,32 @@ +/* + * Copyright Cyberus Technology GmbH. 2025 + * + * ch_pci_addr.h: header file for Cloud-Hypervisor's PCI device address + * management + * + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * . + */ + +#pragma once + +#include "domain_addr.h" +int chDomainPCIAddressSetCreate(virDomainObj *obj); +int chInitPciDevices(virDomainObj *dom); + +int chAssignStaticPciDeviceId(virDomainDeviceInfo *devInfo, + virDomainPCIAddressSet *addrs); + +int chEnsurePciAddress(virDomainObj *obj, virDomainDeviceDef *dev); +void chDomainReleaseDeviceAddress(virDomainObj *vm, virDomainDeviceInfo *info); diff --git a/src/ch/ch_process.c b/src/ch/ch_process.c index e5c13f46f3..391eeae277 100644 --- a/src/ch/ch_process.c +++ b/src/ch/ch_process.c @@ -42,6 +42,7 @@ #include "virstring.h" #include "ch_interface.h" #include "ch_hostdev.h" +#include "ch_pci_addr.h" #define VIR_FROM_THIS VIR_FROM_CH @@ -1016,6 +1017,20 @@ virCHProcessPrepareDomain(virDomainObj *vm) g_atomic_int_set(&priv->shutdown_done, 0); + // Initialize our one and only PCI bus + if (chDomainPCIAddressSetCreate(vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "CHV driver only supports `ethernet` network types!"); + return -1; + } + + // Attach all devices from the config to the PCI bus + if (chInitPciDevices(vm)) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "Failed to assign addresses to PCI devices defined in XML!"); + return -1; + } + return 0; } diff --git a/src/ch/meson.build b/src/ch/meson.build index 488a80ee3d..5ab3798cf8 100644 --- a/src/ch/meson.build +++ b/src/ch/meson.build @@ -21,6 +21,8 @@ ch_driver_sources = [ 'ch_hostdev.h', 'ch_socket.h', 'ch_socket.c', + 'ch_pci_addr.h', + 'ch_pci_addr.c', ] driver_source_files += files(ch_driver_sources)