diff --git a/devicemodel/Makefile b/devicemodel/Makefile index ea34576f7..f4ec7ea68 100644 --- a/devicemodel/Makefile +++ b/devicemodel/Makefile @@ -82,6 +82,8 @@ SRCS += hw/pci/virtio/virtio_block.c SRCS += hw/pci/virtio/virtio_input.c SRCS += hw/pci/ahci.c SRCS += hw/pci/hostbridge.c +SRCS += hw/pci/platform_gsi_info.c +SRCS += hw/pci/gsi_sharing.c SRCS += hw/pci/passthrough.c SRCS += hw/pci/virtio/virtio_audio.c SRCS += hw/pci/virtio/virtio_net.c diff --git a/devicemodel/hw/pci/core.c b/devicemodel/hw/pci/core.c index d88d60180..da0b1831a 100644 --- a/devicemodel/hw/pci/core.c +++ b/devicemodel/hw/pci/core.c @@ -1155,6 +1155,8 @@ init_pci(struct vmctx *ctx) pci_emul_membase32 = vm_get_lowmem_limit(ctx); pci_emul_membase64 = PCI_EMUL_MEMBASE64; + create_gsi_sharing_groups(); + for (bus = 0; bus < MAXBUSES; bus++) { bi = pci_businfo[bus]; if (bi == NULL) @@ -1202,6 +1204,10 @@ init_pci(struct vmctx *ctx) bi->memlimit64 = pci_emul_membase64; } + error = check_gsi_sharing_violation(); + if (error < 0) + return error; + /* * PCI backends are initialized before routing INTx interrupts * so that LPC devices are able to reserve ISA IRQs before diff --git a/devicemodel/hw/pci/gsi_sharing.c b/devicemodel/hw/pci/gsi_sharing.c new file mode 100644 index 000000000..7edb20036 --- /dev/null +++ b/devicemodel/hw/pci/gsi_sharing.c @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2018 Intel Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "pci_core.h" + +#define MAX_DEV_PER_GSI 4 + +/* physical info of the GSI sharing group */ +struct gsi_sharing_group { + uint8_t gsi; + /* the number of devices that are sharing the same GSI */ + int shared_dev_num; + struct { + char *dev_name; + int assigned_to_this_vm; + } dev[MAX_DEV_PER_GSI]; + + LIST_ENTRY(gsi_sharing_group) gsg_list; +}; +static LIST_HEAD(gsg_struct, gsi_sharing_group) gsg_head; + +static int +update_gsi_sharing_info(char *dev_name, uint8_t gsi) +{ + int gsi_shared = 0; + struct gsi_sharing_group *group = NULL; + + LIST_FOREACH(group, &gsg_head, gsg_list) { + if (gsi != group->gsi) + continue; + + if (group->shared_dev_num >= MAX_DEV_PER_GSI) { + warnx("max %d devices share one GSI", MAX_DEV_PER_GSI); + return -EINVAL; + } + + gsi_shared = 1; + break; + } + + if (gsi_shared == 0) { + group = calloc(1, sizeof(struct gsi_sharing_group)); + if (!group) { + warnx("%s: calloc FAIL!", __func__); + return -ENOMEM; + } + + group->gsi = gsi; + group->shared_dev_num = 0; + + LIST_INSERT_HEAD(&gsg_head, group, gsg_list); + } + + if (group != NULL) { + group->dev[group->shared_dev_num].dev_name = dev_name; + group->dev[group->shared_dev_num].assigned_to_this_vm = 0; + group->shared_dev_num++; + } + + return 0; +} + +/* + * return 1 if MSI/MSI-x is supported + * return 0 if MSI/MSI-x is NOT supported + */ +static int +check_msi_capability(char *dev_name) +{ + int bus, slot, func; + uint16_t status; + uint8_t cap_ptr, cap_id; + struct pci_device *phys_dev; + + /* only check the MSI/MSI-x capability for PCI device */ + if (sscanf(dev_name, "%x:%x.%x", &bus, &slot, &func) != 3) + return 0; + + phys_dev = pci_device_find_by_slot(0, bus, slot, func); + if (!phys_dev) + return 0; + + pci_device_cfg_read_u16(phys_dev, &status, PCIR_STATUS); + if (status & PCIM_STATUS_CAPPRESENT) { + pci_device_cfg_read_u8(phys_dev, &cap_ptr, PCIR_CAP_PTR); + + while (cap_ptr != 0 && cap_ptr != 0xff) { + pci_device_cfg_read_u8(phys_dev, &cap_id, + cap_ptr + PCICAP_ID); + + if (cap_id == PCIY_MSI) + return 1; + else if (cap_id == PCIY_MSIX) + return 1; + + pci_device_cfg_read_u8(phys_dev, &cap_ptr, + cap_ptr + PCICAP_NEXTPTR); + } + } + return 0; +} + +int +create_gsi_sharing_groups(void) +{ + uint8_t gsi; + char *dev_name; + int i, error, msi_support; + struct gsi_sharing_group *group = NULL; + + error = pciaccess_init(); + if (error < 0) + return error; + + for (i = 0; i < num_gsi_dev_mapping_tables; i++) { + dev_name = gsi_dev_mapping_tables[i].dev_name; + gsi = gsi_dev_mapping_tables[i].gsi; + + /* skip the devices that support MSI/MSI-x */ + msi_support = check_msi_capability(dev_name); + if (msi_support == 1) + continue; + + /* insert the device into gsi_sharing_group */ + error = update_gsi_sharing_info(dev_name, gsi); + if (error < 0) + return error; + } + + pciaccess_cleanup(); + + /* + * clean up gsg_head - the list for gsi_sharing_group + * delete the element without gsi sharing condition (shared_dev_num < 2) + */ + LIST_FOREACH(group, &gsg_head, gsg_list) { + if (group->shared_dev_num < 2) { + LIST_REMOVE(group, gsg_list); + free(group); + } + } + + return 0; +} + +/* + * update passthrough info in gsi_sharing_group + * set assigned_to_this_vm as 1 if the PCI device is assigned to current VM + */ +void +update_pt_info(uint16_t phys_bdf) +{ + char *name; + int i, bus, slot, func; + struct gsi_sharing_group *group; + + LIST_FOREACH(group, &gsg_head, gsg_list) { + for (i = 0; i < (group->shared_dev_num); i++) { + name = group->dev[i].dev_name; + if (sscanf(name, "%x:%x.%x", &bus, &slot, &func) != 3) + continue; + + if (phys_bdf == (PCI_BDF(bus, slot, func))) + group->dev[i].assigned_to_this_vm = 1; + } + } +} + +/* check if all devices in one GSI sharing group are assigned to same VM */ +int +check_gsi_sharing_violation(void) +{ + struct gsi_sharing_group *group; + int i, error, violation; + + error = 0; + LIST_FOREACH(group, &gsg_head, gsg_list) { + /* + * All the PCI devices that are sharing the same GSI should be + * assigned to same VM to avoid physical GSI sharing between + * multiple VMs. + * + * If the value of 'assigned_to_this_vm' for each device in one + * gsi_sharing_group are all the same, either all 0 or all 1, it + * indicates that there is no GSI sharing between multiple VMs. + * All 0 means that all devices are NOT assigned to current VM. + * All 1 means that all devices are assigned to current VM. + * + * Otherwise, the passthrough will be rejected due to violation. + */ + violation = 0; + for (i = 1; i < (group->shared_dev_num); i++) { + if (group->dev[i].assigned_to_this_vm != + group->dev[i - 1].assigned_to_this_vm) { + violation = 1; + break; + } + } + + if (violation == 0) + continue; + + /* reject the passthrough since gsi sharing violation occurs */ + warnx("GSI SHARING VIOLATION!"); + warnx("following physical devices are sharing same GSI, please " + "assign them to same VM to avoid physical GSI sharing " + "between multiple VMs"); + for (i = 0; i < (group->shared_dev_num); i++) { + warnx("device %s \t assigned_to_this_vm %d", + group->dev[i].dev_name, + group->dev[i].assigned_to_this_vm); + } + error = -EINVAL; + break; + } + + /* destroy the gsg_head after all the checks have been done */ + LIST_FOREACH(group, &gsg_head, gsg_list) { + LIST_REMOVE(group, gsg_list); + free(group); + } + + return error; +} diff --git a/devicemodel/hw/pci/passthrough.c b/devicemodel/hw/pci/passthrough.c index a6ef58a60..c10206954 100644 --- a/devicemodel/hw/pci/passthrough.c +++ b/devicemodel/hw/pci/passthrough.c @@ -67,8 +67,6 @@ #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) #define MSIX_CAPLEN 12 -#define PCI_BDF(bus, dev, func) (((bus & 0xFF)<<8) | ((dev & 0x1F)<<3) \ - | ((func & 0x7))) #define PCI_BDF_GPU 0x00000010 /* 00:02.0 */ @@ -894,7 +892,7 @@ native_pci_system_init() /* * return zero on success or non-zero on failure */ -static int +int pciaccess_init(void) { int error; @@ -957,6 +955,7 @@ passthru_init(struct vmctx *ctx, struct pci_vdev *dev, char *opts) } ptdev->phys_bdf = PCI_BDF(bus, slot, func); + update_pt_info(ptdev->phys_bdf); error = pciaccess_init(); if (error < 0) @@ -1031,7 +1030,7 @@ done: return error; } -static void +void pciaccess_cleanup(void) { pthread_mutex_lock(&ref_cnt_mtx); diff --git a/devicemodel/hw/pci/platform_gsi_info.c b/devicemodel/hw/pci/platform_gsi_info.c new file mode 100644 index 000000000..86d159a2e --- /dev/null +++ b/devicemodel/hw/pci/platform_gsi_info.c @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2018 Intel Corporation. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "pci_core.h" + +#define APL_MRB 1 + +#ifdef APL_MRB +struct gsi_dev gsi_dev_mapping_tables[] = { + {"timer", 0}, + {"keyboard", 1}, + {"hpet0", 2}, + {"00:1b.0", 3}, + {"00:18.0", 4}, + {"00:18.1", 5}, + {"00:18.2", 6}, + {"00:18.3", 7}, + {"rtc", 8}, + {"acpi", 9}, + {"00:15.1", 13}, + {"gpio_ctlr", 14}, + {"00:15.0", 17}, + {"00:02.0", 19}, + {"00:12.0", 19}, + {"00:0f.0", 20}, + {"00:1f.1", 20}, + {"03:00.0", 20}, + {"00:03.0", 21}, + {"04:00.0", 21}, + {"00:00.1", 24}, + {"00:0e.0", 25}, + {"00:11.0", 26}, + {"00:16.0", 27}, + {"00:16.1", 28}, + {"00:16.2", 29}, + {"00:16.3", 30}, + {"00:17.0", 31}, + {"00:17.1", 32}, + {"00:17.2", 33}, + {"00:17.3", 34}, + {"00:19.0", 35}, + {"00:19.1", 36}, + {"00:19.2", 37}, + {"00:1c.0", 39}, + {"pmc_ipc", 40}, + {"00:1e.0", 42}, + {"gpio_pin_00", 50}, + {"gpio_pin_01", 51}, + {"gpio_pin_02", 52}, + {"gpio_pin_03", 53}, + {"gpio_pin_04", 54}, + {"gpio_pin_05", 55}, + {"gpio_pin_06", 56}, + {"gpio_pin_07", 57}, + {"gpio_pin_08", 58}, + {"gpio_pin_09", 59}, + {"gpio_pin_10", 60}, + {"gpio_pin_11", 61}, + {"gpio_pin_12", 62}, + {"gpio_pin_13", 63}, + {"gpio_pin_14", 64}, + {"gpio_pin_15", 65}, + {"gpio_pin_16", 66}, + {"gpio_pin_17", 67}, + {"gpio_pin_18", 68}, + {"gpio_pin_19", 69}, + {"gpio_pin_20", 70}, + {"gpio_pin_21", 71}, + {"gpio_pin_22", 72}, + {"gpio_pin_23", 73}, + {"gpio_pin_24", 74}, + {"gpio_pin_25", 75}, + {"gpio_pin_26", 76}, + {"gpio_pin_27", 77}, + {"gpio_pin_28", 78}, + {"gpio_pin_29", 79}, + {"gpio_pin_30", 80}, + {"gpio_pin_31", 81}, + {"gpio_pin_32", 82}, + {"gpio_pin_33", 83}, + {"gpio_pin_34", 84}, + {"gpio_pin_35", 85}, + {"gpio_pin_36", 86}, + {"gpio_pin_37", 87}, + {"gpio_pin_38", 88}, + {"gpio_pin_39", 89}, + {"gpio_pin_40", 90}, + {"gpio_pin_41", 91}, + {"gpio_pin_42", 92}, + {"gpio_pin_43", 93}, + {"gpio_pin_44", 94}, + {"gpio_pin_45", 95}, + {"gpio_pin_46", 96}, + {"gpio_pin_47", 97}, + {"gpio_pin_48", 98}, + {"gpio_pin_49", 99}, + {"gpio_pin_50", 100}, + {"gpio_pin_51", 101}, + {"gpio_pin_52", 102}, + {"gpio_pin_53", 103}, + {"gpio_pin_54", 104}, + {"gpio_pin_55", 105}, + {"gpio_pin_56", 106}, + {"gpio_pin_57", 107}, + {"gpio_pin_58", 108}, + {"gpio_pin_59", 109}, + {"gpio_pin_60", 110}, + {"gpio_pin_61", 111}, + {"gpio_pin_62", 112}, + {"gpio_pin_63", 113}, + {"gpio_pin_64", 114}, + {"gpio_pin_65", 115}, + {"gpio_pin_66", 116}, + {"gpio_pin_67", 117}, + {"gpio_pin_68", 118}, + {"gpio_pin_69", 119}, +}; +#else +/* + * Assume there is no GSI sharing if no hardcoded info is provided in + * gsi_sharing_tables. + */ +struct gsi_dev gsi_dev_mapping_tables[] = {}; +#endif + +int num_gsi_dev_mapping_tables = ARRAY_SIZE(gsi_dev_mapping_tables); diff --git a/devicemodel/include/pci_core.h b/devicemodel/include/pci_core.h index 88f5c0c21..c8aa4f31f 100644 --- a/devicemodel/include/pci_core.h +++ b/devicemodel/include/pci_core.h @@ -36,6 +36,7 @@ #include "pcireg.h" #define PCI_BARMAX PCIR_MAX_BAR_0 /* BAR registers in a Type 0 header */ +#define PCI_BDF(b, d, f) (((b & 0xFF) << 8) | ((d & 0x1F) << 3) | ((f & 0x7))) struct vmctx; struct pci_vdev; @@ -161,6 +162,20 @@ struct pci_vdev { struct pcibar bar[PCI_BARMAX + 1]; }; +struct gsi_dev { + /* + * For PCI devices, use a string "b:d.f" to stand for the device's BDF, + * such as "00:00.0". + * For non-PCI devices, use the device's name to stand for the device, + * such as "timer". + */ + char *dev_name; + uint8_t gsi; +}; + +extern struct gsi_dev gsi_dev_mapping_tables[]; +extern int num_gsi_dev_mapping_tables; + struct msicap { uint8_t capid; uint8_t nextptr; @@ -260,6 +275,11 @@ uint64_t pci_ecfg_base(void); int pci_bus_configured(int bus); int emulate_pci_cfgrw(struct vmctx *ctx, int vcpu, int in, int bus, int slot, int func, int reg, int bytes, int *value); +int create_gsi_sharing_groups(void); +void update_pt_info(uint16_t phys_bdf); +int check_gsi_sharing_violation(void); +int pciaccess_init(void); +void pciaccess_cleanup(void); static inline void pci_set_cfgdata8(struct pci_vdev *pi, int offset, uint8_t val)