diff --git a/CHANGELOG.md b/CHANGELOG.md index e99c5a5515..7201c484f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Added - On x86: - Option to attempt to fixup p2m page-faults on PVH dom0. + - Resizable BARs is supported for PVH dom0. ### Removed diff --git a/SUPPORT.md b/SUPPORT.md index e1f4769bd8..91cb6f8ed2 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -170,7 +170,7 @@ unexpected behavior or issues on some hardware. At least the following features are missing on a PVH dom0: - * PCI SR-IOV and Resizable BARs. + * PCI SR-IOV. * Native NMI forwarding (nmi=dom0 command line option). diff --git a/xen/drivers/vpci/Makefile b/xen/drivers/vpci/Makefile index 1a1413b93e..a7c8a30a89 100644 --- a/xen/drivers/vpci/Makefile +++ b/xen/drivers/vpci/Makefile @@ -1,2 +1,2 @@ -obj-y += vpci.o header.o +obj-y += vpci.o header.o rebar.o obj-$(CONFIG_HAS_PCI_MSI) += msi.o msix.o diff --git a/xen/drivers/vpci/rebar.c b/xen/drivers/vpci/rebar.c new file mode 100644 index 0000000000..793937449a --- /dev/null +++ b/xen/drivers/vpci/rebar.c @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Author: Jiqian Chen + */ + +#include +#include + +static void cf_check rebar_ctrl_write(const struct pci_dev *pdev, + unsigned int reg, + uint32_t val, + void *data) +{ + struct vpci_bar *bar = data; + const unsigned int index = bar - pdev->vpci->header.bars; + const uint64_t size = PCI_REBAR_CTRL_SIZE(val); + + if ( bar->enabled ) + { + /* + * Refuse to resize a BAR while memory decoding is enabled, as + * otherwise the size of the mapped region in the p2m would become + * stale with the newly set BAR size, and the position of the BAR + * would be reset to undefined. Note the PCIe specification also + * forbids resizing a BAR with memory decoding enabled. + */ + if ( size != bar->size ) + gprintk(XENLOG_ERR, + "%pp: refuse to resize BAR#%u with memory decoding enabled\n", + &pdev->sbdf, index); + return; + } + + if ( !((size >> PCI_REBAR_CTRL_SIZE_BIAS) & bar->resizable_sizes) ) + gprintk(XENLOG_WARNING, + "%pp: new BAR#%u size %#lx is not supported by hardware\n", + &pdev->sbdf, index, size); + + pci_conf_write32(pdev->sbdf, reg, val); + + pci_size_mem_bar(pdev->sbdf, + PCI_BASE_ADDRESS_0 + index * 4, + &bar->addr, + &bar->size, + (index == PCI_HEADER_NORMAL_NR_BARS - 1) ? + PCI_BAR_LAST : 0); + bar->guest_addr = bar->addr; +} + +static int cf_check init_rebar(struct pci_dev *pdev) +{ + uint32_t ctrl; + unsigned int nbars; + unsigned int rebar_offset = pci_find_ext_capability(pdev->sbdf, + PCI_EXT_CAP_ID_REBAR); + + if ( !rebar_offset ) + return 0; + + if ( !is_hardware_domain(pdev->domain) ) + { + printk(XENLOG_ERR "%pp: resizable BARs unsupported for unpriv %pd\n", + &pdev->sbdf, pdev->domain); + return -EOPNOTSUPP; + } + + ctrl = pci_conf_read32(pdev->sbdf, rebar_offset + PCI_REBAR_CTRL(0)); + nbars = MASK_EXTR(ctrl, PCI_REBAR_CTRL_NBAR_MASK); + for ( unsigned int i = 0; i < nbars; i++ ) + { + int rc; + struct vpci_bar *bar; + unsigned int index; + + ctrl = pci_conf_read32(pdev->sbdf, rebar_offset + PCI_REBAR_CTRL(i)); + index = ctrl & PCI_REBAR_CTRL_BAR_IDX; + if ( index >= PCI_HEADER_NORMAL_NR_BARS ) + { + printk(XENLOG_ERR "%pd %pp: too big BAR number %u in REBAR_CTRL\n", + pdev->domain, &pdev->sbdf, index); + continue; + } + + bar = &pdev->vpci->header.bars[index]; + if ( bar->type != VPCI_BAR_MEM64_LO && bar->type != VPCI_BAR_MEM32 ) + { + printk(XENLOG_ERR "%pd %pp: BAR%u is not in memory space\n", + pdev->domain, &pdev->sbdf, index); + continue; + } + + rc = vpci_add_register(pdev->vpci, vpci_hw_read32, rebar_ctrl_write, + rebar_offset + PCI_REBAR_CTRL(i), 4, bar); + if ( rc ) + { + printk(XENLOG_ERR "%pd %pp: BAR%u fail to add reg of REBAR_CTRL rc=%d\n", + pdev->domain, &pdev->sbdf, index, rc); + /* + * Ideally we would hide the ReBar capability on error, but code + * for doing so still needs to be written. Use continue instead + * to keep any already setup register hooks, as returning an + * error will cause the hardware domain to get unmediated access + * to all device registers. + */ + continue; + } + + bar->resizable_sizes = + MASK_EXTR(pci_conf_read32(pdev->sbdf, + rebar_offset + PCI_REBAR_CAP(i)), + PCI_REBAR_CAP_SIZES_MASK); + bar->resizable_sizes |= + (((uint64_t)MASK_EXTR(ctrl, PCI_REBAR_CTRL_SIZES_MASK) << 32) / + ISOLATE_LSB(PCI_REBAR_CAP_SIZES_MASK)); + } + + return 0; +} +REGISTER_VPCI_INIT(init_rebar, VPCI_PRIORITY_LOW); + +/* + * Local variables: + * mode: C + * c-file-style: "BSD" + * c-basic-offset: 4 + * tab-width: 4 + * indent-tabs-mode: nil + * End: + */ diff --git a/xen/include/xen/pci_regs.h b/xen/include/xen/pci_regs.h index 250ba106db..27b4f44eed 100644 --- a/xen/include/xen/pci_regs.h +++ b/xen/include/xen/pci_regs.h @@ -459,6 +459,7 @@ #define PCI_EXT_CAP_ID_ARI 14 #define PCI_EXT_CAP_ID_ATS 15 #define PCI_EXT_CAP_ID_SRIOV 16 +#define PCI_EXT_CAP_ID_REBAR 21 /* Resizable BAR */ /* Advanced Error Reporting */ #define PCI_ERR_UNCOR_STATUS 4 /* Uncorrectable Error Status */ @@ -541,6 +542,19 @@ #define PCI_VNDR_HEADER_REV(x) (((x) >> 16) & 0xf) #define PCI_VNDR_HEADER_LEN(x) (((x) >> 20) & 0xfff) +/* Resizable BARs */ +#define PCI_REBAR_CAP(n) (4 + 8 * (n)) /* capability register */ +#define PCI_REBAR_CAP_SIZES_MASK 0xFFFFFFF0U /* supported BAR sizes in CAP */ +#define PCI_REBAR_CTRL(n) (8 + 8 * (n)) /* control register */ +#define PCI_REBAR_CTRL_BAR_IDX 0x00000007 /* BAR index */ +#define PCI_REBAR_CTRL_NBAR_MASK 0x000000E0 /* # of resizable BARs */ +#define PCI_REBAR_CTRL_BAR_SIZE 0x00003F00 /* BAR size */ +#define PCI_REBAR_CTRL_SIZES_MASK 0xFFFF0000U /* supported BAR sizes in CTRL */ + +#define PCI_REBAR_CTRL_SIZE_BIAS 20 +#define PCI_REBAR_CTRL_SIZE(v) \ + (1ULL << (MASK_EXTR(v, PCI_REBAR_CTRL_BAR_SIZE) + PCI_REBAR_CTRL_SIZE_BIAS)) + /* * Hypertransport sub capability types * diff --git a/xen/include/xen/vpci.h b/xen/include/xen/vpci.h index 41e7c3bc27..807401b2ea 100644 --- a/xen/include/xen/vpci.h +++ b/xen/include/xen/vpci.h @@ -100,6 +100,7 @@ struct vpci { /* Guest address. */ uint64_t guest_addr; uint64_t size; + uint64_t resizable_sizes; struct rangeset *mem; enum { VPCI_BAR_EMPTY,