[linux-nvidia-6.17-next] CXL VFIO: Add CXL Type-2 device passthrough support

Documentation/driver-api/index.rst

@@ -47,6 +47,7 @@ of interest to most developers working on device drivers.
    vfio-mediated-device
    vfio
    vfio-pci-device-specific-driver-acceptance
+   vfio-pci-cxl
 
 Bus-level documentation
 =======================

debian.nvidia-6.17/config/annotations

@@ -255,6 +255,8 @@ CONFIG_UBUNTU_ODM_DRIVERS                       note<'Disable all Ubuntu ODM dri
 CONFIG_ULTRASOC_SMB                             policy<{'arm64': 'n'}>
 CONFIG_ULTRASOC_SMB                             note<'Required for Grace enablement'>
 
+CONFIG_VFIO_CXL_CORE                            policy<{'amd64': 'y', 'arm64': 'y'}>
+CONFIG_VFIO_CXL_CORE                            note<'Enable VFIO CXL core for CXL Type-2 device passthrough support'>
 
 # ---- Annotations without notes ----
 

drivers/crypto/hisilicon/qm.c

@@ -3005,11 +3005,36 @@ static void qm_put_pci_res(struct hisi_qm *qm)
 	pci_release_mem_regions(pdev);
 }
 
+static void hisi_mig_region_clear(struct hisi_qm *qm)
+{
+	u32 val;
+
+	/* Clear migration region set of PF */
+	if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) {
+		val = readl(qm->io_base + QM_MIG_REGION_SEL);
+		val &= ~QM_MIG_REGION_EN;
+		writel(val, qm->io_base + QM_MIG_REGION_SEL);
+	}
+}
+
+static void hisi_mig_region_enable(struct hisi_qm *qm)
+{
+	u32 val;
+
+	/* Select migration region of PF */
+	if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V3) {
+		val = readl(qm->io_base + QM_MIG_REGION_SEL);
+		val |= QM_MIG_REGION_EN;
+		writel(val, qm->io_base + QM_MIG_REGION_SEL);
+	}
+}
+
 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
 {
 	struct pci_dev *pdev = qm->pdev;
 
 	pci_free_irq_vectors(pdev);
+	hisi_mig_region_clear(qm);
 	qm_put_pci_res(qm);
 	pci_disable_device(pdev);
 }
@@ -5696,6 +5721,7 @@ int hisi_qm_init(struct hisi_qm *qm)
 		goto err_free_qm_memory;
 
 	qm_cmd_init(qm);
+	hisi_mig_region_enable(qm);
 
 	return 0;
 
@@ -5834,6 +5860,7 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm)
 	}
 
 	qm_cmd_init(qm);
+	hisi_mig_region_enable(qm);
 	hisi_qm_dev_err_init(qm);
 	/* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */
 	writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);

drivers/cxl/core/pci.c

@@ -147,16 +147,24 @@ static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
 	return 0;
 }
 
-/*
- * Wait up to @media_ready_timeout for the device to report memory
- * active.
+/**
+ * cxl_await_range_active - Wait for all HDM DVSEC memory ranges to be active
+ * @cxlds: CXL device state (DVSEC and HDM count must be valid)
+ *
+ * For each HDM decoder range reported in the CXL DVSEC capability, waits for
+ * the range to report MEM INFO VALID (up to 1s per range), then MEM ACTIVE
+ * (up to media_ready_timeout seconds per range, default 60s). Used by
+ * cxl_await_media_ready() and by callers that only need range readiness
+ * without checking the memory device status register.
+ *
+ * Return: 0 if all ranges become valid and active, -ETIMEDOUT if a timeout
+ * occurs, or a negative errno from config read on failure.
  */
-int cxl_await_media_ready(struct cxl_dev_state *cxlds)
+int cxl_await_range_active(struct cxl_dev_state *cxlds)
 {
 	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
 	int d = cxlds->cxl_dvsec;
 	int rc, i, hdm_count;
-	u64 md_status;
 	u16 cap;
 
 	rc = pci_read_config_word(pdev,
@@ -177,6 +185,23 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds)
 			return rc;
 	}
 
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_await_range_active, "CXL");
+
+/*
+ * Wait up to @media_ready_timeout for the device to report memory
+ * active.
+ */
+int cxl_await_media_ready(struct cxl_dev_state *cxlds)
+{
+	u64 md_status;
+	int rc;
+
+	rc = cxl_await_range_active(cxlds);
+	if (rc)
+		return rc;
+
 	md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
 	if (!CXLMDEV_READY(md_status))
 		return -EIO;
@@ -454,6 +479,35 @@ int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
 }
 EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, "CXL");
 
+/**
+ * cxl_get_hdm_info - Get HDM decoder register block location and count
+ * @cxlds: CXL device state (must have component regs enumerated via
+ *	   cxl_probe_component_regs())
+ * @count:  number of HDM decoders in the block (from HDM Capability bits [3:0])
+ * @offset: byte offset of HDM decoder block within the component register BAR
+ * @size:   size in bytes of the HDM decoder block
+ *
+ * Return: 0 on success. -ENODEV if the HDM decoder block is not present.
+ */
+int cxl_get_hdm_info(struct cxl_dev_state *cxlds, u8 *count,
+		     resource_size_t *offset, resource_size_t *size)
+{
+	struct cxl_reg_map *hdm = &cxlds->reg_map.component_map.hdm_decoder;
+
+	if (WARN_ON(!count || !offset || !size))
+		return -EINVAL;
+
+	if (!hdm->valid)
+		return -ENODEV;
+
+	*count	= hdm->count;
+	*offset = hdm->offset;
+	*size	= hdm->size;
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_get_hdm_info, "CXL");
+
 #define CXL_DOE_TABLE_ACCESS_REQ_CODE		0x000000ff
 #define   CXL_DOE_TABLE_ACCESS_REQ_CODE_READ	0
 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE		0x0000ff00
@@ -1183,7 +1237,7 @@ static void cxl_pci_functions_reset_done(struct cxl_reset_context *ctx)
 /*
  * CXL device reset execution
  */
-static int cxl_dev_reset(struct pci_dev *pdev, int dvsec)
+int cxl_dev_reset(struct pci_dev *pdev, int dvsec, bool mem_clr_en)
 {
 	static const u32 reset_timeout_ms[] = { 10, 100, 1000, 10000, 100000 };
 	u16 cap, ctrl2, status2;
@@ -1253,7 +1307,17 @@ static int cxl_dev_reset(struct pci_dev *pdev, int dvsec)
 		if (rc)
 			return rc;
 
-		ctrl2 |= PCI_DVSEC_CXL_RST_MEM_CLR_EN;
+		/*
+		 * Explicitly set or clear RST_MEM_CLR_EN rather than only
+		 * setting it.  A previous reset may have left the bit set in
+		 * hardware; if mem_clr_en is false we must clear it so that a
+		 * stale bit does not cause an unwanted memory-clearing reset.
+		 */
+		if (mem_clr_en)
+			ctrl2 |= PCI_DVSEC_CXL_RST_MEM_CLR_EN;
+		else
+			ctrl2 &= ~PCI_DVSEC_CXL_RST_MEM_CLR_EN;
+
 		rc = pci_write_config_word(pdev, dvsec + PCI_DVSEC_CXL_CTRL2,
 					   ctrl2);
 		if (rc)
@@ -1302,6 +1366,47 @@ static int cxl_dev_reset(struct pci_dev *pdev, int dvsec)
 
 	return 0;
 }
+EXPORT_SYMBOL_NS_GPL(cxl_dev_reset, "CXL");
+
+/**
+ * cxl_dev_reset_locked() - cxl_dev_reset() under cxl_reset_mutex with sibling
+ *                          CXL.cachemem function save/restore.
+ * @pdev: Target CXL function
+ * @dvsec: CXL DVSEC capability offset (pci_find_dvsec_capability())
+ * @mem_clr_en: Pass-through to cxl_dev_reset() (Mem_Clr_Enable in CTRL2)
+ *
+ * Out-of-tree callers (currently vfio-pci) need the same serialization and
+ * sibling-function coordination that the sysfs reset path applies before
+ * invoking cxl_dev_reset(): take the global cxl_reset_mutex, lock the target,
+ * save_and_disable() target plus every CXL.cachemem sibling function on the
+ * bus, run the reset, then restore in reverse order.  Without this, a guest-
+ * triggered CXL reset could race a host sysfs reset or strand a sibling
+ * function in a half-reset state.
+ *
+ * Return: 0 on success, negative errno from cxl_dev_reset() on failure.
+ */
+int cxl_dev_reset_locked(struct pci_dev *pdev, int dvsec, bool mem_clr_en)
+{
+	struct cxl_reset_context ctx = { .target = pdev };
+	int rc;
+
+	mutex_lock(&cxl_reset_mutex);
+	pci_dev_lock(pdev);
+
+	pci_dev_save_and_disable(pdev);
+	cxl_pci_functions_reset_prepare(&ctx);
+
+	rc = cxl_dev_reset(pdev, dvsec, mem_clr_en);
+
+	cxl_pci_functions_reset_done(&ctx);
+	pci_dev_restore(pdev);
+
+	pci_dev_unlock(pdev);
+	mutex_unlock(&cxl_reset_mutex);
+
+	return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_dev_reset_locked, "CXL");
 
 static int match_memdev_by_parent(struct device *dev, const void *parent)
 {
@@ -1341,7 +1446,7 @@ static int cxl_do_reset(struct pci_dev *pdev)
 	pci_dev_save_and_disable(pdev);
 	cxl_pci_functions_reset_prepare(&ctx);
 
-	rc = cxl_dev_reset(pdev, dvsec);
+	rc = cxl_dev_reset(pdev, dvsec, true);
 
 	cxl_pci_functions_reset_done(&ctx);
 
@@ -1370,7 +1475,7 @@ static int cxl_do_reset(struct pci_dev *pdev)
  * devices under bus core serialization.
  */
 
-static bool pci_cxl_reset_capable(struct pci_dev *pdev)
+bool pci_cxl_reset_capable(struct pci_dev *pdev)
 {
 	int dvsec;
 	u16 cap;
@@ -1389,6 +1494,7 @@ static bool pci_cxl_reset_capable(struct pci_dev *pdev)
 
 	return !!(cap & PCI_DVSEC_CXL_RST_CAPABLE);
 }
+EXPORT_SYMBOL_NS_GPL(pci_cxl_reset_capable, "CXL");
 
 static ssize_t cxl_reset_store(struct device *dev,
 			       struct device_attribute *attr,

drivers/cxl/core/regs.c

@@ -85,6 +85,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
 			decoder_cnt = cxl_hdm_decoder_count(hdr);
 			length = 0x20 * decoder_cnt + 0x10;
 			rmap = &map->hdm_decoder;
+			rmap->count = decoder_cnt;
 			break;
 		}
 		case CXL_CM_CAP_CAP_ID_RAS:
@@ -277,6 +278,19 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
 	u64 offset = ((u64)reg_hi << 32) |
 		     (reg_lo & PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW);
 
+	/*
+	 * The BIR field is 3 bits wide (CXL spec); values 6 and 7 are
+	 * reserved.  PCI only defines BAR 0-5, and pci_resource_*() on a
+	 * higher index reads past the resource array.  Reject those here
+	 * so callers do not get garbage.
+	 */
+	if (bar >= PCI_STD_NUM_BARS) {
+		dev_warn(&pdev->dev,
+			 "Reserved BIR %d in Register Locator entry (type %d)\n",
+			 bar, reg_type);
+		return false;
+	}
+
 	if (offset > pci_resource_len(pdev, bar)) {
 		dev_warn(&pdev->dev,
 			 "BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar,
@@ -287,9 +301,44 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
 	map->reg_type = reg_type;
 	map->resource = pci_resource_start(pdev, bar) + offset;
 	map->max_size = pci_resource_len(pdev, bar) - offset;
+	map->bar_index = bar;
+	map->bar_offset = offset;
 	return true;
 }
 
+/**
+ * cxl_regblock_get_bar_info() - Get BAR index and offset for a BAR-backed
+ * regblock
+ * @map: Register map from cxl_find_regblock() or cxl_find_regblock_instance()
+ * @bar_index: Output BAR index (0-5). Optional, may be NULL.
+ * @bar_offset: Output offset within the BAR. Optional, may be NULL.
+ *
+ * When the register block was found via the Register Locator DVSEC and
+ * lives in a PCI BAR (BIR 0-5), this returns the BAR index and the offset
+ * within that BAR.
+ *
+ * Return: 0 if the regblock is BAR-backed (bar_index <= 5), -EINVAL otherwise.
+ */
+int cxl_regblock_get_bar_info(const struct cxl_register_map *map, u8 *bar_index,
+			      resource_size_t *bar_offset)
+{
+	if (!map || map->bar_index == 0xff)
+		return -EINVAL;
+	/*
+	 * Guard callers against stale or out-of-range bar_index.  Only BAR
+	 * indices 0..5 are valid PCI BARs; anything else means the map was
+	 * not BAR-backed or was filled from a reserved BIR.
+	 */
+	if (map->bar_index >= PCI_STD_NUM_BARS)
+		return -EINVAL;
+	if (bar_index)
+		*bar_index = map->bar_index;
+	if (bar_offset)
+		*bar_offset = map->bar_offset;
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_regblock_get_bar_info, "CXL");
+
 /*
  * __cxl_find_regblock_instance() - Locate a register block or count instances by type / index
  * Use CXL_INSTANCES_COUNT for @index if counting instances.
@@ -308,6 +357,7 @@ static int __cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_ty
 
 	*map = (struct cxl_register_map) {
 		.host = &pdev->dev,
+		.bar_index = 0xFF,
 		.resource = CXL_RESOURCE_NONE,
 	};
 

drivers/cxl/cxl.h

@@ -145,8 +145,6 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
 #define   CXLDEV_MBOX_BG_CMD_COMMAND_VENDOR_MASK GENMASK_ULL(63, 48)
 #define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20
 
-void cxl_probe_component_regs(struct device *dev, void __iomem *base,
-			      struct cxl_component_reg_map *map);
 void cxl_probe_device_regs(struct device *dev, void __iomem *base,
 			   struct cxl_device_reg_map *map);
 int cxl_map_device_regs(const struct cxl_register_map *map,