cxl/pci: Add RCH downstream port AER register discovery

Restricted CXL host (RCH) downstream port AER information is not currently
logged while in the error state. One problem preventing the error logging
is the AER and RAS registers are not accessible. The CXL driver requires
changes to find RCH downstream port AER and RAS registers for purpose of
error logging.

RCH downstream ports are not enumerated during a PCI bus scan and are
instead discovered using system firmware, ACPI in this case.[1] The
downstream port is implemented as a Root Complex Register Block (RCRB).
The RCRB is a 4k memory block containing PCIe registers based on the PCIe
root port.[2] The RCRB includes AER extended capability registers used for
reporting errors. Note, the RCH's AER Capability is located in the RCRB
memory space instead of PCI configuration space, thus its register access
is different. Existing kernel PCIe AER functions can not be used to manage
the downstream port AER capabilities and RAS registers because the port was
not enumerated during PCI scan and the registers are not PCI config
accessible.

Discover RCH downstream port AER extended capability registers. Use MMIO
accesses to search for extended AER capability in RCRB register space.

[1] CXL 3.0 Spec, 9.11.2 - System Firmware View of CXL 1.1 Hierarchy
[2] CXL 3.0 Spec, 8.2.1.1 - RCH Downstream Port RCRB

Co-developed-by: Robert Richter <rrichter@amd.com>
Signed-off-by: Terry Bowman <terry.bowman@amd.com>
Signed-off-by: Robert Richter <rrichter@amd.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Reviewed-by: Dave Jiang <dave.jiang@intel.com>
Link: https://lore.kernel.org/r/20231018171713.1883517-12-rrichter@amd.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 45e7e044cf4a0452b873680200d0379a245b3f49..f470ef5c0a6a23e69312332a806d129aa4cddfda 100644 (file)
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -73,6 +73,7 @@ struct cxl_rcrb_info;
 resource_size_t __rcrb_to_component(struct device *dev,
                                    struct cxl_rcrb_info *ri,
                                    enum cxl_rcrb which);
+u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb);
 
 extern struct rw_semaphore cxl_dpa_rwsem;
 

diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index c7a7887ebdcff859bac675e6f41322aae1b50794..cbccc222bb9187f02346765e8d72ab4ec7ad19c4 100644 (file)
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -718,6 +718,21 @@ static bool cxl_report_and_clear(struct cxl_dev_state *cxlds)
        return true;
 }
 
+#ifdef CONFIG_PCIEAER_CXL
+
+void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport)
+{
+       struct device *dport_dev = dport->dport_dev;
+       struct pci_host_bridge *host_bridge;
+
+       host_bridge = to_pci_host_bridge(dport_dev);
+       if (host_bridge->native_cxl_error)
+               dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL);
+
+#endif
+
 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
                                    pci_channel_state_t state)
 {

diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index e0fbe964f6f0a1b0e6148790fdc67baf1eb0e6aa..9111ceef11270451f8fa34e2b77f2efac033d454 100644 (file)
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -470,6 +470,42 @@ int cxl_setup_regs(struct cxl_register_map *map)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL);
 
+u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb)
+{
+       void __iomem *addr;
+       u16 offset = 0;
+       u32 cap_hdr;
+
+       if (WARN_ON_ONCE(rcrb == CXL_RESOURCE_NONE))
+               return 0;
+
+       if (!request_mem_region(rcrb, SZ_4K, dev_name(dev)))
+               return 0;
+
+       addr = ioremap(rcrb, SZ_4K);
+       if (!addr)
+               goto out;
+
+       cap_hdr = readl(addr + offset);
+       while (PCI_EXT_CAP_ID(cap_hdr) != PCI_EXT_CAP_ID_ERR) {
+               offset = PCI_EXT_CAP_NEXT(cap_hdr);
+
+               /* Offset 0 terminates capability list. */
+               if (!offset)
+                       break;
+               cap_hdr = readl(addr + offset);
+       }
+
+       if (offset)
+               dev_dbg(dev, "found AER extended capability (0x%x)\n", offset);
+
+       iounmap(addr);
+out:
+       release_mem_region(rcrb, SZ_4K);
+
+       return offset;
+}
+
 resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri,
                                    enum cxl_rcrb which)
 {

diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index c07064e0c13698a0ee5463e862751b6f690caf03..cdb2ade6ba29f35d49151b1ea7e10fe2b0c57701 100644 (file)
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -704,6 +704,13 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
                                         struct device *dport_dev, int port_id,
                                         resource_size_t rcrb);
 
+#ifdef CONFIG_PCIEAER_CXL
+void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport);
+#else
+static inline void cxl_setup_parent_dport(struct device *host,
+                                         struct cxl_dport *dport) { }
+#endif
+
 struct cxl_decoder *to_cxl_decoder(struct device *dev);
 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);

diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 04107058739bc2eb3c6e7e2a0879dc03e14e13b4..e087febf9af047c81dfb11b06d91d92d442d586c 100644 (file)
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -157,6 +157,8 @@ static int cxl_mem_probe(struct device *dev)
        else
                endpoint_parent = &parent_port->dev;
 
+       cxl_setup_parent_dport(dev, dport);
+
        device_lock(endpoint_parent);
        if (!endpoint_parent->driver) {
                dev_err(dev, "CXL port topology %s not enabled\n",