cxl: Add post-reset warning if reset results in loss of previously committed HDM...

Secondary Bus Reset (SBR) is equivalent to a device being hot removed and
inserted again. Doing a SBR on a CXL type 3 device is problematic if the
exported device memory is part of system memory that cannot be offlined.
The event is equivalent to violently ripping out that range of memory from
the kernel. While the hardware requires the "Unmask SBR" bit set in the
Port Control Extensions register and the kernel currently does not unmask
it, user can unmask this bit via setpci or similar tool.

The driver does not have a way to detect whether a reset coming from the
PCI subsystem is a Function Level Reset (FLR) or SBR. The only way to
detect is to note if a decoder is marked as enabled in software but the
decoder control register indicates it's not committed.

Add a helper function to find discrepancy between the decoder software
state versus the hardware register state.

Suggested-by: Dan Williams <dan.j.williams@intel.com>
Link: https://lore.kernel.org/r/20240502165851.1948523-6-dave.jiang@intel.com
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>

diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index c496a9710d62ea4587c7e7b1e5a1d2c019331d40..8567dd11eaac7450b56da8883c33d1990fb0daa2 100644 (file)
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -1045,3 +1045,32 @@ long cxl_pci_get_latency(struct pci_dev *pdev)
 
        return cxl_flit_size(pdev) * MEGA / bw;
 }
+
+static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data)
+{
+       struct cxl_port *port = data;
+       struct cxl_decoder *cxld;
+       struct cxl_hdm *cxlhdm;
+       void __iomem *hdm;
+       u32 ctrl;
+
+       if (!is_endpoint_decoder(dev))
+               return 0;
+
+       cxld = to_cxl_decoder(dev);
+       if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
+               return 0;
+
+       cxlhdm = dev_get_drvdata(&port->dev);
+       hdm = cxlhdm->regs.hdm_decoder;
+       ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
+
+       return !FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl);
+}
+
+bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
+{
+       return device_for_each_child(&port->dev, port,
+                                    __cxl_endpoint_decoder_reset_detected);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);

diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 534e25e2f0a48197a0588abd8a46d996bb333ed8..e3c237c50b59213a51d5141b056a30ba544c3a11 100644 (file)
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -895,6 +895,8 @@ void cxl_coordinates_combine(struct access_coordinate *out,
                             struct access_coordinate *c1,
                             struct access_coordinate *c2);
 
+bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
+
 /*
  * Unit test builds overrides this to __weak, find the 'strong' version
  * of these symbols in tools/testing/cxl/.

diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 11047857329609d61aedd89a1f56bba7629f15bb..f9d65f72601b8737f6a55b29b42ccd9c444adf64 100644 (file)
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -957,11 +957,33 @@ static void cxl_error_resume(struct pci_dev *pdev)
                 dev->driver ? "successful" : "failed");
 }
 
+static void cxl_reset_done(struct pci_dev *pdev)
+{
+       struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
+       struct cxl_memdev *cxlmd = cxlds->cxlmd;
+       struct device *dev = &pdev->dev;
+
+       /*
+        * FLR does not expect to touch the HDM decoders and related
+        * registers.  SBR, however, will wipe all device configurations.
+        * Issue a warning if there was an active decoder before the reset
+        * that no longer exists.
+        */
+       guard(device)(&cxlmd->dev);
+       if (cxlmd->endpoint &&
+           cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) {
+               dev_crit(dev, "SBR happened without memory regions removal.\n");
+               dev_crit(dev, "System may be unstable if regions hosted system memory.\n");
+               add_taint(TAINT_USER, LOCKDEP_STILL_OK);
+       }
+}
+
 static const struct pci_error_handlers cxl_error_handlers = {
        .error_detected = cxl_error_detected,
        .slot_reset     = cxl_slot_reset,
        .resume         = cxl_error_resume,
        .cor_error_detected     = cxl_cor_error_detected,
+       .reset_done     = cxl_reset_done,
 };
 
 static struct pci_driver cxl_pci_driver = {