[linux-2.6-block.git] / drivers / cxl / pci.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/moduleparam.h>
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/sizes.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/pci.h>
#include <linux/io.h>
#include "cxlmem.h"
#include "cxlpci.h"
#include "cxl.h"

/**
 * DOC: cxl pci
 *
 * This implements the PCI exclusive functionality for a CXL device as it is
 * defined by the Compute Express Link specification. CXL devices may surface
 * certain functionality even if it isn't CXL enabled. While this driver is
 * focused around the PCI specific aspects of a CXL device, it binds to the
 * specific CXL memory device class code, and therefore the implementation of
 * cxl_pci is focused around CXL memory devices.
 *
 * The driver has several responsibilities, mainly:
 *  - Create the memX device and register on the CXL bus.
 *  - Enumerate device's register interface and map them.
 *  - Registers nvdimm bridge device with cxl_core.
 *  - Registers a CXL mailbox with cxl_core.
 */

#define cxl_doorbell_busy(cxlds)                                                \
	(readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) &                  \
	 CXLDEV_MBOX_CTRL_DOORBELL)

/* CXL 2.0 - 8.2.8.4 */
#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)

/*
 * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to
 * dictate how long to wait for the mailbox to become ready. The new
 * field allows the device to tell software the amount of time to wait
 * before mailbox ready. This field per the spec theoretically allows
 * for up to 255 seconds. 255 seconds is unreasonably long, its longer
 * than the maximum SATA port link recovery wait. Default to 60 seconds
 * until someone builds a CXL device that needs more time in practice.
 */
static unsigned short mbox_ready_timeout = 60;
module_param(mbox_ready_timeout, ushort, 0644);
MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready");

static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
{
	const unsigned long start = jiffies;
	unsigned long end = start;

	while (cxl_doorbell_busy(cxlds)) {
		end = jiffies;

		if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) {
			/* Check again in case preempted before timeout test */
			if (!cxl_doorbell_busy(cxlds))
				break;
			return -ETIMEDOUT;
		}
		cpu_relax();
	}

	dev_dbg(cxlds->dev, "Doorbell wait took %dms",
		jiffies_to_msecs(end) - jiffies_to_msecs(start));
	return 0;
}

#define cxl_err(dev, status, msg)                                        \
	dev_err_ratelimited(dev, msg ", device state %s%s\n",                  \
			    status & CXLMDEV_DEV_FATAL ? " fatal" : "",        \
			    status & CXLMDEV_FW_HALT ? " firmware-halt" : "")

#define cxl_cmd_err(dev, cmd, status, msg)                               \
	dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n",    \
			    (cmd)->opcode,                                     \
			    status & CXLMDEV_DEV_FATAL ? " fatal" : "",        \
			    status & CXLMDEV_FW_HALT ? " firmware-halt" : "")

/**
 * __cxl_pci_mbox_send_cmd() - Execute a mailbox command
 * @cxlds: The device state to communicate with.
 * @mbox_cmd: Command to send to the memory device.
 *
 * Context: Any context. Expects mbox_mutex to be held.
 * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success.
 *         Caller should check the return code in @mbox_cmd to make sure it
 *         succeeded.
 *
 * This is a generic form of the CXL mailbox send command thus only using the
 * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory
 * devices, and perhaps other types of CXL devices may have further information
 * available upon error conditions. Driver facilities wishing to send mailbox
 * commands should use the wrapper command.
 *
 * The CXL spec allows for up to two mailboxes. The intention is for the primary
 * mailbox to be OS controlled and the secondary mailbox to be used by system
 * firmware. This allows the OS and firmware to communicate with the device and
 * not need to coordinate with each other. The driver only uses the primary
 * mailbox.
 */
static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
				   struct cxl_mbox_cmd *mbox_cmd)
{
	void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
	struct device *dev = cxlds->dev;
	u64 cmd_reg, status_reg;
	size_t out_len;
	int rc;

	lockdep_assert_held(&cxlds->mbox_mutex);

	/*
	 * Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
	 *   1. Caller reads MB Control Register to verify doorbell is clear
	 *   2. Caller writes Command Register
	 *   3. Caller writes Command Payload Registers if input payload is non-empty
	 *   4. Caller writes MB Control Register to set doorbell
	 *   5. Caller either polls for doorbell to be clear or waits for interrupt if configured
	 *   6. Caller reads MB Status Register to fetch Return code
	 *   7. If command successful, Caller reads Command Register to get Payload Length
	 *   8. If output payload is non-empty, host reads Command Payload Registers
	 *
	 * Hardware is free to do whatever it wants before the doorbell is rung,
	 * and isn't allowed to change anything after it clears the doorbell. As
	 * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can
	 * also happen in any order (though some orders might not make sense).
	 */

	/* #1 */
	if (cxl_doorbell_busy(cxlds)) {
		u64 md_status =
			readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);

		cxl_cmd_err(cxlds->dev, mbox_cmd, md_status,
			    "mailbox queue busy");
		return -EBUSY;
	}

	cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK,
			     mbox_cmd->opcode);
	if (mbox_cmd->size_in) {
		if (WARN_ON(!mbox_cmd->payload_in))
			return -EINVAL;

		cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK,
				      mbox_cmd->size_in);
		memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in);
	}

	/* #2, #3 */
	writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);

	/* #4 */
	dev_dbg(dev, "Sending command\n");
	writel(CXLDEV_MBOX_CTRL_DOORBELL,
	       cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);

	/* #5 */
	rc = cxl_pci_mbox_wait_for_doorbell(cxlds);
	if (rc == -ETIMEDOUT) {
		u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);

		cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout");
		return rc;
	}

	/* #6 */
	status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET);
	mbox_cmd->return_code =
		FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);

	if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) {
		dev_dbg(dev, "Mailbox operation had an error: %s\n",
			cxl_mbox_cmd_rc2str(mbox_cmd));
		return 0; /* completed but caller must check return_code */
	}

	/* #7 */
	cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
	out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg);

	/* #8 */
	if (out_len && mbox_cmd->payload_out) {
		/*
		 * Sanitize the copy. If hardware misbehaves, out_len per the
		 * spec can actually be greater than the max allowed size (21
		 * bits available but spec defined 1M max). The caller also may
		 * have requested less data than the hardware supplied even
		 * within spec.
		 */
		size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len);

		memcpy_fromio(mbox_cmd->payload_out, payload, n);
		mbox_cmd->size_out = n;
	} else {
		mbox_cmd->size_out = 0;
	}

	return 0;
}

static int cxl_pci_mbox_send(struct cxl_dev_state *cxlds, struct cxl_mbox_cmd *cmd)
{
	int rc;

	mutex_lock_io(&cxlds->mbox_mutex);
	rc = __cxl_pci_mbox_send_cmd(cxlds, cmd);
	mutex_unlock(&cxlds->mbox_mutex);

	return rc;
}

static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
{
	const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
	unsigned long timeout;
	u64 md_status;

	timeout = jiffies + mbox_ready_timeout * HZ;
	do {
		md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
		if (md_status & CXLMDEV_MBOX_IF_READY)
			break;
		if (msleep_interruptible(100))
			break;
	} while (!time_after(jiffies, timeout));

	if (!(md_status & CXLMDEV_MBOX_IF_READY)) {
		cxl_err(cxlds->dev, md_status,
			"timeout awaiting mailbox ready");
		return -ETIMEDOUT;
	}

	/*
	 * A command may be in flight from a previous driver instance,
	 * think kexec, do one doorbell wait so that
	 * __cxl_pci_mbox_send_cmd() can assume that it is the only
	 * source for future doorbell busy events.
	 */
	if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {
		cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle");
		return -ETIMEDOUT;
	}

	cxlds->mbox_send = cxl_pci_mbox_send;
	cxlds->payload_size =
		1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);

	/*
	 * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register
	 *
	 * If the size is too small, mandatory commands will not work and so
	 * there's no point in going forward. If the size is too large, there's
	 * no harm is soft limiting it.
	 */
	cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M);
	if (cxlds->payload_size < 256) {
		dev_err(cxlds->dev, "Mailbox is too small (%zub)",
			cxlds->payload_size);
		return -ENXIO;
	}

	dev_dbg(cxlds->dev, "Mailbox payload sized %zu",
		cxlds->payload_size);

	return 0;
}

static int cxl_map_regblock(struct pci_dev *pdev, struct cxl_register_map *map)
{
	void __iomem *addr;
	int bar = map->barno;
	struct device *dev = &pdev->dev;
	resource_size_t offset = map->block_offset;

	/* Basic sanity check that BAR is big enough */
	if (pci_resource_len(pdev, bar) < offset) {
		dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar,
			&pdev->resource[bar], &offset);
		return -ENXIO;
	}

	addr = pci_iomap(pdev, bar, 0);
	if (!addr) {
		dev_err(dev, "failed to map registers\n");
		return -ENOMEM;
	}

	dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n",
		bar, &offset);

	map->base = addr + map->block_offset;
	return 0;
}

static void cxl_unmap_regblock(struct pci_dev *pdev,
			       struct cxl_register_map *map)
{
	pci_iounmap(pdev, map->base - map->block_offset);
	map->base = NULL;
}

static int cxl_probe_regs(struct pci_dev *pdev, struct cxl_register_map *map)
{
	struct cxl_component_reg_map *comp_map;
	struct cxl_device_reg_map *dev_map;
	struct device *dev = &pdev->dev;
	void __iomem *base = map->base;

	switch (map->reg_type) {
	case CXL_REGLOC_RBI_COMPONENT:
		comp_map = &map->component_map;
		cxl_probe_component_regs(dev, base, comp_map);
		if (!comp_map->hdm_decoder.valid) {
			dev_err(dev, "HDM decoder registers not found\n");
			return -ENXIO;
		}

		dev_dbg(dev, "Set up component registers\n");
		break;
	case CXL_REGLOC_RBI_MEMDEV:
		dev_map = &map->device_map;
		cxl_probe_device_regs(dev, base, dev_map);
		if (!dev_map->status.valid || !dev_map->mbox.valid ||
		    !dev_map->memdev.valid) {
			dev_err(dev, "registers not found: %s%s%s\n",
				!dev_map->status.valid ? "status " : "",
				!dev_map->mbox.valid ? "mbox " : "",
				!dev_map->memdev.valid ? "memdev " : "");
			return -ENXIO;
		}

		dev_dbg(dev, "Probing device registers...\n");
		break;
	default:
		break;
	}

	return 0;
}

static int cxl_map_regs(struct cxl_dev_state *cxlds, struct cxl_register_map *map)
{
	struct device *dev = cxlds->dev;
	struct pci_dev *pdev = to_pci_dev(dev);

	switch (map->reg_type) {
	case CXL_REGLOC_RBI_COMPONENT:
		cxl_map_component_regs(pdev, &cxlds->regs.component, map);
		dev_dbg(dev, "Mapping component registers...\n");
		break;
	case CXL_REGLOC_RBI_MEMDEV:
		cxl_map_device_regs(pdev, &cxlds->regs.device_regs, map);
		dev_dbg(dev, "Probing device registers...\n");
		break;
	default:
		break;
	}

	return 0;
}

static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
			  struct cxl_register_map *map)
{
	int rc;

	rc = cxl_find_regblock(pdev, type, map);
	if (rc)
		return rc;

	rc = cxl_map_regblock(pdev, map);
	if (rc)
		return rc;

	rc = cxl_probe_regs(pdev, map);
	cxl_unmap_regblock(pdev, map);

	return rc;
}

static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	struct cxl_register_map map;
	struct cxl_memdev *cxlmd;
	struct cxl_dev_state *cxlds;
	int rc;

	/*
	 * Double check the anonymous union trickery in struct cxl_regs
	 * FIXME switch to struct_group()
	 */
	BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) !=
		     offsetof(struct cxl_regs, device_regs.memdev));

	rc = pcim_enable_device(pdev);
	if (rc)
		return rc;

	cxlds = cxl_dev_state_create(&pdev->dev);
	if (IS_ERR(cxlds))
		return PTR_ERR(cxlds);

	cxlds->serial = pci_get_dsn(pdev);
	cxlds->cxl_dvsec = pci_find_dvsec_capability(
		pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
	if (!cxlds->cxl_dvsec)
		dev_warn(&pdev->dev,
			 "Device DVSEC not present, skip CXL.mem init\n");

	rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
	if (rc)
		return rc;

	rc = cxl_map_regs(cxlds, &map);
	if (rc)
		return rc;

	/*
	 * If the component registers can't be found, the cxl_pci driver may
	 * still be useful for management functions so don't return an error.
	 */
	cxlds->component_reg_phys = CXL_RESOURCE_NONE;
	rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
	if (rc)
		dev_warn(&pdev->dev, "No component registers (%d)\n", rc);

	cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);

	rc = cxl_pci_setup_mailbox(cxlds);
	if (rc)
		return rc;

	rc = cxl_enumerate_cmds(cxlds);
	if (rc)
		return rc;

	rc = cxl_dev_state_identify(cxlds);
	if (rc)
		return rc;

	rc = cxl_mem_create_range_info(cxlds);
	if (rc)
		return rc;

	cxlmd = devm_cxl_add_memdev(cxlds);
	if (IS_ERR(cxlmd))
		return PTR_ERR(cxlmd);

	if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
		rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);

	return rc;
}

static const struct pci_device_id cxl_mem_pci_tbl[] = {
	/* PCI class code for CXL.mem Type-3 Devices */
	{ PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)},
	{ /* terminate list */ },
};
MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);

static struct pci_driver cxl_pci_driver = {
	.name			= KBUILD_MODNAME,
	.id_table		= cxl_mem_pci_tbl,
	.probe			= cxl_pci_probe,
	.driver	= {
		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
	},
};

MODULE_LICENSE("GPL v2");
module_pci_driver(cxl_pci_driver);
MODULE_IMPORT_NS(CXL);
Commit	Line	Data
4cdadfd5 DW	1	// SPDX-License-Identifier: GPL-2.0-only
4cdadfd5 DW	2	/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
4faf31b4	3	#include <linux/io-64-nonatomic-lo-hi.h>
229e8828	4	#include <linux/moduleparam.h>
4cdadfd5	5	#include <linux/module.h>
229e8828	6	#include <linux/delay.h>
fae8817a	7	#include <linux/sizes.h>
b39cb105	8	#include <linux/mutex.h>
30af9729	9	#include <linux/list.h>
4cdadfd5 DW	10	#include <linux/pci.h>
4cdadfd5 DW	11	#include <linux/io.h>
5161a55c	12	#include "cxlmem.h"
af9cae9f	13	#include "cxlpci.h"
8adaf747 BW	14	#include "cxl.h"
	15
	16	/**
21e9f767	17	* DOC: cxl pci
8adaf747	18	*
21e9f767 BW	19	* This implements the PCI exclusive functionality for a CXL device as it is
21e9f767 BW	20	* defined by the Compute Express Link specification. CXL devices may surface
ed97afb5 BW	21	* certain functionality even if it isn't CXL enabled. While this driver is
	22	* focused around the PCI specific aspects of a CXL device, it binds to the
	23	* specific CXL memory device class code, and therefore the implementation of
	24	* cxl_pci is focused around CXL memory devices.
8adaf747 BW	25	*
	26	* The driver has several responsibilities, mainly:
	27	* - Create the memX device and register on the CXL bus.
	28	* - Enumerate device's register interface and map them.
ed97afb5 BW	29	* - Registers nvdimm bridge device with cxl_core.
ed97afb5 BW	30	* - Registers a CXL mailbox with cxl_core.
8adaf747 BW	31	*/
8adaf747 BW	32
5e2411ae IW	33	#define cxl_doorbell_busy(cxlds) \
5e2411ae IW	34	(readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) & \
8adaf747 BW	35	CXLDEV_MBOX_CTRL_DOORBELL)
	36
	37	/* CXL 2.0 - 8.2.8.4 */
	38	#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)
	39
229e8828 BW	40	/*
	41	* CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to
	42	* dictate how long to wait for the mailbox to become ready. The new
	43	* field allows the device to tell software the amount of time to wait
	44	* before mailbox ready. This field per the spec theoretically allows
	45	* for up to 255 seconds. 255 seconds is unreasonably long, its longer
	46	* than the maximum SATA port link recovery wait. Default to 60 seconds
	47	* until someone builds a CXL device that needs more time in practice.
	48	*/
	49	static unsigned short mbox_ready_timeout = 60;
	50	module_param(mbox_ready_timeout, ushort, 0644);
2e4ba0ec	51	MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready");
229e8828	52
5e2411ae	53	static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
8adaf747 BW	54	{
	55	const unsigned long start = jiffies;
	56	unsigned long end = start;
	57
5e2411ae	58	while (cxl_doorbell_busy(cxlds)) {
8adaf747 BW	59	end = jiffies;
	60
	61	if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) {
	62	/* Check again in case preempted before timeout test */
5e2411ae	63	if (!cxl_doorbell_busy(cxlds))
8adaf747 BW	64	break;
	65	return -ETIMEDOUT;
	66	}
	67	cpu_relax();
	68	}
	69
5e2411ae	70	dev_dbg(cxlds->dev, "Doorbell wait took %dms",
8adaf747 BW	71	jiffies_to_msecs(end) - jiffies_to_msecs(start));
	72	return 0;
	73	}
	74
4f195ee7 DW	75	#define cxl_err(dev, status, msg) \
	76	dev_err_ratelimited(dev, msg ", device state %s%s\n", \
	77	status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
	78	status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
8adaf747	79
4f195ee7 DW	80	#define cxl_cmd_err(dev, cmd, status, msg) \
	81	dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n", \
	82	(cmd)->opcode, \
	83	status & CXLMDEV_DEV_FATAL ? " fatal" : "", \
	84	status & CXLMDEV_FW_HALT ? " firmware-halt" : "")
8adaf747 BW	85
8adaf747 BW	86	/**
ed97afb5	87	* __cxl_pci_mbox_send_cmd() - Execute a mailbox command
5e2411ae	88	* @cxlds: The device state to communicate with.
8adaf747 BW	89	* @mbox_cmd: Command to send to the memory device.
	90	*
	91	* Context: Any context. Expects mbox_mutex to be held.
	92	* Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success.
	93	* Caller should check the return code in @mbox_cmd to make sure it
	94	* succeeded.
	95	*
	96	* This is a generic form of the CXL mailbox send command thus only using the
	97	* registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory
	98	* devices, and perhaps other types of CXL devices may have further information
	99	* available upon error conditions. Driver facilities wishing to send mailbox
	100	* commands should use the wrapper command.
	101	*
	102	* The CXL spec allows for up to two mailboxes. The intention is for the primary
	103	* mailbox to be OS controlled and the secondary mailbox to be used by system
	104	* firmware. This allows the OS and firmware to communicate with the device and
	105	* not need to coordinate with each other. The driver only uses the primary
	106	* mailbox.
	107	*/
5e2411ae	108	static int __cxl_pci_mbox_send_cmd(struct cxl_dev_state *cxlds,
b64955a9	109	struct cxl_mbox_cmd *mbox_cmd)
8adaf747	110	{
5e2411ae IW	111	void __iomem *payload = cxlds->regs.mbox + CXLDEV_MBOX_PAYLOAD_OFFSET;
5e2411ae IW	112	struct device *dev = cxlds->dev;
8adaf747 BW	113	u64 cmd_reg, status_reg;
	114	size_t out_len;
	115	int rc;
	116
5e2411ae	117	lockdep_assert_held(&cxlds->mbox_mutex);
8adaf747 BW	118
	119	/*
	120	* Here are the steps from 8.2.8.4 of the CXL 2.0 spec.
	121	* 1. Caller reads MB Control Register to verify doorbell is clear
	122	* 2. Caller writes Command Register
	123	* 3. Caller writes Command Payload Registers if input payload is non-empty
	124	* 4. Caller writes MB Control Register to set doorbell
	125	* 5. Caller either polls for doorbell to be clear or waits for interrupt if configured
	126	* 6. Caller reads MB Status Register to fetch Return code
	127	* 7. If command successful, Caller reads Command Register to get Payload Length
	128	* 8. If output payload is non-empty, host reads Command Payload Registers
	129	*
	130	* Hardware is free to do whatever it wants before the doorbell is rung,
	131	* and isn't allowed to change anything after it clears the doorbell. As
	132	* such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can
	133	* also happen in any order (though some orders might not make sense).
	134	*/
	135
	136	/* #1 */
5e2411ae	137	if (cxl_doorbell_busy(cxlds)) {
4f195ee7 DW	138	u64 md_status =
	139	readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
	140
	141	cxl_cmd_err(cxlds->dev, mbox_cmd, md_status,
	142	"mailbox queue busy");
8adaf747 BW	143	return -EBUSY;
	144	}
	145
	146	cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK,
	147	mbox_cmd->opcode);
	148	if (mbox_cmd->size_in) {
	149	if (WARN_ON(!mbox_cmd->payload_in))
	150	return -EINVAL;
	151
	152	cmd_reg \|= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK,
	153	mbox_cmd->size_in);
	154	memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in);
	155	}
	156
	157	/* #2, #3 */
5e2411ae	158	writeq(cmd_reg, cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
8adaf747 BW	159
8adaf747 BW	160	/* #4 */
99e222a5	161	dev_dbg(dev, "Sending command\n");
8adaf747	162	writel(CXLDEV_MBOX_CTRL_DOORBELL,
5e2411ae	163	cxlds->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET);
8adaf747 BW	164
8adaf747 BW	165	/* #5 */
5e2411ae	166	rc = cxl_pci_mbox_wait_for_doorbell(cxlds);
8adaf747	167	if (rc == -ETIMEDOUT) {
4f195ee7 DW	168	u64 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
	169
	170	cxl_cmd_err(cxlds->dev, mbox_cmd, md_status, "mailbox timeout");
8adaf747 BW	171	return rc;
	172	}
	173
	174	/* #6 */
5e2411ae	175	status_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_STATUS_OFFSET);
8adaf747 BW	176	mbox_cmd->return_code =
	177	FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg);
	178
92fcc1ab	179	if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS) {
c43e036d DB	180	dev_dbg(dev, "Mailbox operation had an error: %s\n",
c43e036d DB	181	cxl_mbox_cmd_rc2str(mbox_cmd));
cbe83a20	182	return 0; /* completed but caller must check return_code */
8adaf747 BW	183	}
	184
	185	/* #7 */
5e2411ae	186	cmd_reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_CMD_OFFSET);
8adaf747 BW	187	out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg);
	188
	189	/* #8 */
	190	if (out_len && mbox_cmd->payload_out) {
	191	/*
	192	* Sanitize the copy. If hardware misbehaves, out_len per the
	193	* spec can actually be greater than the max allowed size (21
	194	* bits available but spec defined 1M max). The caller also may
	195	* have requested less data than the hardware supplied even
	196	* within spec.
	197	*/
5e2411ae	198	size_t n = min3(mbox_cmd->size_out, cxlds->payload_size, out_len);
8adaf747 BW	199
	200	memcpy_fromio(mbox_cmd->payload_out, payload, n);
	201	mbox_cmd->size_out = n;
	202	} else {
	203	mbox_cmd->size_out = 0;
	204	}
	205
	206	return 0;
	207	}
	208
5e2411ae	209	static int cxl_pci_mbox_send(struct cxl_dev_state cxlds, struct cxl_mbox_cmd cmd)
b64955a9 DW	210	{
	211	int rc;
	212
4f195ee7	213	mutex_lock_io(&cxlds->mbox_mutex);
5e2411ae	214	rc = __cxl_pci_mbox_send_cmd(cxlds, cmd);
4f195ee7	215	mutex_unlock(&cxlds->mbox_mutex);
b64955a9 DW	216
	217	return rc;
	218	}
	219
5e2411ae	220	static int cxl_pci_setup_mailbox(struct cxl_dev_state *cxlds)
8adaf747	221	{
5e2411ae	222	const int cap = readl(cxlds->regs.mbox + CXLDEV_MBOX_CAPS_OFFSET);
229e8828 BW	223	unsigned long timeout;
	224	u64 md_status;
	225
	226	timeout = jiffies + mbox_ready_timeout * HZ;
	227	do {
	228	md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET);
	229	if (md_status & CXLMDEV_MBOX_IF_READY)
	230	break;
	231	if (msleep_interruptible(100))
	232	break;
	233	} while (!time_after(jiffies, timeout));
	234
	235	if (!(md_status & CXLMDEV_MBOX_IF_READY)) {
4f195ee7 DW	236	cxl_err(cxlds->dev, md_status,
	237	"timeout awaiting mailbox ready");
	238	return -ETIMEDOUT;
	239	}
	240
	241	/*
	242	* A command may be in flight from a previous driver instance,
	243	* think kexec, do one doorbell wait so that
	244	* __cxl_pci_mbox_send_cmd() can assume that it is the only
	245	* source for future doorbell busy events.
	246	*/
	247	if (cxl_pci_mbox_wait_for_doorbell(cxlds) != 0) {
	248	cxl_err(cxlds->dev, md_status, "timeout awaiting mailbox idle");
	249	return -ETIMEDOUT;
229e8828	250	}
8adaf747	251
5e2411ae IW	252	cxlds->mbox_send = cxl_pci_mbox_send;
5e2411ae IW	253	cxlds->payload_size =
8adaf747 BW	254	1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap);
	255
	256	/*
	257	* CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register
	258	*
	259	* If the size is too small, mandatory commands will not work and so
	260	* there's no point in going forward. If the size is too large, there's
	261	* no harm is soft limiting it.
	262	*/
5e2411ae IW	263	cxlds->payload_size = min_t(size_t, cxlds->payload_size, SZ_1M);
	264	if (cxlds->payload_size < 256) {
	265	dev_err(cxlds->dev, "Mailbox is too small (%zub)",
	266	cxlds->payload_size);
8adaf747 BW	267	return -ENXIO;
	268	}
	269
5e2411ae IW	270	dev_dbg(cxlds->dev, "Mailbox payload sized %zu",
5e2411ae IW	271	cxlds->payload_size);
8adaf747 BW	272
	273	return 0;
	274	}
	275
a261e9a1	276	static int cxl_map_regblock(struct pci_dev pdev, struct cxl_register_map map)
1b0a1a2a	277	{
f8a7e8c2	278	void __iomem *addr;
7dc7a64d BW	279	int bar = map->barno;
	280	struct device *dev = &pdev->dev;
	281	resource_size_t offset = map->block_offset;
1b0a1a2a	282
8adaf747 BW	283	/* Basic sanity check that BAR is big enough */
8adaf747 BW	284	if (pci_resource_len(pdev, bar) < offset) {
7dc7a64d BW	285	dev_err(dev, "BAR%d: %pr: too small (offset: %pa)\n", bar,
7dc7a64d BW	286	&pdev->resource[bar], &offset);
a261e9a1	287	return -ENXIO;
8adaf747 BW	288	}
8adaf747 BW	289
30af9729	290	addr = pci_iomap(pdev, bar, 0);
f8a7e8c2	291	if (!addr) {
8adaf747	292	dev_err(dev, "failed to map registers\n");
a261e9a1	293	return -ENOMEM;
8adaf747	294	}
8adaf747	295
7dc7a64d BW	296	dev_dbg(dev, "Mapped CXL Memory Device resource bar %u @ %pa\n",
7dc7a64d BW	297	bar, &offset);
6630d31c	298
a261e9a1 DW	299	map->base = addr + map->block_offset;
a261e9a1 DW	300	return 0;
30af9729 IW	301	}
30af9729 IW	302
a261e9a1 DW	303	static void cxl_unmap_regblock(struct pci_dev *pdev,
a261e9a1 DW	304	struct cxl_register_map *map)
30af9729	305	{
a261e9a1 DW	306	pci_iounmap(pdev, map->base - map->block_offset);
a261e9a1 DW	307	map->base = NULL;
8adaf747	308	}
4cdadfd5	309
a261e9a1	310	static int cxl_probe_regs(struct pci_dev pdev, struct cxl_register_map map)
30af9729	311	{
08422378	312	struct cxl_component_reg_map *comp_map;
30af9729	313	struct cxl_device_reg_map *dev_map;
7dc7a64d	314	struct device *dev = &pdev->dev;
a261e9a1	315	void __iomem *base = map->base;
30af9729 IW	316
30af9729 IW	317	switch (map->reg_type) {
08422378 BW	318	case CXL_REGLOC_RBI_COMPONENT:
	319	comp_map = &map->component_map;
	320	cxl_probe_component_regs(dev, base, comp_map);
	321	if (!comp_map->hdm_decoder.valid) {
	322	dev_err(dev, "HDM decoder registers not found\n");
	323	return -ENXIO;
	324	}
	325
	326	dev_dbg(dev, "Set up component registers\n");
	327	break;
30af9729 IW	328	case CXL_REGLOC_RBI_MEMDEV:
	329	dev_map = &map->device_map;
	330	cxl_probe_device_regs(dev, base, dev_map);
	331	if (!dev_map->status.valid \|\| !dev_map->mbox.valid \|\|
	332	!dev_map->memdev.valid) {
	333	dev_err(dev, "registers not found: %s%s%s\n",
	334	!dev_map->status.valid ? "status " : "",
da582aa5 LQJL	335	!dev_map->mbox.valid ? "mbox " : "",
da582aa5 LQJL	336	!dev_map->memdev.valid ? "memdev " : "");
30af9729 IW	337	return -ENXIO;
	338	}
	339
	340	dev_dbg(dev, "Probing device registers...\n");
	341	break;
	342	default:
	343	break;
	344	}
	345
	346	return 0;
	347	}
	348
5e2411ae	349	static int cxl_map_regs(struct cxl_dev_state cxlds, struct cxl_register_map map)
30af9729	350	{
5e2411ae	351	struct device *dev = cxlds->dev;
99e222a5	352	struct pci_dev *pdev = to_pci_dev(dev);
30af9729 IW	353
30af9729 IW	354	switch (map->reg_type) {
08422378	355	case CXL_REGLOC_RBI_COMPONENT:
5e2411ae	356	cxl_map_component_regs(pdev, &cxlds->regs.component, map);
08422378 BW	357	dev_dbg(dev, "Mapping component registers...\n");
08422378 BW	358	break;
30af9729	359	case CXL_REGLOC_RBI_MEMDEV:
5e2411ae	360	cxl_map_device_regs(pdev, &cxlds->regs.device_regs, map);
30af9729 IW	361	dev_dbg(dev, "Probing device registers...\n");
	362	break;
	363	default:
	364	break;
	365	}
	366
	367	return 0;
	368	}
	369
85afc317 BW	370	static int cxl_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
	371	struct cxl_register_map *map)
	372	{
	373	int rc;
5b68705d	374
85afc317 BW	375	rc = cxl_find_regblock(pdev, type, map);
	376	if (rc)
	377	return rc;
1d5a4159	378
85afc317 BW	379	rc = cxl_map_regblock(pdev, map);
	380	if (rc)
	381	return rc;
	382
	383	rc = cxl_probe_regs(pdev, map);
	384	cxl_unmap_regblock(pdev, map);
1d5a4159	385
85afc317	386	return rc;
1d5a4159 BW	387	}
1d5a4159 BW	388
ed97afb5	389	static int cxl_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
4cdadfd5	390	{
85afc317	391	struct cxl_register_map map;
21083f51	392	struct cxl_memdev *cxlmd;
5e2411ae	393	struct cxl_dev_state *cxlds;
1d5a4159	394	int rc;
8adaf747	395
5a2328f4 DW	396	/*
	397	* Double check the anonymous union trickery in struct cxl_regs
	398	* FIXME switch to struct_group()
	399	*/
	400	BUILD_BUG_ON(offsetof(struct cxl_regs, memdev) !=
	401	offsetof(struct cxl_regs, device_regs.memdev));
	402
8adaf747 BW	403	rc = pcim_enable_device(pdev);
	404	if (rc)
	405	return rc;
4cdadfd5	406
5e2411ae IW	407	cxlds = cxl_dev_state_create(&pdev->dev);
	408	if (IS_ERR(cxlds))
	409	return PTR_ERR(cxlds);
1b0a1a2a	410
bcc79ea3	411	cxlds->serial = pci_get_dsn(pdev);
06e279e5 BW	412	cxlds->cxl_dvsec = pci_find_dvsec_capability(
	413	pdev, PCI_DVSEC_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
	414	if (!cxlds->cxl_dvsec)
	415	dev_warn(&pdev->dev,
	416	"Device DVSEC not present, skip CXL.mem init\n");
	417
85afc317 BW	418	rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
	419	if (rc)
	420	return rc;
	421
5e2411ae	422	rc = cxl_map_regs(cxlds, &map);
8adaf747 BW	423	if (rc)
	424	return rc;
	425
4112a08d BW	426	/*
	427	* If the component registers can't be found, the cxl_pci driver may
	428	* still be useful for management functions so don't return an error.
	429	*/
	430	cxlds->component_reg_phys = CXL_RESOURCE_NONE;
	431	rc = cxl_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
	432	if (rc)
	433	dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
	434
	435	cxlds->component_reg_phys = cxl_regmap_to_base(pdev, &map);
	436
5e2411ae	437	rc = cxl_pci_setup_mailbox(cxlds);
8adaf747 BW	438	if (rc)
	439	return rc;
	440
5e2411ae	441	rc = cxl_enumerate_cmds(cxlds);
472b1ce6 BW	442	if (rc)
	443	return rc;
	444
5e2411ae	445	rc = cxl_dev_state_identify(cxlds);
b39cb105 DW	446	if (rc)
	447	return rc;
	448
5e2411ae	449	rc = cxl_mem_create_range_info(cxlds);
f847502a IW	450	if (rc)
	451	return rc;
	452
5e2411ae	453	cxlmd = devm_cxl_add_memdev(cxlds);
21083f51 DW	454	if (IS_ERR(cxlmd))
	455	return PTR_ERR(cxlmd);
	456
5e2411ae	457	if (range_len(&cxlds->pmem_range) && IS_ENABLED(CONFIG_CXL_PMEM))
21083f51 DW	458	rc = devm_cxl_add_nvdimm(&pdev->dev, cxlmd);
	459
	460	return rc;
4cdadfd5 DW	461	}
	462
	463	static const struct pci_device_id cxl_mem_pci_tbl[] = {
	464	/* PCI class code for CXL.mem Type-3 Devices */
	465	{ PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 \| CXL_MEMORY_PROGIF), ~0)},
	466	{ /* terminate list */ },
	467	};
	468	MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
	469
ed97afb5	470	static struct pci_driver cxl_pci_driver = {
4cdadfd5 DW	471	.name = KBUILD_MODNAME,
4cdadfd5 DW	472	.id_table = cxl_mem_pci_tbl,
ed97afb5	473	.probe = cxl_pci_probe,
4cdadfd5 DW	474	.driver = {
	475	.probe_type = PROBE_PREFER_ASYNCHRONOUS,
	476	},
	477	};
	478
	479	MODULE_LICENSE("GPL v2");
ed97afb5	480	module_pci_driver(cxl_pci_driver);
b39cb105	481	MODULE_IMPORT_NS(CXL);