[linux-block.git] / drivers / cxl / pmem.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
#include <linux/libnvdimm.h>
#include <asm/unaligned.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/ndctl.h>
#include <linux/async.h>
#include <linux/slab.h>
#include <linux/nd.h>
#include "cxlmem.h"
#include "cxl.h"

extern const struct nvdimm_security_ops *cxl_security_ops;

static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);

static void clear_exclusive(void *cxlds)
{
	clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
}

static void unregister_nvdimm(void *nvdimm)
{
	nvdimm_delete(nvdimm);
}

static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct nvdimm *nvdimm = to_nvdimm(dev);
	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);

	return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
}
static DEVICE_ATTR_RO(provider);

static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct nvdimm *nvdimm = to_nvdimm(dev);
	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;

	return sysfs_emit(buf, "%lld\n", cxlds->serial);
}
static DEVICE_ATTR_RO(id);

static struct attribute *cxl_dimm_attributes[] = {
	&dev_attr_id.attr,
	&dev_attr_provider.attr,
	NULL
};

static const struct attribute_group cxl_dimm_attribute_group = {
	.name = "cxl",
	.attrs = cxl_dimm_attributes,
};

static const struct attribute_group *cxl_dimm_attribute_groups[] = {
	&cxl_dimm_attribute_group,
	NULL
};

static int cxl_nvdimm_probe(struct device *dev)
{
	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
	unsigned long flags = 0, cmd_mask = 0;
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct nvdimm *nvdimm;
	int rc;

	set_exclusive_cxl_commands(cxlds, exclusive_cmds);
	rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
	if (rc)
		return rc;

	set_bit(NDD_LABELING, &flags);
	set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
	set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
	set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
	nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
				 cxl_dimm_attribute_groups, flags,
				 cmd_mask, 0, NULL, cxl_nvd->dev_id,
				 cxl_security_ops, NULL);
	if (!nvdimm)
		return -ENOMEM;

	dev_set_drvdata(dev, nvdimm);
	return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
}

static struct cxl_driver cxl_nvdimm_driver = {
	.name = "cxl_nvdimm",
	.probe = cxl_nvdimm_probe,
	.id = CXL_DEVICE_NVDIMM,
	.drv = {
		.suppress_bind_attrs = true,
	},
};

static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
				    struct nd_cmd_get_config_size *cmd,
				    unsigned int buf_len)
{
	if (sizeof(*cmd) > buf_len)
		return -EINVAL;

	*cmd = (struct nd_cmd_get_config_size) {
		 .config_size = cxlds->lsa_size,
		 .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
	};

	return 0;
}

static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
				    struct nd_cmd_get_config_data_hdr *cmd,
				    unsigned int buf_len)
{
	struct cxl_mbox_get_lsa get_lsa;
	struct cxl_mbox_cmd mbox_cmd;
	int rc;

	if (sizeof(*cmd) > buf_len)
		return -EINVAL;
	if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
		return -EINVAL;

	get_lsa = (struct cxl_mbox_get_lsa) {
		.offset = cpu_to_le32(cmd->in_offset),
		.length = cpu_to_le32(cmd->in_length),
	};
	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_GET_LSA,
		.payload_in = &get_lsa,
		.size_in = sizeof(get_lsa),
		.size_out = cmd->in_length,
		.payload_out = cmd->out_buf,
	};

	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
	cmd->status = 0;

	return rc;
}

static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
				    struct nd_cmd_set_config_hdr *cmd,
				    unsigned int buf_len)
{
	struct cxl_mbox_set_lsa *set_lsa;
	struct cxl_mbox_cmd mbox_cmd;
	int rc;

	if (sizeof(*cmd) > buf_len)
		return -EINVAL;

	/* 4-byte status follows the input data in the payload */
	if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
		return -EINVAL;

	set_lsa =
		kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
	if (!set_lsa)
		return -ENOMEM;

	*set_lsa = (struct cxl_mbox_set_lsa) {
		.offset = cpu_to_le32(cmd->in_offset),
	};
	memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_SET_LSA,
		.payload_in = set_lsa,
		.size_in = struct_size(set_lsa, data, cmd->in_length),
	};

	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);

	/*
	 * Set "firmware" status (4-packed bytes at the end of the input
	 * payload.
	 */
	put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
	kvfree(set_lsa);

	return rc;
}

static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
			       void *buf, unsigned int buf_len)
{
	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
	struct cxl_dev_state *cxlds = cxlmd->cxlds;

	if (!test_bit(cmd, &cmd_mask))
		return -ENOTTY;

	switch (cmd) {
	case ND_CMD_GET_CONFIG_SIZE:
		return cxl_pmem_get_config_size(cxlds, buf, buf_len);
	case ND_CMD_GET_CONFIG_DATA:
		return cxl_pmem_get_config_data(cxlds, buf, buf_len);
	case ND_CMD_SET_CONFIG_DATA:
		return cxl_pmem_set_config_data(cxlds, buf, buf_len);
	default:
		return -ENOTTY;
	}
}

static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
			struct nvdimm *nvdimm, unsigned int cmd, void *buf,
			unsigned int buf_len, int *cmd_rc)
{
	/*
	 * No firmware response to translate, let the transport error
	 * code take precedence.
	 */
	*cmd_rc = 0;

	if (!nvdimm)
		return -ENOTTY;
	return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
}

static int detach_nvdimm(struct device *dev, void *data)
{
	struct cxl_nvdimm *cxl_nvd;
	bool release = false;

	if (!is_cxl_nvdimm(dev))
		return 0;

	device_lock(dev);
	if (!dev->driver)
		goto out;

	cxl_nvd = to_cxl_nvdimm(dev);
	if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
		release = true;
out:
	device_unlock(dev);
	if (release)
		device_release_driver(dev);
	return 0;
}

static void unregister_nvdimm_bus(void *_cxl_nvb)
{
	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
	struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;

	bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);

	cxl_nvb->nvdimm_bus = NULL;
	nvdimm_bus_unregister(nvdimm_bus);
}

static int cxl_nvdimm_bridge_probe(struct device *dev)
{
	struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);

	cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
		.provider_name = "CXL",
		.module = THIS_MODULE,
		.ndctl = cxl_pmem_ctl,
	};

	cxl_nvb->nvdimm_bus =
		nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);

	if (!cxl_nvb->nvdimm_bus)
		return -ENOMEM;

	return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
}

static struct cxl_driver cxl_nvdimm_bridge_driver = {
	.name = "cxl_nvdimm_bridge",
	.probe = cxl_nvdimm_bridge_probe,
	.id = CXL_DEVICE_NVDIMM_BRIDGE,
	.drv = {
		.suppress_bind_attrs = true,
	},
};

static void unregister_nvdimm_region(void *nd_region)
{
	nvdimm_region_delete(nd_region);
}

static void cxlr_pmem_remove_resource(void *res)
{
	remove_resource(res);
}

struct cxl_pmem_region_info {
	u64 offset;
	u64 serial;
};

static int cxl_pmem_region_probe(struct device *dev)
{
	struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
	struct cxl_region *cxlr = cxlr_pmem->cxlr;
	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
	struct cxl_pmem_region_info *info = NULL;
	struct nd_interleave_set *nd_set;
	struct nd_region_desc ndr_desc;
	struct cxl_nvdimm *cxl_nvd;
	struct nvdimm *nvdimm;
	struct resource *res;
	int rc, i = 0;

	memset(&mappings, 0, sizeof(mappings));
	memset(&ndr_desc, 0, sizeof(ndr_desc));

	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
	if (!res)
		return -ENOMEM;

	res->name = "Persistent Memory";
	res->start = cxlr_pmem->hpa_range.start;
	res->end = cxlr_pmem->hpa_range.end;
	res->flags = IORESOURCE_MEM;
	res->desc = IORES_DESC_PERSISTENT_MEMORY;

	rc = insert_resource(&iomem_resource, res);
	if (rc)
		return rc;

	rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
	if (rc)
		return rc;

	ndr_desc.res = res;
	ndr_desc.provider_data = cxlr_pmem;

	ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
	ndr_desc.target_node = phys_to_target_node(res->start);
	if (ndr_desc.target_node == NUMA_NO_NODE) {
		ndr_desc.target_node = ndr_desc.numa_node;
		dev_dbg(&cxlr->dev, "changing target node from %d to %d",
			NUMA_NO_NODE, ndr_desc.target_node);
	}

	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
	if (!nd_set)
		return -ENOMEM;

	ndr_desc.memregion = cxlr->id;
	set_bit(ND_REGION_CXL, &ndr_desc.flags);
	set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);

	info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;

	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
		struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
		struct cxl_memdev *cxlmd = m->cxlmd;
		struct cxl_dev_state *cxlds = cxlmd->cxlds;

		cxl_nvd = cxlmd->cxl_nvd;
		nvdimm = dev_get_drvdata(&cxl_nvd->dev);
		if (!nvdimm) {
			dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
				dev_name(&cxlmd->dev));
			rc = -ENODEV;
			goto out_nvd;
		}

		m->cxl_nvd = cxl_nvd;
		mappings[i] = (struct nd_mapping_desc) {
			.nvdimm = nvdimm,
			.start = m->start,
			.size = m->size,
			.position = i,
		};
		info[i].offset = m->start;
		info[i].serial = cxlds->serial;
	}
	ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
	ndr_desc.mapping = mappings;

	/*
	 * TODO enable CXL labels which skip the need for 'interleave-set cookie'
	 */
	nd_set->cookie1 =
		nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
	nd_set->cookie2 = nd_set->cookie1;
	ndr_desc.nd_set = nd_set;

	cxlr_pmem->nd_region =
		nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
	if (!cxlr_pmem->nd_region) {
		rc = -ENOMEM;
		goto out_nvd;
	}

	rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
				      cxlr_pmem->nd_region);
out_nvd:
	kfree(info);

	return rc;
}

static struct cxl_driver cxl_pmem_region_driver = {
	.name = "cxl_pmem_region",
	.probe = cxl_pmem_region_probe,
	.id = CXL_DEVICE_PMEM_REGION,
	.drv = {
		.suppress_bind_attrs = true,
	},
};

static __init int cxl_pmem_init(void)
{
	int rc;

	set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
	set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);

	rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
	if (rc)
		return rc;

	rc = cxl_driver_register(&cxl_nvdimm_driver);
	if (rc)
		goto err_nvdimm;

	rc = cxl_driver_register(&cxl_pmem_region_driver);
	if (rc)
		goto err_region;

	return 0;

err_region:
	cxl_driver_unregister(&cxl_nvdimm_driver);
err_nvdimm:
	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
	return rc;
}

static __exit void cxl_pmem_exit(void)
{
	cxl_driver_unregister(&cxl_pmem_region_driver);
	cxl_driver_unregister(&cxl_nvdimm_driver);
	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
}

MODULE_LICENSE("GPL v2");
module_init(cxl_pmem_init);
module_exit(cxl_pmem_exit);
MODULE_IMPORT_NS(CXL);
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);
Commit	Line	Data
8fdcb170 DW	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
	3	#include <linux/libnvdimm.h>
60b8f172	4	#include <asm/unaligned.h>
8fdcb170 DW	5	#include <linux/device.h>
8fdcb170 DW	6	#include <linux/module.h>
21083f51 DW	7	#include <linux/ndctl.h>
21083f51 DW	8	#include <linux/async.h>
8fdcb170	9	#include <linux/slab.h>
04ad63f0	10	#include <linux/nd.h>
5161a55c	11	#include "cxlmem.h"
8fdcb170 DW	12	#include "cxl.h"
8fdcb170 DW	13
32828115 DJ	14	extern const struct nvdimm_security_ops *cxl_security_ops;
32828115 DJ	15
12f3856a DW	16	static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
12f3856a DW	17
5e2411ae	18	static void clear_exclusive(void *cxlds)
12f3856a	19	{
5e2411ae	20	clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
12f3856a DW	21	}
12f3856a DW	22
21083f51 DW	23	static void unregister_nvdimm(void *nvdimm)
	24	{
	25	nvdimm_delete(nvdimm);
	26	}
	27
452996fa DJ	28	static ssize_t provider_show(struct device dev, struct device_attribute attr, char *buf)
	29	{
	30	struct nvdimm *nvdimm = to_nvdimm(dev);
	31	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	32
	33	return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
	34	}
	35	static DEVICE_ATTR_RO(provider);
	36
bd429e53 DJ	37	static ssize_t id_show(struct device dev, struct device_attribute attr, char *buf)
	38	{
	39	struct nvdimm *nvdimm = to_nvdimm(dev);
	40	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	41	struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
	42
	43	return sysfs_emit(buf, "%lld\n", cxlds->serial);
	44	}
	45	static DEVICE_ATTR_RO(id);
	46
	47	static struct attribute *cxl_dimm_attributes[] = {
	48	&dev_attr_id.attr,
452996fa	49	&dev_attr_provider.attr,
bd429e53 DJ	50	NULL
	51	};
	52
	53	static const struct attribute_group cxl_dimm_attribute_group = {
	54	.name = "cxl",
	55	.attrs = cxl_dimm_attributes,
	56	};
	57
	58	static const struct attribute_group *cxl_dimm_attribute_groups[] = {
	59	&cxl_dimm_attribute_group,
	60	NULL
	61	};
	62
21083f51 DW	63	static int cxl_nvdimm_probe(struct device *dev)
	64	{
	65	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
12f3856a	66	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
f17b558d	67	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
60b8f172	68	unsigned long flags = 0, cmd_mask = 0;
5e2411ae	69	struct cxl_dev_state *cxlds = cxlmd->cxlds;
21083f51	70	struct nvdimm *nvdimm;
12f3856a	71	int rc;
21083f51	72
5e2411ae IW	73	set_exclusive_cxl_commands(cxlds, exclusive_cmds);
5e2411ae IW	74	rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
12f3856a	75	if (rc)
f17b558d	76	return rc;
21083f51 DW	77
21083f51 DW	78	set_bit(NDD_LABELING, &flags);
60b8f172 DW	79	set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
	80	set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
	81	set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
bd429e53 DJ	82	nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
bd429e53 DJ	83	cxl_dimm_attribute_groups, flags,
b5807c80 DJ	84	cmd_mask, 0, NULL, cxl_nvd->dev_id,
b5807c80 DJ	85	cxl_security_ops, NULL);
f17b558d DW	86	if (!nvdimm)
f17b558d DW	87	return -ENOMEM;
21083f51	88
12f3856a	89	dev_set_drvdata(dev, nvdimm);
f17b558d	90	return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
21083f51 DW	91	}
	92
	93	static struct cxl_driver cxl_nvdimm_driver = {
	94	.name = "cxl_nvdimm",
	95	.probe = cxl_nvdimm_probe,
	96	.id = CXL_DEVICE_NVDIMM,
cb9cfff8 DW	97	.drv = {
	98	.suppress_bind_attrs = true,
	99	},
21083f51 DW	100	};
21083f51 DW	101
5e2411ae	102	static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
60b8f172 DW	103	struct nd_cmd_get_config_size *cmd,
	104	unsigned int buf_len)
	105	{
	106	if (sizeof(*cmd) > buf_len)
	107	return -EINVAL;
	108
	109	*cmd = (struct nd_cmd_get_config_size) {
5e2411ae	110	.config_size = cxlds->lsa_size,
f010c75c	111	.max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
60b8f172 DW	112	};
	113
	114	return 0;
	115	}
	116
5e2411ae	117	static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
60b8f172 DW	118	struct nd_cmd_get_config_data_hdr *cmd,
	119	unsigned int buf_len)
	120	{
49be6dd8	121	struct cxl_mbox_get_lsa get_lsa;
5331cdf4	122	struct cxl_mbox_cmd mbox_cmd;
60b8f172 DW	123	int rc;
	124
	125	if (sizeof(*cmd) > buf_len)
	126	return -EINVAL;
	127	if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
	128	return -EINVAL;
	129
	130	get_lsa = (struct cxl_mbox_get_lsa) {
8a664875 AS	131	.offset = cpu_to_le32(cmd->in_offset),
8a664875 AS	132	.length = cpu_to_le32(cmd->in_length),
60b8f172	133	};
5331cdf4 DW	134	mbox_cmd = (struct cxl_mbox_cmd) {
	135	.opcode = CXL_MBOX_OP_GET_LSA,
	136	.payload_in = &get_lsa,
	137	.size_in = sizeof(get_lsa),
	138	.size_out = cmd->in_length,
	139	.payload_out = cmd->out_buf,
	140	};
60b8f172	141
5331cdf4	142	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
60b8f172 DW	143	cmd->status = 0;
	144
	145	return rc;
	146	}
	147
5e2411ae	148	static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
60b8f172 DW	149	struct nd_cmd_set_config_hdr *cmd,
	150	unsigned int buf_len)
	151	{
49be6dd8	152	struct cxl_mbox_set_lsa *set_lsa;
5331cdf4	153	struct cxl_mbox_cmd mbox_cmd;
60b8f172 DW	154	int rc;
	155
	156	if (sizeof(*cmd) > buf_len)
	157	return -EINVAL;
	158
	159	/* 4-byte status follows the input data in the payload */
4f1aa35f	160	if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
60b8f172 DW	161	return -EINVAL;
	162
	163	set_lsa =
	164	kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
	165	if (!set_lsa)
	166	return -ENOMEM;
	167
	168	*set_lsa = (struct cxl_mbox_set_lsa) {
8a664875	169	.offset = cpu_to_le32(cmd->in_offset),
60b8f172 DW	170	};
60b8f172 DW	171	memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
5331cdf4 DW	172	mbox_cmd = (struct cxl_mbox_cmd) {
	173	.opcode = CXL_MBOX_OP_SET_LSA,
	174	.payload_in = set_lsa,
	175	.size_in = struct_size(set_lsa, data, cmd->in_length),
	176	};
60b8f172	177
5331cdf4	178	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
60b8f172 DW	179
	180	/*
	181	* Set "firmware" status (4-packed bytes at the end of the input
	182	* payload.
	183	*/
	184	put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
	185	kvfree(set_lsa);
	186
	187	return rc;
	188	}
	189
	190	static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
	191	void *buf, unsigned int buf_len)
	192	{
	193	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	194	unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
	195	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
5e2411ae	196	struct cxl_dev_state *cxlds = cxlmd->cxlds;
60b8f172 DW	197
	198	if (!test_bit(cmd, &cmd_mask))
	199	return -ENOTTY;
	200
	201	switch (cmd) {
	202	case ND_CMD_GET_CONFIG_SIZE:
5e2411ae	203	return cxl_pmem_get_config_size(cxlds, buf, buf_len);
60b8f172	204	case ND_CMD_GET_CONFIG_DATA:
5e2411ae	205	return cxl_pmem_get_config_data(cxlds, buf, buf_len);
60b8f172	206	case ND_CMD_SET_CONFIG_DATA:
5e2411ae	207	return cxl_pmem_set_config_data(cxlds, buf, buf_len);
60b8f172 DW	208	default:
	209	return -ENOTTY;
	210	}
	211	}
	212
8fdcb170 DW	213	static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
	214	struct nvdimm nvdimm, unsigned int cmd, void buf,
	215	unsigned int buf_len, int *cmd_rc)
	216	{
60b8f172 DW	217	/*
	218	* No firmware response to translate, let the transport error
	219	* code take precedence.
	220	*/
	221	*cmd_rc = 0;
	222
	223	if (!nvdimm)
	224	return -ENOTTY;
	225	return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
8fdcb170 DW	226	}
8fdcb170 DW	227
19398821 DW	228	static int detach_nvdimm(struct device dev, void data)
	229	{
	230	struct cxl_nvdimm *cxl_nvd;
	231	bool release = false;
	232
	233	if (!is_cxl_nvdimm(dev))
	234	return 0;
	235
	236	device_lock(dev);
	237	if (!dev->driver)
	238	goto out;
	239
	240	cxl_nvd = to_cxl_nvdimm(dev);
	241	if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
	242	release = true;
	243	out:
	244	device_unlock(dev);
	245	if (release)
	246	device_release_driver(dev);
	247	return 0;
	248	}
	249
f17b558d DW	250	static void unregister_nvdimm_bus(void *_cxl_nvb)
	251	{
	252	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
	253	struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
	254
19398821 DW	255	bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
19398821 DW	256
f17b558d DW	257	cxl_nvb->nvdimm_bus = NULL;
	258	nvdimm_bus_unregister(nvdimm_bus);
	259	}
	260
8fdcb170 DW	261	static int cxl_nvdimm_bridge_probe(struct device *dev)
	262	{
	263	struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
	264
f17b558d DW	265	cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
	266	.provider_name = "CXL",
	267	.module = THIS_MODULE,
	268	.ndctl = cxl_pmem_ctl,
	269	};
8fdcb170	270
f17b558d DW	271	cxl_nvb->nvdimm_bus =
f17b558d DW	272	nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
8fdcb170	273
f17b558d DW	274	if (!cxl_nvb->nvdimm_bus)
f17b558d DW	275	return -ENOMEM;
8fdcb170	276
f17b558d	277	return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
8fdcb170 DW	278	}
	279
	280	static struct cxl_driver cxl_nvdimm_bridge_driver = {
	281	.name = "cxl_nvdimm_bridge",
	282	.probe = cxl_nvdimm_bridge_probe,
8fdcb170	283	.id = CXL_DEVICE_NVDIMM_BRIDGE,
cb9cfff8 DW	284	.drv = {
	285	.suppress_bind_attrs = true,
	286	},
8fdcb170 DW	287	};
8fdcb170 DW	288
04ad63f0 DW	289	static void unregister_nvdimm_region(void *nd_region)
04ad63f0 DW	290	{
4d07ae22 DW	291	nvdimm_region_delete(nd_region);
	292	}
	293
04ad63f0 DW	294	static void cxlr_pmem_remove_resource(void *res)
	295	{
	296	remove_resource(res);
	297	}
	298
	299	struct cxl_pmem_region_info {
	300	u64 offset;
	301	u64 serial;
	302	};
	303
	304	static int cxl_pmem_region_probe(struct device *dev)
	305	{
	306	struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
	307	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
	308	struct cxl_region *cxlr = cxlr_pmem->cxlr;
f17b558d	309	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
04ad63f0	310	struct cxl_pmem_region_info *info = NULL;
04ad63f0 DW	311	struct nd_interleave_set *nd_set;
	312	struct nd_region_desc ndr_desc;
	313	struct cxl_nvdimm *cxl_nvd;
	314	struct nvdimm *nvdimm;
	315	struct resource *res;
	316	int rc, i = 0;
	317
04ad63f0 DW	318	memset(&mappings, 0, sizeof(mappings));
	319	memset(&ndr_desc, 0, sizeof(ndr_desc));
	320
	321	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
f17b558d DW	322	if (!res)
f17b558d DW	323	return -ENOMEM;
04ad63f0 DW	324
	325	res->name = "Persistent Memory";
	326	res->start = cxlr_pmem->hpa_range.start;
	327	res->end = cxlr_pmem->hpa_range.end;
	328	res->flags = IORESOURCE_MEM;
	329	res->desc = IORES_DESC_PERSISTENT_MEMORY;
	330
	331	rc = insert_resource(&iomem_resource, res);
	332	if (rc)
f17b558d	333	return rc;
04ad63f0 DW	334
	335	rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
	336	if (rc)
f17b558d	337	return rc;
04ad63f0 DW	338
	339	ndr_desc.res = res;
	340	ndr_desc.provider_data = cxlr_pmem;
	341
	342	ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
	343	ndr_desc.target_node = phys_to_target_node(res->start);
	344	if (ndr_desc.target_node == NUMA_NO_NODE) {
	345	ndr_desc.target_node = ndr_desc.numa_node;
	346	dev_dbg(&cxlr->dev, "changing target node from %d to %d",
	347	NUMA_NO_NODE, ndr_desc.target_node);
	348	}
	349
	350	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
f17b558d DW	351	if (!nd_set)
f17b558d DW	352	return -ENOMEM;
04ad63f0 DW	353
	354	ndr_desc.memregion = cxlr->id;
	355	set_bit(ND_REGION_CXL, &ndr_desc.flags);
	356	set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
	357
	358	info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
f17b558d DW	359	if (!info)
f17b558d DW	360	return -ENOMEM;
04ad63f0 DW	361
	362	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
	363	struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
	364	struct cxl_memdev *cxlmd = m->cxlmd;
	365	struct cxl_dev_state *cxlds = cxlmd->cxlds;
04ad63f0	366
f17b558d	367	cxl_nvd = cxlmd->cxl_nvd;
04ad63f0 DW	368	nvdimm = dev_get_drvdata(&cxl_nvd->dev);
	369	if (!nvdimm) {
	370	dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
	371	dev_name(&cxlmd->dev));
	372	rc = -ENODEV;
4d07ae22	373	goto out_nvd;
04ad63f0	374	}
4d07ae22	375
04ad63f0 DW	376	m->cxl_nvd = cxl_nvd;
	377	mappings[i] = (struct nd_mapping_desc) {
	378	.nvdimm = nvdimm,
	379	.start = m->start,
	380	.size = m->size,
	381	.position = i,
	382	};
	383	info[i].offset = m->start;
	384	info[i].serial = cxlds->serial;
	385	}
	386	ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
	387	ndr_desc.mapping = mappings;
	388
	389	/*
	390	* TODO enable CXL labels which skip the need for 'interleave-set cookie'
	391	*/
	392	nd_set->cookie1 =
	393	nd_fletcher64(info, sizeof(info) cxlr_pmem->nr_mappings, 0);
	394	nd_set->cookie2 = nd_set->cookie1;
	395	ndr_desc.nd_set = nd_set;
	396
	397	cxlr_pmem->nd_region =
	398	nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
9fd2cf4d DC	399	if (!cxlr_pmem->nd_region) {
9fd2cf4d DC	400	rc = -ENOMEM;
4d07ae22	401	goto out_nvd;
04ad63f0 DW	402	}
	403
	404	rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
	405	cxlr_pmem->nd_region);
4d07ae22	406	out_nvd:
04ad63f0	407	kfree(info);
04ad63f0 DW	408
04ad63f0 DW	409	return rc;
04ad63f0 DW	410	}
	411
	412	static struct cxl_driver cxl_pmem_region_driver = {
	413	.name = "cxl_pmem_region",
	414	.probe = cxl_pmem_region_probe,
	415	.id = CXL_DEVICE_PMEM_REGION,
cb9cfff8 DW	416	.drv = {
	417	.suppress_bind_attrs = true,
	418	},
04ad63f0 DW	419	};
04ad63f0 DW	420
8fdcb170 DW	421	static __init int cxl_pmem_init(void)
	422	{
	423	int rc;
	424
12f3856a DW	425	set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
	426	set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
	427
8fdcb170 DW	428	rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
8fdcb170 DW	429	if (rc)
03ff079a	430	return rc;
21083f51 DW	431
	432	rc = cxl_driver_register(&cxl_nvdimm_driver);
	433	if (rc)
	434	goto err_nvdimm;
8fdcb170	435
04ad63f0 DW	436	rc = cxl_driver_register(&cxl_pmem_region_driver);
	437	if (rc)
	438	goto err_region;
	439
8fdcb170 DW	440	return 0;
8fdcb170 DW	441
04ad63f0 DW	442	err_region:
04ad63f0 DW	443	cxl_driver_unregister(&cxl_nvdimm_driver);
21083f51 DW	444	err_nvdimm:
21083f51 DW	445	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
8fdcb170 DW	446	return rc;
	447	}
	448
	449	static __exit void cxl_pmem_exit(void)
	450	{
04ad63f0	451	cxl_driver_unregister(&cxl_pmem_region_driver);
21083f51	452	cxl_driver_unregister(&cxl_nvdimm_driver);
8fdcb170	453	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
8fdcb170 DW	454	}
	455
	456	MODULE_LICENSE("GPL v2");
	457	module_init(cxl_pmem_init);
	458	module_exit(cxl_pmem_exit);
	459	MODULE_IMPORT_NS(CXL);
	460	MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
21083f51	461	MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
04ad63f0	462	MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);