[linux-block.git] / drivers / cxl / pmem.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
#include <linux/libnvdimm.h>
#include <asm/unaligned.h>
#include <linux/device.h>
#include <linux/module.h>
#include <linux/ndctl.h>
#include <linux/async.h>
#include <linux/slab.h>
#include <linux/nd.h>
#include "cxlmem.h"
#include "cxl.h"

extern const struct nvdimm_security_ops *cxl_security_ops;

static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);

static void clear_exclusive(void *cxlds)
{
	clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
}

static void unregister_nvdimm(void *nvdimm)
{
	nvdimm_delete(nvdimm);
}

static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct nvdimm *nvdimm = to_nvdimm(dev);
	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);

	return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
}
static DEVICE_ATTR_RO(provider);

static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf)
{
	struct nvdimm *nvdimm = to_nvdimm(dev);
	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;

	return sysfs_emit(buf, "%lld\n", cxlds->serial);
}
static DEVICE_ATTR_RO(id);

static struct attribute *cxl_dimm_attributes[] = {
	&dev_attr_id.attr,
	&dev_attr_provider.attr,
	NULL
};

static const struct attribute_group cxl_dimm_attribute_group = {
	.name = "cxl",
	.attrs = cxl_dimm_attributes,
};

static const struct attribute_group *cxl_dimm_attribute_groups[] = {
	&cxl_dimm_attribute_group,
	NULL
};

static int cxl_nvdimm_probe(struct device *dev)
{
	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
	unsigned long flags = 0, cmd_mask = 0;
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct nvdimm *nvdimm;
	int rc;

	set_exclusive_cxl_commands(cxlds, exclusive_cmds);
	rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
	if (rc)
		return rc;

	set_bit(NDD_LABELING, &flags);
	set_bit(NDD_REGISTER_SYNC, &flags);
	set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
	set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
	set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
	nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
				 cxl_dimm_attribute_groups, flags,
				 cmd_mask, 0, NULL, cxl_nvd->dev_id,
				 cxl_security_ops, NULL);
	if (!nvdimm)
		return -ENOMEM;

	dev_set_drvdata(dev, nvdimm);
	return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
}

static struct cxl_driver cxl_nvdimm_driver = {
	.name = "cxl_nvdimm",
	.probe = cxl_nvdimm_probe,
	.id = CXL_DEVICE_NVDIMM,
	.drv = {
		.suppress_bind_attrs = true,
	},
};

static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
				    struct nd_cmd_get_config_size *cmd,
				    unsigned int buf_len)
{
	if (sizeof(*cmd) > buf_len)
		return -EINVAL;

	*cmd = (struct nd_cmd_get_config_size) {
		 .config_size = cxlds->lsa_size,
		 .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
	};

	return 0;
}

static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
				    struct nd_cmd_get_config_data_hdr *cmd,
				    unsigned int buf_len)
{
	struct cxl_mbox_get_lsa get_lsa;
	struct cxl_mbox_cmd mbox_cmd;
	int rc;

	if (sizeof(*cmd) > buf_len)
		return -EINVAL;
	if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
		return -EINVAL;

	get_lsa = (struct cxl_mbox_get_lsa) {
		.offset = cpu_to_le32(cmd->in_offset),
		.length = cpu_to_le32(cmd->in_length),
	};
	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_GET_LSA,
		.payload_in = &get_lsa,
		.size_in = sizeof(get_lsa),
		.size_out = cmd->in_length,
		.payload_out = cmd->out_buf,
	};

	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
	cmd->status = 0;

	return rc;
}

static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
				    struct nd_cmd_set_config_hdr *cmd,
				    unsigned int buf_len)
{
	struct cxl_mbox_set_lsa *set_lsa;
	struct cxl_mbox_cmd mbox_cmd;
	int rc;

	if (sizeof(*cmd) > buf_len)
		return -EINVAL;

	/* 4-byte status follows the input data in the payload */
	if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
		return -EINVAL;

	set_lsa =
		kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
	if (!set_lsa)
		return -ENOMEM;

	*set_lsa = (struct cxl_mbox_set_lsa) {
		.offset = cpu_to_le32(cmd->in_offset),
	};
	memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_SET_LSA,
		.payload_in = set_lsa,
		.size_in = struct_size(set_lsa, data, cmd->in_length),
	};

	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);

	/*
	 * Set "firmware" status (4-packed bytes at the end of the input
	 * payload.
	 */
	put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
	kvfree(set_lsa);

	return rc;
}

static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
			       void *buf, unsigned int buf_len)
{
	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
	struct cxl_dev_state *cxlds = cxlmd->cxlds;

	if (!test_bit(cmd, &cmd_mask))
		return -ENOTTY;

	switch (cmd) {
	case ND_CMD_GET_CONFIG_SIZE:
		return cxl_pmem_get_config_size(cxlds, buf, buf_len);
	case ND_CMD_GET_CONFIG_DATA:
		return cxl_pmem_get_config_data(cxlds, buf, buf_len);
	case ND_CMD_SET_CONFIG_DATA:
		return cxl_pmem_set_config_data(cxlds, buf, buf_len);
	default:
		return -ENOTTY;
	}
}

static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
			struct nvdimm *nvdimm, unsigned int cmd, void *buf,
			unsigned int buf_len, int *cmd_rc)
{
	/*
	 * No firmware response to translate, let the transport error
	 * code take precedence.
	 */
	*cmd_rc = 0;

	if (!nvdimm)
		return -ENOTTY;
	return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
}

static int detach_nvdimm(struct device *dev, void *data)
{
	struct cxl_nvdimm *cxl_nvd;
	bool release = false;

	if (!is_cxl_nvdimm(dev))
		return 0;

	device_lock(dev);
	if (!dev->driver)
		goto out;

	cxl_nvd = to_cxl_nvdimm(dev);
	if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
		release = true;
out:
	device_unlock(dev);
	if (release)
		device_release_driver(dev);
	return 0;
}

static void unregister_nvdimm_bus(void *_cxl_nvb)
{
	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
	struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;

	bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);

	cxl_nvb->nvdimm_bus = NULL;
	nvdimm_bus_unregister(nvdimm_bus);
}

static int cxl_nvdimm_bridge_probe(struct device *dev)
{
	struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);

	cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
		.provider_name = "CXL",
		.module = THIS_MODULE,
		.ndctl = cxl_pmem_ctl,
	};

	cxl_nvb->nvdimm_bus =
		nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);

	if (!cxl_nvb->nvdimm_bus)
		return -ENOMEM;

	return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
}

static struct cxl_driver cxl_nvdimm_bridge_driver = {
	.name = "cxl_nvdimm_bridge",
	.probe = cxl_nvdimm_bridge_probe,
	.id = CXL_DEVICE_NVDIMM_BRIDGE,
	.drv = {
		.suppress_bind_attrs = true,
	},
};

static void unregister_nvdimm_region(void *nd_region)
{
	nvdimm_region_delete(nd_region);
}

static void cxlr_pmem_remove_resource(void *res)
{
	remove_resource(res);
}

struct cxl_pmem_region_info {
	u64 offset;
	u64 serial;
};

static int cxl_pmem_region_probe(struct device *dev)
{
	struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
	struct cxl_region *cxlr = cxlr_pmem->cxlr;
	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
	struct cxl_pmem_region_info *info = NULL;
	struct nd_interleave_set *nd_set;
	struct nd_region_desc ndr_desc;
	struct cxl_nvdimm *cxl_nvd;
	struct nvdimm *nvdimm;
	struct resource *res;
	int rc, i = 0;

	memset(&mappings, 0, sizeof(mappings));
	memset(&ndr_desc, 0, sizeof(ndr_desc));

	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
	if (!res)
		return -ENOMEM;

	res->name = "Persistent Memory";
	res->start = cxlr_pmem->hpa_range.start;
	res->end = cxlr_pmem->hpa_range.end;
	res->flags = IORESOURCE_MEM;
	res->desc = IORES_DESC_PERSISTENT_MEMORY;

	rc = insert_resource(&iomem_resource, res);
	if (rc)
		return rc;

	rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
	if (rc)
		return rc;

	ndr_desc.res = res;
	ndr_desc.provider_data = cxlr_pmem;

	ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
	ndr_desc.target_node = phys_to_target_node(res->start);
	if (ndr_desc.target_node == NUMA_NO_NODE) {
		ndr_desc.target_node = ndr_desc.numa_node;
		dev_dbg(&cxlr->dev, "changing target node from %d to %d",
			NUMA_NO_NODE, ndr_desc.target_node);
	}

	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
	if (!nd_set)
		return -ENOMEM;

	ndr_desc.memregion = cxlr->id;
	set_bit(ND_REGION_CXL, &ndr_desc.flags);
	set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);

	info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;

	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
		struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
		struct cxl_memdev *cxlmd = m->cxlmd;
		struct cxl_dev_state *cxlds = cxlmd->cxlds;

		cxl_nvd = cxlmd->cxl_nvd;
		nvdimm = dev_get_drvdata(&cxl_nvd->dev);
		if (!nvdimm) {
			dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
				dev_name(&cxlmd->dev));
			rc = -ENODEV;
			goto out_nvd;
		}

		m->cxl_nvd = cxl_nvd;
		mappings[i] = (struct nd_mapping_desc) {
			.nvdimm = nvdimm,
			.start = m->start,
			.size = m->size,
			.position = i,
		};
		info[i].offset = m->start;
		info[i].serial = cxlds->serial;
	}
	ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
	ndr_desc.mapping = mappings;

	/*
	 * TODO enable CXL labels which skip the need for 'interleave-set cookie'
	 */
	nd_set->cookie1 =
		nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0);
	nd_set->cookie2 = nd_set->cookie1;
	ndr_desc.nd_set = nd_set;

	cxlr_pmem->nd_region =
		nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
	if (!cxlr_pmem->nd_region) {
		rc = -ENOMEM;
		goto out_nvd;
	}

	rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
				      cxlr_pmem->nd_region);
out_nvd:
	kfree(info);

	return rc;
}

static struct cxl_driver cxl_pmem_region_driver = {
	.name = "cxl_pmem_region",
	.probe = cxl_pmem_region_probe,
	.id = CXL_DEVICE_PMEM_REGION,
	.drv = {
		.suppress_bind_attrs = true,
	},
};

static __init int cxl_pmem_init(void)
{
	int rc;

	set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
	set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);

	rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
	if (rc)
		return rc;

	rc = cxl_driver_register(&cxl_nvdimm_driver);
	if (rc)
		goto err_nvdimm;

	rc = cxl_driver_register(&cxl_pmem_region_driver);
	if (rc)
		goto err_region;

	return 0;

err_region:
	cxl_driver_unregister(&cxl_nvdimm_driver);
err_nvdimm:
	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
	return rc;
}

static __exit void cxl_pmem_exit(void)
{
	cxl_driver_unregister(&cxl_pmem_region_driver);
	cxl_driver_unregister(&cxl_nvdimm_driver);
	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
}

MODULE_LICENSE("GPL v2");
module_init(cxl_pmem_init);
module_exit(cxl_pmem_exit);
MODULE_IMPORT_NS(CXL);
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);
Commit	Line	Data
8fdcb170 DW	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
	3	#include <linux/libnvdimm.h>
60b8f172	4	#include <asm/unaligned.h>
8fdcb170 DW	5	#include <linux/device.h>
8fdcb170 DW	6	#include <linux/module.h>
21083f51 DW	7	#include <linux/ndctl.h>
21083f51 DW	8	#include <linux/async.h>
8fdcb170	9	#include <linux/slab.h>
04ad63f0	10	#include <linux/nd.h>
5161a55c	11	#include "cxlmem.h"
8fdcb170 DW	12	#include "cxl.h"
8fdcb170 DW	13
32828115 DJ	14	extern const struct nvdimm_security_ops *cxl_security_ops;
32828115 DJ	15
12f3856a DW	16	static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
12f3856a DW	17
5e2411ae	18	static void clear_exclusive(void *cxlds)
12f3856a	19	{
5e2411ae	20	clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
12f3856a DW	21	}
12f3856a DW	22
21083f51 DW	23	static void unregister_nvdimm(void *nvdimm)
	24	{
	25	nvdimm_delete(nvdimm);
	26	}
	27
452996fa DJ	28	static ssize_t provider_show(struct device dev, struct device_attribute attr, char *buf)
	29	{
	30	struct nvdimm *nvdimm = to_nvdimm(dev);
	31	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	32
	33	return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev));
	34	}
	35	static DEVICE_ATTR_RO(provider);
	36
bd429e53 DJ	37	static ssize_t id_show(struct device dev, struct device_attribute attr, char *buf)
	38	{
	39	struct nvdimm *nvdimm = to_nvdimm(dev);
	40	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	41	struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds;
	42
	43	return sysfs_emit(buf, "%lld\n", cxlds->serial);
	44	}
	45	static DEVICE_ATTR_RO(id);
	46
	47	static struct attribute *cxl_dimm_attributes[] = {
	48	&dev_attr_id.attr,
452996fa	49	&dev_attr_provider.attr,
bd429e53 DJ	50	NULL
	51	};
	52
	53	static const struct attribute_group cxl_dimm_attribute_group = {
	54	.name = "cxl",
	55	.attrs = cxl_dimm_attributes,
	56	};
	57
	58	static const struct attribute_group *cxl_dimm_attribute_groups[] = {
	59	&cxl_dimm_attribute_group,
	60	NULL
	61	};
	62
21083f51 DW	63	static int cxl_nvdimm_probe(struct device *dev)
	64	{
	65	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
12f3856a	66	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
f17b558d	67	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
60b8f172	68	unsigned long flags = 0, cmd_mask = 0;
5e2411ae	69	struct cxl_dev_state *cxlds = cxlmd->cxlds;
21083f51	70	struct nvdimm *nvdimm;
12f3856a	71	int rc;
21083f51	72
5e2411ae IW	73	set_exclusive_cxl_commands(cxlds, exclusive_cmds);
5e2411ae IW	74	rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
12f3856a	75	if (rc)
f17b558d	76	return rc;
21083f51 DW	77
21083f51 DW	78	set_bit(NDD_LABELING, &flags);
f57aec44	79	set_bit(NDD_REGISTER_SYNC, &flags);
60b8f172 DW	80	set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
	81	set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
	82	set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
bd429e53 DJ	83	nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd,
bd429e53 DJ	84	cxl_dimm_attribute_groups, flags,
b5807c80 DJ	85	cmd_mask, 0, NULL, cxl_nvd->dev_id,
b5807c80 DJ	86	cxl_security_ops, NULL);
f17b558d DW	87	if (!nvdimm)
f17b558d DW	88	return -ENOMEM;
21083f51	89
12f3856a	90	dev_set_drvdata(dev, nvdimm);
f17b558d	91	return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
21083f51 DW	92	}
	93
	94	static struct cxl_driver cxl_nvdimm_driver = {
	95	.name = "cxl_nvdimm",
	96	.probe = cxl_nvdimm_probe,
	97	.id = CXL_DEVICE_NVDIMM,
cb9cfff8 DW	98	.drv = {
	99	.suppress_bind_attrs = true,
	100	},
21083f51 DW	101	};
21083f51 DW	102
5e2411ae	103	static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
60b8f172 DW	104	struct nd_cmd_get_config_size *cmd,
	105	unsigned int buf_len)
	106	{
	107	if (sizeof(*cmd) > buf_len)
	108	return -EINVAL;
	109
	110	*cmd = (struct nd_cmd_get_config_size) {
5e2411ae	111	.config_size = cxlds->lsa_size,
f010c75c	112	.max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa),
60b8f172 DW	113	};
	114
	115	return 0;
	116	}
	117
5e2411ae	118	static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
60b8f172 DW	119	struct nd_cmd_get_config_data_hdr *cmd,
	120	unsigned int buf_len)
	121	{
49be6dd8	122	struct cxl_mbox_get_lsa get_lsa;
5331cdf4	123	struct cxl_mbox_cmd mbox_cmd;
60b8f172 DW	124	int rc;
	125
	126	if (sizeof(*cmd) > buf_len)
	127	return -EINVAL;
	128	if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
	129	return -EINVAL;
	130
	131	get_lsa = (struct cxl_mbox_get_lsa) {
8a664875 AS	132	.offset = cpu_to_le32(cmd->in_offset),
8a664875 AS	133	.length = cpu_to_le32(cmd->in_length),
60b8f172	134	};
5331cdf4 DW	135	mbox_cmd = (struct cxl_mbox_cmd) {
	136	.opcode = CXL_MBOX_OP_GET_LSA,
	137	.payload_in = &get_lsa,
	138	.size_in = sizeof(get_lsa),
	139	.size_out = cmd->in_length,
	140	.payload_out = cmd->out_buf,
	141	};
60b8f172	142
5331cdf4	143	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
60b8f172 DW	144	cmd->status = 0;
	145
	146	return rc;
	147	}
	148
5e2411ae	149	static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
60b8f172 DW	150	struct nd_cmd_set_config_hdr *cmd,
	151	unsigned int buf_len)
	152	{
49be6dd8	153	struct cxl_mbox_set_lsa *set_lsa;
5331cdf4	154	struct cxl_mbox_cmd mbox_cmd;
60b8f172 DW	155	int rc;
	156
	157	if (sizeof(*cmd) > buf_len)
	158	return -EINVAL;
	159
	160	/* 4-byte status follows the input data in the payload */
4f1aa35f	161	if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len)
60b8f172 DW	162	return -EINVAL;
	163
	164	set_lsa =
	165	kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
	166	if (!set_lsa)
	167	return -ENOMEM;
	168
	169	*set_lsa = (struct cxl_mbox_set_lsa) {
8a664875	170	.offset = cpu_to_le32(cmd->in_offset),
60b8f172 DW	171	};
60b8f172 DW	172	memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
5331cdf4 DW	173	mbox_cmd = (struct cxl_mbox_cmd) {
	174	.opcode = CXL_MBOX_OP_SET_LSA,
	175	.payload_in = set_lsa,
	176	.size_in = struct_size(set_lsa, data, cmd->in_length),
	177	};
60b8f172	178
5331cdf4	179	rc = cxl_internal_send_cmd(cxlds, &mbox_cmd);
60b8f172 DW	180
	181	/*
	182	* Set "firmware" status (4-packed bytes at the end of the input
	183	* payload.
	184	*/
	185	put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
	186	kvfree(set_lsa);
	187
	188	return rc;
	189	}
	190
	191	static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
	192	void *buf, unsigned int buf_len)
	193	{
	194	struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
	195	unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
	196	struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
5e2411ae	197	struct cxl_dev_state *cxlds = cxlmd->cxlds;
60b8f172 DW	198
	199	if (!test_bit(cmd, &cmd_mask))
	200	return -ENOTTY;
	201
	202	switch (cmd) {
	203	case ND_CMD_GET_CONFIG_SIZE:
5e2411ae	204	return cxl_pmem_get_config_size(cxlds, buf, buf_len);
60b8f172	205	case ND_CMD_GET_CONFIG_DATA:
5e2411ae	206	return cxl_pmem_get_config_data(cxlds, buf, buf_len);
60b8f172	207	case ND_CMD_SET_CONFIG_DATA:
5e2411ae	208	return cxl_pmem_set_config_data(cxlds, buf, buf_len);
60b8f172 DW	209	default:
	210	return -ENOTTY;
	211	}
	212	}
	213
8fdcb170 DW	214	static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
	215	struct nvdimm nvdimm, unsigned int cmd, void buf,
	216	unsigned int buf_len, int *cmd_rc)
	217	{
60b8f172 DW	218	/*
	219	* No firmware response to translate, let the transport error
	220	* code take precedence.
	221	*/
	222	*cmd_rc = 0;
	223
	224	if (!nvdimm)
	225	return -ENOTTY;
	226	return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
8fdcb170 DW	227	}
8fdcb170 DW	228
19398821 DW	229	static int detach_nvdimm(struct device dev, void data)
	230	{
	231	struct cxl_nvdimm *cxl_nvd;
	232	bool release = false;
	233
	234	if (!is_cxl_nvdimm(dev))
	235	return 0;
	236
	237	device_lock(dev);
	238	if (!dev->driver)
	239	goto out;
	240
	241	cxl_nvd = to_cxl_nvdimm(dev);
	242	if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
	243	release = true;
	244	out:
	245	device_unlock(dev);
	246	if (release)
	247	device_release_driver(dev);
	248	return 0;
	249	}
	250
f17b558d DW	251	static void unregister_nvdimm_bus(void *_cxl_nvb)
	252	{
	253	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
	254	struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus;
	255
19398821 DW	256	bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm);
19398821 DW	257
f17b558d DW	258	cxl_nvb->nvdimm_bus = NULL;
	259	nvdimm_bus_unregister(nvdimm_bus);
	260	}
	261
8fdcb170 DW	262	static int cxl_nvdimm_bridge_probe(struct device *dev)
	263	{
	264	struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
	265
f17b558d DW	266	cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
	267	.provider_name = "CXL",
	268	.module = THIS_MODULE,
	269	.ndctl = cxl_pmem_ctl,
	270	};
8fdcb170	271
f17b558d DW	272	cxl_nvb->nvdimm_bus =
f17b558d DW	273	nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
8fdcb170	274
f17b558d DW	275	if (!cxl_nvb->nvdimm_bus)
f17b558d DW	276	return -ENOMEM;
8fdcb170	277
f17b558d	278	return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb);
8fdcb170 DW	279	}
	280
	281	static struct cxl_driver cxl_nvdimm_bridge_driver = {
	282	.name = "cxl_nvdimm_bridge",
	283	.probe = cxl_nvdimm_bridge_probe,
8fdcb170	284	.id = CXL_DEVICE_NVDIMM_BRIDGE,
cb9cfff8 DW	285	.drv = {
	286	.suppress_bind_attrs = true,
	287	},
8fdcb170 DW	288	};
8fdcb170 DW	289
04ad63f0 DW	290	static void unregister_nvdimm_region(void *nd_region)
04ad63f0 DW	291	{
4d07ae22 DW	292	nvdimm_region_delete(nd_region);
	293	}
	294
04ad63f0 DW	295	static void cxlr_pmem_remove_resource(void *res)
	296	{
	297	remove_resource(res);
	298	}
	299
	300	struct cxl_pmem_region_info {
	301	u64 offset;
	302	u64 serial;
	303	};
	304
	305	static int cxl_pmem_region_probe(struct device *dev)
	306	{
	307	struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE];
	308	struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
	309	struct cxl_region *cxlr = cxlr_pmem->cxlr;
f17b558d	310	struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
04ad63f0	311	struct cxl_pmem_region_info *info = NULL;
04ad63f0 DW	312	struct nd_interleave_set *nd_set;
	313	struct nd_region_desc ndr_desc;
	314	struct cxl_nvdimm *cxl_nvd;
	315	struct nvdimm *nvdimm;
	316	struct resource *res;
	317	int rc, i = 0;
	318
04ad63f0 DW	319	memset(&mappings, 0, sizeof(mappings));
	320	memset(&ndr_desc, 0, sizeof(ndr_desc));
	321
	322	res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL);
f17b558d DW	323	if (!res)
f17b558d DW	324	return -ENOMEM;
04ad63f0 DW	325
	326	res->name = "Persistent Memory";
	327	res->start = cxlr_pmem->hpa_range.start;
	328	res->end = cxlr_pmem->hpa_range.end;
	329	res->flags = IORESOURCE_MEM;
	330	res->desc = IORES_DESC_PERSISTENT_MEMORY;
	331
	332	rc = insert_resource(&iomem_resource, res);
	333	if (rc)
f17b558d	334	return rc;
04ad63f0 DW	335
	336	rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res);
	337	if (rc)
f17b558d	338	return rc;
04ad63f0 DW	339
	340	ndr_desc.res = res;
	341	ndr_desc.provider_data = cxlr_pmem;
	342
	343	ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start);
	344	ndr_desc.target_node = phys_to_target_node(res->start);
	345	if (ndr_desc.target_node == NUMA_NO_NODE) {
	346	ndr_desc.target_node = ndr_desc.numa_node;
	347	dev_dbg(&cxlr->dev, "changing target node from %d to %d",
	348	NUMA_NO_NODE, ndr_desc.target_node);
	349	}
	350
	351	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
f17b558d DW	352	if (!nd_set)
f17b558d DW	353	return -ENOMEM;
04ad63f0 DW	354
	355	ndr_desc.memregion = cxlr->id;
	356	set_bit(ND_REGION_CXL, &ndr_desc.flags);
	357	set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags);
	358
	359	info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL);
f17b558d DW	360	if (!info)
f17b558d DW	361	return -ENOMEM;
04ad63f0 DW	362
	363	for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
	364	struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
	365	struct cxl_memdev *cxlmd = m->cxlmd;
	366	struct cxl_dev_state *cxlds = cxlmd->cxlds;
04ad63f0	367
f17b558d	368	cxl_nvd = cxlmd->cxl_nvd;
04ad63f0 DW	369	nvdimm = dev_get_drvdata(&cxl_nvd->dev);
	370	if (!nvdimm) {
	371	dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i,
	372	dev_name(&cxlmd->dev));
	373	rc = -ENODEV;
4d07ae22	374	goto out_nvd;
04ad63f0	375	}
4d07ae22	376
04ad63f0 DW	377	m->cxl_nvd = cxl_nvd;
	378	mappings[i] = (struct nd_mapping_desc) {
	379	.nvdimm = nvdimm,
	380	.start = m->start,
	381	.size = m->size,
	382	.position = i,
	383	};
	384	info[i].offset = m->start;
	385	info[i].serial = cxlds->serial;
	386	}
	387	ndr_desc.num_mappings = cxlr_pmem->nr_mappings;
	388	ndr_desc.mapping = mappings;
	389
	390	/*
	391	* TODO enable CXL labels which skip the need for 'interleave-set cookie'
	392	*/
	393	nd_set->cookie1 =
	394	nd_fletcher64(info, sizeof(info) cxlr_pmem->nr_mappings, 0);
	395	nd_set->cookie2 = nd_set->cookie1;
	396	ndr_desc.nd_set = nd_set;
	397
	398	cxlr_pmem->nd_region =
	399	nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc);
9fd2cf4d DC	400	if (!cxlr_pmem->nd_region) {
9fd2cf4d DC	401	rc = -ENOMEM;
4d07ae22	402	goto out_nvd;
04ad63f0 DW	403	}
	404
	405	rc = devm_add_action_or_reset(dev, unregister_nvdimm_region,
	406	cxlr_pmem->nd_region);
4d07ae22	407	out_nvd:
04ad63f0	408	kfree(info);
04ad63f0 DW	409
04ad63f0 DW	410	return rc;
04ad63f0 DW	411	}
	412
	413	static struct cxl_driver cxl_pmem_region_driver = {
	414	.name = "cxl_pmem_region",
	415	.probe = cxl_pmem_region_probe,
	416	.id = CXL_DEVICE_PMEM_REGION,
cb9cfff8 DW	417	.drv = {
	418	.suppress_bind_attrs = true,
	419	},
04ad63f0 DW	420	};
04ad63f0 DW	421
8fdcb170 DW	422	static __init int cxl_pmem_init(void)
	423	{
	424	int rc;
	425
12f3856a DW	426	set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
	427	set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
	428
8fdcb170 DW	429	rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
8fdcb170 DW	430	if (rc)
03ff079a	431	return rc;
21083f51 DW	432
	433	rc = cxl_driver_register(&cxl_nvdimm_driver);
	434	if (rc)
	435	goto err_nvdimm;
8fdcb170	436
04ad63f0 DW	437	rc = cxl_driver_register(&cxl_pmem_region_driver);
	438	if (rc)
	439	goto err_region;
	440
8fdcb170 DW	441	return 0;
8fdcb170 DW	442
04ad63f0 DW	443	err_region:
04ad63f0 DW	444	cxl_driver_unregister(&cxl_nvdimm_driver);
21083f51 DW	445	err_nvdimm:
21083f51 DW	446	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
8fdcb170 DW	447	return rc;
	448	}
	449
	450	static __exit void cxl_pmem_exit(void)
	451	{
04ad63f0	452	cxl_driver_unregister(&cxl_pmem_region_driver);
21083f51	453	cxl_driver_unregister(&cxl_nvdimm_driver);
8fdcb170	454	cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
8fdcb170 DW	455	}
	456
	457	MODULE_LICENSE("GPL v2");
	458	module_init(cxl_pmem_init);
	459	module_exit(cxl_pmem_exit);
	460	MODULE_IMPORT_NS(CXL);
	461	MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
21083f51	462	MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
04ad63f0	463	MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION);