Commit | Line | Data |
---|---|---|
4cdadfd5 DW |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright(c) 2020 Intel Corporation. All rights reserved. */ | |
583fa5e7 | 3 | #include <uapi/linux/cxl_mem.h> |
13237183 BW |
4 | #include <linux/security.h> |
5 | #include <linux/debugfs.h> | |
4cdadfd5 | 6 | #include <linux/module.h> |
fae8817a | 7 | #include <linux/sizes.h> |
b39cb105 DW |
8 | #include <linux/mutex.h> |
9 | #include <linux/cdev.h> | |
10 | #include <linux/idr.h> | |
4cdadfd5 DW |
11 | #include <linux/pci.h> |
12 | #include <linux/io.h> | |
8adaf747 | 13 | #include <linux/io-64-nonatomic-lo-hi.h> |
4cdadfd5 | 14 | #include "pci.h" |
8adaf747 BW |
15 | #include "cxl.h" |
16 | ||
17 | /** | |
18 | * DOC: cxl mem | |
19 | * | |
20 | * This implements a CXL memory device ("type-3") as it is defined by the | |
21 | * Compute Express Link specification. | |
22 | * | |
23 | * The driver has several responsibilities, mainly: | |
24 | * - Create the memX device and register on the CXL bus. | |
25 | * - Enumerate device's register interface and map them. | |
26 | * - Probe the device attributes to establish sysfs interface. | |
27 | * - Provide an IOCTL interface to userspace to communicate with the device for | |
28 | * things like firmware update. | |
29 | * - Support management of interleave sets. | |
30 | * - Handle and manage error conditions. | |
31 | */ | |
32 | ||
b39cb105 DW |
33 | /* |
34 | * An entire PCI topology full of devices should be enough for any | |
35 | * config | |
36 | */ | |
37 | #define CXL_MEM_MAX_DEVS 65536 | |
38 | ||
8adaf747 BW |
39 | #define cxl_doorbell_busy(cxlm) \ |
40 | (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ | |
41 | CXLDEV_MBOX_CTRL_DOORBELL) | |
42 | ||
43 | /* CXL 2.0 - 8.2.8.4 */ | |
44 | #define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) | |
45 | ||
46 | enum opcode { | |
583fa5e7 | 47 | CXL_MBOX_OP_INVALID = 0x0000, |
13237183 | 48 | CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, |
57ee605b | 49 | CXL_MBOX_OP_GET_FW_INFO = 0x0200, |
13237183 | 50 | CXL_MBOX_OP_ACTIVATE_FW = 0x0202, |
472b1ce6 BW |
51 | CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, |
52 | CXL_MBOX_OP_GET_LOG = 0x0401, | |
8adaf747 | 53 | CXL_MBOX_OP_IDENTIFY = 0x4000, |
57ee605b | 54 | CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, |
13237183 | 55 | CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, |
57ee605b | 56 | CXL_MBOX_OP_GET_LSA = 0x4102, |
13237183 | 57 | CXL_MBOX_OP_SET_LSA = 0x4103, |
57ee605b | 58 | CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, |
13237183 BW |
59 | CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, |
60 | CXL_MBOX_OP_SCAN_MEDIA = 0x4304, | |
61 | CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, | |
8adaf747 BW |
62 | CXL_MBOX_OP_MAX = 0x10000 |
63 | }; | |
64 | ||
65 | /** | |
66 | * struct mbox_cmd - A command to be submitted to hardware. | |
67 | * @opcode: (input) The command set and command submitted to hardware. | |
68 | * @payload_in: (input) Pointer to the input payload. | |
69 | * @payload_out: (output) Pointer to the output payload. Must be allocated by | |
70 | * the caller. | |
71 | * @size_in: (input) Number of bytes to load from @payload_in. | |
72 | * @size_out: (input) Max number of bytes loaded into @payload_out. | |
73 | * (output) Number of bytes generated by the device. For fixed size | |
74 | * outputs commands this is always expected to be deterministic. For | |
75 | * variable sized output commands, it tells the exact number of bytes | |
76 | * written. | |
77 | * @return_code: (output) Error code returned from hardware. | |
78 | * | |
79 | * This is the primary mechanism used to send commands to the hardware. | |
80 | * All the fields except @payload_* correspond exactly to the fields described in | |
81 | * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and | |
82 | * @payload_out are written to, and read from the Command Payload Registers | |
83 | * defined in CXL 2.0 8.2.8.4.8. | |
84 | */ | |
85 | struct mbox_cmd { | |
86 | u16 opcode; | |
87 | void *payload_in; | |
88 | void *payload_out; | |
89 | size_t size_in; | |
90 | size_t size_out; | |
91 | u16 return_code; | |
92 | #define CXL_MBOX_SUCCESS 0 | |
93 | }; | |
94 | ||
b39cb105 DW |
95 | /** |
96 | * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device | |
97 | * @dev: driver core device object | |
98 | * @cdev: char dev core object for ioctl operations | |
99 | * @cxlm: pointer to the parent device driver data | |
b39cb105 DW |
100 | * @id: id number of this memdev instance. |
101 | */ | |
102 | struct cxl_memdev { | |
103 | struct device dev; | |
104 | struct cdev cdev; | |
105 | struct cxl_mem *cxlm; | |
b39cb105 DW |
106 | int id; |
107 | }; | |
108 | ||
109 | static int cxl_mem_major; | |
110 | static DEFINE_IDA(cxl_memdev_ida); | |
58775159 | 111 | static DECLARE_RWSEM(cxl_memdev_rwsem); |
13237183 BW |
112 | static struct dentry *cxl_debugfs; |
113 | static bool cxl_raw_allow_all; | |
b39cb105 | 114 | |
472b1ce6 BW |
115 | enum { |
116 | CEL_UUID, | |
117 | VENDOR_DEBUG_UUID, | |
118 | }; | |
119 | ||
120 | /* See CXL 2.0 Table 170. Get Log Input Payload */ | |
121 | static const uuid_t log_uuid[] = { | |
122 | [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, | |
123 | 0xb1, 0x62, 0x3b, 0x3f, 0x17), | |
124 | [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, | |
125 | 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), | |
126 | }; | |
127 | ||
583fa5e7 BW |
128 | /** |
129 | * struct cxl_mem_command - Driver representation of a memory device command | |
130 | * @info: Command information as it exists for the UAPI | |
131 | * @opcode: The actual bits used for the mailbox protocol | |
472b1ce6 BW |
132 | * @flags: Set of flags effecting driver behavior. |
133 | * | |
134 | * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag | |
135 | * will be enabled by the driver regardless of what hardware may have | |
136 | * advertised. | |
583fa5e7 BW |
137 | * |
138 | * The cxl_mem_command is the driver's internal representation of commands that | |
139 | * are supported by the driver. Some of these commands may not be supported by | |
140 | * the hardware. The driver will use @info to validate the fields passed in by | |
141 | * the user then submit the @opcode to the hardware. | |
142 | * | |
143 | * See struct cxl_command_info. | |
144 | */ | |
145 | struct cxl_mem_command { | |
146 | struct cxl_command_info info; | |
147 | enum opcode opcode; | |
472b1ce6 BW |
148 | u32 flags; |
149 | #define CXL_CMD_FLAG_NONE 0 | |
150 | #define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) | |
583fa5e7 BW |
151 | }; |
152 | ||
472b1ce6 | 153 | #define CXL_CMD(_id, sin, sout, _flags) \ |
583fa5e7 BW |
154 | [CXL_MEM_COMMAND_ID_##_id] = { \ |
155 | .info = { \ | |
156 | .id = CXL_MEM_COMMAND_ID_##_id, \ | |
157 | .size_in = sin, \ | |
158 | .size_out = sout, \ | |
159 | }, \ | |
160 | .opcode = CXL_MBOX_OP_##_id, \ | |
472b1ce6 | 161 | .flags = _flags, \ |
583fa5e7 BW |
162 | } |
163 | ||
164 | /* | |
165 | * This table defines the supported mailbox commands for the driver. This table | |
166 | * is made up of a UAPI structure. Non-negative values as parameters in the | |
167 | * table will be validated against the user's input. For example, if size_in is | |
168 | * 0, and the user passed in 1, it is an error. | |
169 | */ | |
392be0bd | 170 | static struct cxl_mem_command mem_commands[CXL_MEM_COMMAND_ID_MAX] = { |
472b1ce6 | 171 | CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), |
13237183 | 172 | #ifdef CONFIG_CXL_MEM_RAW_COMMANDS |
472b1ce6 | 173 | CXL_CMD(RAW, ~0, ~0, 0), |
13237183 | 174 | #endif |
472b1ce6 | 175 | CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), |
57ee605b BW |
176 | CXL_CMD(GET_FW_INFO, 0, 0x50, 0), |
177 | CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), | |
178 | CXL_CMD(GET_LSA, 0x8, ~0, 0), | |
179 | CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), | |
180 | CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), | |
13237183 BW |
181 | }; |
182 | ||
183 | /* | |
184 | * Commands that RAW doesn't permit. The rationale for each: | |
185 | * | |
186 | * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / | |
187 | * coordination of transaction timeout values at the root bridge level. | |
188 | * | |
189 | * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live | |
190 | * and needs to be coordinated with HDM updates. | |
191 | * | |
192 | * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the | |
193 | * driver and any writes from userspace invalidates those contents. | |
194 | * | |
195 | * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes | |
196 | * to the device after it is marked clean, userspace can not make that | |
197 | * assertion. | |
198 | * | |
199 | * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that | |
200 | * is kept up to date with patrol notifications and error management. | |
201 | */ | |
202 | static u16 cxl_disabled_raw_commands[] = { | |
203 | CXL_MBOX_OP_ACTIVATE_FW, | |
204 | CXL_MBOX_OP_SET_PARTITION_INFO, | |
205 | CXL_MBOX_OP_SET_LSA, | |
206 | CXL_MBOX_OP_SET_SHUTDOWN_STATE, | |
207 | CXL_MBOX_OP_SCAN_MEDIA, | |
208 | CXL_MBOX_OP_GET_SCAN_MEDIA, | |
209 | }; | |
210 | ||
211 | /* | |
212 | * Command sets that RAW doesn't permit. All opcodes in this set are | |
213 | * disabled because they pass plain text security payloads over the | |
214 | * user/kernel boundary. This functionality is intended to be wrapped | |
215 | * behind the keys ABI which allows for encrypted payloads in the UAPI | |
216 | */ | |
217 | static u8 security_command_sets[] = { | |
218 | 0x44, /* Sanitize */ | |
219 | 0x45, /* Persistent Memory Data-at-rest Security */ | |
220 | 0x46, /* Security Passthrough */ | |
583fa5e7 BW |
221 | }; |
222 | ||
223 | #define cxl_for_each_cmd(cmd) \ | |
224 | for ((cmd) = &mem_commands[0]; \ | |
225 | ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) | |
226 | ||
227 | #define cxl_cmd_count ARRAY_SIZE(mem_commands) | |
228 | ||
8adaf747 BW |
229 | static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) |
230 | { | |
231 | const unsigned long start = jiffies; | |
232 | unsigned long end = start; | |
233 | ||
234 | while (cxl_doorbell_busy(cxlm)) { | |
235 | end = jiffies; | |
236 | ||
237 | if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { | |
238 | /* Check again in case preempted before timeout test */ | |
239 | if (!cxl_doorbell_busy(cxlm)) | |
240 | break; | |
241 | return -ETIMEDOUT; | |
242 | } | |
243 | cpu_relax(); | |
244 | } | |
245 | ||
246 | dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", | |
247 | jiffies_to_msecs(end) - jiffies_to_msecs(start)); | |
248 | return 0; | |
249 | } | |
250 | ||
13237183 BW |
251 | static bool cxl_is_security_command(u16 opcode) |
252 | { | |
253 | int i; | |
254 | ||
255 | for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) | |
256 | if (security_command_sets[i] == (opcode >> 8)) | |
257 | return true; | |
258 | return false; | |
259 | } | |
260 | ||
8adaf747 BW |
261 | static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, |
262 | struct mbox_cmd *mbox_cmd) | |
263 | { | |
264 | struct device *dev = &cxlm->pdev->dev; | |
265 | ||
266 | dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", | |
267 | mbox_cmd->opcode, mbox_cmd->size_in); | |
268 | } | |
269 | ||
270 | /** | |
271 | * __cxl_mem_mbox_send_cmd() - Execute a mailbox command | |
272 | * @cxlm: The CXL memory device to communicate with. | |
273 | * @mbox_cmd: Command to send to the memory device. | |
274 | * | |
275 | * Context: Any context. Expects mbox_mutex to be held. | |
276 | * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. | |
277 | * Caller should check the return code in @mbox_cmd to make sure it | |
278 | * succeeded. | |
279 | * | |
280 | * This is a generic form of the CXL mailbox send command thus only using the | |
281 | * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory | |
282 | * devices, and perhaps other types of CXL devices may have further information | |
283 | * available upon error conditions. Driver facilities wishing to send mailbox | |
284 | * commands should use the wrapper command. | |
285 | * | |
286 | * The CXL spec allows for up to two mailboxes. The intention is for the primary | |
287 | * mailbox to be OS controlled and the secondary mailbox to be used by system | |
288 | * firmware. This allows the OS and firmware to communicate with the device and | |
289 | * not need to coordinate with each other. The driver only uses the primary | |
290 | * mailbox. | |
291 | */ | |
292 | static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, | |
293 | struct mbox_cmd *mbox_cmd) | |
294 | { | |
295 | void __iomem *payload = cxlm->mbox_regs + CXLDEV_MBOX_PAYLOAD_OFFSET; | |
296 | u64 cmd_reg, status_reg; | |
297 | size_t out_len; | |
298 | int rc; | |
299 | ||
300 | lockdep_assert_held(&cxlm->mbox_mutex); | |
301 | ||
302 | /* | |
303 | * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. | |
304 | * 1. Caller reads MB Control Register to verify doorbell is clear | |
305 | * 2. Caller writes Command Register | |
306 | * 3. Caller writes Command Payload Registers if input payload is non-empty | |
307 | * 4. Caller writes MB Control Register to set doorbell | |
308 | * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured | |
309 | * 6. Caller reads MB Status Register to fetch Return code | |
310 | * 7. If command successful, Caller reads Command Register to get Payload Length | |
311 | * 8. If output payload is non-empty, host reads Command Payload Registers | |
312 | * | |
313 | * Hardware is free to do whatever it wants before the doorbell is rung, | |
314 | * and isn't allowed to change anything after it clears the doorbell. As | |
315 | * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can | |
316 | * also happen in any order (though some orders might not make sense). | |
317 | */ | |
318 | ||
319 | /* #1 */ | |
320 | if (cxl_doorbell_busy(cxlm)) { | |
321 | dev_err_ratelimited(&cxlm->pdev->dev, | |
322 | "Mailbox re-busy after acquiring\n"); | |
323 | return -EBUSY; | |
324 | } | |
325 | ||
326 | cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, | |
327 | mbox_cmd->opcode); | |
328 | if (mbox_cmd->size_in) { | |
329 | if (WARN_ON(!mbox_cmd->payload_in)) | |
330 | return -EINVAL; | |
331 | ||
332 | cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, | |
333 | mbox_cmd->size_in); | |
334 | memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); | |
335 | } | |
336 | ||
337 | /* #2, #3 */ | |
338 | writeq(cmd_reg, cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); | |
339 | ||
340 | /* #4 */ | |
341 | dev_dbg(&cxlm->pdev->dev, "Sending command\n"); | |
342 | writel(CXLDEV_MBOX_CTRL_DOORBELL, | |
343 | cxlm->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET); | |
344 | ||
345 | /* #5 */ | |
346 | rc = cxl_mem_wait_for_doorbell(cxlm); | |
347 | if (rc == -ETIMEDOUT) { | |
348 | cxl_mem_mbox_timeout(cxlm, mbox_cmd); | |
349 | return rc; | |
350 | } | |
351 | ||
352 | /* #6 */ | |
353 | status_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_STATUS_OFFSET); | |
354 | mbox_cmd->return_code = | |
355 | FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); | |
356 | ||
357 | if (mbox_cmd->return_code != 0) { | |
358 | dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); | |
359 | return 0; | |
360 | } | |
361 | ||
362 | /* #7 */ | |
363 | cmd_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); | |
364 | out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); | |
365 | ||
366 | /* #8 */ | |
367 | if (out_len && mbox_cmd->payload_out) { | |
368 | /* | |
369 | * Sanitize the copy. If hardware misbehaves, out_len per the | |
370 | * spec can actually be greater than the max allowed size (21 | |
371 | * bits available but spec defined 1M max). The caller also may | |
372 | * have requested less data than the hardware supplied even | |
373 | * within spec. | |
374 | */ | |
375 | size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); | |
376 | ||
377 | memcpy_fromio(mbox_cmd->payload_out, payload, n); | |
378 | mbox_cmd->size_out = n; | |
379 | } else { | |
380 | mbox_cmd->size_out = 0; | |
381 | } | |
382 | ||
383 | return 0; | |
384 | } | |
385 | ||
386 | /** | |
387 | * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. | |
388 | * @cxlm: The memory device to gain access to. | |
389 | * | |
390 | * Context: Any context. Takes the mbox_mutex. | |
391 | * Return: 0 if exclusive access was acquired. | |
392 | */ | |
393 | static int cxl_mem_mbox_get(struct cxl_mem *cxlm) | |
394 | { | |
395 | struct device *dev = &cxlm->pdev->dev; | |
396 | u64 md_status; | |
397 | int rc; | |
398 | ||
399 | mutex_lock_io(&cxlm->mbox_mutex); | |
400 | ||
401 | /* | |
402 | * XXX: There is some amount of ambiguity in the 2.0 version of the spec | |
403 | * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the | |
404 | * bit is to allow firmware running on the device to notify the driver | |
405 | * that it's ready to receive commands. It is unclear if the bit needs | |
406 | * to be read for each transaction mailbox, ie. the firmware can switch | |
407 | * it on and off as needed. Second, there is no defined timeout for | |
408 | * mailbox ready, like there is for the doorbell interface. | |
409 | * | |
410 | * Assumptions: | |
411 | * 1. The firmware might toggle the Mailbox Interface Ready bit, check | |
412 | * it for every command. | |
413 | * | |
414 | * 2. If the doorbell is clear, the firmware should have first set the | |
415 | * Mailbox Interface Ready bit. Therefore, waiting for the doorbell | |
416 | * to be ready is sufficient. | |
417 | */ | |
418 | rc = cxl_mem_wait_for_doorbell(cxlm); | |
419 | if (rc) { | |
420 | dev_warn(dev, "Mailbox interface not ready\n"); | |
421 | goto out; | |
422 | } | |
423 | ||
424 | md_status = readq(cxlm->memdev_regs + CXLMDEV_STATUS_OFFSET); | |
425 | if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { | |
426 | dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); | |
427 | rc = -EBUSY; | |
428 | goto out; | |
429 | } | |
430 | ||
431 | /* | |
432 | * Hardware shouldn't allow a ready status but also have failure bits | |
433 | * set. Spit out an error, this should be a bug report | |
434 | */ | |
435 | rc = -EFAULT; | |
436 | if (md_status & CXLMDEV_DEV_FATAL) { | |
437 | dev_err(dev, "mbox: reported ready, but fatal\n"); | |
438 | goto out; | |
439 | } | |
440 | if (md_status & CXLMDEV_FW_HALT) { | |
441 | dev_err(dev, "mbox: reported ready, but halted\n"); | |
442 | goto out; | |
443 | } | |
444 | if (CXLMDEV_RESET_NEEDED(md_status)) { | |
445 | dev_err(dev, "mbox: reported ready, but reset needed\n"); | |
446 | goto out; | |
447 | } | |
448 | ||
449 | /* with lock held */ | |
450 | return 0; | |
451 | ||
452 | out: | |
453 | mutex_unlock(&cxlm->mbox_mutex); | |
454 | return rc; | |
455 | } | |
456 | ||
457 | /** | |
458 | * cxl_mem_mbox_put() - Release exclusive access to the mailbox. | |
459 | * @cxlm: The CXL memory device to communicate with. | |
460 | * | |
461 | * Context: Any context. Expects mbox_mutex to be held. | |
462 | */ | |
463 | static void cxl_mem_mbox_put(struct cxl_mem *cxlm) | |
464 | { | |
465 | mutex_unlock(&cxlm->mbox_mutex); | |
466 | } | |
467 | ||
583fa5e7 BW |
468 | /** |
469 | * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. | |
470 | * @cxlm: The CXL memory device to communicate with. | |
471 | * @cmd: The validated command. | |
472 | * @in_payload: Pointer to userspace's input payload. | |
473 | * @out_payload: Pointer to userspace's output payload. | |
474 | * @size_out: (Input) Max payload size to copy out. | |
475 | * (Output) Payload size hardware generated. | |
476 | * @retval: Hardware generated return code from the operation. | |
477 | * | |
478 | * Return: | |
479 | * * %0 - Mailbox transaction succeeded. This implies the mailbox | |
480 | * protocol completed successfully not that the operation itself | |
481 | * was successful. | |
482 | * * %-ENOMEM - Couldn't allocate a bounce buffer. | |
483 | * * %-EFAULT - Something happened with copy_to/from_user. | |
484 | * * %-EINTR - Mailbox acquisition interrupted. | |
485 | * * %-EXXX - Transaction level failures. | |
486 | * | |
487 | * Creates the appropriate mailbox command and dispatches it on behalf of a | |
488 | * userspace request. The input and output payloads are copied between | |
489 | * userspace. | |
490 | * | |
491 | * See cxl_send_cmd(). | |
492 | */ | |
493 | static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, | |
494 | const struct cxl_mem_command *cmd, | |
495 | u64 in_payload, u64 out_payload, | |
496 | s32 *size_out, u32 *retval) | |
497 | { | |
498 | struct device *dev = &cxlm->pdev->dev; | |
499 | struct mbox_cmd mbox_cmd = { | |
500 | .opcode = cmd->opcode, | |
501 | .size_in = cmd->info.size_in, | |
502 | .size_out = cmd->info.size_out, | |
503 | }; | |
504 | int rc; | |
505 | ||
506 | if (cmd->info.size_out) { | |
507 | mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); | |
508 | if (!mbox_cmd.payload_out) | |
509 | return -ENOMEM; | |
510 | } | |
511 | ||
512 | if (cmd->info.size_in) { | |
513 | mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), | |
514 | cmd->info.size_in); | |
88ff5d46 BW |
515 | if (IS_ERR(mbox_cmd.payload_in)) { |
516 | kvfree(mbox_cmd.payload_out); | |
583fa5e7 | 517 | return PTR_ERR(mbox_cmd.payload_in); |
88ff5d46 | 518 | } |
583fa5e7 BW |
519 | } |
520 | ||
521 | rc = cxl_mem_mbox_get(cxlm); | |
522 | if (rc) | |
523 | goto out; | |
524 | ||
525 | dev_dbg(dev, | |
526 | "Submitting %s command for user\n" | |
527 | "\topcode: %x\n" | |
528 | "\tsize: %ub\n", | |
529 | cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, | |
530 | cmd->info.size_in); | |
531 | ||
13237183 BW |
532 | dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, |
533 | "raw command path used\n"); | |
534 | ||
583fa5e7 BW |
535 | rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); |
536 | cxl_mem_mbox_put(cxlm); | |
537 | if (rc) | |
538 | goto out; | |
539 | ||
540 | /* | |
541 | * @size_out contains the max size that's allowed to be written back out | |
542 | * to userspace. While the payload may have written more output than | |
543 | * this it will have to be ignored. | |
544 | */ | |
545 | if (mbox_cmd.size_out) { | |
546 | dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, | |
547 | "Invalid return size\n"); | |
548 | if (copy_to_user(u64_to_user_ptr(out_payload), | |
549 | mbox_cmd.payload_out, mbox_cmd.size_out)) { | |
550 | rc = -EFAULT; | |
551 | goto out; | |
552 | } | |
553 | } | |
554 | ||
555 | *size_out = mbox_cmd.size_out; | |
556 | *retval = mbox_cmd.return_code; | |
557 | ||
558 | out: | |
559 | kvfree(mbox_cmd.payload_in); | |
560 | kvfree(mbox_cmd.payload_out); | |
561 | return rc; | |
562 | } | |
563 | ||
13237183 BW |
564 | static bool cxl_mem_raw_command_allowed(u16 opcode) |
565 | { | |
566 | int i; | |
567 | ||
568 | if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) | |
569 | return false; | |
570 | ||
571 | if (security_locked_down(LOCKDOWN_NONE)) | |
572 | return false; | |
573 | ||
574 | if (cxl_raw_allow_all) | |
575 | return true; | |
576 | ||
577 | if (cxl_is_security_command(opcode)) | |
578 | return false; | |
579 | ||
580 | for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) | |
581 | if (cxl_disabled_raw_commands[i] == opcode) | |
582 | return false; | |
583 | ||
584 | return true; | |
585 | } | |
586 | ||
583fa5e7 BW |
587 | /** |
588 | * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. | |
589 | * @cxlm: &struct cxl_mem device whose mailbox will be used. | |
590 | * @send_cmd: &struct cxl_send_command copied in from userspace. | |
591 | * @out_cmd: Sanitized and populated &struct cxl_mem_command. | |
592 | * | |
593 | * Return: | |
594 | * * %0 - @out_cmd is ready to send. | |
595 | * * %-ENOTTY - Invalid command specified. | |
596 | * * %-EINVAL - Reserved fields or invalid values were used. | |
597 | * * %-ENOMEM - Input or output buffer wasn't sized properly. | |
13237183 | 598 | * * %-EPERM - Attempted to use a protected command. |
583fa5e7 BW |
599 | * |
600 | * The result of this command is a fully validated command in @out_cmd that is | |
601 | * safe to send to the hardware. | |
602 | * | |
603 | * See handle_mailbox_cmd_from_user() | |
604 | */ | |
605 | static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, | |
606 | const struct cxl_send_command *send_cmd, | |
607 | struct cxl_mem_command *out_cmd) | |
608 | { | |
609 | const struct cxl_command_info *info; | |
610 | struct cxl_mem_command *c; | |
611 | ||
612 | if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) | |
613 | return -ENOTTY; | |
614 | ||
615 | /* | |
616 | * The user can never specify an input payload larger than what hardware | |
617 | * supports, but output can be arbitrarily large (simply write out as | |
618 | * much data as the hardware provides). | |
619 | */ | |
620 | if (send_cmd->in.size > cxlm->payload_size) | |
621 | return -EINVAL; | |
622 | ||
13237183 BW |
623 | /* |
624 | * Checks are bypassed for raw commands but a WARN/taint will occur | |
625 | * later in the callchain | |
626 | */ | |
627 | if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { | |
628 | const struct cxl_mem_command temp = { | |
629 | .info = { | |
630 | .id = CXL_MEM_COMMAND_ID_RAW, | |
631 | .flags = 0, | |
632 | .size_in = send_cmd->in.size, | |
633 | .size_out = send_cmd->out.size, | |
634 | }, | |
635 | .opcode = send_cmd->raw.opcode | |
636 | }; | |
637 | ||
638 | if (send_cmd->raw.rsvd) | |
639 | return -EINVAL; | |
640 | ||
641 | /* | |
642 | * Unlike supported commands, the output size of RAW commands | |
643 | * gets passed along without further checking, so it must be | |
644 | * validated here. | |
645 | */ | |
646 | if (send_cmd->out.size > cxlm->payload_size) | |
647 | return -EINVAL; | |
648 | ||
649 | if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) | |
650 | return -EPERM; | |
651 | ||
652 | memcpy(out_cmd, &temp, sizeof(temp)); | |
653 | ||
654 | return 0; | |
655 | } | |
656 | ||
583fa5e7 BW |
657 | if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) |
658 | return -EINVAL; | |
659 | ||
660 | if (send_cmd->rsvd) | |
661 | return -EINVAL; | |
662 | ||
663 | if (send_cmd->in.rsvd || send_cmd->out.rsvd) | |
664 | return -EINVAL; | |
665 | ||
666 | /* Convert user's command into the internal representation */ | |
667 | c = &mem_commands[send_cmd->id]; | |
668 | info = &c->info; | |
669 | ||
472b1ce6 BW |
670 | /* Check that the command is enabled for hardware */ |
671 | if (!test_bit(info->id, cxlm->enabled_cmds)) | |
672 | return -ENOTTY; | |
673 | ||
583fa5e7 BW |
674 | /* Check the input buffer is the expected size */ |
675 | if (info->size_in >= 0 && info->size_in != send_cmd->in.size) | |
676 | return -ENOMEM; | |
677 | ||
678 | /* Check the output buffer is at least large enough */ | |
679 | if (info->size_out >= 0 && send_cmd->out.size < info->size_out) | |
680 | return -ENOMEM; | |
681 | ||
682 | memcpy(out_cmd, c, sizeof(*c)); | |
683 | out_cmd->info.size_in = send_cmd->in.size; | |
684 | /* | |
685 | * XXX: out_cmd->info.size_out will be controlled by the driver, and the | |
686 | * specified number of bytes @send_cmd->out.size will be copied back out | |
687 | * to userspace. | |
688 | */ | |
689 | ||
690 | return 0; | |
691 | } | |
692 | ||
693 | static int cxl_query_cmd(struct cxl_memdev *cxlmd, | |
694 | struct cxl_mem_query_commands __user *q) | |
695 | { | |
696 | struct device *dev = &cxlmd->dev; | |
697 | struct cxl_mem_command *cmd; | |
698 | u32 n_commands; | |
699 | int j = 0; | |
700 | ||
701 | dev_dbg(dev, "Query IOCTL\n"); | |
702 | ||
703 | if (get_user(n_commands, &q->n_commands)) | |
704 | return -EFAULT; | |
705 | ||
706 | /* returns the total number if 0 elements are requested. */ | |
707 | if (n_commands == 0) | |
708 | return put_user(cxl_cmd_count, &q->n_commands); | |
709 | ||
710 | /* | |
711 | * otherwise, return max(n_commands, total commands) cxl_command_info | |
712 | * structures. | |
713 | */ | |
714 | cxl_for_each_cmd(cmd) { | |
715 | const struct cxl_command_info *info = &cmd->info; | |
716 | ||
717 | if (copy_to_user(&q->commands[j++], info, sizeof(*info))) | |
718 | return -EFAULT; | |
719 | ||
720 | if (j == n_commands) | |
721 | break; | |
722 | } | |
723 | ||
724 | return 0; | |
725 | } | |
726 | ||
727 | static int cxl_send_cmd(struct cxl_memdev *cxlmd, | |
728 | struct cxl_send_command __user *s) | |
729 | { | |
730 | struct cxl_mem *cxlm = cxlmd->cxlm; | |
731 | struct device *dev = &cxlmd->dev; | |
732 | struct cxl_send_command send; | |
733 | struct cxl_mem_command c; | |
734 | int rc; | |
735 | ||
736 | dev_dbg(dev, "Send IOCTL\n"); | |
737 | ||
738 | if (copy_from_user(&send, s, sizeof(send))) | |
739 | return -EFAULT; | |
740 | ||
741 | rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); | |
742 | if (rc) | |
743 | return rc; | |
744 | ||
745 | /* Prepare to handle a full payload for variable sized output */ | |
746 | if (c.info.size_out < 0) | |
747 | c.info.size_out = cxlm->payload_size; | |
748 | ||
749 | rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, | |
750 | send.out.payload, &send.out.size, | |
751 | &send.retval); | |
752 | if (rc) | |
753 | return rc; | |
754 | ||
58294927 DC |
755 | if (copy_to_user(s, &send, sizeof(send))) |
756 | return -EFAULT; | |
757 | ||
758 | return 0; | |
583fa5e7 BW |
759 | } |
760 | ||
761 | static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, | |
762 | unsigned long arg) | |
763 | { | |
764 | switch (cmd) { | |
765 | case CXL_MEM_QUERY_COMMANDS: | |
766 | return cxl_query_cmd(cxlmd, (void __user *)arg); | |
767 | case CXL_MEM_SEND_COMMAND: | |
768 | return cxl_send_cmd(cxlmd, (void __user *)arg); | |
769 | default: | |
770 | return -ENOTTY; | |
771 | } | |
772 | } | |
773 | ||
b39cb105 DW |
774 | static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, |
775 | unsigned long arg) | |
776 | { | |
58775159 DW |
777 | struct cxl_memdev *cxlmd = file->private_data; |
778 | int rc = -ENXIO; | |
b39cb105 | 779 | |
58775159 DW |
780 | down_read(&cxl_memdev_rwsem); |
781 | if (cxlmd->cxlm) | |
782 | rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); | |
783 | up_read(&cxl_memdev_rwsem); | |
b39cb105 | 784 | |
58775159 DW |
785 | return rc; |
786 | } | |
b39cb105 | 787 | |
58775159 DW |
788 | static int cxl_memdev_open(struct inode *inode, struct file *file) |
789 | { | |
790 | struct cxl_memdev *cxlmd = | |
791 | container_of(inode->i_cdev, typeof(*cxlmd), cdev); | |
b39cb105 | 792 | |
58775159 DW |
793 | get_device(&cxlmd->dev); |
794 | file->private_data = cxlmd; | |
b39cb105 | 795 | |
58775159 DW |
796 | return 0; |
797 | } | |
798 | ||
799 | static int cxl_memdev_release_file(struct inode *inode, struct file *file) | |
800 | { | |
801 | struct cxl_memdev *cxlmd = | |
802 | container_of(inode->i_cdev, typeof(*cxlmd), cdev); | |
803 | ||
804 | put_device(&cxlmd->dev); | |
805 | ||
806 | return 0; | |
b39cb105 DW |
807 | } |
808 | ||
809 | static const struct file_operations cxl_memdev_fops = { | |
810 | .owner = THIS_MODULE, | |
811 | .unlocked_ioctl = cxl_memdev_ioctl, | |
58775159 DW |
812 | .open = cxl_memdev_open, |
813 | .release = cxl_memdev_release_file, | |
b39cb105 DW |
814 | .compat_ioctl = compat_ptr_ioctl, |
815 | .llseek = noop_llseek, | |
816 | }; | |
817 | ||
472b1ce6 BW |
818 | static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) |
819 | { | |
820 | struct cxl_mem_command *c; | |
821 | ||
822 | cxl_for_each_cmd(c) | |
823 | if (c->opcode == opcode) | |
824 | return c; | |
825 | ||
826 | return NULL; | |
827 | } | |
828 | ||
8adaf747 BW |
829 | /** |
830 | * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. | |
831 | * @cxlm: The CXL memory device to communicate with. | |
832 | * @opcode: Opcode for the mailbox command. | |
833 | * @in: The input payload for the mailbox command. | |
834 | * @in_size: The length of the input payload | |
835 | * @out: Caller allocated buffer for the output. | |
836 | * @out_size: Expected size of output. | |
837 | * | |
838 | * Context: Any context. Will acquire and release mbox_mutex. | |
839 | * Return: | |
840 | * * %>=0 - Number of bytes returned in @out. | |
841 | * * %-E2BIG - Payload is too large for hardware. | |
842 | * * %-EBUSY - Couldn't acquire exclusive mailbox access. | |
843 | * * %-EFAULT - Hardware error occurred. | |
844 | * * %-ENXIO - Command completed, but device reported an error. | |
845 | * * %-EIO - Unexpected output size. | |
846 | * | |
847 | * Mailbox commands may execute successfully yet the device itself reported an | |
848 | * error. While this distinction can be useful for commands from userspace, the | |
472b1ce6 | 849 | * kernel will only be able to use results when both are successful. |
8adaf747 BW |
850 | * |
851 | * See __cxl_mem_mbox_send_cmd() | |
852 | */ | |
853 | static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, | |
854 | void *in, size_t in_size, | |
855 | void *out, size_t out_size) | |
856 | { | |
472b1ce6 | 857 | const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); |
8adaf747 BW |
858 | struct mbox_cmd mbox_cmd = { |
859 | .opcode = opcode, | |
860 | .payload_in = in, | |
861 | .size_in = in_size, | |
862 | .size_out = out_size, | |
863 | .payload_out = out, | |
864 | }; | |
865 | int rc; | |
866 | ||
867 | if (out_size > cxlm->payload_size) | |
868 | return -E2BIG; | |
869 | ||
870 | rc = cxl_mem_mbox_get(cxlm); | |
871 | if (rc) | |
872 | return rc; | |
873 | ||
874 | rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); | |
875 | cxl_mem_mbox_put(cxlm); | |
876 | if (rc) | |
877 | return rc; | |
878 | ||
879 | /* TODO: Map return code to proper kernel style errno */ | |
880 | if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) | |
881 | return -ENXIO; | |
882 | ||
472b1ce6 BW |
883 | /* |
884 | * Variable sized commands can't be validated and so it's up to the | |
885 | * caller to do that if they wish. | |
886 | */ | |
887 | if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) | |
8adaf747 BW |
888 | return -EIO; |
889 | ||
890 | return 0; | |
891 | } | |
892 | ||
893 | /** | |
894 | * cxl_mem_setup_regs() - Setup necessary MMIO. | |
895 | * @cxlm: The CXL memory device to communicate with. | |
896 | * | |
897 | * Return: 0 if all necessary registers mapped. | |
898 | * | |
899 | * A memory device is required by spec to implement a certain set of MMIO | |
900 | * regions. The purpose of this function is to enumerate and map those | |
901 | * registers. | |
902 | */ | |
903 | static int cxl_mem_setup_regs(struct cxl_mem *cxlm) | |
904 | { | |
905 | struct device *dev = &cxlm->pdev->dev; | |
906 | int cap, cap_count; | |
907 | u64 cap_array; | |
908 | ||
909 | cap_array = readq(cxlm->regs + CXLDEV_CAP_ARRAY_OFFSET); | |
910 | if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != | |
911 | CXLDEV_CAP_ARRAY_CAP_ID) | |
912 | return -ENODEV; | |
913 | ||
914 | cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); | |
915 | ||
916 | for (cap = 1; cap <= cap_count; cap++) { | |
917 | void __iomem *register_block; | |
918 | u32 offset; | |
919 | u16 cap_id; | |
920 | ||
921 | cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, | |
922 | readl(cxlm->regs + cap * 0x10)); | |
923 | offset = readl(cxlm->regs + cap * 0x10 + 0x4); | |
924 | register_block = cxlm->regs + offset; | |
925 | ||
926 | switch (cap_id) { | |
927 | case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: | |
928 | dev_dbg(dev, "found Status capability (0x%x)\n", offset); | |
929 | cxlm->status_regs = register_block; | |
930 | break; | |
931 | case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: | |
932 | dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); | |
933 | cxlm->mbox_regs = register_block; | |
934 | break; | |
935 | case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: | |
936 | dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); | |
937 | break; | |
938 | case CXLDEV_CAP_CAP_ID_MEMDEV: | |
939 | dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); | |
940 | cxlm->memdev_regs = register_block; | |
941 | break; | |
942 | default: | |
943 | dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); | |
944 | break; | |
945 | } | |
946 | } | |
947 | ||
948 | if (!cxlm->status_regs || !cxlm->mbox_regs || !cxlm->memdev_regs) { | |
949 | dev_err(dev, "registers not found: %s%s%s\n", | |
950 | !cxlm->status_regs ? "status " : "", | |
951 | !cxlm->mbox_regs ? "mbox " : "", | |
952 | !cxlm->memdev_regs ? "memdev" : ""); | |
953 | return -ENXIO; | |
954 | } | |
955 | ||
956 | return 0; | |
957 | } | |
958 | ||
959 | static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) | |
960 | { | |
961 | const int cap = readl(cxlm->mbox_regs + CXLDEV_MBOX_CAPS_OFFSET); | |
962 | ||
963 | cxlm->payload_size = | |
964 | 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); | |
965 | ||
966 | /* | |
967 | * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register | |
968 | * | |
969 | * If the size is too small, mandatory commands will not work and so | |
970 | * there's no point in going forward. If the size is too large, there's | |
971 | * no harm is soft limiting it. | |
972 | */ | |
973 | cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); | |
974 | if (cxlm->payload_size < 256) { | |
975 | dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", | |
976 | cxlm->payload_size); | |
977 | return -ENXIO; | |
978 | } | |
979 | ||
980 | dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", | |
981 | cxlm->payload_size); | |
982 | ||
983 | return 0; | |
984 | } | |
985 | ||
986 | static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, | |
987 | u32 reg_hi) | |
988 | { | |
989 | struct device *dev = &pdev->dev; | |
990 | struct cxl_mem *cxlm; | |
991 | void __iomem *regs; | |
992 | u64 offset; | |
993 | u8 bar; | |
994 | int rc; | |
995 | ||
996 | cxlm = devm_kzalloc(&pdev->dev, sizeof(*cxlm), GFP_KERNEL); | |
997 | if (!cxlm) { | |
998 | dev_err(dev, "No memory available\n"); | |
999 | return NULL; | |
1000 | } | |
1001 | ||
b21bb4cd | 1002 | offset = ((u64)reg_hi << 32) | (reg_lo & CXL_REGLOC_ADDR_MASK); |
8adaf747 BW |
1003 | bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); |
1004 | ||
1005 | /* Basic sanity check that BAR is big enough */ | |
1006 | if (pci_resource_len(pdev, bar) < offset) { | |
1007 | dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, | |
1008 | &pdev->resource[bar], (unsigned long long)offset); | |
1009 | return NULL; | |
1010 | } | |
1011 | ||
1012 | rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); | |
1013 | if (rc) { | |
1014 | dev_err(dev, "failed to map registers\n"); | |
1015 | return NULL; | |
1016 | } | |
1017 | regs = pcim_iomap_table(pdev)[bar]; | |
1018 | ||
1019 | mutex_init(&cxlm->mbox_mutex); | |
1020 | cxlm->pdev = pdev; | |
1021 | cxlm->regs = regs + offset; | |
472b1ce6 BW |
1022 | cxlm->enabled_cmds = |
1023 | devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), | |
1024 | sizeof(unsigned long), | |
1025 | GFP_KERNEL | __GFP_ZERO); | |
1026 | if (!cxlm->enabled_cmds) { | |
1027 | dev_err(dev, "No memory available for bitmap\n"); | |
1028 | return NULL; | |
1029 | } | |
8adaf747 BW |
1030 | |
1031 | dev_dbg(dev, "Mapped CXL Memory Device resource\n"); | |
1032 | return cxlm; | |
1033 | } | |
4cdadfd5 DW |
1034 | |
1035 | static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) | |
1036 | { | |
1037 | int pos; | |
1038 | ||
1039 | pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); | |
1040 | if (!pos) | |
1041 | return 0; | |
1042 | ||
1043 | while (pos) { | |
1044 | u16 vendor, id; | |
1045 | ||
1046 | pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); | |
1047 | pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); | |
1048 | if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) | |
1049 | return pos; | |
1050 | ||
1051 | pos = pci_find_next_ext_capability(pdev, pos, | |
1052 | PCI_EXT_CAP_ID_DVSEC); | |
1053 | } | |
1054 | ||
1055 | return 0; | |
1056 | } | |
1057 | ||
b39cb105 DW |
1058 | static struct cxl_memdev *to_cxl_memdev(struct device *dev) |
1059 | { | |
1060 | return container_of(dev, struct cxl_memdev, dev); | |
1061 | } | |
1062 | ||
1063 | static void cxl_memdev_release(struct device *dev) | |
1064 | { | |
1065 | struct cxl_memdev *cxlmd = to_cxl_memdev(dev); | |
1066 | ||
b39cb105 DW |
1067 | ida_free(&cxl_memdev_ida, cxlmd->id); |
1068 | kfree(cxlmd); | |
1069 | } | |
1070 | ||
1071 | static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, | |
1072 | kgid_t *gid) | |
1073 | { | |
1074 | return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); | |
1075 | } | |
1076 | ||
1077 | static ssize_t firmware_version_show(struct device *dev, | |
1078 | struct device_attribute *attr, char *buf) | |
1079 | { | |
1080 | struct cxl_memdev *cxlmd = to_cxl_memdev(dev); | |
1081 | struct cxl_mem *cxlm = cxlmd->cxlm; | |
1082 | ||
6eff5721 | 1083 | return sysfs_emit(buf, "%.16s\n", cxlm->firmware_version); |
b39cb105 DW |
1084 | } |
1085 | static DEVICE_ATTR_RO(firmware_version); | |
1086 | ||
1087 | static ssize_t payload_max_show(struct device *dev, | |
1088 | struct device_attribute *attr, char *buf) | |
1089 | { | |
1090 | struct cxl_memdev *cxlmd = to_cxl_memdev(dev); | |
1091 | struct cxl_mem *cxlm = cxlmd->cxlm; | |
1092 | ||
6eff5721 | 1093 | return sysfs_emit(buf, "%zu\n", cxlm->payload_size); |
b39cb105 DW |
1094 | } |
1095 | static DEVICE_ATTR_RO(payload_max); | |
1096 | ||
1097 | static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, | |
1098 | char *buf) | |
1099 | { | |
1100 | struct cxl_memdev *cxlmd = to_cxl_memdev(dev); | |
1101 | struct cxl_mem *cxlm = cxlmd->cxlm; | |
1102 | unsigned long long len = range_len(&cxlm->ram_range); | |
1103 | ||
6eff5721 | 1104 | return sysfs_emit(buf, "%#llx\n", len); |
b39cb105 DW |
1105 | } |
1106 | ||
1107 | static struct device_attribute dev_attr_ram_size = | |
1108 | __ATTR(size, 0444, ram_size_show, NULL); | |
1109 | ||
1110 | static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, | |
1111 | char *buf) | |
1112 | { | |
1113 | struct cxl_memdev *cxlmd = to_cxl_memdev(dev); | |
1114 | struct cxl_mem *cxlm = cxlmd->cxlm; | |
1115 | unsigned long long len = range_len(&cxlm->pmem_range); | |
1116 | ||
6eff5721 | 1117 | return sysfs_emit(buf, "%#llx\n", len); |
b39cb105 DW |
1118 | } |
1119 | ||
1120 | static struct device_attribute dev_attr_pmem_size = | |
1121 | __ATTR(size, 0444, pmem_size_show, NULL); | |
1122 | ||
1123 | static struct attribute *cxl_memdev_attributes[] = { | |
1124 | &dev_attr_firmware_version.attr, | |
1125 | &dev_attr_payload_max.attr, | |
1126 | NULL, | |
1127 | }; | |
1128 | ||
1129 | static struct attribute *cxl_memdev_pmem_attributes[] = { | |
1130 | &dev_attr_pmem_size.attr, | |
1131 | NULL, | |
1132 | }; | |
1133 | ||
1134 | static struct attribute *cxl_memdev_ram_attributes[] = { | |
1135 | &dev_attr_ram_size.attr, | |
1136 | NULL, | |
1137 | }; | |
1138 | ||
1139 | static struct attribute_group cxl_memdev_attribute_group = { | |
1140 | .attrs = cxl_memdev_attributes, | |
1141 | }; | |
1142 | ||
1143 | static struct attribute_group cxl_memdev_ram_attribute_group = { | |
1144 | .name = "ram", | |
1145 | .attrs = cxl_memdev_ram_attributes, | |
1146 | }; | |
1147 | ||
1148 | static struct attribute_group cxl_memdev_pmem_attribute_group = { | |
1149 | .name = "pmem", | |
1150 | .attrs = cxl_memdev_pmem_attributes, | |
1151 | }; | |
1152 | ||
1153 | static const struct attribute_group *cxl_memdev_attribute_groups[] = { | |
1154 | &cxl_memdev_attribute_group, | |
1155 | &cxl_memdev_ram_attribute_group, | |
1156 | &cxl_memdev_pmem_attribute_group, | |
1157 | NULL, | |
1158 | }; | |
1159 | ||
1160 | static const struct device_type cxl_memdev_type = { | |
1161 | .name = "cxl_memdev", | |
1162 | .release = cxl_memdev_release, | |
1163 | .devnode = cxl_memdev_devnode, | |
1164 | .groups = cxl_memdev_attribute_groups, | |
1165 | }; | |
1166 | ||
58775159 | 1167 | static void cxl_memdev_shutdown(struct cxl_memdev *cxlmd) |
b39cb105 | 1168 | { |
58775159 | 1169 | down_write(&cxl_memdev_rwsem); |
b39cb105 | 1170 | cxlmd->cxlm = NULL; |
58775159 | 1171 | up_write(&cxl_memdev_rwsem); |
b39cb105 DW |
1172 | } |
1173 | ||
58775159 | 1174 | static void cxl_memdev_unregister(void *_cxlmd) |
b39cb105 | 1175 | { |
58775159 DW |
1176 | struct cxl_memdev *cxlmd = _cxlmd; |
1177 | struct device *dev = &cxlmd->dev; | |
b39cb105 | 1178 | |
58775159 DW |
1179 | cdev_device_del(&cxlmd->cdev, dev); |
1180 | cxl_memdev_shutdown(cxlmd); | |
1181 | put_device(dev); | |
b39cb105 DW |
1182 | } |
1183 | ||
1c3333a2 | 1184 | static struct cxl_memdev *cxl_memdev_alloc(struct cxl_mem *cxlm) |
b39cb105 DW |
1185 | { |
1186 | struct pci_dev *pdev = cxlm->pdev; | |
1187 | struct cxl_memdev *cxlmd; | |
1188 | struct device *dev; | |
1189 | struct cdev *cdev; | |
1190 | int rc; | |
1191 | ||
1192 | cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); | |
1193 | if (!cxlmd) | |
1c3333a2 | 1194 | return ERR_PTR(-ENOMEM); |
b39cb105 DW |
1195 | |
1196 | rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); | |
1197 | if (rc < 0) | |
1c3333a2 | 1198 | goto err; |
b39cb105 DW |
1199 | cxlmd->id = rc; |
1200 | ||
1201 | dev = &cxlmd->dev; | |
1202 | device_initialize(dev); | |
1203 | dev->parent = &pdev->dev; | |
1204 | dev->bus = &cxl_bus_type; | |
1205 | dev->devt = MKDEV(cxl_mem_major, cxlmd->id); | |
1206 | dev->type = &cxl_memdev_type; | |
7eda6457 | 1207 | device_set_pm_not_required(dev); |
b39cb105 DW |
1208 | |
1209 | cdev = &cxlmd->cdev; | |
1210 | cdev_init(cdev, &cxl_memdev_fops); | |
1c3333a2 DW |
1211 | return cxlmd; |
1212 | ||
1213 | err: | |
1214 | kfree(cxlmd); | |
1215 | return ERR_PTR(rc); | |
1216 | } | |
1217 | ||
1218 | static int cxl_mem_add_memdev(struct cxl_mem *cxlm) | |
1219 | { | |
1220 | struct cxl_memdev *cxlmd; | |
1221 | struct device *dev; | |
1222 | struct cdev *cdev; | |
1223 | int rc; | |
1224 | ||
1225 | cxlmd = cxl_memdev_alloc(cxlm); | |
1226 | if (IS_ERR(cxlmd)) | |
1227 | return PTR_ERR(cxlmd); | |
1228 | ||
1229 | dev = &cxlmd->dev; | |
1230 | rc = dev_set_name(dev, "mem%d", cxlmd->id); | |
1231 | if (rc) | |
1232 | goto err; | |
b39cb105 | 1233 | |
58775159 DW |
1234 | /* |
1235 | * Activate ioctl operations, no cxl_memdev_rwsem manipulation | |
1236 | * needed as this is ordered with cdev_add() publishing the device. | |
1237 | */ | |
1238 | cxlmd->cxlm = cxlm; | |
1239 | ||
1c3333a2 | 1240 | cdev = &cxlmd->cdev; |
b39cb105 DW |
1241 | rc = cdev_device_add(cdev, dev); |
1242 | if (rc) | |
1c3333a2 | 1243 | goto err; |
b39cb105 | 1244 | |
58775159 DW |
1245 | return devm_add_action_or_reset(dev->parent, cxl_memdev_unregister, |
1246 | cxlmd); | |
b39cb105 | 1247 | |
1c3333a2 | 1248 | err: |
b39cb105 | 1249 | /* |
58775159 DW |
1250 | * The cdev was briefly live, shutdown any ioctl operations that |
1251 | * saw that state. | |
b39cb105 | 1252 | */ |
58775159 | 1253 | cxl_memdev_shutdown(cxlmd); |
1c3333a2 | 1254 | put_device(dev); |
b39cb105 DW |
1255 | return rc; |
1256 | } | |
1257 | ||
472b1ce6 BW |
1258 | static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) |
1259 | { | |
1260 | u32 remaining = size; | |
1261 | u32 offset = 0; | |
1262 | ||
1263 | while (remaining) { | |
1264 | u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); | |
1265 | struct cxl_mbox_get_log { | |
1266 | uuid_t uuid; | |
1267 | __le32 offset; | |
1268 | __le32 length; | |
1269 | } __packed log = { | |
1270 | .uuid = *uuid, | |
1271 | .offset = cpu_to_le32(offset), | |
1272 | .length = cpu_to_le32(xfer_size) | |
1273 | }; | |
1274 | int rc; | |
1275 | ||
1276 | rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, | |
1277 | sizeof(log), out, xfer_size); | |
1278 | if (rc < 0) | |
1279 | return rc; | |
1280 | ||
1281 | out += xfer_size; | |
1282 | remaining -= xfer_size; | |
1283 | offset += xfer_size; | |
1284 | } | |
1285 | ||
1286 | return 0; | |
1287 | } | |
1288 | ||
1289 | /** | |
1290 | * cxl_walk_cel() - Walk through the Command Effects Log. | |
1291 | * @cxlm: Device. | |
1292 | * @size: Length of the Command Effects Log. | |
1293 | * @cel: CEL | |
1294 | * | |
1295 | * Iterate over each entry in the CEL and determine if the driver supports the | |
1296 | * command. If so, the command is enabled for the device and can be used later. | |
1297 | */ | |
1298 | static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) | |
1299 | { | |
1300 | struct cel_entry { | |
1301 | __le16 opcode; | |
1302 | __le16 effect; | |
1303 | } __packed * cel_entry; | |
1304 | const int cel_entries = size / sizeof(*cel_entry); | |
1305 | int i; | |
1306 | ||
1307 | cel_entry = (struct cel_entry *)cel; | |
1308 | ||
1309 | for (i = 0; i < cel_entries; i++) { | |
1310 | u16 opcode = le16_to_cpu(cel_entry[i].opcode); | |
1311 | struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); | |
1312 | ||
1313 | if (!cmd) { | |
1314 | dev_dbg(&cxlm->pdev->dev, | |
1315 | "Opcode 0x%04x unsupported by driver", opcode); | |
1316 | continue; | |
1317 | } | |
1318 | ||
1319 | set_bit(cmd->info.id, cxlm->enabled_cmds); | |
1320 | } | |
1321 | } | |
1322 | ||
1323 | struct cxl_mbox_get_supported_logs { | |
1324 | __le16 entries; | |
1325 | u8 rsvd[6]; | |
1326 | struct gsl_entry { | |
1327 | uuid_t uuid; | |
1328 | __le32 size; | |
1329 | } __packed entry[]; | |
1330 | } __packed; | |
1331 | ||
1332 | static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) | |
1333 | { | |
1334 | struct cxl_mbox_get_supported_logs *ret; | |
1335 | int rc; | |
1336 | ||
1337 | ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); | |
1338 | if (!ret) | |
1339 | return ERR_PTR(-ENOMEM); | |
1340 | ||
1341 | rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, | |
1342 | 0, ret, cxlm->payload_size); | |
1343 | if (rc < 0) { | |
1344 | kvfree(ret); | |
1345 | return ERR_PTR(rc); | |
1346 | } | |
1347 | ||
1348 | return ret; | |
1349 | } | |
1350 | ||
1351 | /** | |
1352 | * cxl_mem_enumerate_cmds() - Enumerate commands for a device. | |
1353 | * @cxlm: The device. | |
1354 | * | |
1355 | * Returns 0 if enumerate completed successfully. | |
1356 | * | |
1357 | * CXL devices have optional support for certain commands. This function will | |
1358 | * determine the set of supported commands for the hardware and update the | |
1359 | * enabled_cmds bitmap in the @cxlm. | |
1360 | */ | |
1361 | static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) | |
1362 | { | |
1363 | struct cxl_mbox_get_supported_logs *gsl; | |
1364 | struct device *dev = &cxlm->pdev->dev; | |
1365 | struct cxl_mem_command *cmd; | |
1366 | int i, rc; | |
1367 | ||
1368 | gsl = cxl_get_gsl(cxlm); | |
1369 | if (IS_ERR(gsl)) | |
1370 | return PTR_ERR(gsl); | |
1371 | ||
1372 | rc = -ENOENT; | |
1373 | for (i = 0; i < le16_to_cpu(gsl->entries); i++) { | |
1374 | u32 size = le32_to_cpu(gsl->entry[i].size); | |
1375 | uuid_t uuid = gsl->entry[i].uuid; | |
1376 | u8 *log; | |
1377 | ||
1378 | dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); | |
1379 | ||
1380 | if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) | |
1381 | continue; | |
1382 | ||
1383 | log = kvmalloc(size, GFP_KERNEL); | |
1384 | if (!log) { | |
1385 | rc = -ENOMEM; | |
1386 | goto out; | |
1387 | } | |
1388 | ||
1389 | rc = cxl_xfer_log(cxlm, &uuid, size, log); | |
1390 | if (rc) { | |
1391 | kvfree(log); | |
1392 | goto out; | |
1393 | } | |
1394 | ||
1395 | cxl_walk_cel(cxlm, size, log); | |
1396 | kvfree(log); | |
1397 | ||
1398 | /* In case CEL was bogus, enable some default commands. */ | |
1399 | cxl_for_each_cmd(cmd) | |
1400 | if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) | |
1401 | set_bit(cmd->info.id, cxlm->enabled_cmds); | |
1402 | ||
1403 | /* Found the required CEL */ | |
1404 | rc = 0; | |
1405 | } | |
1406 | ||
1407 | out: | |
1408 | kvfree(gsl); | |
1409 | return rc; | |
1410 | } | |
1411 | ||
8adaf747 BW |
1412 | /** |
1413 | * cxl_mem_identify() - Send the IDENTIFY command to the device. | |
1414 | * @cxlm: The device to identify. | |
1415 | * | |
1416 | * Return: 0 if identify was executed successfully. | |
1417 | * | |
1418 | * This will dispatch the identify command to the device and on success populate | |
1419 | * structures to be exported to sysfs. | |
1420 | */ | |
1421 | static int cxl_mem_identify(struct cxl_mem *cxlm) | |
1422 | { | |
fae8817a | 1423 | /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ |
8adaf747 BW |
1424 | struct cxl_mbox_identify { |
1425 | char fw_revision[0x10]; | |
1426 | __le64 total_capacity; | |
1427 | __le64 volatile_capacity; | |
1428 | __le64 persistent_capacity; | |
1429 | __le64 partition_align; | |
1430 | __le16 info_event_log_size; | |
1431 | __le16 warning_event_log_size; | |
1432 | __le16 failure_event_log_size; | |
1433 | __le16 fatal_event_log_size; | |
1434 | __le32 lsa_size; | |
1435 | u8 poison_list_max_mer[3]; | |
1436 | __le16 inject_poison_limit; | |
1437 | u8 poison_caps; | |
1438 | u8 qos_telemetry_caps; | |
1439 | } __packed id; | |
1440 | int rc; | |
1441 | ||
1442 | rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, | |
1443 | sizeof(id)); | |
1444 | if (rc < 0) | |
1445 | return rc; | |
1446 | ||
1447 | /* | |
1448 | * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. | |
1449 | * For now, only the capacity is exported in sysfs | |
1450 | */ | |
1451 | cxlm->ram_range.start = 0; | |
fae8817a | 1452 | cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) * SZ_256M - 1; |
8adaf747 BW |
1453 | |
1454 | cxlm->pmem_range.start = 0; | |
fae8817a DW |
1455 | cxlm->pmem_range.end = |
1456 | le64_to_cpu(id.persistent_capacity) * SZ_256M - 1; | |
8adaf747 BW |
1457 | |
1458 | memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); | |
1459 | ||
1460 | return 0; | |
1461 | } | |
1462 | ||
4cdadfd5 DW |
1463 | static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
1464 | { | |
1465 | struct device *dev = &pdev->dev; | |
8adaf747 BW |
1466 | struct cxl_mem *cxlm = NULL; |
1467 | u32 regloc_size, regblocks; | |
1468 | int rc, regloc, i; | |
1469 | ||
1470 | rc = pcim_enable_device(pdev); | |
1471 | if (rc) | |
1472 | return rc; | |
4cdadfd5 DW |
1473 | |
1474 | regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); | |
1475 | if (!regloc) { | |
1476 | dev_err(dev, "register location dvsec not found\n"); | |
1477 | return -ENXIO; | |
1478 | } | |
1479 | ||
8adaf747 BW |
1480 | /* Get the size of the Register Locator DVSEC */ |
1481 | pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); | |
1482 | regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); | |
1483 | ||
1484 | regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; | |
1485 | regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; | |
1486 | ||
1487 | for (i = 0; i < regblocks; i++, regloc += 8) { | |
1488 | u32 reg_lo, reg_hi; | |
1489 | u8 reg_type; | |
1490 | ||
1491 | /* "register low and high" contain other bits */ | |
1492 | pci_read_config_dword(pdev, regloc, ®_lo); | |
1493 | pci_read_config_dword(pdev, regloc + 4, ®_hi); | |
1494 | ||
1495 | reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); | |
1496 | ||
1497 | if (reg_type == CXL_REGLOC_RBI_MEMDEV) { | |
1498 | cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); | |
1499 | break; | |
1500 | } | |
1501 | } | |
1502 | ||
1503 | if (!cxlm) | |
1504 | return -ENODEV; | |
1505 | ||
1506 | rc = cxl_mem_setup_regs(cxlm); | |
1507 | if (rc) | |
1508 | return rc; | |
1509 | ||
1510 | rc = cxl_mem_setup_mailbox(cxlm); | |
1511 | if (rc) | |
1512 | return rc; | |
1513 | ||
472b1ce6 BW |
1514 | rc = cxl_mem_enumerate_cmds(cxlm); |
1515 | if (rc) | |
1516 | return rc; | |
1517 | ||
b39cb105 DW |
1518 | rc = cxl_mem_identify(cxlm); |
1519 | if (rc) | |
1520 | return rc; | |
1521 | ||
1522 | return cxl_mem_add_memdev(cxlm); | |
4cdadfd5 DW |
1523 | } |
1524 | ||
1525 | static const struct pci_device_id cxl_mem_pci_tbl[] = { | |
1526 | /* PCI class code for CXL.mem Type-3 Devices */ | |
1527 | { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, | |
1528 | { /* terminate list */ }, | |
1529 | }; | |
1530 | MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); | |
1531 | ||
1532 | static struct pci_driver cxl_mem_driver = { | |
1533 | .name = KBUILD_MODNAME, | |
1534 | .id_table = cxl_mem_pci_tbl, | |
1535 | .probe = cxl_mem_probe, | |
1536 | .driver = { | |
1537 | .probe_type = PROBE_PREFER_ASYNCHRONOUS, | |
1538 | }, | |
1539 | }; | |
1540 | ||
b39cb105 DW |
1541 | static __init int cxl_mem_init(void) |
1542 | { | |
13237183 | 1543 | struct dentry *mbox_debugfs; |
b39cb105 DW |
1544 | dev_t devt; |
1545 | int rc; | |
1546 | ||
1547 | rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); | |
1548 | if (rc) | |
1549 | return rc; | |
1550 | ||
1551 | cxl_mem_major = MAJOR(devt); | |
1552 | ||
1553 | rc = pci_register_driver(&cxl_mem_driver); | |
1554 | if (rc) { | |
1555 | unregister_chrdev_region(MKDEV(cxl_mem_major, 0), | |
1556 | CXL_MEM_MAX_DEVS); | |
1557 | return rc; | |
1558 | } | |
1559 | ||
13237183 BW |
1560 | cxl_debugfs = debugfs_create_dir("cxl", NULL); |
1561 | mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); | |
1562 | debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, | |
1563 | &cxl_raw_allow_all); | |
1564 | ||
b39cb105 DW |
1565 | return 0; |
1566 | } | |
1567 | ||
1568 | static __exit void cxl_mem_exit(void) | |
1569 | { | |
13237183 | 1570 | debugfs_remove_recursive(cxl_debugfs); |
b39cb105 DW |
1571 | pci_unregister_driver(&cxl_mem_driver); |
1572 | unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); | |
1573 | } | |
1574 | ||
4cdadfd5 | 1575 | MODULE_LICENSE("GPL v2"); |
b39cb105 DW |
1576 | module_init(cxl_mem_init); |
1577 | module_exit(cxl_mem_exit); | |
1578 | MODULE_IMPORT_NS(CXL); |