drivers/cxl/core/mbox.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3 #include <linux/security.h>
   4 #include <linux/debugfs.h>
   5 #include <linux/ktime.h>
   6 #include <linux/mutex.h>
   7 #include <asm/unaligned.h>
   8 #include <cxlpci.h>
   9 #include <cxlmem.h>
  10 #include <cxl.h>
  11
  12 #include "core.h"
  13 #include "trace.h"
  14
  15 static bool cxl_raw_allow_all;
  16
  17 /**
  18  * DOC: cxl mbox
  19  *
  20  * Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The
  21  * implementation is used by the cxl_pci driver to initialize the device
  22  * and implement the cxl_mem.h IOCTL UAPI. It also implements the
  23  * backend of the cxl_pmem_ctl() transport for LIBNVDIMM.
  24  */
  25
  26 #define cxl_for_each_cmd(cmd)                                                  \
  27         for ((cmd) = &cxl_mem_commands[0];                                     \
  28              ((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)
  29
  30 #define CXL_CMD(_id, sin, sout, _flags)                                        \
  31         [CXL_MEM_COMMAND_ID_##_id] = {                                         \
  32         .info = {                                                              \
  33                         .id = CXL_MEM_COMMAND_ID_##_id,                        \
  34                         .size_in = sin,                                        \
  35                         .size_out = sout,                                      \
  36                 },                                                             \
  37         .opcode = CXL_MBOX_OP_##_id,                                           \
  38         .flags = _flags,                                                       \
  39         }
  40
  41 #define CXL_VARIABLE_PAYLOAD    ~0U
  42 /*
  43  * This table defines the supported mailbox commands for the driver. This table
  44  * is made up of a UAPI structure. Non-negative values as parameters in the
  45  * table will be validated against the user's input. For example, if size_in is
  46  * 0, and the user passed in 1, it is an error.
  47  */
  48 static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
  49         CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
  50 #ifdef CONFIG_CXL_MEM_RAW_COMMANDS
  51         CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
  52 #endif
  53         CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
  54         CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
  55         CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
  56         CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),
  57         CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
  58         CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
  59         CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
  60         CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),
  61         CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
  62         CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
  63         CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
  64         CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
  65         CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
  66 };
  67
  68 /*
  69  * Commands that RAW doesn't permit. The rationale for each:
  70  *
  71  * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
  72  * coordination of transaction timeout values at the root bridge level.
  73  *
  74  * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
  75  * and needs to be coordinated with HDM updates.
  76  *
  77  * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
  78  * driver and any writes from userspace invalidates those contents.
  79  *
  80  * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
  81  * to the device after it is marked clean, userspace can not make that
  82  * assertion.
  83  *
  84  * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
  85  * is kept up to date with patrol notifications and error management.
  86  *
  87  * CXL_MBOX_OP_[GET_,INJECT_,CLEAR_]POISON: These commands require kernel
  88  * driver orchestration for safety.
  89  */
  90 static u16 cxl_disabled_raw_commands[] = {
  91         CXL_MBOX_OP_ACTIVATE_FW,
  92         CXL_MBOX_OP_SET_PARTITION_INFO,
  93         CXL_MBOX_OP_SET_LSA,
  94         CXL_MBOX_OP_SET_SHUTDOWN_STATE,
  95         CXL_MBOX_OP_SCAN_MEDIA,
  96         CXL_MBOX_OP_GET_SCAN_MEDIA,
  97         CXL_MBOX_OP_GET_POISON,
  98         CXL_MBOX_OP_INJECT_POISON,
  99         CXL_MBOX_OP_CLEAR_POISON,
 100 };
 101
 102 /*
 103  * Command sets that RAW doesn't permit. All opcodes in this set are
 104  * disabled because they pass plain text security payloads over the
 105  * user/kernel boundary. This functionality is intended to be wrapped
 106  * behind the keys ABI which allows for encrypted payloads in the UAPI
 107  */
 108 static u8 security_command_sets[] = {
 109         0x44, /* Sanitize */
 110         0x45, /* Persistent Memory Data-at-rest Security */
 111         0x46, /* Security Passthrough */
 112 };
 113
 114 static bool cxl_is_security_command(u16 opcode)
 115 {
 116         int i;
 117
 118         for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
 119                 if (security_command_sets[i] == (opcode >> 8))
 120                         return true;
 121         return false;
 122 }
 123
 124 static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,
 125                                          u16 opcode)
 126 {
 127         switch (opcode) {
 128         case CXL_MBOX_OP_SANITIZE:
 129                 set_bit(CXL_SEC_ENABLED_SANITIZE, security->enabled_cmds);
 130                 break;
 131         case CXL_MBOX_OP_SECURE_ERASE:
 132                 set_bit(CXL_SEC_ENABLED_SECURE_ERASE,
 133                         security->enabled_cmds);
 134                 break;
 135         case CXL_MBOX_OP_GET_SECURITY_STATE:
 136                 set_bit(CXL_SEC_ENABLED_GET_SECURITY_STATE,
 137                         security->enabled_cmds);
 138                 break;
 139         case CXL_MBOX_OP_SET_PASSPHRASE:
 140                 set_bit(CXL_SEC_ENABLED_SET_PASSPHRASE,
 141                         security->enabled_cmds);
 142                 break;
 143         case CXL_MBOX_OP_DISABLE_PASSPHRASE:
 144                 set_bit(CXL_SEC_ENABLED_DISABLE_PASSPHRASE,
 145                         security->enabled_cmds);
 146                 break;
 147         case CXL_MBOX_OP_UNLOCK:
 148                 set_bit(CXL_SEC_ENABLED_UNLOCK, security->enabled_cmds);
 149                 break;
 150         case CXL_MBOX_OP_FREEZE_SECURITY:
 151                 set_bit(CXL_SEC_ENABLED_FREEZE_SECURITY,
 152                         security->enabled_cmds);
 153                 break;
 154         case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:
 155                 set_bit(CXL_SEC_ENABLED_PASSPHRASE_SECURE_ERASE,
 156                         security->enabled_cmds);
 157                 break;
 158         default:
 159                 break;
 160         }
 161 }
 162
 163 static bool cxl_is_poison_command(u16 opcode)
 164 {
 165 #define CXL_MBOX_OP_POISON_CMDS 0x43
 166
 167         if ((opcode >> 8) == CXL_MBOX_OP_POISON_CMDS)
 168                 return true;
 169
 170         return false;
 171 }
 172
 173 static void cxl_set_poison_cmd_enabled(struct cxl_poison_state *poison,
 174                                        u16 opcode)
 175 {
 176         switch (opcode) {
 177         case CXL_MBOX_OP_GET_POISON:
 178                 set_bit(CXL_POISON_ENABLED_LIST, poison->enabled_cmds);
 179                 break;
 180         case CXL_MBOX_OP_INJECT_POISON:
 181                 set_bit(CXL_POISON_ENABLED_INJECT, poison->enabled_cmds);
 182                 break;
 183         case CXL_MBOX_OP_CLEAR_POISON:
 184                 set_bit(CXL_POISON_ENABLED_CLEAR, poison->enabled_cmds);
 185                 break;
 186         case CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS:
 187                 set_bit(CXL_POISON_ENABLED_SCAN_CAPS, poison->enabled_cmds);
 188                 break;
 189         case CXL_MBOX_OP_SCAN_MEDIA:
 190                 set_bit(CXL_POISON_ENABLED_SCAN_MEDIA, poison->enabled_cmds);
 191                 break;
 192         case CXL_MBOX_OP_GET_SCAN_MEDIA:
 193                 set_bit(CXL_POISON_ENABLED_SCAN_RESULTS, poison->enabled_cmds);
 194                 break;
 195         default:
 196                 break;
 197         }
 198 }
 199
 200 static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
 201 {
 202         struct cxl_mem_command *c;
 203
 204         cxl_for_each_cmd(c)
 205                 if (c->opcode == opcode)
 206                         return c;
 207
 208         return NULL;
 209 }
 210
 211 static const char *cxl_mem_opcode_to_name(u16 opcode)
 212 {
 213         struct cxl_mem_command *c;
 214
 215         c = cxl_mem_find_command(opcode);
 216         if (!c)
 217                 return NULL;
 218
 219         return cxl_command_names[c->info.id].name;
 220 }
 221
 222 /**
 223  * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
 224  * @mds: The driver data for the operation
 225  * @mbox_cmd: initialized command to execute
 226  *
 227  * Context: Any context.
 228  * Return:
 229  *  * %>=0      - Number of bytes returned in @out.
 230  *  * %-E2BIG   - Payload is too large for hardware.
 231  *  * %-EBUSY   - Couldn't acquire exclusive mailbox access.
 232  *  * %-EFAULT  - Hardware error occurred.
 233  *  * %-ENXIO   - Command completed, but device reported an error.
 234  *  * %-EIO     - Unexpected output size.
 235  *
 236  * Mailbox commands may execute successfully yet the device itself reported an
 237  * error. While this distinction can be useful for commands from userspace, the
 238  * kernel will only be able to use results when both are successful.
 239  */
 240 int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
 241                           struct cxl_mbox_cmd *mbox_cmd)
 242 {
 243         size_t out_size, min_out;
 244         int rc;
 245
 246         if (mbox_cmd->size_in > mds->payload_size ||
 247             mbox_cmd->size_out > mds->payload_size)
 248                 return -E2BIG;
 249
 250         out_size = mbox_cmd->size_out;
 251         min_out = mbox_cmd->min_out;
 252         rc = mds->mbox_send(mds, mbox_cmd);
 253         /*
 254          * EIO is reserved for a payload size mismatch and mbox_send()
 255          * may not return this error.
 256          */
 257         if (WARN_ONCE(rc == -EIO, "Bad return code: -EIO"))
 258                 return -ENXIO;
 259         if (rc)
 260                 return rc;
 261
 262         if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&
 263             mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)
 264                 return cxl_mbox_cmd_rc2errno(mbox_cmd);
 265
 266         if (!out_size)
 267                 return 0;
 268
 269         /*
 270          * Variable sized output needs to at least satisfy the caller's
 271          * minimum if not the fully requested size.
 272          */
 273         if (min_out == 0)
 274                 min_out = out_size;
 275
 276         if (mbox_cmd->size_out < min_out)
 277                 return -EIO;
 278         return 0;
 279 }
 280 EXPORT_SYMBOL_NS_GPL(cxl_internal_send_cmd, CXL);
 281
 282 static bool cxl_mem_raw_command_allowed(u16 opcode)
 283 {
 284         int i;
 285
 286         if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
 287                 return false;
 288
 289         if (security_locked_down(LOCKDOWN_PCI_ACCESS))
 290                 return false;
 291
 292         if (cxl_raw_allow_all)
 293                 return true;
 294
 295         if (cxl_is_security_command(opcode))
 296                 return false;
 297
 298         for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
 299                 if (cxl_disabled_raw_commands[i] == opcode)
 300                         return false;
 301
 302         return true;
 303 }
 304
 305 /**
 306  * cxl_payload_from_user_allowed() - Check contents of in_payload.
 307  * @opcode: The mailbox command opcode.
 308  * @payload_in: Pointer to the input payload passed in from user space.
 309  *
 310  * Return:
 311  *  * true      - payload_in passes check for @opcode.
 312  *  * false     - payload_in contains invalid or unsupported values.
 313  *
 314  * The driver may inspect payload contents before sending a mailbox
 315  * command from user space to the device. The intent is to reject
 316  * commands with input payloads that are known to be unsafe. This
 317  * check is not intended to replace the users careful selection of
 318  * mailbox command parameters and makes no guarantee that the user
 319  * command will succeed, nor that it is appropriate.
 320  *
 321  * The specific checks are determined by the opcode.
 322  */
 323 static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)
 324 {
 325         switch (opcode) {
 326         case CXL_MBOX_OP_SET_PARTITION_INFO: {
 327                 struct cxl_mbox_set_partition_info *pi = payload_in;
 328
 329                 if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)
 330                         return false;
 331                 break;
 332         }
 333         default:
 334                 break;
 335         }
 336         return true;
 337 }
 338
 339 static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
 340                              struct cxl_memdev_state *mds, u16 opcode,
 341                              size_t in_size, size_t out_size, u64 in_payload)
 342 {
 343         *mbox = (struct cxl_mbox_cmd) {
 344                 .opcode = opcode,
 345                 .size_in = in_size,
 346         };
 347
 348         if (in_size) {
 349                 mbox->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
 350                                                 in_size);
 351                 if (IS_ERR(mbox->payload_in))
 352                         return PTR_ERR(mbox->payload_in);
 353
 354                 if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {
 355                         dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n",
 356                                 cxl_mem_opcode_to_name(opcode));
 357                         kvfree(mbox->payload_in);
 358                         return -EBUSY;
 359                 }
 360         }
 361
 362         /* Prepare to handle a full payload for variable sized output */
 363         if (out_size == CXL_VARIABLE_PAYLOAD)
 364                 mbox->size_out = mds->payload_size;
 365         else
 366                 mbox->size_out = out_size;
 367
 368         if (mbox->size_out) {
 369                 mbox->payload_out = kvzalloc(mbox->size_out, GFP_KERNEL);
 370                 if (!mbox->payload_out) {
 371                         kvfree(mbox->payload_in);
 372                         return -ENOMEM;
 373                 }
 374         }
 375         return 0;
 376 }
 377
 378 static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
 379 {
 380         kvfree(mbox->payload_in);
 381         kvfree(mbox->payload_out);
 382 }
 383
 384 static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
 385                               const struct cxl_send_command *send_cmd,
 386                               struct cxl_memdev_state *mds)
 387 {
 388         if (send_cmd->raw.rsvd)
 389                 return -EINVAL;
 390
 391         /*
 392          * Unlike supported commands, the output size of RAW commands
 393          * gets passed along without further checking, so it must be
 394          * validated here.
 395          */
 396         if (send_cmd->out.size > mds->payload_size)
 397                 return -EINVAL;
 398
 399         if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
 400                 return -EPERM;
 401
 402         dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n");
 403
 404         *mem_cmd = (struct cxl_mem_command) {
 405                 .info = {
 406                         .id = CXL_MEM_COMMAND_ID_RAW,
 407                         .size_in = send_cmd->in.size,
 408                         .size_out = send_cmd->out.size,
 409                 },
 410                 .opcode = send_cmd->raw.opcode
 411         };
 412
 413         return 0;
 414 }
 415
 416 static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
 417                           const struct cxl_send_command *send_cmd,
 418                           struct cxl_memdev_state *mds)
 419 {
 420         struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
 421         const struct cxl_command_info *info = &c->info;
 422
 423         if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
 424                 return -EINVAL;
 425
 426         if (send_cmd->rsvd)
 427                 return -EINVAL;
 428
 429         if (send_cmd->in.rsvd || send_cmd->out.rsvd)
 430                 return -EINVAL;
 431
 432         /* Check that the command is enabled for hardware */
 433         if (!test_bit(info->id, mds->enabled_cmds))
 434                 return -ENOTTY;
 435
 436         /* Check that the command is not claimed for exclusive kernel use */
 437         if (test_bit(info->id, mds->exclusive_cmds))
 438                 return -EBUSY;
 439
 440         /* Check the input buffer is the expected size */
 441         if ((info->size_in != CXL_VARIABLE_PAYLOAD) &&
 442             (info->size_in != send_cmd->in.size))
 443                 return -ENOMEM;
 444
 445         /* Check the output buffer is at least large enough */
 446         if ((info->size_out != CXL_VARIABLE_PAYLOAD) &&
 447             (send_cmd->out.size < info->size_out))
 448                 return -ENOMEM;
 449
 450         *mem_cmd = (struct cxl_mem_command) {
 451                 .info = {
 452                         .id = info->id,
 453                         .flags = info->flags,
 454                         .size_in = send_cmd->in.size,
 455                         .size_out = send_cmd->out.size,
 456                 },
 457                 .opcode = c->opcode
 458         };
 459
 460         return 0;
 461 }
 462
 463 /**
 464  * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
 465  * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
 466  * @mds: The driver data for the operation
 467  * @send_cmd: &struct cxl_send_command copied in from userspace.
 468  *
 469  * Return:
 470  *  * %0        - @out_cmd is ready to send.
 471  *  * %-ENOTTY  - Invalid command specified.
 472  *  * %-EINVAL  - Reserved fields or invalid values were used.
 473  *  * %-ENOMEM  - Input or output buffer wasn't sized properly.
 474  *  * %-EPERM   - Attempted to use a protected command.
 475  *  * %-EBUSY   - Kernel has claimed exclusive access to this opcode
 476  *
 477  * The result of this command is a fully validated command in @mbox_cmd that is
 478  * safe to send to the hardware.
 479  */
 480 static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
 481                                       struct cxl_memdev_state *mds,
 482                                       const struct cxl_send_command *send_cmd)
 483 {
 484         struct cxl_mem_command mem_cmd;
 485         int rc;
 486
 487         if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
 488                 return -ENOTTY;
 489
 490         /*
 491          * The user can never specify an input payload larger than what hardware
 492          * supports, but output can be arbitrarily large (simply write out as
 493          * much data as the hardware provides).
 494          */
 495         if (send_cmd->in.size > mds->payload_size)
 496                 return -EINVAL;
 497
 498         /* Sanitize and construct a cxl_mem_command */
 499         if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
 500                 rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds);
 501         else
 502                 rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds);
 503
 504         if (rc)
 505                 return rc;
 506
 507         /* Sanitize and construct a cxl_mbox_cmd */
 508         return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode,
 509                                  mem_cmd.info.size_in, mem_cmd.info.size_out,
 510                                  send_cmd->in.payload);
 511 }
 512
 513 int cxl_query_cmd(struct cxl_memdev *cxlmd,
 514                   struct cxl_mem_query_commands __user *q)
 515 {
 516         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 517         struct device *dev = &cxlmd->dev;
 518         struct cxl_mem_command *cmd;
 519         u32 n_commands;
 520         int j = 0;
 521
 522         dev_dbg(dev, "Query IOCTL\n");
 523
 524         if (get_user(n_commands, &q->n_commands))
 525                 return -EFAULT;
 526
 527         /* returns the total number if 0 elements are requested. */
 528         if (n_commands == 0)
 529                 return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
 530
 531         /*
 532          * otherwise, return max(n_commands, total commands) cxl_command_info
 533          * structures.
 534          */
 535         cxl_for_each_cmd(cmd) {
 536                 struct cxl_command_info info = cmd->info;
 537
 538                 if (test_bit(info.id, mds->enabled_cmds))
 539                         info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
 540                 if (test_bit(info.id, mds->exclusive_cmds))
 541                         info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
 542
 543                 if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
 544                         return -EFAULT;
 545
 546                 if (j == n_commands)
 547                         break;
 548         }
 549
 550         return 0;
 551 }
 552
 553 /**
 554  * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
 555  * @mds: The driver data for the operation
 556  * @mbox_cmd: The validated mailbox command.
 557  * @out_payload: Pointer to userspace's output payload.
 558  * @size_out: (Input) Max payload size to copy out.
 559  *            (Output) Payload size hardware generated.
 560  * @retval: Hardware generated return code from the operation.
 561  *
 562  * Return:
 563  *  * %0        - Mailbox transaction succeeded. This implies the mailbox
 564  *                protocol completed successfully not that the operation itself
 565  *                was successful.
 566  *  * %-ENOMEM  - Couldn't allocate a bounce buffer.
 567  *  * %-EFAULT  - Something happened with copy_to/from_user.
 568  *  * %-EINTR   - Mailbox acquisition interrupted.
 569  *  * %-EXXX    - Transaction level failures.
 570  *
 571  * Dispatches a mailbox command on behalf of a userspace request.
 572  * The output payload is copied to userspace.
 573  *
 574  * See cxl_send_cmd().
 575  */
 576 static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
 577                                         struct cxl_mbox_cmd *mbox_cmd,
 578                                         u64 out_payload, s32 *size_out,
 579                                         u32 *retval)
 580 {
 581         struct device *dev = mds->cxlds.dev;
 582         int rc;
 583
 584         dev_dbg(dev,
 585                 "Submitting %s command for user\n"
 586                 "\topcode: %x\n"
 587                 "\tsize: %zx\n",
 588                 cxl_mem_opcode_to_name(mbox_cmd->opcode),
 589                 mbox_cmd->opcode, mbox_cmd->size_in);
 590
 591         rc = mds->mbox_send(mds, mbox_cmd);
 592         if (rc)
 593                 goto out;
 594
 595         /*
 596          * @size_out contains the max size that's allowed to be written back out
 597          * to userspace. While the payload may have written more output than
 598          * this it will have to be ignored.
 599          */
 600         if (mbox_cmd->size_out) {
 601                 dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,
 602                               "Invalid return size\n");
 603                 if (copy_to_user(u64_to_user_ptr(out_payload),
 604                                  mbox_cmd->payload_out, mbox_cmd->size_out)) {
 605                         rc = -EFAULT;
 606                         goto out;
 607                 }
 608         }
 609
 610         *size_out = mbox_cmd->size_out;
 611         *retval = mbox_cmd->return_code;
 612
 613 out:
 614         cxl_mbox_cmd_dtor(mbox_cmd);
 615         return rc;
 616 }
 617
 618 int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
 619 {
 620         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 621         struct device *dev = &cxlmd->dev;
 622         struct cxl_send_command send;
 623         struct cxl_mbox_cmd mbox_cmd;
 624         int rc;
 625
 626         dev_dbg(dev, "Send IOCTL\n");
 627
 628         if (copy_from_user(&send, s, sizeof(send)))
 629                 return -EFAULT;
 630
 631         rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send);
 632         if (rc)
 633                 return rc;
 634
 635         rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload,
 636                                           &send.out.size, &send.retval);
 637         if (rc)
 638                 return rc;
 639
 640         if (copy_to_user(s, &send, sizeof(send)))
 641                 return -EFAULT;
 642
 643         return 0;
 644 }
 645
 646 static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
 647                         u32 *size, u8 *out)
 648 {
 649         u32 remaining = *size;
 650         u32 offset = 0;
 651
 652         while (remaining) {
 653                 u32 xfer_size = min_t(u32, remaining, mds->payload_size);
 654                 struct cxl_mbox_cmd mbox_cmd;
 655                 struct cxl_mbox_get_log log;
 656                 int rc;
 657
 658                 log = (struct cxl_mbox_get_log) {
 659                         .uuid = *uuid,
 660                         .offset = cpu_to_le32(offset),
 661                         .length = cpu_to_le32(xfer_size),
 662                 };
 663
 664                 mbox_cmd = (struct cxl_mbox_cmd) {
 665                         .opcode = CXL_MBOX_OP_GET_LOG,
 666                         .size_in = sizeof(log),
 667                         .payload_in = &log,
 668                         .size_out = xfer_size,
 669                         .payload_out = out,
 670                 };
 671
 672                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 673
 674                 /*
 675                  * The output payload length that indicates the number
 676                  * of valid bytes can be smaller than the Log buffer
 677                  * size.
 678                  */
 679                 if (rc == -EIO && mbox_cmd.size_out < xfer_size) {
 680                         offset += mbox_cmd.size_out;
 681                         break;
 682                 }
 683
 684                 if (rc < 0)
 685                         return rc;
 686
 687                 out += xfer_size;
 688                 remaining -= xfer_size;
 689                 offset += xfer_size;
 690         }
 691
 692         *size = offset;
 693
 694         return 0;
 695 }
 696
 697 /**
 698  * cxl_walk_cel() - Walk through the Command Effects Log.
 699  * @mds: The driver data for the operation
 700  * @size: Length of the Command Effects Log.
 701  * @cel: CEL
 702  *
 703  * Iterate over each entry in the CEL and determine if the driver supports the
 704  * command. If so, the command is enabled for the device and can be used later.
 705  */
 706 static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
 707 {
 708         struct cxl_cel_entry *cel_entry;
 709         const int cel_entries = size / sizeof(*cel_entry);
 710         struct device *dev = mds->cxlds.dev;
 711         int i;
 712
 713         cel_entry = (struct cxl_cel_entry *) cel;
 714
 715         for (i = 0; i < cel_entries; i++) {
 716                 u16 opcode = le16_to_cpu(cel_entry[i].opcode);
 717                 struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
 718
 719                 if (!cmd && (!cxl_is_poison_command(opcode) ||
 720                              !cxl_is_security_command(opcode))) {
 721                         dev_dbg(dev,
 722                                 "Opcode 0x%04x unsupported by driver\n", opcode);
 723                         continue;
 724                 }
 725
 726                 if (cmd)
 727                         set_bit(cmd->info.id, mds->enabled_cmds);
 728
 729                 if (cxl_is_poison_command(opcode))
 730                         cxl_set_poison_cmd_enabled(&mds->poison, opcode);
 731
 732                 if (cxl_is_security_command(opcode))
 733                         cxl_set_security_cmd_enabled(&mds->security, opcode);
 734
 735                 dev_dbg(dev, "Opcode 0x%04x enabled\n", opcode);
 736         }
 737 }
 738
 739 static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
 740 {
 741         struct cxl_mbox_get_supported_logs *ret;
 742         struct cxl_mbox_cmd mbox_cmd;
 743         int rc;
 744
 745         ret = kvmalloc(mds->payload_size, GFP_KERNEL);
 746         if (!ret)
 747                 return ERR_PTR(-ENOMEM);
 748
 749         mbox_cmd = (struct cxl_mbox_cmd) {
 750                 .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
 751                 .size_out = mds->payload_size,
 752                 .payload_out = ret,
 753                 /* At least the record number field must be valid */
 754                 .min_out = 2,
 755         };
 756         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 757         if (rc < 0) {
 758                 kvfree(ret);
 759                 return ERR_PTR(rc);
 760         }
 761
 762
 763         return ret;
 764 }
 765
 766 enum {
 767         CEL_UUID,
 768         VENDOR_DEBUG_UUID,
 769 };
 770
 771 /* See CXL 2.0 Table 170. Get Log Input Payload */
 772 static const uuid_t log_uuid[] = {
 773         [CEL_UUID] = DEFINE_CXL_CEL_UUID,
 774         [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
 775 };
 776
 777 /**
 778  * cxl_enumerate_cmds() - Enumerate commands for a device.
 779  * @mds: The driver data for the operation
 780  *
 781  * Returns 0 if enumerate completed successfully.
 782  *
 783  * CXL devices have optional support for certain commands. This function will
 784  * determine the set of supported commands for the hardware and update the
 785  * enabled_cmds bitmap in the @mds.
 786  */
 787 int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
 788 {
 789         struct cxl_mbox_get_supported_logs *gsl;
 790         struct device *dev = mds->cxlds.dev;
 791         struct cxl_mem_command *cmd;
 792         int i, rc;
 793
 794         gsl = cxl_get_gsl(mds);
 795         if (IS_ERR(gsl))
 796                 return PTR_ERR(gsl);
 797
 798         rc = -ENOENT;
 799         for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
 800                 u32 size = le32_to_cpu(gsl->entry[i].size);
 801                 uuid_t uuid = gsl->entry[i].uuid;
 802                 u8 *log;
 803
 804                 dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
 805
 806                 if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
 807                         continue;
 808
 809                 log = kvmalloc(size, GFP_KERNEL);
 810                 if (!log) {
 811                         rc = -ENOMEM;
 812                         goto out;
 813                 }
 814
 815                 rc = cxl_xfer_log(mds, &uuid, &size, log);
 816                 if (rc) {
 817                         kvfree(log);
 818                         goto out;
 819                 }
 820
 821                 cxl_walk_cel(mds, size, log);
 822                 kvfree(log);
 823
 824                 /* In case CEL was bogus, enable some default commands. */
 825                 cxl_for_each_cmd(cmd)
 826                         if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
 827                                 set_bit(cmd->info.id, mds->enabled_cmds);
 828
 829                 /* Found the required CEL */
 830                 rc = 0;
 831         }
 832 out:
 833         kvfree(gsl);
 834         return rc;
 835 }
 836 EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL);
 837
 838 /*
 839  * General Media Event Record
 840  * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43
 841  */
 842 static const uuid_t gen_media_event_uuid =
 843         UUID_INIT(0xfbcd0a77, 0xc260, 0x417f,
 844                   0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6);
 845
 846 /*
 847  * DRAM Event Record
 848  * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44
 849  */
 850 static const uuid_t dram_event_uuid =
 851         UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab,
 852                   0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24);
 853
 854 /*
 855  * Memory Module Event Record
 856  * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45
 857  */
 858 static const uuid_t mem_mod_event_uuid =
 859         UUID_INIT(0xfe927475, 0xdd59, 0x4339,
 860                   0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74);
 861
 862 static void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
 863                                    enum cxl_event_log_type type,
 864                                    struct cxl_event_record_raw *record)
 865 {
 866         uuid_t *id = &record->hdr.id;
 867
 868         if (uuid_equal(id, &gen_media_event_uuid)) {
 869                 struct cxl_event_gen_media *rec =
 870                                 (struct cxl_event_gen_media *)record;
 871
 872                 trace_cxl_general_media(cxlmd, type, rec);
 873         } else if (uuid_equal(id, &dram_event_uuid)) {
 874                 struct cxl_event_dram *rec = (struct cxl_event_dram *)record;
 875
 876                 trace_cxl_dram(cxlmd, type, rec);
 877         } else if (uuid_equal(id, &mem_mod_event_uuid)) {
 878                 struct cxl_event_mem_module *rec =
 879                                 (struct cxl_event_mem_module *)record;
 880
 881                 trace_cxl_memory_module(cxlmd, type, rec);
 882         } else {
 883                 /* For unknown record types print just the header */
 884                 trace_cxl_generic_event(cxlmd, type, record);
 885         }
 886 }
 887
 888 static int cxl_clear_event_record(struct cxl_memdev_state *mds,
 889                                   enum cxl_event_log_type log,
 890                                   struct cxl_get_event_payload *get_pl)
 891 {
 892         struct cxl_mbox_clear_event_payload *payload;
 893         u16 total = le16_to_cpu(get_pl->record_count);
 894         u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
 895         size_t pl_size = struct_size(payload, handles, max_handles);
 896         struct cxl_mbox_cmd mbox_cmd;
 897         u16 cnt;
 898         int rc = 0;
 899         int i;
 900
 901         /* Payload size may limit the max handles */
 902         if (pl_size > mds->payload_size) {
 903                 max_handles = (mds->payload_size - sizeof(*payload)) /
 904                               sizeof(__le16);
 905                 pl_size = struct_size(payload, handles, max_handles);
 906         }
 907
 908         payload = kvzalloc(pl_size, GFP_KERNEL);
 909         if (!payload)
 910                 return -ENOMEM;
 911
 912         *payload = (struct cxl_mbox_clear_event_payload) {
 913                 .event_log = log,
 914         };
 915
 916         mbox_cmd = (struct cxl_mbox_cmd) {
 917                 .opcode = CXL_MBOX_OP_CLEAR_EVENT_RECORD,
 918                 .payload_in = payload,
 919                 .size_in = pl_size,
 920         };
 921
 922         /*
 923          * Clear Event Records uses u8 for the handle cnt while Get Event
 924          * Record can return up to 0xffff records.
 925          */
 926         i = 0;
 927         for (cnt = 0; cnt < total; cnt++) {
 928                 payload->handles[i++] = get_pl->records[cnt].hdr.handle;
 929                 dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,
 930                         le16_to_cpu(payload->handles[i]));
 931
 932                 if (i == max_handles) {
 933                         payload->nr_recs = i;
 934                         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 935                         if (rc)
 936                                 goto free_pl;
 937                         i = 0;
 938                 }
 939         }
 940
 941         /* Clear what is left if any */
 942         if (i) {
 943                 payload->nr_recs = i;
 944                 mbox_cmd.size_in = struct_size(payload, handles, i);
 945                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 946                 if (rc)
 947                         goto free_pl;
 948         }
 949
 950 free_pl:
 951         kvfree(payload);
 952         return rc;
 953 }
 954
 955 static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
 956                                     enum cxl_event_log_type type)
 957 {
 958         struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
 959         struct device *dev = mds->cxlds.dev;
 960         struct cxl_get_event_payload *payload;
 961         struct cxl_mbox_cmd mbox_cmd;
 962         u8 log_type = type;
 963         u16 nr_rec;
 964
 965         mutex_lock(&mds->event.log_lock);
 966         payload = mds->event.buf;
 967
 968         mbox_cmd = (struct cxl_mbox_cmd) {
 969                 .opcode = CXL_MBOX_OP_GET_EVENT_RECORD,
 970                 .payload_in = &log_type,
 971                 .size_in = sizeof(log_type),
 972                 .payload_out = payload,
 973                 .size_out = mds->payload_size,
 974                 .min_out = struct_size(payload, records, 0),
 975         };
 976
 977         do {
 978                 int rc, i;
 979
 980                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 981                 if (rc) {
 982                         dev_err_ratelimited(dev,
 983                                 "Event log '%d': Failed to query event records : %d",
 984                                 type, rc);
 985                         break;
 986                 }
 987
 988                 nr_rec = le16_to_cpu(payload->record_count);
 989                 if (!nr_rec)
 990                         break;
 991
 992                 for (i = 0; i < nr_rec; i++)
 993                         cxl_event_trace_record(cxlmd, type,
 994                                                &payload->records[i]);
 995
 996                 if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)
 997                         trace_cxl_overflow(cxlmd, type, payload);
 998
 999                 rc = cxl_clear_event_record(mds, type, payload);
1000                 if (rc) {
1001                         dev_err_ratelimited(dev,
1002                                 "Event log '%d': Failed to clear events : %d",
1003                                 type, rc);
1004                         break;
1005                 }
1006         } while (nr_rec);
1007
1008         mutex_unlock(&mds->event.log_lock);
1009 }
1010
1011 /**
1012  * cxl_mem_get_event_records - Get Event Records from the device
1013  * @mds: The driver data for the operation
1014  * @status: Event Status register value identifying which events are available.
1015  *
1016  * Retrieve all event records available on the device, report them as trace
1017  * events, and clear them.
1018  *
1019  * See CXL rev 3.0 @8.2.9.2.2 Get Event Records
1020  * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records
1021  */
1022 void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)
1023 {
1024         dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);
1025
1026         if (status & CXLDEV_EVENT_STATUS_FATAL)
1027                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);
1028         if (status & CXLDEV_EVENT_STATUS_FAIL)
1029                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);
1030         if (status & CXLDEV_EVENT_STATUS_WARN)
1031                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);
1032         if (status & CXLDEV_EVENT_STATUS_INFO)
1033                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);
1034 }
1035 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);
1036
1037 /**
1038  * cxl_mem_get_partition_info - Get partition info
1039  * @mds: The driver data for the operation
1040  *
1041  * Retrieve the current partition info for the device specified.  The active
1042  * values are the current capacity in bytes.  If not 0, the 'next' values are
1043  * the pending values, in bytes, which take affect on next cold reset.
1044  *
1045  * Return: 0 if no error: or the result of the mailbox command.
1046  *
1047  * See CXL @8.2.9.5.2.1 Get Partition Info
1048  */
1049 static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
1050 {
1051         struct cxl_mbox_get_partition_info pi;
1052         struct cxl_mbox_cmd mbox_cmd;
1053         int rc;
1054
1055         mbox_cmd = (struct cxl_mbox_cmd) {
1056                 .opcode = CXL_MBOX_OP_GET_PARTITION_INFO,
1057                 .size_out = sizeof(pi),
1058                 .payload_out = &pi,
1059         };
1060         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1061         if (rc)
1062                 return rc;
1063
1064         mds->active_volatile_bytes =
1065                 le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1066         mds->active_persistent_bytes =
1067                 le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
1068         mds->next_volatile_bytes =
1069                 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1070         mds->next_persistent_bytes =
1071                 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1072
1073         return 0;
1074 }
1075
1076 /**
1077  * cxl_dev_state_identify() - Send the IDENTIFY command to the device.
1078  * @mds: The driver data for the operation
1079  *
1080  * Return: 0 if identify was executed successfully or media not ready.
1081  *
1082  * This will dispatch the identify command to the device and on success populate
1083  * structures to be exported to sysfs.
1084  */
1085 int cxl_dev_state_identify(struct cxl_memdev_state *mds)
1086 {
1087         /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
1088         struct cxl_mbox_identify id;
1089         struct cxl_mbox_cmd mbox_cmd;
1090         u32 val;
1091         int rc;
1092
1093         if (!mds->cxlds.media_ready)
1094                 return 0;
1095
1096         mbox_cmd = (struct cxl_mbox_cmd) {
1097                 .opcode = CXL_MBOX_OP_IDENTIFY,
1098                 .size_out = sizeof(id),
1099                 .payload_out = &id,
1100         };
1101         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1102         if (rc < 0)
1103                 return rc;
1104
1105         mds->total_bytes =
1106                 le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
1107         mds->volatile_only_bytes =
1108                 le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
1109         mds->persistent_only_bytes =
1110                 le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
1111         mds->partition_align_bytes =
1112                 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
1113
1114         mds->lsa_size = le32_to_cpu(id.lsa_size);
1115         memcpy(mds->firmware_version, id.fw_revision,
1116                sizeof(id.fw_revision));
1117
1118         if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {
1119                 val = get_unaligned_le24(id.poison_list_max_mer);
1120                 mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
1121         }
1122
1123         return 0;
1124 }
1125 EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);
1126
1127 /**
1128  * cxl_mem_sanitize() - Send a sanitization command to the device.
1129  * @mds: The device data for the operation
1130  * @cmd: The specific sanitization command opcode
1131  *
1132  * Return: 0 if the command was executed successfully, regardless of
1133  * whether or not the actual security operation is done in the background,
1134  * such as for the Sanitize case.
1135  * Error return values can be the result of the mailbox command, -EINVAL
1136  * when security requirements are not met or invalid contexts.
1137  *
1138  * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.
1139  */
1140 int cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
1141 {
1142         int rc;
1143         u32 sec_out = 0;
1144         struct cxl_get_security_output {
1145                 __le32 flags;
1146         } out;
1147         struct cxl_mbox_cmd sec_cmd = {
1148                 .opcode = CXL_MBOX_OP_GET_SECURITY_STATE,
1149                 .payload_out = &out,
1150                 .size_out = sizeof(out),
1151         };
1152         struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
1153         struct cxl_dev_state *cxlds = &mds->cxlds;
1154
1155         if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
1156                 return -EINVAL;
1157
1158         rc = cxl_internal_send_cmd(mds, &sec_cmd);
1159         if (rc < 0) {
1160                 dev_err(cxlds->dev, "Failed to get security state : %d", rc);
1161                 return rc;
1162         }
1163
1164         /*
1165          * Prior to using these commands, any security applied to
1166          * the user data areas of the device shall be DISABLED (or
1167          * UNLOCKED for secure erase case).
1168          */
1169         sec_out = le32_to_cpu(out.flags);
1170         if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)
1171                 return -EINVAL;
1172
1173         if (cmd == CXL_MBOX_OP_SECURE_ERASE &&
1174             sec_out & CXL_PMEM_SEC_STATE_LOCKED)
1175                 return -EINVAL;
1176
1177         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1178         if (rc < 0) {
1179                 dev_err(cxlds->dev, "Failed to sanitize device : %d", rc);
1180                 return rc;
1181         }
1182
1183         return 0;
1184 }
1185 EXPORT_SYMBOL_NS_GPL(cxl_mem_sanitize, CXL);
1186
1187 static int add_dpa_res(struct device *dev, struct resource *parent,
1188                        struct resource *res, resource_size_t start,
1189                        resource_size_t size, const char *type)
1190 {
1191         int rc;
1192
1193         res->name = type;
1194         res->start = start;
1195         res->end = start + size - 1;
1196         res->flags = IORESOURCE_MEM;
1197         if (resource_size(res) == 0) {
1198                 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
1199                 return 0;
1200         }
1201         rc = request_resource(parent, res);
1202         if (rc) {
1203                 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
1204                         res, rc);
1205                 return rc;
1206         }
1207
1208         dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
1209
1210         return 0;
1211 }
1212
1213 int cxl_mem_create_range_info(struct cxl_memdev_state *mds)
1214 {
1215         struct cxl_dev_state *cxlds = &mds->cxlds;
1216         struct device *dev = cxlds->dev;
1217         int rc;
1218
1219         if (!cxlds->media_ready) {
1220                 cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
1221                 cxlds->ram_res = DEFINE_RES_MEM(0, 0);
1222                 cxlds->pmem_res = DEFINE_RES_MEM(0, 0);
1223                 return 0;
1224         }
1225
1226         cxlds->dpa_res =
1227                 (struct resource)DEFINE_RES_MEM(0, mds->total_bytes);
1228
1229         if (mds->partition_align_bytes == 0) {
1230                 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
1231                                  mds->volatile_only_bytes, "ram");
1232                 if (rc)
1233                         return rc;
1234                 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
1235                                    mds->volatile_only_bytes,
1236                                    mds->persistent_only_bytes, "pmem");
1237         }
1238
1239         rc = cxl_mem_get_partition_info(mds);
1240         if (rc) {
1241                 dev_err(dev, "Failed to query partition information\n");
1242                 return rc;
1243         }
1244
1245         rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
1246                          mds->active_volatile_bytes, "ram");
1247         if (rc)
1248                 return rc;
1249         return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
1250                            mds->active_volatile_bytes,
1251                            mds->active_persistent_bytes, "pmem");
1252 }
1253 EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
1254
1255 int cxl_set_timestamp(struct cxl_memdev_state *mds)
1256 {
1257         struct cxl_mbox_cmd mbox_cmd;
1258         struct cxl_mbox_set_timestamp_in pi;
1259         int rc;
1260
1261         pi.timestamp = cpu_to_le64(ktime_get_real_ns());
1262         mbox_cmd = (struct cxl_mbox_cmd) {
1263                 .opcode = CXL_MBOX_OP_SET_TIMESTAMP,
1264                 .size_in = sizeof(pi),
1265                 .payload_in = &pi,
1266         };
1267
1268         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1269         /*
1270          * Command is optional. Devices may have another way of providing
1271          * a timestamp, or may return all 0s in timestamp fields.
1272          * Don't report an error if this command isn't supported
1273          */
1274         if (rc && (mbox_cmd.return_code != CXL_MBOX_CMD_RC_UNSUPPORTED))
1275                 return rc;
1276
1277         return 0;
1278 }
1279 EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, CXL);
1280
1281 int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
1282                        struct cxl_region *cxlr)
1283 {
1284         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1285         struct cxl_mbox_poison_out *po;
1286         struct cxl_mbox_poison_in pi;
1287         struct cxl_mbox_cmd mbox_cmd;
1288         int nr_records = 0;
1289         int rc;
1290
1291         rc = mutex_lock_interruptible(&mds->poison.lock);
1292         if (rc)
1293                 return rc;
1294
1295         po = mds->poison.list_out;
1296         pi.offset = cpu_to_le64(offset);
1297         pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);
1298
1299         mbox_cmd = (struct cxl_mbox_cmd) {
1300                 .opcode = CXL_MBOX_OP_GET_POISON,
1301                 .size_in = sizeof(pi),
1302                 .payload_in = &pi,
1303                 .size_out = mds->payload_size,
1304                 .payload_out = po,
1305                 .min_out = struct_size(po, record, 0),
1306         };
1307
1308         do {
1309                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1310                 if (rc)
1311                         break;
1312
1313                 for (int i = 0; i < le16_to_cpu(po->count); i++)
1314                         trace_cxl_poison(cxlmd, cxlr, &po->record[i],
1315                                          po->flags, po->overflow_ts,
1316                                          CXL_POISON_TRACE_LIST);
1317
1318                 /* Protect against an uncleared _FLAG_MORE */
1319                 nr_records = nr_records + le16_to_cpu(po->count);
1320                 if (nr_records >= mds->poison.max_errors) {
1321                         dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",
1322                                 nr_records);
1323                         break;
1324                 }
1325         } while (po->flags & CXL_POISON_FLAG_MORE);
1326
1327         mutex_unlock(&mds->poison.lock);
1328         return rc;
1329 }
1330 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL);
1331
1332 static void free_poison_buf(void *buf)
1333 {
1334         kvfree(buf);
1335 }
1336
1337 /* Get Poison List output buffer is protected by mds->poison.lock */
1338 static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
1339 {
1340         mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL);
1341         if (!mds->poison.list_out)
1342                 return -ENOMEM;
1343
1344         return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,
1345                                         mds->poison.list_out);
1346 }
1347
1348 int cxl_poison_state_init(struct cxl_memdev_state *mds)
1349 {
1350         int rc;
1351
1352         if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))
1353                 return 0;
1354
1355         rc = cxl_poison_alloc_buf(mds);
1356         if (rc) {
1357                 clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);
1358                 return rc;
1359         }
1360
1361         mutex_init(&mds->poison.lock);
1362         return 0;
1363 }
1364 EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
1365
1366 struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
1367 {
1368         struct cxl_memdev_state *mds;
1369
1370         mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
1371         if (!mds) {
1372                 dev_err(dev, "No memory available\n");
1373                 return ERR_PTR(-ENOMEM);
1374         }
1375
1376         mutex_init(&mds->mbox_mutex);
1377         mutex_init(&mds->event.log_lock);
1378         mds->cxlds.dev = dev;
1379         mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
1380
1381         return mds;
1382 }
1383 EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, CXL);
1384
1385 void __init cxl_mbox_init(void)
1386 {
1387         struct dentry *mbox_debugfs;
1388
1389         mbox_debugfs = cxl_debugfs_create_dir("mbox");
1390         debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
1391                             &cxl_raw_allow_all);
1392 }