drivers/cxl/core/cdat.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2023 Intel Corporation. All rights reserved. */
   3 #include <linux/acpi.h>
   4 #include <linux/xarray.h>
   5 #include <linux/fw_table.h>
   6 #include <linux/node.h>
   7 #include <linux/overflow.h>
   8 #include "cxlpci.h"
   9 #include "cxlmem.h"
  10 #include "core.h"
  11 #include "cxl.h"
  12
  13 struct dsmas_entry {
  14         struct range dpa_range;
  15         u8 handle;
  16         struct access_coordinate coord;
  17
  18         int entries;
  19         int qos_class;
  20 };
  21
  22 static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,
  23                               const unsigned long end)
  24 {
  25         struct acpi_cdat_header *hdr = &header->cdat;
  26         struct acpi_cdat_dsmas *dsmas;
  27         int size = sizeof(*hdr) + sizeof(*dsmas);
  28         struct xarray *dsmas_xa = arg;
  29         struct dsmas_entry *dent;
  30         u16 len;
  31         int rc;
  32
  33         len = le16_to_cpu((__force __le16)hdr->length);
  34         if (len != size || (unsigned long)hdr + len > end) {
  35                 pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len);
  36                 return -EINVAL;
  37         }
  38
  39         /* Skip common header */
  40         dsmas = (struct acpi_cdat_dsmas *)(hdr + 1);
  41
  42         dent = kzalloc(sizeof(*dent), GFP_KERNEL);
  43         if (!dent)
  44                 return -ENOMEM;
  45
  46         dent->handle = dsmas->dsmad_handle;
  47         dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);
  48         dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +
  49                               le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;
  50
  51         rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL);
  52         if (rc) {
  53                 kfree(dent);
  54                 return rc;
  55         }
  56
  57         return 0;
  58 }
  59
  60 static void cxl_access_coordinate_set(struct access_coordinate *coord,
  61                                       int access, unsigned int val)
  62 {
  63         switch (access) {
  64         case ACPI_HMAT_ACCESS_LATENCY:
  65                 coord->read_latency = val;
  66                 coord->write_latency = val;
  67                 break;
  68         case ACPI_HMAT_READ_LATENCY:
  69                 coord->read_latency = val;
  70                 break;
  71         case ACPI_HMAT_WRITE_LATENCY:
  72                 coord->write_latency = val;
  73                 break;
  74         case ACPI_HMAT_ACCESS_BANDWIDTH:
  75                 coord->read_bandwidth = val;
  76                 coord->write_bandwidth = val;
  77                 break;
  78         case ACPI_HMAT_READ_BANDWIDTH:
  79                 coord->read_bandwidth = val;
  80                 break;
  81         case ACPI_HMAT_WRITE_BANDWIDTH:
  82                 coord->write_bandwidth = val;
  83                 break;
  84         }
  85 }
  86
  87 static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg,
  88                                const unsigned long end)
  89 {
  90         struct acpi_cdat_header *hdr = &header->cdat;
  91         struct acpi_cdat_dslbis *dslbis;
  92         int size = sizeof(*hdr) + sizeof(*dslbis);
  93         struct xarray *dsmas_xa = arg;
  94         struct dsmas_entry *dent;
  95         __le64 le_base;
  96         __le16 le_val;
  97         u64 val;
  98         u16 len;
  99         int rc;
 100
 101         len = le16_to_cpu((__force __le16)hdr->length);
 102         if (len != size || (unsigned long)hdr + len > end) {
 103                 pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len);
 104                 return -EINVAL;
 105         }
 106
 107         /* Skip common header */
 108         dslbis = (struct acpi_cdat_dslbis *)(hdr + 1);
 109
 110         /* Skip unrecognized data type */
 111         if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
 112                 return 0;
 113
 114         /* Not a memory type, skip */
 115         if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY)
 116                 return 0;
 117
 118         dent = xa_load(dsmas_xa, dslbis->handle);
 119         if (!dent) {
 120                 pr_warn("No matching DSMAS entry for DSLBIS entry.\n");
 121                 return 0;
 122         }
 123
 124         le_base = (__force __le64)dslbis->entry_base_unit;
 125         le_val = (__force __le16)dslbis->entry[0];
 126         rc = check_mul_overflow(le64_to_cpu(le_base),
 127                                 le16_to_cpu(le_val), &val);
 128         if (rc)
 129                 pr_warn("DSLBIS value overflowed.\n");
 130
 131         cxl_access_coordinate_set(&dent->coord, dslbis->data_type, val);
 132
 133         return 0;
 134 }
 135
 136 static int cdat_table_parse_output(int rc)
 137 {
 138         if (rc < 0)
 139                 return rc;
 140         if (rc == 0)
 141                 return -ENOENT;
 142
 143         return 0;
 144 }
 145
 146 static int cxl_cdat_endpoint_process(struct cxl_port *port,
 147                                      struct xarray *dsmas_xa)
 148 {
 149         int rc;
 150
 151         rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler,
 152                               dsmas_xa, port->cdat.table);
 153         rc = cdat_table_parse_output(rc);
 154         if (rc)
 155                 return rc;
 156
 157         rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,
 158                               dsmas_xa, port->cdat.table);
 159         return cdat_table_parse_output(rc);
 160 }
 161
 162 static int cxl_port_perf_data_calculate(struct cxl_port *port,
 163                                         struct xarray *dsmas_xa)
 164 {
 165         struct access_coordinate c;
 166         struct dsmas_entry *dent;
 167         int valid_entries = 0;
 168         unsigned long index;
 169         int rc;
 170
 171         rc = cxl_endpoint_get_perf_coordinates(port, &c);
 172         if (rc) {
 173                 dev_dbg(&port->dev, "Failed to retrieve perf coordinates.\n");
 174                 return rc;
 175         }
 176
 177         struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);
 178
 179         if (!cxl_root)
 180                 return -ENODEV;
 181
 182         if (!cxl_root->ops || !cxl_root->ops->qos_class)
 183                 return -EOPNOTSUPP;
 184
 185         xa_for_each(dsmas_xa, index, dent) {
 186                 int qos_class;
 187
 188                 dent->coord.read_latency = dent->coord.read_latency +
 189                                            c.read_latency;
 190                 dent->coord.write_latency = dent->coord.write_latency +
 191                                             c.write_latency;
 192                 dent->coord.read_bandwidth = min_t(int, c.read_bandwidth,
 193                                                    dent->coord.read_bandwidth);
 194                 dent->coord.write_bandwidth = min_t(int, c.write_bandwidth,
 195                                                     dent->coord.write_bandwidth);
 196
 197                 dent->entries = 1;
 198                 rc = cxl_root->ops->qos_class(cxl_root, &dent->coord, 1,
 199                                               &qos_class);
 200                 if (rc != 1)
 201                         continue;
 202
 203                 valid_entries++;
 204                 dent->qos_class = qos_class;
 205         }
 206
 207         if (!valid_entries)
 208                 return -ENOENT;
 209
 210         return 0;
 211 }
 212
 213 static void add_perf_entry(struct device *dev, struct dsmas_entry *dent,
 214                            struct list_head *list)
 215 {
 216         struct cxl_dpa_perf *dpa_perf;
 217
 218         dpa_perf = kzalloc(sizeof(*dpa_perf), GFP_KERNEL);
 219         if (!dpa_perf)
 220                 return;
 221
 222         dpa_perf->dpa_range = dent->dpa_range;
 223         dpa_perf->coord = dent->coord;
 224         dpa_perf->qos_class = dent->qos_class;
 225         list_add_tail(&dpa_perf->list, list);
 226         dev_dbg(dev,
 227                 "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n",
 228                 dent->dpa_range.start, dpa_perf->qos_class,
 229                 dent->coord.read_bandwidth, dent->coord.write_bandwidth,
 230                 dent->coord.read_latency, dent->coord.write_latency);
 231 }
 232
 233 static void free_perf_ents(void *data)
 234 {
 235         struct cxl_memdev_state *mds = data;
 236         struct cxl_dpa_perf *dpa_perf, *n;
 237         LIST_HEAD(discard);
 238
 239         list_splice_tail_init(&mds->ram_perf_list, &discard);
 240         list_splice_tail_init(&mds->pmem_perf_list, &discard);
 241         list_for_each_entry_safe(dpa_perf, n, &discard, list) {
 242                 list_del(&dpa_perf->list);
 243                 kfree(dpa_perf);
 244         }
 245 }
 246
 247 static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,
 248                                      struct xarray *dsmas_xa)
 249 {
 250         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 251         struct device *dev = cxlds->dev;
 252         struct range pmem_range = {
 253                 .start = cxlds->pmem_res.start,
 254                 .end = cxlds->pmem_res.end,
 255         };
 256         struct range ram_range = {
 257                 .start = cxlds->ram_res.start,
 258                 .end = cxlds->ram_res.end,
 259         };
 260         struct dsmas_entry *dent;
 261         unsigned long index;
 262
 263         xa_for_each(dsmas_xa, index, dent) {
 264                 if (resource_size(&cxlds->ram_res) &&
 265                     range_contains(&ram_range, &dent->dpa_range))
 266                         add_perf_entry(dev, dent, &mds->ram_perf_list);
 267                 else if (resource_size(&cxlds->pmem_res) &&
 268                          range_contains(&pmem_range, &dent->dpa_range))
 269                         add_perf_entry(dev, dent, &mds->pmem_perf_list);
 270                 else
 271                         dev_dbg(dev, "no partition for dsmas dpa: %#llx\n",
 272                                 dent->dpa_range.start);
 273         }
 274
 275         devm_add_action_or_reset(&cxlds->cxlmd->dev, free_perf_ents, mds);
 276 }
 277
 278 static int match_cxlrd_qos_class(struct device *dev, void *data)
 279 {
 280         int dev_qos_class = *(int *)data;
 281         struct cxl_root_decoder *cxlrd;
 282
 283         if (!is_root_decoder(dev))
 284                 return 0;
 285
 286         cxlrd = to_cxl_root_decoder(dev);
 287         if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID)
 288                 return 0;
 289
 290         if (cxlrd->qos_class == dev_qos_class)
 291                 return 1;
 292
 293         return 0;
 294 }
 295
 296 static void cxl_qos_match(struct cxl_port *root_port,
 297                           struct list_head *work_list,
 298                           struct list_head *discard_list)
 299 {
 300         struct cxl_dpa_perf *dpa_perf, *n;
 301
 302         list_for_each_entry_safe(dpa_perf, n, work_list, list) {
 303                 int rc;
 304
 305                 if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID)
 306                         return;
 307
 308                 rc = device_for_each_child(&root_port->dev,
 309                                            (void *)&dpa_perf->qos_class,
 310                                            match_cxlrd_qos_class);
 311                 if (!rc)
 312                         list_move_tail(&dpa_perf->list, discard_list);
 313         }
 314 }
 315
 316 static int match_cxlrd_hb(struct device *dev, void *data)
 317 {
 318         struct device *host_bridge = data;
 319         struct cxl_switch_decoder *cxlsd;
 320         struct cxl_root_decoder *cxlrd;
 321
 322         if (!is_root_decoder(dev))
 323                 return 0;
 324
 325         cxlrd = to_cxl_root_decoder(dev);
 326         cxlsd = &cxlrd->cxlsd;
 327
 328         guard(rwsem_read)(&cxl_region_rwsem);
 329         for (int i = 0; i < cxlsd->nr_targets; i++) {
 330                 if (host_bridge == cxlsd->target[i]->dport_dev)
 331                         return 1;
 332         }
 333
 334         return 0;
 335 }
 336
 337 static void discard_dpa_perf(struct list_head *list)
 338 {
 339         struct cxl_dpa_perf *dpa_perf, *n;
 340
 341         list_for_each_entry_safe(dpa_perf, n, list, list) {
 342                 list_del(&dpa_perf->list);
 343                 kfree(dpa_perf);
 344         }
 345 }
 346 DEFINE_FREE(dpa_perf, struct list_head *, if (!list_empty(_T)) discard_dpa_perf(_T))
 347
 348 static int cxl_qos_class_verify(struct cxl_memdev *cxlmd)
 349 {
 350         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 351         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
 352         LIST_HEAD(__discard);
 353         struct list_head *discard __free(dpa_perf) = &__discard;
 354         struct cxl_port *root_port;
 355         int rc;
 356
 357         struct cxl_root *cxl_root __free(put_cxl_root) =
 358                 find_cxl_root(cxlmd->endpoint);
 359
 360         if (!cxl_root)
 361                 return -ENODEV;
 362
 363         root_port = &cxl_root->port;
 364
 365         /* Check that the QTG IDs are all sane between end device and root decoders */
 366         cxl_qos_match(root_port, &mds->ram_perf_list, discard);
 367         cxl_qos_match(root_port, &mds->pmem_perf_list, discard);
 368
 369         /* Check to make sure that the device's host bridge is under a root decoder */
 370         rc = device_for_each_child(&root_port->dev,
 371                                    (void *)cxlmd->endpoint->host_bridge,
 372                                    match_cxlrd_hb);
 373         if (!rc) {
 374                 list_splice_tail_init(&mds->ram_perf_list, discard);
 375                 list_splice_tail_init(&mds->pmem_perf_list, discard);
 376         }
 377
 378         return rc;
 379 }
 380
 381 static void discard_dsmas(struct xarray *xa)
 382 {
 383         unsigned long index;
 384         void *ent;
 385
 386         xa_for_each(xa, index, ent) {
 387                 xa_erase(xa, index);
 388                 kfree(ent);
 389         }
 390         xa_destroy(xa);
 391 }
 392 DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T))
 393
 394 void cxl_endpoint_parse_cdat(struct cxl_port *port)
 395 {
 396         struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
 397         struct cxl_dev_state *cxlds = cxlmd->cxlds;
 398         struct xarray __dsmas_xa;
 399         struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa;
 400         int rc;
 401
 402         xa_init(&__dsmas_xa);
 403         if (!port->cdat.table)
 404                 return;
 405
 406         rc = cxl_cdat_endpoint_process(port, dsmas_xa);
 407         if (rc < 0) {
 408                 dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);
 409                 return;
 410         }
 411
 412         rc = cxl_port_perf_data_calculate(port, dsmas_xa);
 413         if (rc) {
 414                 dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");
 415                 return;
 416         }
 417
 418         cxl_memdev_set_qos_class(cxlds, dsmas_xa);
 419         cxl_qos_class_verify(cxlmd);
 420 }
 421 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL);
 422
 423 static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg,
 424                                const unsigned long end)
 425 {
 426         struct acpi_cdat_sslbis *sslbis;
 427         int size = sizeof(header->cdat) + sizeof(*sslbis);
 428         struct cxl_port *port = arg;
 429         struct device *dev = &port->dev;
 430         struct acpi_cdat_sslbe *entry;
 431         int remain, entries, i;
 432         u16 len;
 433
 434         len = le16_to_cpu((__force __le16)header->cdat.length);
 435         remain = len - size;
 436         if (!remain || remain % sizeof(*entry) ||
 437             (unsigned long)header + len > end) {
 438                 dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len);
 439                 return -EINVAL;
 440         }
 441
 442         /* Skip common header */
 443         sslbis = (struct acpi_cdat_sslbis *)((unsigned long)header +
 444                                              sizeof(header->cdat));
 445
 446         /* Unrecognized data type, we can skip */
 447         if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
 448                 return 0;
 449
 450         entries = remain / sizeof(*entry);
 451         entry = (struct acpi_cdat_sslbe *)((unsigned long)header + sizeof(*sslbis));
 452
 453         for (i = 0; i < entries; i++) {
 454                 u16 x = le16_to_cpu((__force __le16)entry->portx_id);
 455                 u16 y = le16_to_cpu((__force __le16)entry->porty_id);
 456                 __le64 le_base;
 457                 __le16 le_val;
 458                 struct cxl_dport *dport;
 459                 unsigned long index;
 460                 u16 dsp_id;
 461                 u64 val;
 462
 463                 switch (x) {
 464                 case ACPI_CDAT_SSLBIS_US_PORT:
 465                         dsp_id = y;
 466                         break;
 467                 case ACPI_CDAT_SSLBIS_ANY_PORT:
 468                         switch (y) {
 469                         case ACPI_CDAT_SSLBIS_US_PORT:
 470                                 dsp_id = x;
 471                                 break;
 472                         case ACPI_CDAT_SSLBIS_ANY_PORT:
 473                                 dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT;
 474                                 break;
 475                         default:
 476                                 dsp_id = y;
 477                                 break;
 478                         }
 479                         break;
 480                 default:
 481                         dsp_id = x;
 482                         break;
 483                 }
 484
 485                 le_base = (__force __le64)sslbis->entry_base_unit;
 486                 le_val = (__force __le16)entry->latency_or_bandwidth;
 487
 488                 if (check_mul_overflow(le64_to_cpu(le_base),
 489                                        le16_to_cpu(le_val), &val))
 490                         dev_warn(dev, "SSLBIS value overflowed!\n");
 491
 492                 xa_for_each(&port->dports, index, dport) {
 493                         if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT ||
 494                             dsp_id == dport->port_id)
 495                                 cxl_access_coordinate_set(&dport->sw_coord,
 496                                                           sslbis->data_type,
 497                                                           val);
 498                 }
 499
 500                 entry++;
 501         }
 502
 503         return 0;
 504 }
 505
 506 void cxl_switch_parse_cdat(struct cxl_port *port)
 507 {
 508         int rc;
 509
 510         if (!port->cdat.table)
 511                 return;
 512
 513         rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler,
 514                               port, port->cdat.table);
 515         rc = cdat_table_parse_output(rc);
 516         if (rc)
 517                 dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc);
 518 }
 519 EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL);
 520
 521 MODULE_IMPORT_NS(CXL);