drivers/cxl/core/region.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/memregion.h>
   4 #include <linux/genalloc.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/slab.h>
   8 #include <linux/uuid.h>
   9 #include <linux/sort.h>
  10 #include <linux/idr.h>
  11 #include <cxlmem.h>
  12 #include <cxl.h>
  13 #include "core.h"
  14
  15 /**
  16  * DOC: cxl core region
  17  *
  18  * CXL Regions represent mapped memory capacity in system physical address
  19  * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
  20  * Memory ranges, Regions represent the active mapped capacity by the HDM
  21  * Decoder Capability structures throughout the Host Bridges, Switches, and
  22  * Endpoints in the topology.
  23  *
  24  * Region configuration has ordering constraints. UUID may be set at any time
  25  * but is only visible for persistent regions.
  26  * 1. Interleave granularity
  27  * 2. Interleave size
  28  * 3. Decoder targets
  29  */
  30
  31 /*
  32  * All changes to the interleave configuration occur with this lock held
  33  * for write.
  34  */
  35 static DECLARE_RWSEM(cxl_region_rwsem);
  36
  37 static struct cxl_region *to_cxl_region(struct device *dev);
  38
  39 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
  40                          char *buf)
  41 {
  42         struct cxl_region *cxlr = to_cxl_region(dev);
  43         struct cxl_region_params *p = &cxlr->params;
  44         ssize_t rc;
  45
  46         rc = down_read_interruptible(&cxl_region_rwsem);
  47         if (rc)
  48                 return rc;
  49         if (cxlr->mode != CXL_DECODER_PMEM)
  50                 rc = sysfs_emit(buf, "\n");
  51         else
  52                 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
  53         up_read(&cxl_region_rwsem);
  54
  55         return rc;
  56 }
  57
  58 static int is_dup(struct device *match, void *data)
  59 {
  60         struct cxl_region_params *p;
  61         struct cxl_region *cxlr;
  62         uuid_t *uuid = data;
  63
  64         if (!is_cxl_region(match))
  65                 return 0;
  66
  67         lockdep_assert_held(&cxl_region_rwsem);
  68         cxlr = to_cxl_region(match);
  69         p = &cxlr->params;
  70
  71         if (uuid_equal(&p->uuid, uuid)) {
  72                 dev_dbg(match, "already has uuid: %pUb\n", uuid);
  73                 return -EBUSY;
  74         }
  75
  76         return 0;
  77 }
  78
  79 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
  80                           const char *buf, size_t len)
  81 {
  82         struct cxl_region *cxlr = to_cxl_region(dev);
  83         struct cxl_region_params *p = &cxlr->params;
  84         uuid_t temp;
  85         ssize_t rc;
  86
  87         if (len != UUID_STRING_LEN + 1)
  88                 return -EINVAL;
  89
  90         rc = uuid_parse(buf, &temp);
  91         if (rc)
  92                 return rc;
  93
  94         if (uuid_is_null(&temp))
  95                 return -EINVAL;
  96
  97         rc = down_write_killable(&cxl_region_rwsem);
  98         if (rc)
  99                 return rc;
 100
 101         if (uuid_equal(&p->uuid, &temp))
 102                 goto out;
 103
 104         rc = -EBUSY;
 105         if (p->state >= CXL_CONFIG_ACTIVE)
 106                 goto out;
 107
 108         rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
 109         if (rc < 0)
 110                 goto out;
 111
 112         uuid_copy(&p->uuid, &temp);
 113 out:
 114         up_write(&cxl_region_rwsem);
 115
 116         if (rc)
 117                 return rc;
 118         return len;
 119 }
 120 static DEVICE_ATTR_RW(uuid);
 121
 122 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
 123                                           struct cxl_region *cxlr)
 124 {
 125         return xa_load(&port->regions, (unsigned long)cxlr);
 126 }
 127
 128 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
 129 {
 130         if (!cpu_cache_has_invalidate_memregion()) {
 131                 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
 132                         dev_warn_once(
 133                                 &cxlr->dev,
 134                                 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
 135                         return 0;
 136                 } else {
 137                         dev_err(&cxlr->dev,
 138                                 "Failed to synchronize CPU cache state\n");
 139                         return -ENXIO;
 140                 }
 141         }
 142
 143         cpu_cache_invalidate_memregion(IORES_DESC_CXL);
 144         return 0;
 145 }
 146
 147 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
 148 {
 149         struct cxl_region_params *p = &cxlr->params;
 150         int i, rc = 0;
 151
 152         /*
 153          * Before region teardown attempt to flush, and if the flush
 154          * fails cancel the region teardown for data consistency
 155          * concerns
 156          */
 157         rc = cxl_region_invalidate_memregion(cxlr);
 158         if (rc)
 159                 return rc;
 160
 161         for (i = count - 1; i >= 0; i--) {
 162                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 163                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 164                 struct cxl_port *iter = cxled_to_port(cxled);
 165                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 166                 struct cxl_ep *ep;
 167
 168                 if (cxlds->rcd)
 169                         goto endpoint_reset;
 170
 171                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
 172                         iter = to_cxl_port(iter->dev.parent);
 173
 174                 for (ep = cxl_ep_load(iter, cxlmd); iter;
 175                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 176                         struct cxl_region_ref *cxl_rr;
 177                         struct cxl_decoder *cxld;
 178
 179                         cxl_rr = cxl_rr_load(iter, cxlr);
 180                         cxld = cxl_rr->decoder;
 181                         if (cxld->reset)
 182                                 rc = cxld->reset(cxld);
 183                         if (rc)
 184                                 return rc;
 185                         set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 186                 }
 187
 188 endpoint_reset:
 189                 rc = cxled->cxld.reset(&cxled->cxld);
 190                 if (rc)
 191                         return rc;
 192                 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 193         }
 194
 195         /* all decoders associated with this region have been torn down */
 196         clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 197
 198         return 0;
 199 }
 200
 201 static int commit_decoder(struct cxl_decoder *cxld)
 202 {
 203         struct cxl_switch_decoder *cxlsd = NULL;
 204
 205         if (cxld->commit)
 206                 return cxld->commit(cxld);
 207
 208         if (is_switch_decoder(&cxld->dev))
 209                 cxlsd = to_cxl_switch_decoder(&cxld->dev);
 210
 211         if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
 212                           "->commit() is required\n"))
 213                 return -ENXIO;
 214         return 0;
 215 }
 216
 217 static int cxl_region_decode_commit(struct cxl_region *cxlr)
 218 {
 219         struct cxl_region_params *p = &cxlr->params;
 220         int i, rc = 0;
 221
 222         for (i = 0; i < p->nr_targets; i++) {
 223                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 224                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 225                 struct cxl_region_ref *cxl_rr;
 226                 struct cxl_decoder *cxld;
 227                 struct cxl_port *iter;
 228                 struct cxl_ep *ep;
 229
 230                 /* commit bottom up */
 231                 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
 232                      iter = to_cxl_port(iter->dev.parent)) {
 233                         cxl_rr = cxl_rr_load(iter, cxlr);
 234                         cxld = cxl_rr->decoder;
 235                         rc = commit_decoder(cxld);
 236                         if (rc)
 237                                 break;
 238                 }
 239
 240                 if (rc) {
 241                         /* programming @iter failed, teardown */
 242                         for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
 243                              iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 244                                 cxl_rr = cxl_rr_load(iter, cxlr);
 245                                 cxld = cxl_rr->decoder;
 246                                 if (cxld->reset)
 247                                         cxld->reset(cxld);
 248                         }
 249
 250                         cxled->cxld.reset(&cxled->cxld);
 251                         goto err;
 252                 }
 253         }
 254
 255         return 0;
 256
 257 err:
 258         /* undo the targets that were successfully committed */
 259         cxl_region_decode_reset(cxlr, i);
 260         return rc;
 261 }
 262
 263 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
 264                             const char *buf, size_t len)
 265 {
 266         struct cxl_region *cxlr = to_cxl_region(dev);
 267         struct cxl_region_params *p = &cxlr->params;
 268         bool commit;
 269         ssize_t rc;
 270
 271         rc = kstrtobool(buf, &commit);
 272         if (rc)
 273                 return rc;
 274
 275         rc = down_write_killable(&cxl_region_rwsem);
 276         if (rc)
 277                 return rc;
 278
 279         /* Already in the requested state? */
 280         if (commit && p->state >= CXL_CONFIG_COMMIT)
 281                 goto out;
 282         if (!commit && p->state < CXL_CONFIG_COMMIT)
 283                 goto out;
 284
 285         /* Not ready to commit? */
 286         if (commit && p->state < CXL_CONFIG_ACTIVE) {
 287                 rc = -ENXIO;
 288                 goto out;
 289         }
 290
 291         /*
 292          * Invalidate caches before region setup to drop any speculative
 293          * consumption of this address space
 294          */
 295         rc = cxl_region_invalidate_memregion(cxlr);
 296         if (rc)
 297                 return rc;
 298
 299         if (commit) {
 300                 rc = cxl_region_decode_commit(cxlr);
 301                 if (rc == 0)
 302                         p->state = CXL_CONFIG_COMMIT;
 303         } else {
 304                 p->state = CXL_CONFIG_RESET_PENDING;
 305                 up_write(&cxl_region_rwsem);
 306                 device_release_driver(&cxlr->dev);
 307                 down_write(&cxl_region_rwsem);
 308
 309                 /*
 310                  * The lock was dropped, so need to revalidate that the reset is
 311                  * still pending.
 312                  */
 313                 if (p->state == CXL_CONFIG_RESET_PENDING) {
 314                         rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
 315                         /*
 316                          * Revert to committed since there may still be active
 317                          * decoders associated with this region, or move forward
 318                          * to active to mark the reset successful
 319                          */
 320                         if (rc)
 321                                 p->state = CXL_CONFIG_COMMIT;
 322                         else
 323                                 p->state = CXL_CONFIG_ACTIVE;
 324                 }
 325         }
 326
 327 out:
 328         up_write(&cxl_region_rwsem);
 329
 330         if (rc)
 331                 return rc;
 332         return len;
 333 }
 334
 335 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
 336                            char *buf)
 337 {
 338         struct cxl_region *cxlr = to_cxl_region(dev);
 339         struct cxl_region_params *p = &cxlr->params;
 340         ssize_t rc;
 341
 342         rc = down_read_interruptible(&cxl_region_rwsem);
 343         if (rc)
 344                 return rc;
 345         rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
 346         up_read(&cxl_region_rwsem);
 347
 348         return rc;
 349 }
 350 static DEVICE_ATTR_RW(commit);
 351
 352 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
 353                                   int n)
 354 {
 355         struct device *dev = kobj_to_dev(kobj);
 356         struct cxl_region *cxlr = to_cxl_region(dev);
 357
 358         /*
 359          * Support tooling that expects to find a 'uuid' attribute for all
 360          * regions regardless of mode.
 361          */
 362         if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
 363                 return 0444;
 364         return a->mode;
 365 }
 366
 367 static ssize_t interleave_ways_show(struct device *dev,
 368                                     struct device_attribute *attr, char *buf)
 369 {
 370         struct cxl_region *cxlr = to_cxl_region(dev);
 371         struct cxl_region_params *p = &cxlr->params;
 372         ssize_t rc;
 373
 374         rc = down_read_interruptible(&cxl_region_rwsem);
 375         if (rc)
 376                 return rc;
 377         rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
 378         up_read(&cxl_region_rwsem);
 379
 380         return rc;
 381 }
 382
 383 static const struct attribute_group *get_cxl_region_target_group(void);
 384
 385 static ssize_t interleave_ways_store(struct device *dev,
 386                                      struct device_attribute *attr,
 387                                      const char *buf, size_t len)
 388 {
 389         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 390         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 391         struct cxl_region *cxlr = to_cxl_region(dev);
 392         struct cxl_region_params *p = &cxlr->params;
 393         unsigned int val, save;
 394         int rc;
 395         u8 iw;
 396
 397         rc = kstrtouint(buf, 0, &val);
 398         if (rc)
 399                 return rc;
 400
 401         rc = ways_to_eiw(val, &iw);
 402         if (rc)
 403                 return rc;
 404
 405         /*
 406          * Even for x3, x9, and x12 interleaves the region interleave must be a
 407          * power of 2 multiple of the host bridge interleave.
 408          */
 409         if (!is_power_of_2(val / cxld->interleave_ways) ||
 410             (val % cxld->interleave_ways)) {
 411                 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
 412                 return -EINVAL;
 413         }
 414
 415         rc = down_write_killable(&cxl_region_rwsem);
 416         if (rc)
 417                 return rc;
 418         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 419                 rc = -EBUSY;
 420                 goto out;
 421         }
 422
 423         save = p->interleave_ways;
 424         p->interleave_ways = val;
 425         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
 426         if (rc)
 427                 p->interleave_ways = save;
 428 out:
 429         up_write(&cxl_region_rwsem);
 430         if (rc)
 431                 return rc;
 432         return len;
 433 }
 434 static DEVICE_ATTR_RW(interleave_ways);
 435
 436 static ssize_t interleave_granularity_show(struct device *dev,
 437                                            struct device_attribute *attr,
 438                                            char *buf)
 439 {
 440         struct cxl_region *cxlr = to_cxl_region(dev);
 441         struct cxl_region_params *p = &cxlr->params;
 442         ssize_t rc;
 443
 444         rc = down_read_interruptible(&cxl_region_rwsem);
 445         if (rc)
 446                 return rc;
 447         rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
 448         up_read(&cxl_region_rwsem);
 449
 450         return rc;
 451 }
 452
 453 static ssize_t interleave_granularity_store(struct device *dev,
 454                                             struct device_attribute *attr,
 455                                             const char *buf, size_t len)
 456 {
 457         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 458         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 459         struct cxl_region *cxlr = to_cxl_region(dev);
 460         struct cxl_region_params *p = &cxlr->params;
 461         int rc, val;
 462         u16 ig;
 463
 464         rc = kstrtoint(buf, 0, &val);
 465         if (rc)
 466                 return rc;
 467
 468         rc = granularity_to_eig(val, &ig);
 469         if (rc)
 470                 return rc;
 471
 472         /*
 473          * When the host-bridge is interleaved, disallow region granularity !=
 474          * root granularity. Regions with a granularity less than the root
 475          * interleave result in needing multiple endpoints to support a single
 476          * slot in the interleave (possible to support in the future). Regions
 477          * with a granularity greater than the root interleave result in invalid
 478          * DPA translations (invalid to support).
 479          */
 480         if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
 481                 return -EINVAL;
 482
 483         rc = down_write_killable(&cxl_region_rwsem);
 484         if (rc)
 485                 return rc;
 486         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 487                 rc = -EBUSY;
 488                 goto out;
 489         }
 490
 491         p->interleave_granularity = val;
 492 out:
 493         up_write(&cxl_region_rwsem);
 494         if (rc)
 495                 return rc;
 496         return len;
 497 }
 498 static DEVICE_ATTR_RW(interleave_granularity);
 499
 500 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 501                              char *buf)
 502 {
 503         struct cxl_region *cxlr = to_cxl_region(dev);
 504         struct cxl_region_params *p = &cxlr->params;
 505         u64 resource = -1ULL;
 506         ssize_t rc;
 507
 508         rc = down_read_interruptible(&cxl_region_rwsem);
 509         if (rc)
 510                 return rc;
 511         if (p->res)
 512                 resource = p->res->start;
 513         rc = sysfs_emit(buf, "%#llx\n", resource);
 514         up_read(&cxl_region_rwsem);
 515
 516         return rc;
 517 }
 518 static DEVICE_ATTR_RO(resource);
 519
 520 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 521                          char *buf)
 522 {
 523         struct cxl_region *cxlr = to_cxl_region(dev);
 524
 525         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
 526 }
 527 static DEVICE_ATTR_RO(mode);
 528
 529 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
 530 {
 531         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
 532         struct cxl_region_params *p = &cxlr->params;
 533         struct resource *res;
 534         u32 remainder = 0;
 535
 536         lockdep_assert_held_write(&cxl_region_rwsem);
 537
 538         /* Nothing to do... */
 539         if (p->res && resource_size(p->res) == size)
 540                 return 0;
 541
 542         /* To change size the old size must be freed first */
 543         if (p->res)
 544                 return -EBUSY;
 545
 546         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 547                 return -EBUSY;
 548
 549         /* ways, granularity and uuid (if PMEM) need to be set before HPA */
 550         if (!p->interleave_ways || !p->interleave_granularity ||
 551             (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
 552                 return -ENXIO;
 553
 554         div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
 555         if (remainder)
 556                 return -EINVAL;
 557
 558         res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
 559                                     dev_name(&cxlr->dev));
 560         if (IS_ERR(res)) {
 561                 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
 562                         PTR_ERR(res));
 563                 return PTR_ERR(res);
 564         }
 565
 566         p->res = res;
 567         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
 568
 569         return 0;
 570 }
 571
 572 static void cxl_region_iomem_release(struct cxl_region *cxlr)
 573 {
 574         struct cxl_region_params *p = &cxlr->params;
 575
 576         if (device_is_registered(&cxlr->dev))
 577                 lockdep_assert_held_write(&cxl_region_rwsem);
 578         if (p->res) {
 579                 /*
 580                  * Autodiscovered regions may not have been able to insert their
 581                  * resource.
 582                  */
 583                 if (p->res->parent)
 584                         remove_resource(p->res);
 585                 kfree(p->res);
 586                 p->res = NULL;
 587         }
 588 }
 589
 590 static int free_hpa(struct cxl_region *cxlr)
 591 {
 592         struct cxl_region_params *p = &cxlr->params;
 593
 594         lockdep_assert_held_write(&cxl_region_rwsem);
 595
 596         if (!p->res)
 597                 return 0;
 598
 599         if (p->state >= CXL_CONFIG_ACTIVE)
 600                 return -EBUSY;
 601
 602         cxl_region_iomem_release(cxlr);
 603         p->state = CXL_CONFIG_IDLE;
 604         return 0;
 605 }
 606
 607 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
 608                           const char *buf, size_t len)
 609 {
 610         struct cxl_region *cxlr = to_cxl_region(dev);
 611         u64 val;
 612         int rc;
 613
 614         rc = kstrtou64(buf, 0, &val);
 615         if (rc)
 616                 return rc;
 617
 618         rc = down_write_killable(&cxl_region_rwsem);
 619         if (rc)
 620                 return rc;
 621
 622         if (val)
 623                 rc = alloc_hpa(cxlr, val);
 624         else
 625                 rc = free_hpa(cxlr);
 626         up_write(&cxl_region_rwsem);
 627
 628         if (rc)
 629                 return rc;
 630
 631         return len;
 632 }
 633
 634 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 635                          char *buf)
 636 {
 637         struct cxl_region *cxlr = to_cxl_region(dev);
 638         struct cxl_region_params *p = &cxlr->params;
 639         u64 size = 0;
 640         ssize_t rc;
 641
 642         rc = down_read_interruptible(&cxl_region_rwsem);
 643         if (rc)
 644                 return rc;
 645         if (p->res)
 646                 size = resource_size(p->res);
 647         rc = sysfs_emit(buf, "%#llx\n", size);
 648         up_read(&cxl_region_rwsem);
 649
 650         return rc;
 651 }
 652 static DEVICE_ATTR_RW(size);
 653
 654 static struct attribute *cxl_region_attrs[] = {
 655         &dev_attr_uuid.attr,
 656         &dev_attr_commit.attr,
 657         &dev_attr_interleave_ways.attr,
 658         &dev_attr_interleave_granularity.attr,
 659         &dev_attr_resource.attr,
 660         &dev_attr_size.attr,
 661         &dev_attr_mode.attr,
 662         NULL,
 663 };
 664
 665 static const struct attribute_group cxl_region_group = {
 666         .attrs = cxl_region_attrs,
 667         .is_visible = cxl_region_visible,
 668 };
 669
 670 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
 671 {
 672         struct cxl_region_params *p = &cxlr->params;
 673         struct cxl_endpoint_decoder *cxled;
 674         int rc;
 675
 676         rc = down_read_interruptible(&cxl_region_rwsem);
 677         if (rc)
 678                 return rc;
 679
 680         if (pos >= p->interleave_ways) {
 681                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
 682                         p->interleave_ways);
 683                 rc = -ENXIO;
 684                 goto out;
 685         }
 686
 687         cxled = p->targets[pos];
 688         if (!cxled)
 689                 rc = sysfs_emit(buf, "\n");
 690         else
 691                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
 692 out:
 693         up_read(&cxl_region_rwsem);
 694
 695         return rc;
 696 }
 697
 698 static int match_free_decoder(struct device *dev, void *data)
 699 {
 700         struct cxl_decoder *cxld;
 701         int *id = data;
 702
 703         if (!is_switch_decoder(dev))
 704                 return 0;
 705
 706         cxld = to_cxl_decoder(dev);
 707
 708         /* enforce ordered allocation */
 709         if (cxld->id != *id)
 710                 return 0;
 711
 712         if (!cxld->region)
 713                 return 1;
 714
 715         (*id)++;
 716
 717         return 0;
 718 }
 719
 720 static int match_auto_decoder(struct device *dev, void *data)
 721 {
 722         struct cxl_region_params *p = data;
 723         struct cxl_decoder *cxld;
 724         struct range *r;
 725
 726         if (!is_switch_decoder(dev))
 727                 return 0;
 728
 729         cxld = to_cxl_decoder(dev);
 730         r = &cxld->hpa_range;
 731
 732         if (p->res && p->res->start == r->start && p->res->end == r->end)
 733                 return 1;
 734
 735         return 0;
 736 }
 737
 738 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
 739                                                    struct cxl_region *cxlr)
 740 {
 741         struct device *dev;
 742         int id = 0;
 743
 744         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
 745                 dev = device_find_child(&port->dev, &cxlr->params,
 746                                         match_auto_decoder);
 747         else
 748                 dev = device_find_child(&port->dev, &id, match_free_decoder);
 749         if (!dev)
 750                 return NULL;
 751         /*
 752          * This decoder is pinned registered as long as the endpoint decoder is
 753          * registered, and endpoint decoder unregistration holds the
 754          * cxl_region_rwsem over unregister events, so no need to hold on to
 755          * this extra reference.
 756          */
 757         put_device(dev);
 758         return to_cxl_decoder(dev);
 759 }
 760
 761 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
 762                                                struct cxl_region *cxlr)
 763 {
 764         struct cxl_region_params *p = &cxlr->params;
 765         struct cxl_region_ref *cxl_rr, *iter;
 766         unsigned long index;
 767         int rc;
 768
 769         xa_for_each(&port->regions, index, iter) {
 770                 struct cxl_region_params *ip = &iter->region->params;
 771
 772                 if (!ip->res)
 773                         continue;
 774
 775                 if (ip->res->start > p->res->start) {
 776                         dev_dbg(&cxlr->dev,
 777                                 "%s: HPA order violation %s:%pr vs %pr\n",
 778                                 dev_name(&port->dev),
 779                                 dev_name(&iter->region->dev), ip->res, p->res);
 780                         return ERR_PTR(-EBUSY);
 781                 }
 782         }
 783
 784         cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
 785         if (!cxl_rr)
 786                 return ERR_PTR(-ENOMEM);
 787         cxl_rr->port = port;
 788         cxl_rr->region = cxlr;
 789         cxl_rr->nr_targets = 1;
 790         xa_init(&cxl_rr->endpoints);
 791
 792         rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
 793         if (rc) {
 794                 dev_dbg(&cxlr->dev,
 795                         "%s: failed to track region reference: %d\n",
 796                         dev_name(&port->dev), rc);
 797                 kfree(cxl_rr);
 798                 return ERR_PTR(rc);
 799         }
 800
 801         return cxl_rr;
 802 }
 803
 804 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
 805 {
 806         struct cxl_region *cxlr = cxl_rr->region;
 807         struct cxl_decoder *cxld = cxl_rr->decoder;
 808
 809         if (!cxld)
 810                 return;
 811
 812         dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
 813         if (cxld->region == cxlr) {
 814                 cxld->region = NULL;
 815                 put_device(&cxlr->dev);
 816         }
 817 }
 818
 819 static void free_region_ref(struct cxl_region_ref *cxl_rr)
 820 {
 821         struct cxl_port *port = cxl_rr->port;
 822         struct cxl_region *cxlr = cxl_rr->region;
 823
 824         cxl_rr_free_decoder(cxl_rr);
 825         xa_erase(&port->regions, (unsigned long)cxlr);
 826         xa_destroy(&cxl_rr->endpoints);
 827         kfree(cxl_rr);
 828 }
 829
 830 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
 831                          struct cxl_endpoint_decoder *cxled)
 832 {
 833         int rc;
 834         struct cxl_port *port = cxl_rr->port;
 835         struct cxl_region *cxlr = cxl_rr->region;
 836         struct cxl_decoder *cxld = cxl_rr->decoder;
 837         struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
 838
 839         if (ep) {
 840                 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
 841                                GFP_KERNEL);
 842                 if (rc)
 843                         return rc;
 844         }
 845         cxl_rr->nr_eps++;
 846
 847         if (!cxld->region) {
 848                 cxld->region = cxlr;
 849                 get_device(&cxlr->dev);
 850         }
 851
 852         return 0;
 853 }
 854
 855 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
 856                                 struct cxl_endpoint_decoder *cxled,
 857                                 struct cxl_region_ref *cxl_rr)
 858 {
 859         struct cxl_decoder *cxld;
 860
 861         if (port == cxled_to_port(cxled))
 862                 cxld = &cxled->cxld;
 863         else
 864                 cxld = cxl_region_find_decoder(port, cxlr);
 865         if (!cxld) {
 866                 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
 867                         dev_name(&port->dev));
 868                 return -EBUSY;
 869         }
 870
 871         if (cxld->region) {
 872                 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
 873                         dev_name(&port->dev), dev_name(&cxld->dev),
 874                         dev_name(&cxld->region->dev));
 875                 return -EBUSY;
 876         }
 877
 878         /*
 879          * Endpoints should already match the region type, but backstop that
 880          * assumption with an assertion. Switch-decoders change mapping-type
 881          * based on what is mapped when they are assigned to a region.
 882          */
 883         dev_WARN_ONCE(&cxlr->dev,
 884                       port == cxled_to_port(cxled) &&
 885                               cxld->target_type != cxlr->type,
 886                       "%s:%s mismatch decoder type %d -> %d\n",
 887                       dev_name(&cxled_to_memdev(cxled)->dev),
 888                       dev_name(&cxld->dev), cxld->target_type, cxlr->type);
 889         cxld->target_type = cxlr->type;
 890         cxl_rr->decoder = cxld;
 891         return 0;
 892 }
 893
 894 /**
 895  * cxl_port_attach_region() - track a region's interest in a port by endpoint
 896  * @port: port to add a new region reference 'struct cxl_region_ref'
 897  * @cxlr: region to attach to @port
 898  * @cxled: endpoint decoder used to create or further pin a region reference
 899  * @pos: interleave position of @cxled in @cxlr
 900  *
 901  * The attach event is an opportunity to validate CXL decode setup
 902  * constraints and record metadata needed for programming HDM decoders,
 903  * in particular decoder target lists.
 904  *
 905  * The steps are:
 906  *
 907  * - validate that there are no other regions with a higher HPA already
 908  *   associated with @port
 909  * - establish a region reference if one is not already present
 910  *
 911  *   - additionally allocate a decoder instance that will host @cxlr on
 912  *     @port
 913  *
 914  * - pin the region reference by the endpoint
 915  * - account for how many entries in @port's target list are needed to
 916  *   cover all of the added endpoints.
 917  */
 918 static int cxl_port_attach_region(struct cxl_port *port,
 919                                   struct cxl_region *cxlr,
 920                                   struct cxl_endpoint_decoder *cxled, int pos)
 921 {
 922         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 923         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
 924         struct cxl_region_ref *cxl_rr;
 925         bool nr_targets_inc = false;
 926         struct cxl_decoder *cxld;
 927         unsigned long index;
 928         int rc = -EBUSY;
 929
 930         lockdep_assert_held_write(&cxl_region_rwsem);
 931
 932         cxl_rr = cxl_rr_load(port, cxlr);
 933         if (cxl_rr) {
 934                 struct cxl_ep *ep_iter;
 935                 int found = 0;
 936
 937                 /*
 938                  * Walk the existing endpoints that have been attached to
 939                  * @cxlr at @port and see if they share the same 'next' port
 940                  * in the downstream direction. I.e. endpoints that share common
 941                  * upstream switch.
 942                  */
 943                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
 944                         if (ep_iter == ep)
 945                                 continue;
 946                         if (ep_iter->next == ep->next) {
 947                                 found++;
 948                                 break;
 949                         }
 950                 }
 951
 952                 /*
 953                  * New target port, or @port is an endpoint port that always
 954                  * accounts its own local decode as a target.
 955                  */
 956                 if (!found || !ep->next) {
 957                         cxl_rr->nr_targets++;
 958                         nr_targets_inc = true;
 959                 }
 960         } else {
 961                 cxl_rr = alloc_region_ref(port, cxlr);
 962                 if (IS_ERR(cxl_rr)) {
 963                         dev_dbg(&cxlr->dev,
 964                                 "%s: failed to allocate region reference\n",
 965                                 dev_name(&port->dev));
 966                         return PTR_ERR(cxl_rr);
 967                 }
 968                 nr_targets_inc = true;
 969
 970                 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
 971                 if (rc)
 972                         goto out_erase;
 973         }
 974         cxld = cxl_rr->decoder;
 975
 976         rc = cxl_rr_ep_add(cxl_rr, cxled);
 977         if (rc) {
 978                 dev_dbg(&cxlr->dev,
 979                         "%s: failed to track endpoint %s:%s reference\n",
 980                         dev_name(&port->dev), dev_name(&cxlmd->dev),
 981                         dev_name(&cxld->dev));
 982                 goto out_erase;
 983         }
 984
 985         dev_dbg(&cxlr->dev,
 986                 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
 987                 dev_name(port->uport_dev), dev_name(&port->dev),
 988                 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
 989                 dev_name(&cxled->cxld.dev), pos,
 990                 ep ? ep->next ? dev_name(ep->next->uport_dev) :
 991                                       dev_name(&cxlmd->dev) :
 992                            "none",
 993                 cxl_rr->nr_eps, cxl_rr->nr_targets);
 994
 995         return 0;
 996 out_erase:
 997         if (nr_targets_inc)
 998                 cxl_rr->nr_targets--;
 999         if (cxl_rr->nr_eps == 0)
1000                 free_region_ref(cxl_rr);
1001         return rc;
1002 }
1003
1004 static void cxl_port_detach_region(struct cxl_port *port,
1005                                    struct cxl_region *cxlr,
1006                                    struct cxl_endpoint_decoder *cxled)
1007 {
1008         struct cxl_region_ref *cxl_rr;
1009         struct cxl_ep *ep = NULL;
1010
1011         lockdep_assert_held_write(&cxl_region_rwsem);
1012
1013         cxl_rr = cxl_rr_load(port, cxlr);
1014         if (!cxl_rr)
1015                 return;
1016
1017         /*
1018          * Endpoint ports do not carry cxl_ep references, and they
1019          * never target more than one endpoint by definition
1020          */
1021         if (cxl_rr->decoder == &cxled->cxld)
1022                 cxl_rr->nr_eps--;
1023         else
1024                 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1025         if (ep) {
1026                 struct cxl_ep *ep_iter;
1027                 unsigned long index;
1028                 int found = 0;
1029
1030                 cxl_rr->nr_eps--;
1031                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1032                         if (ep_iter->next == ep->next) {
1033                                 found++;
1034                                 break;
1035                         }
1036                 }
1037                 if (!found)
1038                         cxl_rr->nr_targets--;
1039         }
1040
1041         if (cxl_rr->nr_eps == 0)
1042                 free_region_ref(cxl_rr);
1043 }
1044
1045 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1046                            struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1047                            int distance)
1048 {
1049         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1050         struct cxl_region *cxlr = cxl_rr->region;
1051         struct cxl_region_params *p = &cxlr->params;
1052         struct cxl_endpoint_decoder *cxled_peer;
1053         struct cxl_port *port = cxl_rr->port;
1054         struct cxl_memdev *cxlmd_peer;
1055         struct cxl_ep *ep_peer;
1056         int pos = cxled->pos;
1057
1058         /*
1059          * If this position wants to share a dport with the last endpoint mapped
1060          * then that endpoint, at index 'position - distance', must also be
1061          * mapped by this dport.
1062          */
1063         if (pos < distance) {
1064                 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1065                         dev_name(port->uport_dev), dev_name(&port->dev),
1066                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1067                 return -ENXIO;
1068         }
1069         cxled_peer = p->targets[pos - distance];
1070         cxlmd_peer = cxled_to_memdev(cxled_peer);
1071         ep_peer = cxl_ep_load(port, cxlmd_peer);
1072         if (ep->dport != ep_peer->dport) {
1073                 dev_dbg(&cxlr->dev,
1074                         "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1075                         dev_name(port->uport_dev), dev_name(&port->dev),
1076                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1077                         dev_name(&cxlmd_peer->dev),
1078                         dev_name(&cxled_peer->cxld.dev));
1079                 return -ENXIO;
1080         }
1081
1082         return 0;
1083 }
1084
1085 static int cxl_port_setup_targets(struct cxl_port *port,
1086                                   struct cxl_region *cxlr,
1087                                   struct cxl_endpoint_decoder *cxled)
1088 {
1089         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1090         int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1091         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1092         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1093         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1094         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1095         struct cxl_region_params *p = &cxlr->params;
1096         struct cxl_decoder *cxld = cxl_rr->decoder;
1097         struct cxl_switch_decoder *cxlsd;
1098         u16 eig, peig;
1099         u8 eiw, peiw;
1100
1101         /*
1102          * While root level decoders support x3, x6, x12, switch level
1103          * decoders only support powers of 2 up to x16.
1104          */
1105         if (!is_power_of_2(cxl_rr->nr_targets)) {
1106                 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1107                         dev_name(port->uport_dev), dev_name(&port->dev),
1108                         cxl_rr->nr_targets);
1109                 return -EINVAL;
1110         }
1111
1112         cxlsd = to_cxl_switch_decoder(&cxld->dev);
1113         if (cxl_rr->nr_targets_set) {
1114                 int i, distance;
1115
1116                 /*
1117                  * Passthrough decoders impose no distance requirements between
1118                  * peers
1119                  */
1120                 if (cxl_rr->nr_targets == 1)
1121                         distance = 0;
1122                 else
1123                         distance = p->nr_targets / cxl_rr->nr_targets;
1124                 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1125                         if (ep->dport == cxlsd->target[i]) {
1126                                 rc = check_last_peer(cxled, ep, cxl_rr,
1127                                                      distance);
1128                                 if (rc)
1129                                         return rc;
1130                                 goto out_target_set;
1131                         }
1132                 goto add_target;
1133         }
1134
1135         if (is_cxl_root(parent_port)) {
1136                 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1137                 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1138                 /*
1139                  * For purposes of address bit routing, use power-of-2 math for
1140                  * switch ports.
1141                  */
1142                 if (!is_power_of_2(parent_iw))
1143                         parent_iw /= 3;
1144         } else {
1145                 struct cxl_region_ref *parent_rr;
1146                 struct cxl_decoder *parent_cxld;
1147
1148                 parent_rr = cxl_rr_load(parent_port, cxlr);
1149                 parent_cxld = parent_rr->decoder;
1150                 parent_ig = parent_cxld->interleave_granularity;
1151                 parent_iw = parent_cxld->interleave_ways;
1152         }
1153
1154         rc = granularity_to_eig(parent_ig, &peig);
1155         if (rc) {
1156                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1157                         dev_name(parent_port->uport_dev),
1158                         dev_name(&parent_port->dev), parent_ig);
1159                 return rc;
1160         }
1161
1162         rc = ways_to_eiw(parent_iw, &peiw);
1163         if (rc) {
1164                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1165                         dev_name(parent_port->uport_dev),
1166                         dev_name(&parent_port->dev), parent_iw);
1167                 return rc;
1168         }
1169
1170         iw = cxl_rr->nr_targets;
1171         rc = ways_to_eiw(iw, &eiw);
1172         if (rc) {
1173                 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1174                         dev_name(port->uport_dev), dev_name(&port->dev), iw);
1175                 return rc;
1176         }
1177
1178         /*
1179          * Interleave granularity is a multiple of @parent_port granularity.
1180          * Multiplier is the parent port interleave ways.
1181          */
1182         rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1183         if (rc) {
1184                 dev_dbg(&cxlr->dev,
1185                         "%s: invalid granularity calculation (%d * %d)\n",
1186                         dev_name(&parent_port->dev), parent_ig, parent_iw);
1187                 return rc;
1188         }
1189
1190         rc = eig_to_granularity(eig, &ig);
1191         if (rc) {
1192                 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1193                         dev_name(port->uport_dev), dev_name(&port->dev),
1194                         256 << eig);
1195                 return rc;
1196         }
1197
1198         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1199                 if (cxld->interleave_ways != iw ||
1200                     cxld->interleave_granularity != ig ||
1201                     cxld->hpa_range.start != p->res->start ||
1202                     cxld->hpa_range.end != p->res->end ||
1203                     ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1204                         dev_err(&cxlr->dev,
1205                                 "%s:%s %s expected iw: %d ig: %d %pr\n",
1206                                 dev_name(port->uport_dev), dev_name(&port->dev),
1207                                 __func__, iw, ig, p->res);
1208                         dev_err(&cxlr->dev,
1209                                 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1210                                 dev_name(port->uport_dev), dev_name(&port->dev),
1211                                 __func__, cxld->interleave_ways,
1212                                 cxld->interleave_granularity,
1213                                 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1214                                         "enabled" :
1215                                         "disabled",
1216                                 cxld->hpa_range.start, cxld->hpa_range.end);
1217                         return -ENXIO;
1218                 }
1219         } else {
1220                 cxld->interleave_ways = iw;
1221                 cxld->interleave_granularity = ig;
1222                 cxld->hpa_range = (struct range) {
1223                         .start = p->res->start,
1224                         .end = p->res->end,
1225                 };
1226         }
1227         dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1228                 dev_name(&port->dev), iw, ig);
1229 add_target:
1230         if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1231                 dev_dbg(&cxlr->dev,
1232                         "%s:%s: targets full trying to add %s:%s at %d\n",
1233                         dev_name(port->uport_dev), dev_name(&port->dev),
1234                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1235                 return -ENXIO;
1236         }
1237         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1238                 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1239                         dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1240                                 dev_name(port->uport_dev), dev_name(&port->dev),
1241                                 dev_name(&cxlsd->cxld.dev),
1242                                 dev_name(ep->dport->dport_dev),
1243                                 cxl_rr->nr_targets_set);
1244                         return -ENXIO;
1245                 }
1246         } else
1247                 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1248         inc = 1;
1249 out_target_set:
1250         cxl_rr->nr_targets_set += inc;
1251         dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1252                 dev_name(port->uport_dev), dev_name(&port->dev),
1253                 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1254                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1255
1256         return 0;
1257 }
1258
1259 static void cxl_port_reset_targets(struct cxl_port *port,
1260                                    struct cxl_region *cxlr)
1261 {
1262         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1263         struct cxl_decoder *cxld;
1264
1265         /*
1266          * After the last endpoint has been detached the entire cxl_rr may now
1267          * be gone.
1268          */
1269         if (!cxl_rr)
1270                 return;
1271         cxl_rr->nr_targets_set = 0;
1272
1273         cxld = cxl_rr->decoder;
1274         cxld->hpa_range = (struct range) {
1275                 .start = 0,
1276                 .end = -1,
1277         };
1278 }
1279
1280 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1281 {
1282         struct cxl_region_params *p = &cxlr->params;
1283         struct cxl_endpoint_decoder *cxled;
1284         struct cxl_dev_state *cxlds;
1285         struct cxl_memdev *cxlmd;
1286         struct cxl_port *iter;
1287         struct cxl_ep *ep;
1288         int i;
1289
1290         /*
1291          * In the auto-discovery case skip automatic teardown since the
1292          * address space is already active
1293          */
1294         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1295                 return;
1296
1297         for (i = 0; i < p->nr_targets; i++) {
1298                 cxled = p->targets[i];
1299                 cxlmd = cxled_to_memdev(cxled);
1300                 cxlds = cxlmd->cxlds;
1301
1302                 if (cxlds->rcd)
1303                         continue;
1304
1305                 iter = cxled_to_port(cxled);
1306                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1307                         iter = to_cxl_port(iter->dev.parent);
1308
1309                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1310                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1311                         cxl_port_reset_targets(iter, cxlr);
1312         }
1313 }
1314
1315 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1316 {
1317         struct cxl_region_params *p = &cxlr->params;
1318         struct cxl_endpoint_decoder *cxled;
1319         struct cxl_dev_state *cxlds;
1320         int i, rc, rch = 0, vh = 0;
1321         struct cxl_memdev *cxlmd;
1322         struct cxl_port *iter;
1323         struct cxl_ep *ep;
1324
1325         for (i = 0; i < p->nr_targets; i++) {
1326                 cxled = p->targets[i];
1327                 cxlmd = cxled_to_memdev(cxled);
1328                 cxlds = cxlmd->cxlds;
1329
1330                 /* validate that all targets agree on topology */
1331                 if (!cxlds->rcd) {
1332                         vh++;
1333                 } else {
1334                         rch++;
1335                         continue;
1336                 }
1337
1338                 iter = cxled_to_port(cxled);
1339                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1340                         iter = to_cxl_port(iter->dev.parent);
1341
1342                 /*
1343                  * Descend the topology tree programming / validating
1344                  * targets while looking for conflicts.
1345                  */
1346                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1347                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1348                         rc = cxl_port_setup_targets(iter, cxlr, cxled);
1349                         if (rc) {
1350                                 cxl_region_teardown_targets(cxlr);
1351                                 return rc;
1352                         }
1353                 }
1354         }
1355
1356         if (rch && vh) {
1357                 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1358                 cxl_region_teardown_targets(cxlr);
1359                 return -ENXIO;
1360         }
1361
1362         return 0;
1363 }
1364
1365 static int cxl_region_validate_position(struct cxl_region *cxlr,
1366                                         struct cxl_endpoint_decoder *cxled,
1367                                         int pos)
1368 {
1369         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1370         struct cxl_region_params *p = &cxlr->params;
1371         int i;
1372
1373         if (pos < 0 || pos >= p->interleave_ways) {
1374                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1375                         p->interleave_ways);
1376                 return -ENXIO;
1377         }
1378
1379         if (p->targets[pos] == cxled)
1380                 return 0;
1381
1382         if (p->targets[pos]) {
1383                 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1384                 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1385
1386                 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1387                         pos, dev_name(&cxlmd_target->dev),
1388                         dev_name(&cxled_target->cxld.dev));
1389                 return -EBUSY;
1390         }
1391
1392         for (i = 0; i < p->interleave_ways; i++) {
1393                 struct cxl_endpoint_decoder *cxled_target;
1394                 struct cxl_memdev *cxlmd_target;
1395
1396                 cxled_target = p->targets[i];
1397                 if (!cxled_target)
1398                         continue;
1399
1400                 cxlmd_target = cxled_to_memdev(cxled_target);
1401                 if (cxlmd_target == cxlmd) {
1402                         dev_dbg(&cxlr->dev,
1403                                 "%s already specified at position %d via: %s\n",
1404                                 dev_name(&cxlmd->dev), pos,
1405                                 dev_name(&cxled_target->cxld.dev));
1406                         return -EBUSY;
1407                 }
1408         }
1409
1410         return 0;
1411 }
1412
1413 static int cxl_region_attach_position(struct cxl_region *cxlr,
1414                                       struct cxl_root_decoder *cxlrd,
1415                                       struct cxl_endpoint_decoder *cxled,
1416                                       const struct cxl_dport *dport, int pos)
1417 {
1418         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1419         struct cxl_port *iter;
1420         int rc;
1421
1422         if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1423                 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1424                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1425                         dev_name(&cxlrd->cxlsd.cxld.dev));
1426                 return -ENXIO;
1427         }
1428
1429         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1430              iter = to_cxl_port(iter->dev.parent)) {
1431                 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1432                 if (rc)
1433                         goto err;
1434         }
1435
1436         return 0;
1437
1438 err:
1439         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1440              iter = to_cxl_port(iter->dev.parent))
1441                 cxl_port_detach_region(iter, cxlr, cxled);
1442         return rc;
1443 }
1444
1445 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1446                                   struct cxl_endpoint_decoder *cxled, int pos)
1447 {
1448         struct cxl_region_params *p = &cxlr->params;
1449
1450         if (cxled->state != CXL_DECODER_STATE_AUTO) {
1451                 dev_err(&cxlr->dev,
1452                         "%s: unable to add decoder to autodetected region\n",
1453                         dev_name(&cxled->cxld.dev));
1454                 return -EINVAL;
1455         }
1456
1457         if (pos >= 0) {
1458                 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1459                         dev_name(&cxled->cxld.dev), pos);
1460                 return -EINVAL;
1461         }
1462
1463         if (p->nr_targets >= p->interleave_ways) {
1464                 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1465                         dev_name(&cxled->cxld.dev));
1466                 return -ENXIO;
1467         }
1468
1469         /*
1470          * Temporarily record the endpoint decoder into the target array. Yes,
1471          * this means that userspace can view devices in the wrong position
1472          * before the region activates, and must be careful to understand when
1473          * it might be racing region autodiscovery.
1474          */
1475         pos = p->nr_targets;
1476         p->targets[pos] = cxled;
1477         cxled->pos = pos;
1478         p->nr_targets++;
1479
1480         return 0;
1481 }
1482
1483 static struct cxl_port *next_port(struct cxl_port *port)
1484 {
1485         if (!port->parent_dport)
1486                 return NULL;
1487         return port->parent_dport->port;
1488 }
1489
1490 static int decoder_match_range(struct device *dev, void *data)
1491 {
1492         struct cxl_endpoint_decoder *cxled = data;
1493         struct cxl_switch_decoder *cxlsd;
1494
1495         if (!is_switch_decoder(dev))
1496                 return 0;
1497
1498         cxlsd = to_cxl_switch_decoder(dev);
1499         return range_contains(&cxlsd->cxld.hpa_range, &cxled->cxld.hpa_range);
1500 }
1501
1502 static void find_positions(const struct cxl_switch_decoder *cxlsd,
1503                            const struct cxl_port *iter_a,
1504                            const struct cxl_port *iter_b, int *a_pos,
1505                            int *b_pos)
1506 {
1507         int i;
1508
1509         for (i = 0, *a_pos = -1, *b_pos = -1; i < cxlsd->nr_targets; i++) {
1510                 if (cxlsd->target[i] == iter_a->parent_dport)
1511                         *a_pos = i;
1512                 else if (cxlsd->target[i] == iter_b->parent_dport)
1513                         *b_pos = i;
1514                 if (*a_pos >= 0 && *b_pos >= 0)
1515                         break;
1516         }
1517 }
1518
1519 static int cmp_decode_pos(const void *a, const void *b)
1520 {
1521         struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1522         struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1523         struct cxl_memdev *cxlmd_a = cxled_to_memdev(cxled_a);
1524         struct cxl_memdev *cxlmd_b = cxled_to_memdev(cxled_b);
1525         struct cxl_port *port_a = cxled_to_port(cxled_a);
1526         struct cxl_port *port_b = cxled_to_port(cxled_b);
1527         struct cxl_port *iter_a, *iter_b, *port = NULL;
1528         struct cxl_switch_decoder *cxlsd;
1529         struct device *dev;
1530         int a_pos, b_pos;
1531         unsigned int seq;
1532
1533         /* Exit early if any prior sorting failed */
1534         if (cxled_a->pos < 0 || cxled_b->pos < 0)
1535                 return 0;
1536
1537         /*
1538          * Walk up the hierarchy to find a shared port, find the decoder that
1539          * maps the range, compare the relative position of those dport
1540          * mappings.
1541          */
1542         for (iter_a = port_a; iter_a; iter_a = next_port(iter_a)) {
1543                 struct cxl_port *next_a, *next_b;
1544
1545                 next_a = next_port(iter_a);
1546                 if (!next_a)
1547                         break;
1548
1549                 for (iter_b = port_b; iter_b; iter_b = next_port(iter_b)) {
1550                         next_b = next_port(iter_b);
1551                         if (next_a != next_b)
1552                                 continue;
1553                         port = next_a;
1554                         break;
1555                 }
1556
1557                 if (port)
1558                         break;
1559         }
1560
1561         if (!port) {
1562                 dev_err(cxlmd_a->dev.parent,
1563                         "failed to find shared port with %s\n",
1564                         dev_name(cxlmd_b->dev.parent));
1565                 goto err;
1566         }
1567
1568         dev = device_find_child(&port->dev, cxled_a, decoder_match_range);
1569         if (!dev) {
1570                 struct range *range = &cxled_a->cxld.hpa_range;
1571
1572                 dev_err(port->uport_dev,
1573                         "failed to find decoder that maps %#llx-%#llx\n",
1574                         range->start, range->end);
1575                 goto err;
1576         }
1577
1578         cxlsd = to_cxl_switch_decoder(dev);
1579         do {
1580                 seq = read_seqbegin(&cxlsd->target_lock);
1581                 find_positions(cxlsd, iter_a, iter_b, &a_pos, &b_pos);
1582         } while (read_seqretry(&cxlsd->target_lock, seq));
1583
1584         put_device(dev);
1585
1586         if (a_pos < 0 || b_pos < 0) {
1587                 dev_err(port->uport_dev,
1588                         "failed to find shared decoder for %s and %s\n",
1589                         dev_name(cxlmd_a->dev.parent),
1590                         dev_name(cxlmd_b->dev.parent));
1591                 goto err;
1592         }
1593
1594         dev_dbg(port->uport_dev, "%s comes %s %s\n",
1595                 dev_name(cxlmd_a->dev.parent),
1596                 a_pos - b_pos < 0 ? "before" : "after",
1597                 dev_name(cxlmd_b->dev.parent));
1598
1599         return a_pos - b_pos;
1600 err:
1601         cxled_a->pos = -1;
1602         return 0;
1603 }
1604
1605 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1606 {
1607         struct cxl_region_params *p = &cxlr->params;
1608         int i, rc = 0;
1609
1610         sort(p->targets, p->nr_targets, sizeof(p->targets[0]), cmp_decode_pos,
1611              NULL);
1612
1613         for (i = 0; i < p->nr_targets; i++) {
1614                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1615
1616                 /*
1617                  * Record that sorting failed, but still continue to restore
1618                  * cxled->pos with its ->targets[] position so that follow-on
1619                  * code paths can reliably do p->targets[cxled->pos] to
1620                  * self-reference their entry.
1621                  */
1622                 if (cxled->pos < 0)
1623                         rc = -ENXIO;
1624                 cxled->pos = i;
1625         }
1626
1627         dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1628         return rc;
1629 }
1630
1631 static int cxl_region_attach(struct cxl_region *cxlr,
1632                              struct cxl_endpoint_decoder *cxled, int pos)
1633 {
1634         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1635         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1636         struct cxl_region_params *p = &cxlr->params;
1637         struct cxl_port *ep_port, *root_port;
1638         struct cxl_dport *dport;
1639         int rc = -ENXIO;
1640
1641         if (cxled->mode != cxlr->mode) {
1642                 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1643                         dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1644                 return -EINVAL;
1645         }
1646
1647         if (cxled->mode == CXL_DECODER_DEAD) {
1648                 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1649                 return -ENODEV;
1650         }
1651
1652         /* all full of members, or interleave config not established? */
1653         if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1654                 dev_dbg(&cxlr->dev, "region already active\n");
1655                 return -EBUSY;
1656         } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1657                 dev_dbg(&cxlr->dev, "interleave config missing\n");
1658                 return -ENXIO;
1659         }
1660
1661         ep_port = cxled_to_port(cxled);
1662         root_port = cxlrd_to_port(cxlrd);
1663         dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1664         if (!dport) {
1665                 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1666                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1667                         dev_name(cxlr->dev.parent));
1668                 return -ENXIO;
1669         }
1670
1671         if (cxled->cxld.target_type != cxlr->type) {
1672                 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1673                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1674                         cxled->cxld.target_type, cxlr->type);
1675                 return -ENXIO;
1676         }
1677
1678         if (!cxled->dpa_res) {
1679                 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1680                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1681                 return -ENXIO;
1682         }
1683
1684         if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1685             resource_size(p->res)) {
1686                 dev_dbg(&cxlr->dev,
1687                         "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1688                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1689                         (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1690                         (u64)resource_size(p->res));
1691                 return -EINVAL;
1692         }
1693
1694         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1695                 int i;
1696
1697                 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1698                 if (rc)
1699                         return rc;
1700
1701                 /* await more targets to arrive... */
1702                 if (p->nr_targets < p->interleave_ways)
1703                         return 0;
1704
1705                 /*
1706                  * All targets are here, which implies all PCI enumeration that
1707                  * affects this region has been completed. Walk the topology to
1708                  * sort the devices into their relative region decode position.
1709                  */
1710                 rc = cxl_region_sort_targets(cxlr);
1711                 if (rc)
1712                         return rc;
1713
1714                 for (i = 0; i < p->nr_targets; i++) {
1715                         cxled = p->targets[i];
1716                         ep_port = cxled_to_port(cxled);
1717                         dport = cxl_find_dport_by_dev(root_port,
1718                                                       ep_port->host_bridge);
1719                         rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1720                                                         dport, i);
1721                         if (rc)
1722                                 return rc;
1723                 }
1724
1725                 rc = cxl_region_setup_targets(cxlr);
1726                 if (rc)
1727                         return rc;
1728
1729                 /*
1730                  * If target setup succeeds in the autodiscovery case
1731                  * then the region is already committed.
1732                  */
1733                 p->state = CXL_CONFIG_COMMIT;
1734
1735                 return 0;
1736         }
1737
1738         rc = cxl_region_validate_position(cxlr, cxled, pos);
1739         if (rc)
1740                 return rc;
1741
1742         rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1743         if (rc)
1744                 return rc;
1745
1746         p->targets[pos] = cxled;
1747         cxled->pos = pos;
1748         p->nr_targets++;
1749
1750         if (p->nr_targets == p->interleave_ways) {
1751                 rc = cxl_region_setup_targets(cxlr);
1752                 if (rc)
1753                         goto err_decrement;
1754                 p->state = CXL_CONFIG_ACTIVE;
1755         }
1756
1757         cxled->cxld.interleave_ways = p->interleave_ways;
1758         cxled->cxld.interleave_granularity = p->interleave_granularity;
1759         cxled->cxld.hpa_range = (struct range) {
1760                 .start = p->res->start,
1761                 .end = p->res->end,
1762         };
1763
1764         return 0;
1765
1766 err_decrement:
1767         p->nr_targets--;
1768         cxled->pos = -1;
1769         p->targets[pos] = NULL;
1770         return rc;
1771 }
1772
1773 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1774 {
1775         struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1776         struct cxl_region *cxlr = cxled->cxld.region;
1777         struct cxl_region_params *p;
1778         int rc = 0;
1779
1780         lockdep_assert_held_write(&cxl_region_rwsem);
1781
1782         if (!cxlr)
1783                 return 0;
1784
1785         p = &cxlr->params;
1786         get_device(&cxlr->dev);
1787
1788         if (p->state > CXL_CONFIG_ACTIVE) {
1789                 /*
1790                  * TODO: tear down all impacted regions if a device is
1791                  * removed out of order
1792                  */
1793                 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1794                 if (rc)
1795                         goto out;
1796                 p->state = CXL_CONFIG_ACTIVE;
1797         }
1798
1799         for (iter = ep_port; !is_cxl_root(iter);
1800              iter = to_cxl_port(iter->dev.parent))
1801                 cxl_port_detach_region(iter, cxlr, cxled);
1802
1803         if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1804             p->targets[cxled->pos] != cxled) {
1805                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1806
1807                 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1808                               dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1809                               cxled->pos);
1810                 goto out;
1811         }
1812
1813         if (p->state == CXL_CONFIG_ACTIVE) {
1814                 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1815                 cxl_region_teardown_targets(cxlr);
1816         }
1817         p->targets[cxled->pos] = NULL;
1818         p->nr_targets--;
1819         cxled->cxld.hpa_range = (struct range) {
1820                 .start = 0,
1821                 .end = -1,
1822         };
1823
1824         /* notify the region driver that one of its targets has departed */
1825         up_write(&cxl_region_rwsem);
1826         device_release_driver(&cxlr->dev);
1827         down_write(&cxl_region_rwsem);
1828 out:
1829         put_device(&cxlr->dev);
1830         return rc;
1831 }
1832
1833 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1834 {
1835         down_write(&cxl_region_rwsem);
1836         cxled->mode = CXL_DECODER_DEAD;
1837         cxl_region_detach(cxled);
1838         up_write(&cxl_region_rwsem);
1839 }
1840
1841 static int attach_target(struct cxl_region *cxlr,
1842                          struct cxl_endpoint_decoder *cxled, int pos,
1843                          unsigned int state)
1844 {
1845         int rc = 0;
1846
1847         if (state == TASK_INTERRUPTIBLE)
1848                 rc = down_write_killable(&cxl_region_rwsem);
1849         else
1850                 down_write(&cxl_region_rwsem);
1851         if (rc)
1852                 return rc;
1853
1854         down_read(&cxl_dpa_rwsem);
1855         rc = cxl_region_attach(cxlr, cxled, pos);
1856         up_read(&cxl_dpa_rwsem);
1857         up_write(&cxl_region_rwsem);
1858         return rc;
1859 }
1860
1861 static int detach_target(struct cxl_region *cxlr, int pos)
1862 {
1863         struct cxl_region_params *p = &cxlr->params;
1864         int rc;
1865
1866         rc = down_write_killable(&cxl_region_rwsem);
1867         if (rc)
1868                 return rc;
1869
1870         if (pos >= p->interleave_ways) {
1871                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1872                         p->interleave_ways);
1873                 rc = -ENXIO;
1874                 goto out;
1875         }
1876
1877         if (!p->targets[pos]) {
1878                 rc = 0;
1879                 goto out;
1880         }
1881
1882         rc = cxl_region_detach(p->targets[pos]);
1883 out:
1884         up_write(&cxl_region_rwsem);
1885         return rc;
1886 }
1887
1888 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1889                             size_t len)
1890 {
1891         int rc;
1892
1893         if (sysfs_streq(buf, "\n"))
1894                 rc = detach_target(cxlr, pos);
1895         else {
1896                 struct device *dev;
1897
1898                 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1899                 if (!dev)
1900                         return -ENODEV;
1901
1902                 if (!is_endpoint_decoder(dev)) {
1903                         rc = -EINVAL;
1904                         goto out;
1905                 }
1906
1907                 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1908                                    TASK_INTERRUPTIBLE);
1909 out:
1910                 put_device(dev);
1911         }
1912
1913         if (rc < 0)
1914                 return rc;
1915         return len;
1916 }
1917
1918 #define TARGET_ATTR_RW(n)                                              \
1919 static ssize_t target##n##_show(                                       \
1920         struct device *dev, struct device_attribute *attr, char *buf)  \
1921 {                                                                      \
1922         return show_targetN(to_cxl_region(dev), buf, (n));             \
1923 }                                                                      \
1924 static ssize_t target##n##_store(struct device *dev,                   \
1925                                  struct device_attribute *attr,        \
1926                                  const char *buf, size_t len)          \
1927 {                                                                      \
1928         return store_targetN(to_cxl_region(dev), buf, (n), len);       \
1929 }                                                                      \
1930 static DEVICE_ATTR_RW(target##n)
1931
1932 TARGET_ATTR_RW(0);
1933 TARGET_ATTR_RW(1);
1934 TARGET_ATTR_RW(2);
1935 TARGET_ATTR_RW(3);
1936 TARGET_ATTR_RW(4);
1937 TARGET_ATTR_RW(5);
1938 TARGET_ATTR_RW(6);
1939 TARGET_ATTR_RW(7);
1940 TARGET_ATTR_RW(8);
1941 TARGET_ATTR_RW(9);
1942 TARGET_ATTR_RW(10);
1943 TARGET_ATTR_RW(11);
1944 TARGET_ATTR_RW(12);
1945 TARGET_ATTR_RW(13);
1946 TARGET_ATTR_RW(14);
1947 TARGET_ATTR_RW(15);
1948
1949 static struct attribute *target_attrs[] = {
1950         &dev_attr_target0.attr,
1951         &dev_attr_target1.attr,
1952         &dev_attr_target2.attr,
1953         &dev_attr_target3.attr,
1954         &dev_attr_target4.attr,
1955         &dev_attr_target5.attr,
1956         &dev_attr_target6.attr,
1957         &dev_attr_target7.attr,
1958         &dev_attr_target8.attr,
1959         &dev_attr_target9.attr,
1960         &dev_attr_target10.attr,
1961         &dev_attr_target11.attr,
1962         &dev_attr_target12.attr,
1963         &dev_attr_target13.attr,
1964         &dev_attr_target14.attr,
1965         &dev_attr_target15.attr,
1966         NULL,
1967 };
1968
1969 static umode_t cxl_region_target_visible(struct kobject *kobj,
1970                                          struct attribute *a, int n)
1971 {
1972         struct device *dev = kobj_to_dev(kobj);
1973         struct cxl_region *cxlr = to_cxl_region(dev);
1974         struct cxl_region_params *p = &cxlr->params;
1975
1976         if (n < p->interleave_ways)
1977                 return a->mode;
1978         return 0;
1979 }
1980
1981 static const struct attribute_group cxl_region_target_group = {
1982         .attrs = target_attrs,
1983         .is_visible = cxl_region_target_visible,
1984 };
1985
1986 static const struct attribute_group *get_cxl_region_target_group(void)
1987 {
1988         return &cxl_region_target_group;
1989 }
1990
1991 static const struct attribute_group *region_groups[] = {
1992         &cxl_base_attribute_group,
1993         &cxl_region_group,
1994         &cxl_region_target_group,
1995         NULL,
1996 };
1997
1998 static void cxl_region_release(struct device *dev)
1999 {
2000         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2001         struct cxl_region *cxlr = to_cxl_region(dev);
2002         int id = atomic_read(&cxlrd->region_id);
2003
2004         /*
2005          * Try to reuse the recently idled id rather than the cached
2006          * next id to prevent the region id space from increasing
2007          * unnecessarily.
2008          */
2009         if (cxlr->id < id)
2010                 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2011                         memregion_free(id);
2012                         goto out;
2013                 }
2014
2015         memregion_free(cxlr->id);
2016 out:
2017         put_device(dev->parent);
2018         kfree(cxlr);
2019 }
2020
2021 const struct device_type cxl_region_type = {
2022         .name = "cxl_region",
2023         .release = cxl_region_release,
2024         .groups = region_groups
2025 };
2026
2027 bool is_cxl_region(struct device *dev)
2028 {
2029         return dev->type == &cxl_region_type;
2030 }
2031 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2032
2033 static struct cxl_region *to_cxl_region(struct device *dev)
2034 {
2035         if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2036                           "not a cxl_region device\n"))
2037                 return NULL;
2038
2039         return container_of(dev, struct cxl_region, dev);
2040 }
2041
2042 static void unregister_region(void *dev)
2043 {
2044         struct cxl_region *cxlr = to_cxl_region(dev);
2045         struct cxl_region_params *p = &cxlr->params;
2046         int i;
2047
2048         device_del(dev);
2049
2050         /*
2051          * Now that region sysfs is shutdown, the parameter block is now
2052          * read-only, so no need to hold the region rwsem to access the
2053          * region parameters.
2054          */
2055         for (i = 0; i < p->interleave_ways; i++)
2056                 detach_target(cxlr, i);
2057
2058         cxl_region_iomem_release(cxlr);
2059         put_device(dev);
2060 }
2061
2062 static struct lock_class_key cxl_region_key;
2063
2064 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2065 {
2066         struct cxl_region *cxlr;
2067         struct device *dev;
2068
2069         cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2070         if (!cxlr) {
2071                 memregion_free(id);
2072                 return ERR_PTR(-ENOMEM);
2073         }
2074
2075         dev = &cxlr->dev;
2076         device_initialize(dev);
2077         lockdep_set_class(&dev->mutex, &cxl_region_key);
2078         dev->parent = &cxlrd->cxlsd.cxld.dev;
2079         /*
2080          * Keep root decoder pinned through cxl_region_release to fixup
2081          * region id allocations
2082          */
2083         get_device(dev->parent);
2084         device_set_pm_not_required(dev);
2085         dev->bus = &cxl_bus_type;
2086         dev->type = &cxl_region_type;
2087         cxlr->id = id;
2088
2089         return cxlr;
2090 }
2091
2092 /**
2093  * devm_cxl_add_region - Adds a region to a decoder
2094  * @cxlrd: root decoder
2095  * @id: memregion id to create, or memregion_free() on failure
2096  * @mode: mode for the endpoint decoders of this region
2097  * @type: select whether this is an expander or accelerator (type-2 or type-3)
2098  *
2099  * This is the second step of region initialization. Regions exist within an
2100  * address space which is mapped by a @cxlrd.
2101  *
2102  * Return: 0 if the region was added to the @cxlrd, else returns negative error
2103  * code. The region will be named "regionZ" where Z is the unique region number.
2104  */
2105 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2106                                               int id,
2107                                               enum cxl_decoder_mode mode,
2108                                               enum cxl_decoder_type type)
2109 {
2110         struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2111         struct cxl_region *cxlr;
2112         struct device *dev;
2113         int rc;
2114
2115         switch (mode) {
2116         case CXL_DECODER_RAM:
2117         case CXL_DECODER_PMEM:
2118                 break;
2119         default:
2120                 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2121                 return ERR_PTR(-EINVAL);
2122         }
2123
2124         cxlr = cxl_region_alloc(cxlrd, id);
2125         if (IS_ERR(cxlr))
2126                 return cxlr;
2127         cxlr->mode = mode;
2128         cxlr->type = type;
2129
2130         dev = &cxlr->dev;
2131         rc = dev_set_name(dev, "region%d", id);
2132         if (rc)
2133                 goto err;
2134
2135         rc = device_add(dev);
2136         if (rc)
2137                 goto err;
2138
2139         rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2140         if (rc)
2141                 return ERR_PTR(rc);
2142
2143         dev_dbg(port->uport_dev, "%s: created %s\n",
2144                 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2145         return cxlr;
2146
2147 err:
2148         put_device(dev);
2149         return ERR_PTR(rc);
2150 }
2151
2152 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2153 {
2154         return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2155 }
2156
2157 static ssize_t create_pmem_region_show(struct device *dev,
2158                                        struct device_attribute *attr, char *buf)
2159 {
2160         return __create_region_show(to_cxl_root_decoder(dev), buf);
2161 }
2162
2163 static ssize_t create_ram_region_show(struct device *dev,
2164                                       struct device_attribute *attr, char *buf)
2165 {
2166         return __create_region_show(to_cxl_root_decoder(dev), buf);
2167 }
2168
2169 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2170                                           enum cxl_decoder_mode mode, int id)
2171 {
2172         int rc;
2173
2174         rc = memregion_alloc(GFP_KERNEL);
2175         if (rc < 0)
2176                 return ERR_PTR(rc);
2177
2178         if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2179                 memregion_free(rc);
2180                 return ERR_PTR(-EBUSY);
2181         }
2182
2183         return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2184 }
2185
2186 static ssize_t create_pmem_region_store(struct device *dev,
2187                                         struct device_attribute *attr,
2188                                         const char *buf, size_t len)
2189 {
2190         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2191         struct cxl_region *cxlr;
2192         int rc, id;
2193
2194         rc = sscanf(buf, "region%d\n", &id);
2195         if (rc != 1)
2196                 return -EINVAL;
2197
2198         cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2199         if (IS_ERR(cxlr))
2200                 return PTR_ERR(cxlr);
2201
2202         return len;
2203 }
2204 DEVICE_ATTR_RW(create_pmem_region);
2205
2206 static ssize_t create_ram_region_store(struct device *dev,
2207                                        struct device_attribute *attr,
2208                                        const char *buf, size_t len)
2209 {
2210         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2211         struct cxl_region *cxlr;
2212         int rc, id;
2213
2214         rc = sscanf(buf, "region%d\n", &id);
2215         if (rc != 1)
2216                 return -EINVAL;
2217
2218         cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2219         if (IS_ERR(cxlr))
2220                 return PTR_ERR(cxlr);
2221
2222         return len;
2223 }
2224 DEVICE_ATTR_RW(create_ram_region);
2225
2226 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2227                            char *buf)
2228 {
2229         struct cxl_decoder *cxld = to_cxl_decoder(dev);
2230         ssize_t rc;
2231
2232         rc = down_read_interruptible(&cxl_region_rwsem);
2233         if (rc)
2234                 return rc;
2235
2236         if (cxld->region)
2237                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2238         else
2239                 rc = sysfs_emit(buf, "\n");
2240         up_read(&cxl_region_rwsem);
2241
2242         return rc;
2243 }
2244 DEVICE_ATTR_RO(region);
2245
2246 static struct cxl_region *
2247 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2248 {
2249         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2250         struct device *region_dev;
2251
2252         region_dev = device_find_child_by_name(&cxld->dev, name);
2253         if (!region_dev)
2254                 return ERR_PTR(-ENODEV);
2255
2256         return to_cxl_region(region_dev);
2257 }
2258
2259 static ssize_t delete_region_store(struct device *dev,
2260                                    struct device_attribute *attr,
2261                                    const char *buf, size_t len)
2262 {
2263         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2264         struct cxl_port *port = to_cxl_port(dev->parent);
2265         struct cxl_region *cxlr;
2266
2267         cxlr = cxl_find_region_by_name(cxlrd, buf);
2268         if (IS_ERR(cxlr))
2269                 return PTR_ERR(cxlr);
2270
2271         devm_release_action(port->uport_dev, unregister_region, cxlr);
2272         put_device(&cxlr->dev);
2273
2274         return len;
2275 }
2276 DEVICE_ATTR_WO(delete_region);
2277
2278 static void cxl_pmem_region_release(struct device *dev)
2279 {
2280         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2281         int i;
2282
2283         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2284                 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2285
2286                 put_device(&cxlmd->dev);
2287         }
2288
2289         kfree(cxlr_pmem);
2290 }
2291
2292 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2293         &cxl_base_attribute_group,
2294         NULL,
2295 };
2296
2297 const struct device_type cxl_pmem_region_type = {
2298         .name = "cxl_pmem_region",
2299         .release = cxl_pmem_region_release,
2300         .groups = cxl_pmem_region_attribute_groups,
2301 };
2302
2303 bool is_cxl_pmem_region(struct device *dev)
2304 {
2305         return dev->type == &cxl_pmem_region_type;
2306 }
2307 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2308
2309 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2310 {
2311         if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2312                           "not a cxl_pmem_region device\n"))
2313                 return NULL;
2314         return container_of(dev, struct cxl_pmem_region, dev);
2315 }
2316 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2317
2318 struct cxl_poison_context {
2319         struct cxl_port *port;
2320         enum cxl_decoder_mode mode;
2321         u64 offset;
2322 };
2323
2324 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2325                                    struct cxl_poison_context *ctx)
2326 {
2327         struct cxl_dev_state *cxlds = cxlmd->cxlds;
2328         u64 offset, length;
2329         int rc = 0;
2330
2331         /*
2332          * Collect poison for the remaining unmapped resources
2333          * after poison is collected by committed endpoints.
2334          *
2335          * Knowing that PMEM must always follow RAM, get poison
2336          * for unmapped resources based on the last decoder's mode:
2337          *      ram: scan remains of ram range, then any pmem range
2338          *      pmem: scan remains of pmem range
2339          */
2340
2341         if (ctx->mode == CXL_DECODER_RAM) {
2342                 offset = ctx->offset;
2343                 length = resource_size(&cxlds->ram_res) - offset;
2344                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2345                 if (rc == -EFAULT)
2346                         rc = 0;
2347                 if (rc)
2348                         return rc;
2349         }
2350         if (ctx->mode == CXL_DECODER_PMEM) {
2351                 offset = ctx->offset;
2352                 length = resource_size(&cxlds->dpa_res) - offset;
2353                 if (!length)
2354                         return 0;
2355         } else if (resource_size(&cxlds->pmem_res)) {
2356                 offset = cxlds->pmem_res.start;
2357                 length = resource_size(&cxlds->pmem_res);
2358         } else {
2359                 return 0;
2360         }
2361
2362         return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2363 }
2364
2365 static int poison_by_decoder(struct device *dev, void *arg)
2366 {
2367         struct cxl_poison_context *ctx = arg;
2368         struct cxl_endpoint_decoder *cxled;
2369         struct cxl_memdev *cxlmd;
2370         u64 offset, length;
2371         int rc = 0;
2372
2373         if (!is_endpoint_decoder(dev))
2374                 return rc;
2375
2376         cxled = to_cxl_endpoint_decoder(dev);
2377         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2378                 return rc;
2379
2380         /*
2381          * Regions are only created with single mode decoders: pmem or ram.
2382          * Linux does not support mixed mode decoders. This means that
2383          * reading poison per endpoint decoder adheres to the requirement
2384          * that poison reads of pmem and ram must be separated.
2385          * CXL 3.0 Spec 8.2.9.8.4.1
2386          */
2387         if (cxled->mode == CXL_DECODER_MIXED) {
2388                 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2389                 return rc;
2390         }
2391
2392         cxlmd = cxled_to_memdev(cxled);
2393         if (cxled->skip) {
2394                 offset = cxled->dpa_res->start - cxled->skip;
2395                 length = cxled->skip;
2396                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2397                 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2398                         rc = 0;
2399                 if (rc)
2400                         return rc;
2401         }
2402
2403         offset = cxled->dpa_res->start;
2404         length = cxled->dpa_res->end - offset + 1;
2405         rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2406         if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2407                 rc = 0;
2408         if (rc)
2409                 return rc;
2410
2411         /* Iterate until commit_end is reached */
2412         if (cxled->cxld.id == ctx->port->commit_end) {
2413                 ctx->offset = cxled->dpa_res->end + 1;
2414                 ctx->mode = cxled->mode;
2415                 return 1;
2416         }
2417
2418         return 0;
2419 }
2420
2421 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2422 {
2423         struct cxl_poison_context ctx;
2424         int rc = 0;
2425
2426         rc = down_read_interruptible(&cxl_region_rwsem);
2427         if (rc)
2428                 return rc;
2429
2430         ctx = (struct cxl_poison_context) {
2431                 .port = port
2432         };
2433
2434         rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2435         if (rc == 1)
2436                 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2437                                              &ctx);
2438
2439         up_read(&cxl_region_rwsem);
2440         return rc;
2441 }
2442
2443 static struct lock_class_key cxl_pmem_region_key;
2444
2445 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2446 {
2447         struct cxl_region_params *p = &cxlr->params;
2448         struct cxl_nvdimm_bridge *cxl_nvb;
2449         struct cxl_pmem_region *cxlr_pmem;
2450         struct device *dev;
2451         int i;
2452
2453         down_read(&cxl_region_rwsem);
2454         if (p->state != CXL_CONFIG_COMMIT) {
2455                 cxlr_pmem = ERR_PTR(-ENXIO);
2456                 goto out;
2457         }
2458
2459         cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2460                             GFP_KERNEL);
2461         if (!cxlr_pmem) {
2462                 cxlr_pmem = ERR_PTR(-ENOMEM);
2463                 goto out;
2464         }
2465
2466         cxlr_pmem->hpa_range.start = p->res->start;
2467         cxlr_pmem->hpa_range.end = p->res->end;
2468
2469         /* Snapshot the region configuration underneath the cxl_region_rwsem */
2470         cxlr_pmem->nr_mappings = p->nr_targets;
2471         for (i = 0; i < p->nr_targets; i++) {
2472                 struct cxl_endpoint_decoder *cxled = p->targets[i];
2473                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2474                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2475
2476                 /*
2477                  * Regions never span CXL root devices, so by definition the
2478                  * bridge for one device is the same for all.
2479                  */
2480                 if (i == 0) {
2481                         cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2482                         if (!cxl_nvb) {
2483                                 cxlr_pmem = ERR_PTR(-ENODEV);
2484                                 goto out;
2485                         }
2486                         cxlr->cxl_nvb = cxl_nvb;
2487                 }
2488                 m->cxlmd = cxlmd;
2489                 get_device(&cxlmd->dev);
2490                 m->start = cxled->dpa_res->start;
2491                 m->size = resource_size(cxled->dpa_res);
2492                 m->position = i;
2493         }
2494
2495         dev = &cxlr_pmem->dev;
2496         cxlr_pmem->cxlr = cxlr;
2497         cxlr->cxlr_pmem = cxlr_pmem;
2498         device_initialize(dev);
2499         lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2500         device_set_pm_not_required(dev);
2501         dev->parent = &cxlr->dev;
2502         dev->bus = &cxl_bus_type;
2503         dev->type = &cxl_pmem_region_type;
2504 out:
2505         up_read(&cxl_region_rwsem);
2506
2507         return cxlr_pmem;
2508 }
2509
2510 static void cxl_dax_region_release(struct device *dev)
2511 {
2512         struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2513
2514         kfree(cxlr_dax);
2515 }
2516
2517 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2518         &cxl_base_attribute_group,
2519         NULL,
2520 };
2521
2522 const struct device_type cxl_dax_region_type = {
2523         .name = "cxl_dax_region",
2524         .release = cxl_dax_region_release,
2525         .groups = cxl_dax_region_attribute_groups,
2526 };
2527
2528 static bool is_cxl_dax_region(struct device *dev)
2529 {
2530         return dev->type == &cxl_dax_region_type;
2531 }
2532
2533 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2534 {
2535         if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2536                           "not a cxl_dax_region device\n"))
2537                 return NULL;
2538         return container_of(dev, struct cxl_dax_region, dev);
2539 }
2540 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2541
2542 static struct lock_class_key cxl_dax_region_key;
2543
2544 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2545 {
2546         struct cxl_region_params *p = &cxlr->params;
2547         struct cxl_dax_region *cxlr_dax;
2548         struct device *dev;
2549
2550         down_read(&cxl_region_rwsem);
2551         if (p->state != CXL_CONFIG_COMMIT) {
2552                 cxlr_dax = ERR_PTR(-ENXIO);
2553                 goto out;
2554         }
2555
2556         cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2557         if (!cxlr_dax) {
2558                 cxlr_dax = ERR_PTR(-ENOMEM);
2559                 goto out;
2560         }
2561
2562         cxlr_dax->hpa_range.start = p->res->start;
2563         cxlr_dax->hpa_range.end = p->res->end;
2564
2565         dev = &cxlr_dax->dev;
2566         cxlr_dax->cxlr = cxlr;
2567         device_initialize(dev);
2568         lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2569         device_set_pm_not_required(dev);
2570         dev->parent = &cxlr->dev;
2571         dev->bus = &cxl_bus_type;
2572         dev->type = &cxl_dax_region_type;
2573 out:
2574         up_read(&cxl_region_rwsem);
2575
2576         return cxlr_dax;
2577 }
2578
2579 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2580 {
2581         struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2582         struct cxl_region *cxlr = cxlr_pmem->cxlr;
2583         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2584
2585         /*
2586          * Either the bridge is in ->remove() context under the device_lock(),
2587          * or cxlr_release_nvdimm() is cancelling the bridge's release action
2588          * for @cxlr_pmem and doing it itself (while manually holding the bridge
2589          * lock).
2590          */
2591         device_lock_assert(&cxl_nvb->dev);
2592         cxlr->cxlr_pmem = NULL;
2593         cxlr_pmem->cxlr = NULL;
2594         device_unregister(&cxlr_pmem->dev);
2595 }
2596
2597 static void cxlr_release_nvdimm(void *_cxlr)
2598 {
2599         struct cxl_region *cxlr = _cxlr;
2600         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2601
2602         device_lock(&cxl_nvb->dev);
2603         if (cxlr->cxlr_pmem)
2604                 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2605                                     cxlr->cxlr_pmem);
2606         device_unlock(&cxl_nvb->dev);
2607         cxlr->cxl_nvb = NULL;
2608         put_device(&cxl_nvb->dev);
2609 }
2610
2611 /**
2612  * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2613  * @cxlr: parent CXL region for this pmem region bridge device
2614  *
2615  * Return: 0 on success negative error code on failure.
2616  */
2617 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2618 {
2619         struct cxl_pmem_region *cxlr_pmem;
2620         struct cxl_nvdimm_bridge *cxl_nvb;
2621         struct device *dev;
2622         int rc;
2623
2624         cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2625         if (IS_ERR(cxlr_pmem))
2626                 return PTR_ERR(cxlr_pmem);
2627         cxl_nvb = cxlr->cxl_nvb;
2628
2629         dev = &cxlr_pmem->dev;
2630         rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2631         if (rc)
2632                 goto err;
2633
2634         rc = device_add(dev);
2635         if (rc)
2636                 goto err;
2637
2638         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2639                 dev_name(dev));
2640
2641         device_lock(&cxl_nvb->dev);
2642         if (cxl_nvb->dev.driver)
2643                 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2644                                               cxlr_pmem_unregister, cxlr_pmem);
2645         else
2646                 rc = -ENXIO;
2647         device_unlock(&cxl_nvb->dev);
2648
2649         if (rc)
2650                 goto err_bridge;
2651
2652         /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2653         return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2654
2655 err:
2656         put_device(dev);
2657 err_bridge:
2658         put_device(&cxl_nvb->dev);
2659         cxlr->cxl_nvb = NULL;
2660         return rc;
2661 }
2662
2663 static void cxlr_dax_unregister(void *_cxlr_dax)
2664 {
2665         struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2666
2667         device_unregister(&cxlr_dax->dev);
2668 }
2669
2670 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2671 {
2672         struct cxl_dax_region *cxlr_dax;
2673         struct device *dev;
2674         int rc;
2675
2676         cxlr_dax = cxl_dax_region_alloc(cxlr);
2677         if (IS_ERR(cxlr_dax))
2678                 return PTR_ERR(cxlr_dax);
2679
2680         dev = &cxlr_dax->dev;
2681         rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2682         if (rc)
2683                 goto err;
2684
2685         rc = device_add(dev);
2686         if (rc)
2687                 goto err;
2688
2689         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2690                 dev_name(dev));
2691
2692         return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2693                                         cxlr_dax);
2694 err:
2695         put_device(dev);
2696         return rc;
2697 }
2698
2699 static int match_decoder_by_range(struct device *dev, void *data)
2700 {
2701         struct range *r1, *r2 = data;
2702         struct cxl_root_decoder *cxlrd;
2703
2704         if (!is_root_decoder(dev))
2705                 return 0;
2706
2707         cxlrd = to_cxl_root_decoder(dev);
2708         r1 = &cxlrd->cxlsd.cxld.hpa_range;
2709         return range_contains(r1, r2);
2710 }
2711
2712 static int match_region_by_range(struct device *dev, void *data)
2713 {
2714         struct cxl_region_params *p;
2715         struct cxl_region *cxlr;
2716         struct range *r = data;
2717         int rc = 0;
2718
2719         if (!is_cxl_region(dev))
2720                 return 0;
2721
2722         cxlr = to_cxl_region(dev);
2723         p = &cxlr->params;
2724
2725         down_read(&cxl_region_rwsem);
2726         if (p->res && p->res->start == r->start && p->res->end == r->end)
2727                 rc = 1;
2728         up_read(&cxl_region_rwsem);
2729
2730         return rc;
2731 }
2732
2733 /* Establish an empty region covering the given HPA range */
2734 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2735                                            struct cxl_endpoint_decoder *cxled)
2736 {
2737         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2738         struct cxl_port *port = cxlrd_to_port(cxlrd);
2739         struct range *hpa = &cxled->cxld.hpa_range;
2740         struct cxl_region_params *p;
2741         struct cxl_region *cxlr;
2742         struct resource *res;
2743         int rc;
2744
2745         do {
2746                 cxlr = __create_region(cxlrd, cxled->mode,
2747                                        atomic_read(&cxlrd->region_id));
2748         } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2749
2750         if (IS_ERR(cxlr)) {
2751                 dev_err(cxlmd->dev.parent,
2752                         "%s:%s: %s failed assign region: %ld\n",
2753                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2754                         __func__, PTR_ERR(cxlr));
2755                 return cxlr;
2756         }
2757
2758         down_write(&cxl_region_rwsem);
2759         p = &cxlr->params;
2760         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2761                 dev_err(cxlmd->dev.parent,
2762                         "%s:%s: %s autodiscovery interrupted\n",
2763                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2764                         __func__);
2765                 rc = -EBUSY;
2766                 goto err;
2767         }
2768
2769         set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2770
2771         res = kmalloc(sizeof(*res), GFP_KERNEL);
2772         if (!res) {
2773                 rc = -ENOMEM;
2774                 goto err;
2775         }
2776
2777         *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2778                                     dev_name(&cxlr->dev));
2779         rc = insert_resource(cxlrd->res, res);
2780         if (rc) {
2781                 /*
2782                  * Platform-firmware may not have split resources like "System
2783                  * RAM" on CXL window boundaries see cxl_region_iomem_release()
2784                  */
2785                 dev_warn(cxlmd->dev.parent,
2786                          "%s:%s: %s %s cannot insert resource\n",
2787                          dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2788                          __func__, dev_name(&cxlr->dev));
2789         }
2790
2791         p->res = res;
2792         p->interleave_ways = cxled->cxld.interleave_ways;
2793         p->interleave_granularity = cxled->cxld.interleave_granularity;
2794         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2795
2796         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2797         if (rc)
2798                 goto err;
2799
2800         dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2801                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2802                 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2803                 p->interleave_granularity);
2804
2805         /* ...to match put_device() in cxl_add_to_region() */
2806         get_device(&cxlr->dev);
2807         up_write(&cxl_region_rwsem);
2808
2809         return cxlr;
2810
2811 err:
2812         up_write(&cxl_region_rwsem);
2813         devm_release_action(port->uport_dev, unregister_region, cxlr);
2814         return ERR_PTR(rc);
2815 }
2816
2817 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2818 {
2819         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2820         struct range *hpa = &cxled->cxld.hpa_range;
2821         struct cxl_decoder *cxld = &cxled->cxld;
2822         struct device *cxlrd_dev, *region_dev;
2823         struct cxl_root_decoder *cxlrd;
2824         struct cxl_region_params *p;
2825         struct cxl_region *cxlr;
2826         bool attach = false;
2827         int rc;
2828
2829         cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2830                                       match_decoder_by_range);
2831         if (!cxlrd_dev) {
2832                 dev_err(cxlmd->dev.parent,
2833                         "%s:%s no CXL window for range %#llx:%#llx\n",
2834                         dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2835                         cxld->hpa_range.start, cxld->hpa_range.end);
2836                 return -ENXIO;
2837         }
2838
2839         cxlrd = to_cxl_root_decoder(cxlrd_dev);
2840
2841         /*
2842          * Ensure that if multiple threads race to construct_region() for @hpa
2843          * one does the construction and the others add to that.
2844          */
2845         mutex_lock(&cxlrd->range_lock);
2846         region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2847                                        match_region_by_range);
2848         if (!region_dev) {
2849                 cxlr = construct_region(cxlrd, cxled);
2850                 region_dev = &cxlr->dev;
2851         } else
2852                 cxlr = to_cxl_region(region_dev);
2853         mutex_unlock(&cxlrd->range_lock);
2854
2855         rc = PTR_ERR_OR_ZERO(cxlr);
2856         if (rc)
2857                 goto out;
2858
2859         attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2860
2861         down_read(&cxl_region_rwsem);
2862         p = &cxlr->params;
2863         attach = p->state == CXL_CONFIG_COMMIT;
2864         up_read(&cxl_region_rwsem);
2865
2866         if (attach) {
2867                 /*
2868                  * If device_attach() fails the range may still be active via
2869                  * the platform-firmware memory map, otherwise the driver for
2870                  * regions is local to this file, so driver matching can't fail.
2871                  */
2872                 if (device_attach(&cxlr->dev) < 0)
2873                         dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2874                                 p->res);
2875         }
2876
2877         put_device(region_dev);
2878 out:
2879         put_device(cxlrd_dev);
2880         return rc;
2881 }
2882 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2883
2884 static int is_system_ram(struct resource *res, void *arg)
2885 {
2886         struct cxl_region *cxlr = arg;
2887         struct cxl_region_params *p = &cxlr->params;
2888
2889         dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2890         return 1;
2891 }
2892
2893 static int cxl_region_probe(struct device *dev)
2894 {
2895         struct cxl_region *cxlr = to_cxl_region(dev);
2896         struct cxl_region_params *p = &cxlr->params;
2897         int rc;
2898
2899         rc = down_read_interruptible(&cxl_region_rwsem);
2900         if (rc) {
2901                 dev_dbg(&cxlr->dev, "probe interrupted\n");
2902                 return rc;
2903         }
2904
2905         if (p->state < CXL_CONFIG_COMMIT) {
2906                 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2907                 rc = -ENXIO;
2908                 goto out;
2909         }
2910
2911         if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
2912                 dev_err(&cxlr->dev,
2913                         "failed to activate, re-commit region and retry\n");
2914                 rc = -ENXIO;
2915                 goto out;
2916         }
2917
2918         /*
2919          * From this point on any path that changes the region's state away from
2920          * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2921          */
2922 out:
2923         up_read(&cxl_region_rwsem);
2924
2925         if (rc)
2926                 return rc;
2927
2928         switch (cxlr->mode) {
2929         case CXL_DECODER_PMEM:
2930                 return devm_cxl_add_pmem_region(cxlr);
2931         case CXL_DECODER_RAM:
2932                 /*
2933                  * The region can not be manged by CXL if any portion of
2934                  * it is already online as 'System RAM'
2935                  */
2936                 if (walk_iomem_res_desc(IORES_DESC_NONE,
2937                                         IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2938                                         p->res->start, p->res->end, cxlr,
2939                                         is_system_ram) > 0)
2940                         return 0;
2941                 return devm_cxl_add_dax_region(cxlr);
2942         default:
2943                 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2944                         cxlr->mode);
2945                 return -ENXIO;
2946         }
2947 }
2948
2949 static struct cxl_driver cxl_region_driver = {
2950         .name = "cxl_region",
2951         .probe = cxl_region_probe,
2952         .id = CXL_DEVICE_REGION,
2953 };
2954
2955 int cxl_region_init(void)
2956 {
2957         return cxl_driver_register(&cxl_region_driver);
2958 }
2959
2960 void cxl_region_exit(void)
2961 {
2962         cxl_driver_unregister(&cxl_region_driver);
2963 }
2964
2965 MODULE_IMPORT_NS(CXL);
2966 MODULE_IMPORT_NS(DEVMEM);
2967 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);