drivers/cxl/core/region.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/memregion.h>
   4 #include <linux/genalloc.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/slab.h>
   8 #include <linux/uuid.h>
   9 #include <linux/sort.h>
  10 #include <linux/idr.h>
  11 #include <cxlmem.h>
  12 #include <cxl.h>
  13 #include "core.h"
  14
  15 /**
  16  * DOC: cxl core region
  17  *
  18  * CXL Regions represent mapped memory capacity in system physical address
  19  * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
  20  * Memory ranges, Regions represent the active mapped capacity by the HDM
  21  * Decoder Capability structures throughout the Host Bridges, Switches, and
  22  * Endpoints in the topology.
  23  *
  24  * Region configuration has ordering constraints. UUID may be set at any time
  25  * but is only visible for persistent regions.
  26  * 1. Interleave granularity
  27  * 2. Interleave size
  28  * 3. Decoder targets
  29  */
  30
  31 /*
  32  * All changes to the interleave configuration occur with this lock held
  33  * for write.
  34  */
  35 static DECLARE_RWSEM(cxl_region_rwsem);
  36
  37 static struct cxl_region *to_cxl_region(struct device *dev);
  38
  39 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
  40                          char *buf)
  41 {
  42         struct cxl_region *cxlr = to_cxl_region(dev);
  43         struct cxl_region_params *p = &cxlr->params;
  44         ssize_t rc;
  45
  46         rc = down_read_interruptible(&cxl_region_rwsem);
  47         if (rc)
  48                 return rc;
  49         if (cxlr->mode != CXL_DECODER_PMEM)
  50                 rc = sysfs_emit(buf, "\n");
  51         else
  52                 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
  53         up_read(&cxl_region_rwsem);
  54
  55         return rc;
  56 }
  57
  58 static int is_dup(struct device *match, void *data)
  59 {
  60         struct cxl_region_params *p;
  61         struct cxl_region *cxlr;
  62         uuid_t *uuid = data;
  63
  64         if (!is_cxl_region(match))
  65                 return 0;
  66
  67         lockdep_assert_held(&cxl_region_rwsem);
  68         cxlr = to_cxl_region(match);
  69         p = &cxlr->params;
  70
  71         if (uuid_equal(&p->uuid, uuid)) {
  72                 dev_dbg(match, "already has uuid: %pUb\n", uuid);
  73                 return -EBUSY;
  74         }
  75
  76         return 0;
  77 }
  78
  79 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
  80                           const char *buf, size_t len)
  81 {
  82         struct cxl_region *cxlr = to_cxl_region(dev);
  83         struct cxl_region_params *p = &cxlr->params;
  84         uuid_t temp;
  85         ssize_t rc;
  86
  87         if (len != UUID_STRING_LEN + 1)
  88                 return -EINVAL;
  89
  90         rc = uuid_parse(buf, &temp);
  91         if (rc)
  92                 return rc;
  93
  94         if (uuid_is_null(&temp))
  95                 return -EINVAL;
  96
  97         rc = down_write_killable(&cxl_region_rwsem);
  98         if (rc)
  99                 return rc;
 100
 101         if (uuid_equal(&p->uuid, &temp))
 102                 goto out;
 103
 104         rc = -EBUSY;
 105         if (p->state >= CXL_CONFIG_ACTIVE)
 106                 goto out;
 107
 108         rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
 109         if (rc < 0)
 110                 goto out;
 111
 112         uuid_copy(&p->uuid, &temp);
 113 out:
 114         up_write(&cxl_region_rwsem);
 115
 116         if (rc)
 117                 return rc;
 118         return len;
 119 }
 120 static DEVICE_ATTR_RW(uuid);
 121
 122 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
 123                                           struct cxl_region *cxlr)
 124 {
 125         return xa_load(&port->regions, (unsigned long)cxlr);
 126 }
 127
 128 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
 129 {
 130         if (!cpu_cache_has_invalidate_memregion()) {
 131                 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
 132                         dev_warn_once(
 133                                 &cxlr->dev,
 134                                 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
 135                         return 0;
 136                 } else {
 137                         dev_err(&cxlr->dev,
 138                                 "Failed to synchronize CPU cache state\n");
 139                         return -ENXIO;
 140                 }
 141         }
 142
 143         cpu_cache_invalidate_memregion(IORES_DESC_CXL);
 144         return 0;
 145 }
 146
 147 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
 148 {
 149         struct cxl_region_params *p = &cxlr->params;
 150         int i, rc = 0;
 151
 152         /*
 153          * Before region teardown attempt to flush, and if the flush
 154          * fails cancel the region teardown for data consistency
 155          * concerns
 156          */
 157         rc = cxl_region_invalidate_memregion(cxlr);
 158         if (rc)
 159                 return rc;
 160
 161         for (i = count - 1; i >= 0; i--) {
 162                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 163                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 164                 struct cxl_port *iter = cxled_to_port(cxled);
 165                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 166                 struct cxl_ep *ep;
 167
 168                 if (cxlds->rcd)
 169                         goto endpoint_reset;
 170
 171                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
 172                         iter = to_cxl_port(iter->dev.parent);
 173
 174                 for (ep = cxl_ep_load(iter, cxlmd); iter;
 175                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 176                         struct cxl_region_ref *cxl_rr;
 177                         struct cxl_decoder *cxld;
 178
 179                         cxl_rr = cxl_rr_load(iter, cxlr);
 180                         cxld = cxl_rr->decoder;
 181                         if (cxld->reset)
 182                                 rc = cxld->reset(cxld);
 183                         if (rc)
 184                                 return rc;
 185                         set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 186                 }
 187
 188 endpoint_reset:
 189                 rc = cxled->cxld.reset(&cxled->cxld);
 190                 if (rc)
 191                         return rc;
 192                 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 193         }
 194
 195         /* all decoders associated with this region have been torn down */
 196         clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 197
 198         return 0;
 199 }
 200
 201 static int commit_decoder(struct cxl_decoder *cxld)
 202 {
 203         struct cxl_switch_decoder *cxlsd = NULL;
 204
 205         if (cxld->commit)
 206                 return cxld->commit(cxld);
 207
 208         if (is_switch_decoder(&cxld->dev))
 209                 cxlsd = to_cxl_switch_decoder(&cxld->dev);
 210
 211         if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
 212                           "->commit() is required\n"))
 213                 return -ENXIO;
 214         return 0;
 215 }
 216
 217 static int cxl_region_decode_commit(struct cxl_region *cxlr)
 218 {
 219         struct cxl_region_params *p = &cxlr->params;
 220         int i, rc = 0;
 221
 222         for (i = 0; i < p->nr_targets; i++) {
 223                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 224                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 225                 struct cxl_region_ref *cxl_rr;
 226                 struct cxl_decoder *cxld;
 227                 struct cxl_port *iter;
 228                 struct cxl_ep *ep;
 229
 230                 /* commit bottom up */
 231                 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
 232                      iter = to_cxl_port(iter->dev.parent)) {
 233                         cxl_rr = cxl_rr_load(iter, cxlr);
 234                         cxld = cxl_rr->decoder;
 235                         rc = commit_decoder(cxld);
 236                         if (rc)
 237                                 break;
 238                 }
 239
 240                 if (rc) {
 241                         /* programming @iter failed, teardown */
 242                         for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
 243                              iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 244                                 cxl_rr = cxl_rr_load(iter, cxlr);
 245                                 cxld = cxl_rr->decoder;
 246                                 if (cxld->reset)
 247                                         cxld->reset(cxld);
 248                         }
 249
 250                         cxled->cxld.reset(&cxled->cxld);
 251                         goto err;
 252                 }
 253         }
 254
 255         return 0;
 256
 257 err:
 258         /* undo the targets that were successfully committed */
 259         cxl_region_decode_reset(cxlr, i);
 260         return rc;
 261 }
 262
 263 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
 264                             const char *buf, size_t len)
 265 {
 266         struct cxl_region *cxlr = to_cxl_region(dev);
 267         struct cxl_region_params *p = &cxlr->params;
 268         bool commit;
 269         ssize_t rc;
 270
 271         rc = kstrtobool(buf, &commit);
 272         if (rc)
 273                 return rc;
 274
 275         rc = down_write_killable(&cxl_region_rwsem);
 276         if (rc)
 277                 return rc;
 278
 279         /* Already in the requested state? */
 280         if (commit && p->state >= CXL_CONFIG_COMMIT)
 281                 goto out;
 282         if (!commit && p->state < CXL_CONFIG_COMMIT)
 283                 goto out;
 284
 285         /* Not ready to commit? */
 286         if (commit && p->state < CXL_CONFIG_ACTIVE) {
 287                 rc = -ENXIO;
 288                 goto out;
 289         }
 290
 291         /*
 292          * Invalidate caches before region setup to drop any speculative
 293          * consumption of this address space
 294          */
 295         rc = cxl_region_invalidate_memregion(cxlr);
 296         if (rc)
 297                 return rc;
 298
 299         if (commit) {
 300                 rc = cxl_region_decode_commit(cxlr);
 301                 if (rc == 0)
 302                         p->state = CXL_CONFIG_COMMIT;
 303         } else {
 304                 p->state = CXL_CONFIG_RESET_PENDING;
 305                 up_write(&cxl_region_rwsem);
 306                 device_release_driver(&cxlr->dev);
 307                 down_write(&cxl_region_rwsem);
 308
 309                 /*
 310                  * The lock was dropped, so need to revalidate that the reset is
 311                  * still pending.
 312                  */
 313                 if (p->state == CXL_CONFIG_RESET_PENDING) {
 314                         rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
 315                         /*
 316                          * Revert to committed since there may still be active
 317                          * decoders associated with this region, or move forward
 318                          * to active to mark the reset successful
 319                          */
 320                         if (rc)
 321                                 p->state = CXL_CONFIG_COMMIT;
 322                         else
 323                                 p->state = CXL_CONFIG_ACTIVE;
 324                 }
 325         }
 326
 327 out:
 328         up_write(&cxl_region_rwsem);
 329
 330         if (rc)
 331                 return rc;
 332         return len;
 333 }
 334
 335 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
 336                            char *buf)
 337 {
 338         struct cxl_region *cxlr = to_cxl_region(dev);
 339         struct cxl_region_params *p = &cxlr->params;
 340         ssize_t rc;
 341
 342         rc = down_read_interruptible(&cxl_region_rwsem);
 343         if (rc)
 344                 return rc;
 345         rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
 346         up_read(&cxl_region_rwsem);
 347
 348         return rc;
 349 }
 350 static DEVICE_ATTR_RW(commit);
 351
 352 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
 353                                   int n)
 354 {
 355         struct device *dev = kobj_to_dev(kobj);
 356         struct cxl_region *cxlr = to_cxl_region(dev);
 357
 358         /*
 359          * Support tooling that expects to find a 'uuid' attribute for all
 360          * regions regardless of mode.
 361          */
 362         if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
 363                 return 0444;
 364         return a->mode;
 365 }
 366
 367 static ssize_t interleave_ways_show(struct device *dev,
 368                                     struct device_attribute *attr, char *buf)
 369 {
 370         struct cxl_region *cxlr = to_cxl_region(dev);
 371         struct cxl_region_params *p = &cxlr->params;
 372         ssize_t rc;
 373
 374         rc = down_read_interruptible(&cxl_region_rwsem);
 375         if (rc)
 376                 return rc;
 377         rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
 378         up_read(&cxl_region_rwsem);
 379
 380         return rc;
 381 }
 382
 383 static const struct attribute_group *get_cxl_region_target_group(void);
 384
 385 static ssize_t interleave_ways_store(struct device *dev,
 386                                      struct device_attribute *attr,
 387                                      const char *buf, size_t len)
 388 {
 389         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 390         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 391         struct cxl_region *cxlr = to_cxl_region(dev);
 392         struct cxl_region_params *p = &cxlr->params;
 393         unsigned int val, save;
 394         int rc;
 395         u8 iw;
 396
 397         rc = kstrtouint(buf, 0, &val);
 398         if (rc)
 399                 return rc;
 400
 401         rc = ways_to_eiw(val, &iw);
 402         if (rc)
 403                 return rc;
 404
 405         /*
 406          * Even for x3, x9, and x12 interleaves the region interleave must be a
 407          * power of 2 multiple of the host bridge interleave.
 408          */
 409         if (!is_power_of_2(val / cxld->interleave_ways) ||
 410             (val % cxld->interleave_ways)) {
 411                 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
 412                 return -EINVAL;
 413         }
 414
 415         rc = down_write_killable(&cxl_region_rwsem);
 416         if (rc)
 417                 return rc;
 418         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 419                 rc = -EBUSY;
 420                 goto out;
 421         }
 422
 423         save = p->interleave_ways;
 424         p->interleave_ways = val;
 425         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
 426         if (rc)
 427                 p->interleave_ways = save;
 428 out:
 429         up_write(&cxl_region_rwsem);
 430         if (rc)
 431                 return rc;
 432         return len;
 433 }
 434 static DEVICE_ATTR_RW(interleave_ways);
 435
 436 static ssize_t interleave_granularity_show(struct device *dev,
 437                                            struct device_attribute *attr,
 438                                            char *buf)
 439 {
 440         struct cxl_region *cxlr = to_cxl_region(dev);
 441         struct cxl_region_params *p = &cxlr->params;
 442         ssize_t rc;
 443
 444         rc = down_read_interruptible(&cxl_region_rwsem);
 445         if (rc)
 446                 return rc;
 447         rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
 448         up_read(&cxl_region_rwsem);
 449
 450         return rc;
 451 }
 452
 453 static ssize_t interleave_granularity_store(struct device *dev,
 454                                             struct device_attribute *attr,
 455                                             const char *buf, size_t len)
 456 {
 457         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 458         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 459         struct cxl_region *cxlr = to_cxl_region(dev);
 460         struct cxl_region_params *p = &cxlr->params;
 461         int rc, val;
 462         u16 ig;
 463
 464         rc = kstrtoint(buf, 0, &val);
 465         if (rc)
 466                 return rc;
 467
 468         rc = granularity_to_eig(val, &ig);
 469         if (rc)
 470                 return rc;
 471
 472         /*
 473          * When the host-bridge is interleaved, disallow region granularity !=
 474          * root granularity. Regions with a granularity less than the root
 475          * interleave result in needing multiple endpoints to support a single
 476          * slot in the interleave (possible to support in the future). Regions
 477          * with a granularity greater than the root interleave result in invalid
 478          * DPA translations (invalid to support).
 479          */
 480         if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
 481                 return -EINVAL;
 482
 483         rc = down_write_killable(&cxl_region_rwsem);
 484         if (rc)
 485                 return rc;
 486         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 487                 rc = -EBUSY;
 488                 goto out;
 489         }
 490
 491         p->interleave_granularity = val;
 492 out:
 493         up_write(&cxl_region_rwsem);
 494         if (rc)
 495                 return rc;
 496         return len;
 497 }
 498 static DEVICE_ATTR_RW(interleave_granularity);
 499
 500 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 501                              char *buf)
 502 {
 503         struct cxl_region *cxlr = to_cxl_region(dev);
 504         struct cxl_region_params *p = &cxlr->params;
 505         u64 resource = -1ULL;
 506         ssize_t rc;
 507
 508         rc = down_read_interruptible(&cxl_region_rwsem);
 509         if (rc)
 510                 return rc;
 511         if (p->res)
 512                 resource = p->res->start;
 513         rc = sysfs_emit(buf, "%#llx\n", resource);
 514         up_read(&cxl_region_rwsem);
 515
 516         return rc;
 517 }
 518 static DEVICE_ATTR_RO(resource);
 519
 520 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 521                          char *buf)
 522 {
 523         struct cxl_region *cxlr = to_cxl_region(dev);
 524
 525         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
 526 }
 527 static DEVICE_ATTR_RO(mode);
 528
 529 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
 530 {
 531         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
 532         struct cxl_region_params *p = &cxlr->params;
 533         struct resource *res;
 534         u32 remainder = 0;
 535
 536         lockdep_assert_held_write(&cxl_region_rwsem);
 537
 538         /* Nothing to do... */
 539         if (p->res && resource_size(p->res) == size)
 540                 return 0;
 541
 542         /* To change size the old size must be freed first */
 543         if (p->res)
 544                 return -EBUSY;
 545
 546         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 547                 return -EBUSY;
 548
 549         /* ways, granularity and uuid (if PMEM) need to be set before HPA */
 550         if (!p->interleave_ways || !p->interleave_granularity ||
 551             (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
 552                 return -ENXIO;
 553
 554         div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
 555         if (remainder)
 556                 return -EINVAL;
 557
 558         res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
 559                                     dev_name(&cxlr->dev));
 560         if (IS_ERR(res)) {
 561                 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
 562                         PTR_ERR(res));
 563                 return PTR_ERR(res);
 564         }
 565
 566         p->res = res;
 567         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
 568
 569         return 0;
 570 }
 571
 572 static void cxl_region_iomem_release(struct cxl_region *cxlr)
 573 {
 574         struct cxl_region_params *p = &cxlr->params;
 575
 576         if (device_is_registered(&cxlr->dev))
 577                 lockdep_assert_held_write(&cxl_region_rwsem);
 578         if (p->res) {
 579                 /*
 580                  * Autodiscovered regions may not have been able to insert their
 581                  * resource.
 582                  */
 583                 if (p->res->parent)
 584                         remove_resource(p->res);
 585                 kfree(p->res);
 586                 p->res = NULL;
 587         }
 588 }
 589
 590 static int free_hpa(struct cxl_region *cxlr)
 591 {
 592         struct cxl_region_params *p = &cxlr->params;
 593
 594         lockdep_assert_held_write(&cxl_region_rwsem);
 595
 596         if (!p->res)
 597                 return 0;
 598
 599         if (p->state >= CXL_CONFIG_ACTIVE)
 600                 return -EBUSY;
 601
 602         cxl_region_iomem_release(cxlr);
 603         p->state = CXL_CONFIG_IDLE;
 604         return 0;
 605 }
 606
 607 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
 608                           const char *buf, size_t len)
 609 {
 610         struct cxl_region *cxlr = to_cxl_region(dev);
 611         u64 val;
 612         int rc;
 613
 614         rc = kstrtou64(buf, 0, &val);
 615         if (rc)
 616                 return rc;
 617
 618         rc = down_write_killable(&cxl_region_rwsem);
 619         if (rc)
 620                 return rc;
 621
 622         if (val)
 623                 rc = alloc_hpa(cxlr, val);
 624         else
 625                 rc = free_hpa(cxlr);
 626         up_write(&cxl_region_rwsem);
 627
 628         if (rc)
 629                 return rc;
 630
 631         return len;
 632 }
 633
 634 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 635                          char *buf)
 636 {
 637         struct cxl_region *cxlr = to_cxl_region(dev);
 638         struct cxl_region_params *p = &cxlr->params;
 639         u64 size = 0;
 640         ssize_t rc;
 641
 642         rc = down_read_interruptible(&cxl_region_rwsem);
 643         if (rc)
 644                 return rc;
 645         if (p->res)
 646                 size = resource_size(p->res);
 647         rc = sysfs_emit(buf, "%#llx\n", size);
 648         up_read(&cxl_region_rwsem);
 649
 650         return rc;
 651 }
 652 static DEVICE_ATTR_RW(size);
 653
 654 static struct attribute *cxl_region_attrs[] = {
 655         &dev_attr_uuid.attr,
 656         &dev_attr_commit.attr,
 657         &dev_attr_interleave_ways.attr,
 658         &dev_attr_interleave_granularity.attr,
 659         &dev_attr_resource.attr,
 660         &dev_attr_size.attr,
 661         &dev_attr_mode.attr,
 662         NULL,
 663 };
 664
 665 static const struct attribute_group cxl_region_group = {
 666         .attrs = cxl_region_attrs,
 667         .is_visible = cxl_region_visible,
 668 };
 669
 670 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
 671 {
 672         struct cxl_region_params *p = &cxlr->params;
 673         struct cxl_endpoint_decoder *cxled;
 674         int rc;
 675
 676         rc = down_read_interruptible(&cxl_region_rwsem);
 677         if (rc)
 678                 return rc;
 679
 680         if (pos >= p->interleave_ways) {
 681                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
 682                         p->interleave_ways);
 683                 rc = -ENXIO;
 684                 goto out;
 685         }
 686
 687         cxled = p->targets[pos];
 688         if (!cxled)
 689                 rc = sysfs_emit(buf, "\n");
 690         else
 691                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
 692 out:
 693         up_read(&cxl_region_rwsem);
 694
 695         return rc;
 696 }
 697
 698 static int match_free_decoder(struct device *dev, void *data)
 699 {
 700         struct cxl_decoder *cxld;
 701         int *id = data;
 702
 703         if (!is_switch_decoder(dev))
 704                 return 0;
 705
 706         cxld = to_cxl_decoder(dev);
 707
 708         /* enforce ordered allocation */
 709         if (cxld->id != *id)
 710                 return 0;
 711
 712         if (!cxld->region)
 713                 return 1;
 714
 715         (*id)++;
 716
 717         return 0;
 718 }
 719
 720 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
 721                                                    struct cxl_region *cxlr)
 722 {
 723         struct device *dev;
 724         int id = 0;
 725
 726         dev = device_find_child(&port->dev, &id, match_free_decoder);
 727         if (!dev)
 728                 return NULL;
 729         /*
 730          * This decoder is pinned registered as long as the endpoint decoder is
 731          * registered, and endpoint decoder unregistration holds the
 732          * cxl_region_rwsem over unregister events, so no need to hold on to
 733          * this extra reference.
 734          */
 735         put_device(dev);
 736         return to_cxl_decoder(dev);
 737 }
 738
 739 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
 740                                                struct cxl_region *cxlr)
 741 {
 742         struct cxl_region_params *p = &cxlr->params;
 743         struct cxl_region_ref *cxl_rr, *iter;
 744         unsigned long index;
 745         int rc;
 746
 747         xa_for_each(&port->regions, index, iter) {
 748                 struct cxl_region_params *ip = &iter->region->params;
 749
 750                 if (!ip->res)
 751                         continue;
 752
 753                 if (ip->res->start > p->res->start) {
 754                         dev_dbg(&cxlr->dev,
 755                                 "%s: HPA order violation %s:%pr vs %pr\n",
 756                                 dev_name(&port->dev),
 757                                 dev_name(&iter->region->dev), ip->res, p->res);
 758                         return ERR_PTR(-EBUSY);
 759                 }
 760         }
 761
 762         cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
 763         if (!cxl_rr)
 764                 return ERR_PTR(-ENOMEM);
 765         cxl_rr->port = port;
 766         cxl_rr->region = cxlr;
 767         cxl_rr->nr_targets = 1;
 768         xa_init(&cxl_rr->endpoints);
 769
 770         rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
 771         if (rc) {
 772                 dev_dbg(&cxlr->dev,
 773                         "%s: failed to track region reference: %d\n",
 774                         dev_name(&port->dev), rc);
 775                 kfree(cxl_rr);
 776                 return ERR_PTR(rc);
 777         }
 778
 779         return cxl_rr;
 780 }
 781
 782 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
 783 {
 784         struct cxl_region *cxlr = cxl_rr->region;
 785         struct cxl_decoder *cxld = cxl_rr->decoder;
 786
 787         if (!cxld)
 788                 return;
 789
 790         dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
 791         if (cxld->region == cxlr) {
 792                 cxld->region = NULL;
 793                 put_device(&cxlr->dev);
 794         }
 795 }
 796
 797 static void free_region_ref(struct cxl_region_ref *cxl_rr)
 798 {
 799         struct cxl_port *port = cxl_rr->port;
 800         struct cxl_region *cxlr = cxl_rr->region;
 801
 802         cxl_rr_free_decoder(cxl_rr);
 803         xa_erase(&port->regions, (unsigned long)cxlr);
 804         xa_destroy(&cxl_rr->endpoints);
 805         kfree(cxl_rr);
 806 }
 807
 808 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
 809                          struct cxl_endpoint_decoder *cxled)
 810 {
 811         int rc;
 812         struct cxl_port *port = cxl_rr->port;
 813         struct cxl_region *cxlr = cxl_rr->region;
 814         struct cxl_decoder *cxld = cxl_rr->decoder;
 815         struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
 816
 817         if (ep) {
 818                 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
 819                                GFP_KERNEL);
 820                 if (rc)
 821                         return rc;
 822         }
 823         cxl_rr->nr_eps++;
 824
 825         if (!cxld->region) {
 826                 cxld->region = cxlr;
 827                 get_device(&cxlr->dev);
 828         }
 829
 830         return 0;
 831 }
 832
 833 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
 834                                 struct cxl_endpoint_decoder *cxled,
 835                                 struct cxl_region_ref *cxl_rr)
 836 {
 837         struct cxl_decoder *cxld;
 838
 839         if (port == cxled_to_port(cxled))
 840                 cxld = &cxled->cxld;
 841         else
 842                 cxld = cxl_region_find_decoder(port, cxlr);
 843         if (!cxld) {
 844                 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
 845                         dev_name(&port->dev));
 846                 return -EBUSY;
 847         }
 848
 849         if (cxld->region) {
 850                 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
 851                         dev_name(&port->dev), dev_name(&cxld->dev),
 852                         dev_name(&cxld->region->dev));
 853                 return -EBUSY;
 854         }
 855
 856         /*
 857          * Endpoints should already match the region type, but backstop that
 858          * assumption with an assertion. Switch-decoders change mapping-type
 859          * based on what is mapped when they are assigned to a region.
 860          */
 861         dev_WARN_ONCE(&cxlr->dev,
 862                       port == cxled_to_port(cxled) &&
 863                               cxld->target_type != cxlr->type,
 864                       "%s:%s mismatch decoder type %d -> %d\n",
 865                       dev_name(&cxled_to_memdev(cxled)->dev),
 866                       dev_name(&cxld->dev), cxld->target_type, cxlr->type);
 867         cxld->target_type = cxlr->type;
 868         cxl_rr->decoder = cxld;
 869         return 0;
 870 }
 871
 872 /**
 873  * cxl_port_attach_region() - track a region's interest in a port by endpoint
 874  * @port: port to add a new region reference 'struct cxl_region_ref'
 875  * @cxlr: region to attach to @port
 876  * @cxled: endpoint decoder used to create or further pin a region reference
 877  * @pos: interleave position of @cxled in @cxlr
 878  *
 879  * The attach event is an opportunity to validate CXL decode setup
 880  * constraints and record metadata needed for programming HDM decoders,
 881  * in particular decoder target lists.
 882  *
 883  * The steps are:
 884  *
 885  * - validate that there are no other regions with a higher HPA already
 886  *   associated with @port
 887  * - establish a region reference if one is not already present
 888  *
 889  *   - additionally allocate a decoder instance that will host @cxlr on
 890  *     @port
 891  *
 892  * - pin the region reference by the endpoint
 893  * - account for how many entries in @port's target list are needed to
 894  *   cover all of the added endpoints.
 895  */
 896 static int cxl_port_attach_region(struct cxl_port *port,
 897                                   struct cxl_region *cxlr,
 898                                   struct cxl_endpoint_decoder *cxled, int pos)
 899 {
 900         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 901         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
 902         struct cxl_region_ref *cxl_rr;
 903         bool nr_targets_inc = false;
 904         struct cxl_decoder *cxld;
 905         unsigned long index;
 906         int rc = -EBUSY;
 907
 908         lockdep_assert_held_write(&cxl_region_rwsem);
 909
 910         cxl_rr = cxl_rr_load(port, cxlr);
 911         if (cxl_rr) {
 912                 struct cxl_ep *ep_iter;
 913                 int found = 0;
 914
 915                 /*
 916                  * Walk the existing endpoints that have been attached to
 917                  * @cxlr at @port and see if they share the same 'next' port
 918                  * in the downstream direction. I.e. endpoints that share common
 919                  * upstream switch.
 920                  */
 921                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
 922                         if (ep_iter == ep)
 923                                 continue;
 924                         if (ep_iter->next == ep->next) {
 925                                 found++;
 926                                 break;
 927                         }
 928                 }
 929
 930                 /*
 931                  * New target port, or @port is an endpoint port that always
 932                  * accounts its own local decode as a target.
 933                  */
 934                 if (!found || !ep->next) {
 935                         cxl_rr->nr_targets++;
 936                         nr_targets_inc = true;
 937                 }
 938         } else {
 939                 cxl_rr = alloc_region_ref(port, cxlr);
 940                 if (IS_ERR(cxl_rr)) {
 941                         dev_dbg(&cxlr->dev,
 942                                 "%s: failed to allocate region reference\n",
 943                                 dev_name(&port->dev));
 944                         return PTR_ERR(cxl_rr);
 945                 }
 946                 nr_targets_inc = true;
 947
 948                 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
 949                 if (rc)
 950                         goto out_erase;
 951         }
 952         cxld = cxl_rr->decoder;
 953
 954         rc = cxl_rr_ep_add(cxl_rr, cxled);
 955         if (rc) {
 956                 dev_dbg(&cxlr->dev,
 957                         "%s: failed to track endpoint %s:%s reference\n",
 958                         dev_name(&port->dev), dev_name(&cxlmd->dev),
 959                         dev_name(&cxld->dev));
 960                 goto out_erase;
 961         }
 962
 963         dev_dbg(&cxlr->dev,
 964                 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
 965                 dev_name(port->uport_dev), dev_name(&port->dev),
 966                 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
 967                 dev_name(&cxled->cxld.dev), pos,
 968                 ep ? ep->next ? dev_name(ep->next->uport_dev) :
 969                                       dev_name(&cxlmd->dev) :
 970                            "none",
 971                 cxl_rr->nr_eps, cxl_rr->nr_targets);
 972
 973         return 0;
 974 out_erase:
 975         if (nr_targets_inc)
 976                 cxl_rr->nr_targets--;
 977         if (cxl_rr->nr_eps == 0)
 978                 free_region_ref(cxl_rr);
 979         return rc;
 980 }
 981
 982 static void cxl_port_detach_region(struct cxl_port *port,
 983                                    struct cxl_region *cxlr,
 984                                    struct cxl_endpoint_decoder *cxled)
 985 {
 986         struct cxl_region_ref *cxl_rr;
 987         struct cxl_ep *ep = NULL;
 988
 989         lockdep_assert_held_write(&cxl_region_rwsem);
 990
 991         cxl_rr = cxl_rr_load(port, cxlr);
 992         if (!cxl_rr)
 993                 return;
 994
 995         /*
 996          * Endpoint ports do not carry cxl_ep references, and they
 997          * never target more than one endpoint by definition
 998          */
 999         if (cxl_rr->decoder == &cxled->cxld)
1000                 cxl_rr->nr_eps--;
1001         else
1002                 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1003         if (ep) {
1004                 struct cxl_ep *ep_iter;
1005                 unsigned long index;
1006                 int found = 0;
1007
1008                 cxl_rr->nr_eps--;
1009                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1010                         if (ep_iter->next == ep->next) {
1011                                 found++;
1012                                 break;
1013                         }
1014                 }
1015                 if (!found)
1016                         cxl_rr->nr_targets--;
1017         }
1018
1019         if (cxl_rr->nr_eps == 0)
1020                 free_region_ref(cxl_rr);
1021 }
1022
1023 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1024                            struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1025                            int distance)
1026 {
1027         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1028         struct cxl_region *cxlr = cxl_rr->region;
1029         struct cxl_region_params *p = &cxlr->params;
1030         struct cxl_endpoint_decoder *cxled_peer;
1031         struct cxl_port *port = cxl_rr->port;
1032         struct cxl_memdev *cxlmd_peer;
1033         struct cxl_ep *ep_peer;
1034         int pos = cxled->pos;
1035
1036         /*
1037          * If this position wants to share a dport with the last endpoint mapped
1038          * then that endpoint, at index 'position - distance', must also be
1039          * mapped by this dport.
1040          */
1041         if (pos < distance) {
1042                 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1043                         dev_name(port->uport_dev), dev_name(&port->dev),
1044                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1045                 return -ENXIO;
1046         }
1047         cxled_peer = p->targets[pos - distance];
1048         cxlmd_peer = cxled_to_memdev(cxled_peer);
1049         ep_peer = cxl_ep_load(port, cxlmd_peer);
1050         if (ep->dport != ep_peer->dport) {
1051                 dev_dbg(&cxlr->dev,
1052                         "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1053                         dev_name(port->uport_dev), dev_name(&port->dev),
1054                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1055                         dev_name(&cxlmd_peer->dev),
1056                         dev_name(&cxled_peer->cxld.dev));
1057                 return -ENXIO;
1058         }
1059
1060         return 0;
1061 }
1062
1063 static int cxl_port_setup_targets(struct cxl_port *port,
1064                                   struct cxl_region *cxlr,
1065                                   struct cxl_endpoint_decoder *cxled)
1066 {
1067         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1068         int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1069         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1070         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1071         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1072         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1073         struct cxl_region_params *p = &cxlr->params;
1074         struct cxl_decoder *cxld = cxl_rr->decoder;
1075         struct cxl_switch_decoder *cxlsd;
1076         u16 eig, peig;
1077         u8 eiw, peiw;
1078
1079         /*
1080          * While root level decoders support x3, x6, x12, switch level
1081          * decoders only support powers of 2 up to x16.
1082          */
1083         if (!is_power_of_2(cxl_rr->nr_targets)) {
1084                 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1085                         dev_name(port->uport_dev), dev_name(&port->dev),
1086                         cxl_rr->nr_targets);
1087                 return -EINVAL;
1088         }
1089
1090         cxlsd = to_cxl_switch_decoder(&cxld->dev);
1091         if (cxl_rr->nr_targets_set) {
1092                 int i, distance;
1093
1094                 /*
1095                  * Passthrough decoders impose no distance requirements between
1096                  * peers
1097                  */
1098                 if (cxl_rr->nr_targets == 1)
1099                         distance = 0;
1100                 else
1101                         distance = p->nr_targets / cxl_rr->nr_targets;
1102                 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1103                         if (ep->dport == cxlsd->target[i]) {
1104                                 rc = check_last_peer(cxled, ep, cxl_rr,
1105                                                      distance);
1106                                 if (rc)
1107                                         return rc;
1108                                 goto out_target_set;
1109                         }
1110                 goto add_target;
1111         }
1112
1113         if (is_cxl_root(parent_port)) {
1114                 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1115                 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1116                 /*
1117                  * For purposes of address bit routing, use power-of-2 math for
1118                  * switch ports.
1119                  */
1120                 if (!is_power_of_2(parent_iw))
1121                         parent_iw /= 3;
1122         } else {
1123                 struct cxl_region_ref *parent_rr;
1124                 struct cxl_decoder *parent_cxld;
1125
1126                 parent_rr = cxl_rr_load(parent_port, cxlr);
1127                 parent_cxld = parent_rr->decoder;
1128                 parent_ig = parent_cxld->interleave_granularity;
1129                 parent_iw = parent_cxld->interleave_ways;
1130         }
1131
1132         rc = granularity_to_eig(parent_ig, &peig);
1133         if (rc) {
1134                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1135                         dev_name(parent_port->uport_dev),
1136                         dev_name(&parent_port->dev), parent_ig);
1137                 return rc;
1138         }
1139
1140         rc = ways_to_eiw(parent_iw, &peiw);
1141         if (rc) {
1142                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1143                         dev_name(parent_port->uport_dev),
1144                         dev_name(&parent_port->dev), parent_iw);
1145                 return rc;
1146         }
1147
1148         iw = cxl_rr->nr_targets;
1149         rc = ways_to_eiw(iw, &eiw);
1150         if (rc) {
1151                 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1152                         dev_name(port->uport_dev), dev_name(&port->dev), iw);
1153                 return rc;
1154         }
1155
1156         /*
1157          * If @parent_port is masking address bits, pick the next unused address
1158          * bit to route @port's targets.
1159          */
1160         if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
1161                 u32 address_bit = max(peig + peiw, eiw + peig);
1162
1163                 eig = address_bit - eiw + 1;
1164         } else {
1165                 eiw = peiw;
1166                 eig = peig;
1167         }
1168
1169         rc = eig_to_granularity(eig, &ig);
1170         if (rc) {
1171                 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1172                         dev_name(port->uport_dev), dev_name(&port->dev),
1173                         256 << eig);
1174                 return rc;
1175         }
1176
1177         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1178                 if (cxld->interleave_ways != iw ||
1179                     cxld->interleave_granularity != ig ||
1180                     cxld->hpa_range.start != p->res->start ||
1181                     cxld->hpa_range.end != p->res->end ||
1182                     ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1183                         dev_err(&cxlr->dev,
1184                                 "%s:%s %s expected iw: %d ig: %d %pr\n",
1185                                 dev_name(port->uport_dev), dev_name(&port->dev),
1186                                 __func__, iw, ig, p->res);
1187                         dev_err(&cxlr->dev,
1188                                 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1189                                 dev_name(port->uport_dev), dev_name(&port->dev),
1190                                 __func__, cxld->interleave_ways,
1191                                 cxld->interleave_granularity,
1192                                 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1193                                         "enabled" :
1194                                         "disabled",
1195                                 cxld->hpa_range.start, cxld->hpa_range.end);
1196                         return -ENXIO;
1197                 }
1198         } else {
1199                 cxld->interleave_ways = iw;
1200                 cxld->interleave_granularity = ig;
1201                 cxld->hpa_range = (struct range) {
1202                         .start = p->res->start,
1203                         .end = p->res->end,
1204                 };
1205         }
1206         dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1207                 dev_name(&port->dev), iw, ig);
1208 add_target:
1209         if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1210                 dev_dbg(&cxlr->dev,
1211                         "%s:%s: targets full trying to add %s:%s at %d\n",
1212                         dev_name(port->uport_dev), dev_name(&port->dev),
1213                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1214                 return -ENXIO;
1215         }
1216         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1217                 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1218                         dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1219                                 dev_name(port->uport_dev), dev_name(&port->dev),
1220                                 dev_name(&cxlsd->cxld.dev),
1221                                 dev_name(ep->dport->dport_dev),
1222                                 cxl_rr->nr_targets_set);
1223                         return -ENXIO;
1224                 }
1225         } else
1226                 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1227         inc = 1;
1228 out_target_set:
1229         cxl_rr->nr_targets_set += inc;
1230         dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1231                 dev_name(port->uport_dev), dev_name(&port->dev),
1232                 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1233                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1234
1235         return 0;
1236 }
1237
1238 static void cxl_port_reset_targets(struct cxl_port *port,
1239                                    struct cxl_region *cxlr)
1240 {
1241         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1242         struct cxl_decoder *cxld;
1243
1244         /*
1245          * After the last endpoint has been detached the entire cxl_rr may now
1246          * be gone.
1247          */
1248         if (!cxl_rr)
1249                 return;
1250         cxl_rr->nr_targets_set = 0;
1251
1252         cxld = cxl_rr->decoder;
1253         cxld->hpa_range = (struct range) {
1254                 .start = 0,
1255                 .end = -1,
1256         };
1257 }
1258
1259 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1260 {
1261         struct cxl_region_params *p = &cxlr->params;
1262         struct cxl_endpoint_decoder *cxled;
1263         struct cxl_dev_state *cxlds;
1264         struct cxl_memdev *cxlmd;
1265         struct cxl_port *iter;
1266         struct cxl_ep *ep;
1267         int i;
1268
1269         /*
1270          * In the auto-discovery case skip automatic teardown since the
1271          * address space is already active
1272          */
1273         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1274                 return;
1275
1276         for (i = 0; i < p->nr_targets; i++) {
1277                 cxled = p->targets[i];
1278                 cxlmd = cxled_to_memdev(cxled);
1279                 cxlds = cxlmd->cxlds;
1280
1281                 if (cxlds->rcd)
1282                         continue;
1283
1284                 iter = cxled_to_port(cxled);
1285                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1286                         iter = to_cxl_port(iter->dev.parent);
1287
1288                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1289                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1290                         cxl_port_reset_targets(iter, cxlr);
1291         }
1292 }
1293
1294 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1295 {
1296         struct cxl_region_params *p = &cxlr->params;
1297         struct cxl_endpoint_decoder *cxled;
1298         struct cxl_dev_state *cxlds;
1299         int i, rc, rch = 0, vh = 0;
1300         struct cxl_memdev *cxlmd;
1301         struct cxl_port *iter;
1302         struct cxl_ep *ep;
1303
1304         for (i = 0; i < p->nr_targets; i++) {
1305                 cxled = p->targets[i];
1306                 cxlmd = cxled_to_memdev(cxled);
1307                 cxlds = cxlmd->cxlds;
1308
1309                 /* validate that all targets agree on topology */
1310                 if (!cxlds->rcd) {
1311                         vh++;
1312                 } else {
1313                         rch++;
1314                         continue;
1315                 }
1316
1317                 iter = cxled_to_port(cxled);
1318                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1319                         iter = to_cxl_port(iter->dev.parent);
1320
1321                 /*
1322                  * Descend the topology tree programming / validating
1323                  * targets while looking for conflicts.
1324                  */
1325                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1326                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1327                         rc = cxl_port_setup_targets(iter, cxlr, cxled);
1328                         if (rc) {
1329                                 cxl_region_teardown_targets(cxlr);
1330                                 return rc;
1331                         }
1332                 }
1333         }
1334
1335         if (rch && vh) {
1336                 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1337                 cxl_region_teardown_targets(cxlr);
1338                 return -ENXIO;
1339         }
1340
1341         return 0;
1342 }
1343
1344 static int cxl_region_validate_position(struct cxl_region *cxlr,
1345                                         struct cxl_endpoint_decoder *cxled,
1346                                         int pos)
1347 {
1348         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1349         struct cxl_region_params *p = &cxlr->params;
1350         int i;
1351
1352         if (pos < 0 || pos >= p->interleave_ways) {
1353                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1354                         p->interleave_ways);
1355                 return -ENXIO;
1356         }
1357
1358         if (p->targets[pos] == cxled)
1359                 return 0;
1360
1361         if (p->targets[pos]) {
1362                 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1363                 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1364
1365                 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1366                         pos, dev_name(&cxlmd_target->dev),
1367                         dev_name(&cxled_target->cxld.dev));
1368                 return -EBUSY;
1369         }
1370
1371         for (i = 0; i < p->interleave_ways; i++) {
1372                 struct cxl_endpoint_decoder *cxled_target;
1373                 struct cxl_memdev *cxlmd_target;
1374
1375                 cxled_target = p->targets[i];
1376                 if (!cxled_target)
1377                         continue;
1378
1379                 cxlmd_target = cxled_to_memdev(cxled_target);
1380                 if (cxlmd_target == cxlmd) {
1381                         dev_dbg(&cxlr->dev,
1382                                 "%s already specified at position %d via: %s\n",
1383                                 dev_name(&cxlmd->dev), pos,
1384                                 dev_name(&cxled_target->cxld.dev));
1385                         return -EBUSY;
1386                 }
1387         }
1388
1389         return 0;
1390 }
1391
1392 static int cxl_region_attach_position(struct cxl_region *cxlr,
1393                                       struct cxl_root_decoder *cxlrd,
1394                                       struct cxl_endpoint_decoder *cxled,
1395                                       const struct cxl_dport *dport, int pos)
1396 {
1397         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1398         struct cxl_port *iter;
1399         int rc;
1400
1401         if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1402                 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1403                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1404                         dev_name(&cxlrd->cxlsd.cxld.dev));
1405                 return -ENXIO;
1406         }
1407
1408         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1409              iter = to_cxl_port(iter->dev.parent)) {
1410                 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1411                 if (rc)
1412                         goto err;
1413         }
1414
1415         return 0;
1416
1417 err:
1418         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1419              iter = to_cxl_port(iter->dev.parent))
1420                 cxl_port_detach_region(iter, cxlr, cxled);
1421         return rc;
1422 }
1423
1424 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1425                                   struct cxl_endpoint_decoder *cxled, int pos)
1426 {
1427         struct cxl_region_params *p = &cxlr->params;
1428
1429         if (cxled->state != CXL_DECODER_STATE_AUTO) {
1430                 dev_err(&cxlr->dev,
1431                         "%s: unable to add decoder to autodetected region\n",
1432                         dev_name(&cxled->cxld.dev));
1433                 return -EINVAL;
1434         }
1435
1436         if (pos >= 0) {
1437                 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1438                         dev_name(&cxled->cxld.dev), pos);
1439                 return -EINVAL;
1440         }
1441
1442         if (p->nr_targets >= p->interleave_ways) {
1443                 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1444                         dev_name(&cxled->cxld.dev));
1445                 return -ENXIO;
1446         }
1447
1448         /*
1449          * Temporarily record the endpoint decoder into the target array. Yes,
1450          * this means that userspace can view devices in the wrong position
1451          * before the region activates, and must be careful to understand when
1452          * it might be racing region autodiscovery.
1453          */
1454         pos = p->nr_targets;
1455         p->targets[pos] = cxled;
1456         cxled->pos = pos;
1457         p->nr_targets++;
1458
1459         return 0;
1460 }
1461
1462 static struct cxl_port *next_port(struct cxl_port *port)
1463 {
1464         if (!port->parent_dport)
1465                 return NULL;
1466         return port->parent_dport->port;
1467 }
1468
1469 static int decoder_match_range(struct device *dev, void *data)
1470 {
1471         struct cxl_endpoint_decoder *cxled = data;
1472         struct cxl_switch_decoder *cxlsd;
1473
1474         if (!is_switch_decoder(dev))
1475                 return 0;
1476
1477         cxlsd = to_cxl_switch_decoder(dev);
1478         return range_contains(&cxlsd->cxld.hpa_range, &cxled->cxld.hpa_range);
1479 }
1480
1481 static void find_positions(const struct cxl_switch_decoder *cxlsd,
1482                            const struct cxl_port *iter_a,
1483                            const struct cxl_port *iter_b, int *a_pos,
1484                            int *b_pos)
1485 {
1486         int i;
1487
1488         for (i = 0, *a_pos = -1, *b_pos = -1; i < cxlsd->nr_targets; i++) {
1489                 if (cxlsd->target[i] == iter_a->parent_dport)
1490                         *a_pos = i;
1491                 else if (cxlsd->target[i] == iter_b->parent_dport)
1492                         *b_pos = i;
1493                 if (*a_pos >= 0 && *b_pos >= 0)
1494                         break;
1495         }
1496 }
1497
1498 static int cmp_decode_pos(const void *a, const void *b)
1499 {
1500         struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1501         struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1502         struct cxl_memdev *cxlmd_a = cxled_to_memdev(cxled_a);
1503         struct cxl_memdev *cxlmd_b = cxled_to_memdev(cxled_b);
1504         struct cxl_port *port_a = cxled_to_port(cxled_a);
1505         struct cxl_port *port_b = cxled_to_port(cxled_b);
1506         struct cxl_port *iter_a, *iter_b, *port = NULL;
1507         struct cxl_switch_decoder *cxlsd;
1508         struct device *dev;
1509         int a_pos, b_pos;
1510         unsigned int seq;
1511
1512         /* Exit early if any prior sorting failed */
1513         if (cxled_a->pos < 0 || cxled_b->pos < 0)
1514                 return 0;
1515
1516         /*
1517          * Walk up the hierarchy to find a shared port, find the decoder that
1518          * maps the range, compare the relative position of those dport
1519          * mappings.
1520          */
1521         for (iter_a = port_a; iter_a; iter_a = next_port(iter_a)) {
1522                 struct cxl_port *next_a, *next_b;
1523
1524                 next_a = next_port(iter_a);
1525                 if (!next_a)
1526                         break;
1527
1528                 for (iter_b = port_b; iter_b; iter_b = next_port(iter_b)) {
1529                         next_b = next_port(iter_b);
1530                         if (next_a != next_b)
1531                                 continue;
1532                         port = next_a;
1533                         break;
1534                 }
1535
1536                 if (port)
1537                         break;
1538         }
1539
1540         if (!port) {
1541                 dev_err(cxlmd_a->dev.parent,
1542                         "failed to find shared port with %s\n",
1543                         dev_name(cxlmd_b->dev.parent));
1544                 goto err;
1545         }
1546
1547         dev = device_find_child(&port->dev, cxled_a, decoder_match_range);
1548         if (!dev) {
1549                 struct range *range = &cxled_a->cxld.hpa_range;
1550
1551                 dev_err(port->uport_dev,
1552                         "failed to find decoder that maps %#llx-%#llx\n",
1553                         range->start, range->end);
1554                 goto err;
1555         }
1556
1557         cxlsd = to_cxl_switch_decoder(dev);
1558         do {
1559                 seq = read_seqbegin(&cxlsd->target_lock);
1560                 find_positions(cxlsd, iter_a, iter_b, &a_pos, &b_pos);
1561         } while (read_seqretry(&cxlsd->target_lock, seq));
1562
1563         put_device(dev);
1564
1565         if (a_pos < 0 || b_pos < 0) {
1566                 dev_err(port->uport_dev,
1567                         "failed to find shared decoder for %s and %s\n",
1568                         dev_name(cxlmd_a->dev.parent),
1569                         dev_name(cxlmd_b->dev.parent));
1570                 goto err;
1571         }
1572
1573         dev_dbg(port->uport_dev, "%s comes %s %s\n",
1574                 dev_name(cxlmd_a->dev.parent),
1575                 a_pos - b_pos < 0 ? "before" : "after",
1576                 dev_name(cxlmd_b->dev.parent));
1577
1578         return a_pos - b_pos;
1579 err:
1580         cxled_a->pos = -1;
1581         return 0;
1582 }
1583
1584 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1585 {
1586         struct cxl_region_params *p = &cxlr->params;
1587         int i, rc = 0;
1588
1589         sort(p->targets, p->nr_targets, sizeof(p->targets[0]), cmp_decode_pos,
1590              NULL);
1591
1592         for (i = 0; i < p->nr_targets; i++) {
1593                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1594
1595                 /*
1596                  * Record that sorting failed, but still continue to restore
1597                  * cxled->pos with its ->targets[] position so that follow-on
1598                  * code paths can reliably do p->targets[cxled->pos] to
1599                  * self-reference their entry.
1600                  */
1601                 if (cxled->pos < 0)
1602                         rc = -ENXIO;
1603                 cxled->pos = i;
1604         }
1605
1606         dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1607         return rc;
1608 }
1609
1610 static int cxl_region_attach(struct cxl_region *cxlr,
1611                              struct cxl_endpoint_decoder *cxled, int pos)
1612 {
1613         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1614         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1615         struct cxl_region_params *p = &cxlr->params;
1616         struct cxl_port *ep_port, *root_port;
1617         struct cxl_dport *dport;
1618         int rc = -ENXIO;
1619
1620         if (cxled->mode != cxlr->mode) {
1621                 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1622                         dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1623                 return -EINVAL;
1624         }
1625
1626         if (cxled->mode == CXL_DECODER_DEAD) {
1627                 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1628                 return -ENODEV;
1629         }
1630
1631         /* all full of members, or interleave config not established? */
1632         if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1633                 dev_dbg(&cxlr->dev, "region already active\n");
1634                 return -EBUSY;
1635         } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1636                 dev_dbg(&cxlr->dev, "interleave config missing\n");
1637                 return -ENXIO;
1638         }
1639
1640         ep_port = cxled_to_port(cxled);
1641         root_port = cxlrd_to_port(cxlrd);
1642         dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1643         if (!dport) {
1644                 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1645                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1646                         dev_name(cxlr->dev.parent));
1647                 return -ENXIO;
1648         }
1649
1650         if (cxled->cxld.target_type != cxlr->type) {
1651                 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1652                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1653                         cxled->cxld.target_type, cxlr->type);
1654                 return -ENXIO;
1655         }
1656
1657         if (!cxled->dpa_res) {
1658                 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1659                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1660                 return -ENXIO;
1661         }
1662
1663         if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1664             resource_size(p->res)) {
1665                 dev_dbg(&cxlr->dev,
1666                         "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1667                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1668                         (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1669                         (u64)resource_size(p->res));
1670                 return -EINVAL;
1671         }
1672
1673         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1674                 int i;
1675
1676                 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1677                 if (rc)
1678                         return rc;
1679
1680                 /* await more targets to arrive... */
1681                 if (p->nr_targets < p->interleave_ways)
1682                         return 0;
1683
1684                 /*
1685                  * All targets are here, which implies all PCI enumeration that
1686                  * affects this region has been completed. Walk the topology to
1687                  * sort the devices into their relative region decode position.
1688                  */
1689                 rc = cxl_region_sort_targets(cxlr);
1690                 if (rc)
1691                         return rc;
1692
1693                 for (i = 0; i < p->nr_targets; i++) {
1694                         cxled = p->targets[i];
1695                         ep_port = cxled_to_port(cxled);
1696                         dport = cxl_find_dport_by_dev(root_port,
1697                                                       ep_port->host_bridge);
1698                         rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1699                                                         dport, i);
1700                         if (rc)
1701                                 return rc;
1702                 }
1703
1704                 rc = cxl_region_setup_targets(cxlr);
1705                 if (rc)
1706                         return rc;
1707
1708                 /*
1709                  * If target setup succeeds in the autodiscovery case
1710                  * then the region is already committed.
1711                  */
1712                 p->state = CXL_CONFIG_COMMIT;
1713
1714                 return 0;
1715         }
1716
1717         rc = cxl_region_validate_position(cxlr, cxled, pos);
1718         if (rc)
1719                 return rc;
1720
1721         rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1722         if (rc)
1723                 return rc;
1724
1725         p->targets[pos] = cxled;
1726         cxled->pos = pos;
1727         p->nr_targets++;
1728
1729         if (p->nr_targets == p->interleave_ways) {
1730                 rc = cxl_region_setup_targets(cxlr);
1731                 if (rc)
1732                         goto err_decrement;
1733                 p->state = CXL_CONFIG_ACTIVE;
1734         }
1735
1736         cxled->cxld.interleave_ways = p->interleave_ways;
1737         cxled->cxld.interleave_granularity = p->interleave_granularity;
1738         cxled->cxld.hpa_range = (struct range) {
1739                 .start = p->res->start,
1740                 .end = p->res->end,
1741         };
1742
1743         return 0;
1744
1745 err_decrement:
1746         p->nr_targets--;
1747         cxled->pos = -1;
1748         p->targets[pos] = NULL;
1749         return rc;
1750 }
1751
1752 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1753 {
1754         struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1755         struct cxl_region *cxlr = cxled->cxld.region;
1756         struct cxl_region_params *p;
1757         int rc = 0;
1758
1759         lockdep_assert_held_write(&cxl_region_rwsem);
1760
1761         if (!cxlr)
1762                 return 0;
1763
1764         p = &cxlr->params;
1765         get_device(&cxlr->dev);
1766
1767         if (p->state > CXL_CONFIG_ACTIVE) {
1768                 /*
1769                  * TODO: tear down all impacted regions if a device is
1770                  * removed out of order
1771                  */
1772                 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1773                 if (rc)
1774                         goto out;
1775                 p->state = CXL_CONFIG_ACTIVE;
1776         }
1777
1778         for (iter = ep_port; !is_cxl_root(iter);
1779              iter = to_cxl_port(iter->dev.parent))
1780                 cxl_port_detach_region(iter, cxlr, cxled);
1781
1782         if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1783             p->targets[cxled->pos] != cxled) {
1784                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1785
1786                 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1787                               dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1788                               cxled->pos);
1789                 goto out;
1790         }
1791
1792         if (p->state == CXL_CONFIG_ACTIVE) {
1793                 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1794                 cxl_region_teardown_targets(cxlr);
1795         }
1796         p->targets[cxled->pos] = NULL;
1797         p->nr_targets--;
1798         cxled->cxld.hpa_range = (struct range) {
1799                 .start = 0,
1800                 .end = -1,
1801         };
1802
1803         /* notify the region driver that one of its targets has departed */
1804         up_write(&cxl_region_rwsem);
1805         device_release_driver(&cxlr->dev);
1806         down_write(&cxl_region_rwsem);
1807 out:
1808         put_device(&cxlr->dev);
1809         return rc;
1810 }
1811
1812 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1813 {
1814         down_write(&cxl_region_rwsem);
1815         cxled->mode = CXL_DECODER_DEAD;
1816         cxl_region_detach(cxled);
1817         up_write(&cxl_region_rwsem);
1818 }
1819
1820 static int attach_target(struct cxl_region *cxlr,
1821                          struct cxl_endpoint_decoder *cxled, int pos,
1822                          unsigned int state)
1823 {
1824         int rc = 0;
1825
1826         if (state == TASK_INTERRUPTIBLE)
1827                 rc = down_write_killable(&cxl_region_rwsem);
1828         else
1829                 down_write(&cxl_region_rwsem);
1830         if (rc)
1831                 return rc;
1832
1833         down_read(&cxl_dpa_rwsem);
1834         rc = cxl_region_attach(cxlr, cxled, pos);
1835         up_read(&cxl_dpa_rwsem);
1836         up_write(&cxl_region_rwsem);
1837         return rc;
1838 }
1839
1840 static int detach_target(struct cxl_region *cxlr, int pos)
1841 {
1842         struct cxl_region_params *p = &cxlr->params;
1843         int rc;
1844
1845         rc = down_write_killable(&cxl_region_rwsem);
1846         if (rc)
1847                 return rc;
1848
1849         if (pos >= p->interleave_ways) {
1850                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1851                         p->interleave_ways);
1852                 rc = -ENXIO;
1853                 goto out;
1854         }
1855
1856         if (!p->targets[pos]) {
1857                 rc = 0;
1858                 goto out;
1859         }
1860
1861         rc = cxl_region_detach(p->targets[pos]);
1862 out:
1863         up_write(&cxl_region_rwsem);
1864         return rc;
1865 }
1866
1867 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1868                             size_t len)
1869 {
1870         int rc;
1871
1872         if (sysfs_streq(buf, "\n"))
1873                 rc = detach_target(cxlr, pos);
1874         else {
1875                 struct device *dev;
1876
1877                 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1878                 if (!dev)
1879                         return -ENODEV;
1880
1881                 if (!is_endpoint_decoder(dev)) {
1882                         rc = -EINVAL;
1883                         goto out;
1884                 }
1885
1886                 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1887                                    TASK_INTERRUPTIBLE);
1888 out:
1889                 put_device(dev);
1890         }
1891
1892         if (rc < 0)
1893                 return rc;
1894         return len;
1895 }
1896
1897 #define TARGET_ATTR_RW(n)                                              \
1898 static ssize_t target##n##_show(                                       \
1899         struct device *dev, struct device_attribute *attr, char *buf)  \
1900 {                                                                      \
1901         return show_targetN(to_cxl_region(dev), buf, (n));             \
1902 }                                                                      \
1903 static ssize_t target##n##_store(struct device *dev,                   \
1904                                  struct device_attribute *attr,        \
1905                                  const char *buf, size_t len)          \
1906 {                                                                      \
1907         return store_targetN(to_cxl_region(dev), buf, (n), len);       \
1908 }                                                                      \
1909 static DEVICE_ATTR_RW(target##n)
1910
1911 TARGET_ATTR_RW(0);
1912 TARGET_ATTR_RW(1);
1913 TARGET_ATTR_RW(2);
1914 TARGET_ATTR_RW(3);
1915 TARGET_ATTR_RW(4);
1916 TARGET_ATTR_RW(5);
1917 TARGET_ATTR_RW(6);
1918 TARGET_ATTR_RW(7);
1919 TARGET_ATTR_RW(8);
1920 TARGET_ATTR_RW(9);
1921 TARGET_ATTR_RW(10);
1922 TARGET_ATTR_RW(11);
1923 TARGET_ATTR_RW(12);
1924 TARGET_ATTR_RW(13);
1925 TARGET_ATTR_RW(14);
1926 TARGET_ATTR_RW(15);
1927
1928 static struct attribute *target_attrs[] = {
1929         &dev_attr_target0.attr,
1930         &dev_attr_target1.attr,
1931         &dev_attr_target2.attr,
1932         &dev_attr_target3.attr,
1933         &dev_attr_target4.attr,
1934         &dev_attr_target5.attr,
1935         &dev_attr_target6.attr,
1936         &dev_attr_target7.attr,
1937         &dev_attr_target8.attr,
1938         &dev_attr_target9.attr,
1939         &dev_attr_target10.attr,
1940         &dev_attr_target11.attr,
1941         &dev_attr_target12.attr,
1942         &dev_attr_target13.attr,
1943         &dev_attr_target14.attr,
1944         &dev_attr_target15.attr,
1945         NULL,
1946 };
1947
1948 static umode_t cxl_region_target_visible(struct kobject *kobj,
1949                                          struct attribute *a, int n)
1950 {
1951         struct device *dev = kobj_to_dev(kobj);
1952         struct cxl_region *cxlr = to_cxl_region(dev);
1953         struct cxl_region_params *p = &cxlr->params;
1954
1955         if (n < p->interleave_ways)
1956                 return a->mode;
1957         return 0;
1958 }
1959
1960 static const struct attribute_group cxl_region_target_group = {
1961         .attrs = target_attrs,
1962         .is_visible = cxl_region_target_visible,
1963 };
1964
1965 static const struct attribute_group *get_cxl_region_target_group(void)
1966 {
1967         return &cxl_region_target_group;
1968 }
1969
1970 static const struct attribute_group *region_groups[] = {
1971         &cxl_base_attribute_group,
1972         &cxl_region_group,
1973         &cxl_region_target_group,
1974         NULL,
1975 };
1976
1977 static void cxl_region_release(struct device *dev)
1978 {
1979         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
1980         struct cxl_region *cxlr = to_cxl_region(dev);
1981         int id = atomic_read(&cxlrd->region_id);
1982
1983         /*
1984          * Try to reuse the recently idled id rather than the cached
1985          * next id to prevent the region id space from increasing
1986          * unnecessarily.
1987          */
1988         if (cxlr->id < id)
1989                 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
1990                         memregion_free(id);
1991                         goto out;
1992                 }
1993
1994         memregion_free(cxlr->id);
1995 out:
1996         put_device(dev->parent);
1997         kfree(cxlr);
1998 }
1999
2000 const struct device_type cxl_region_type = {
2001         .name = "cxl_region",
2002         .release = cxl_region_release,
2003         .groups = region_groups
2004 };
2005
2006 bool is_cxl_region(struct device *dev)
2007 {
2008         return dev->type == &cxl_region_type;
2009 }
2010 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2011
2012 static struct cxl_region *to_cxl_region(struct device *dev)
2013 {
2014         if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2015                           "not a cxl_region device\n"))
2016                 return NULL;
2017
2018         return container_of(dev, struct cxl_region, dev);
2019 }
2020
2021 static void unregister_region(void *dev)
2022 {
2023         struct cxl_region *cxlr = to_cxl_region(dev);
2024         struct cxl_region_params *p = &cxlr->params;
2025         int i;
2026
2027         device_del(dev);
2028
2029         /*
2030          * Now that region sysfs is shutdown, the parameter block is now
2031          * read-only, so no need to hold the region rwsem to access the
2032          * region parameters.
2033          */
2034         for (i = 0; i < p->interleave_ways; i++)
2035                 detach_target(cxlr, i);
2036
2037         cxl_region_iomem_release(cxlr);
2038         put_device(dev);
2039 }
2040
2041 static struct lock_class_key cxl_region_key;
2042
2043 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2044 {
2045         struct cxl_region *cxlr;
2046         struct device *dev;
2047
2048         cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2049         if (!cxlr) {
2050                 memregion_free(id);
2051                 return ERR_PTR(-ENOMEM);
2052         }
2053
2054         dev = &cxlr->dev;
2055         device_initialize(dev);
2056         lockdep_set_class(&dev->mutex, &cxl_region_key);
2057         dev->parent = &cxlrd->cxlsd.cxld.dev;
2058         /*
2059          * Keep root decoder pinned through cxl_region_release to fixup
2060          * region id allocations
2061          */
2062         get_device(dev->parent);
2063         device_set_pm_not_required(dev);
2064         dev->bus = &cxl_bus_type;
2065         dev->type = &cxl_region_type;
2066         cxlr->id = id;
2067
2068         return cxlr;
2069 }
2070
2071 /**
2072  * devm_cxl_add_region - Adds a region to a decoder
2073  * @cxlrd: root decoder
2074  * @id: memregion id to create, or memregion_free() on failure
2075  * @mode: mode for the endpoint decoders of this region
2076  * @type: select whether this is an expander or accelerator (type-2 or type-3)
2077  *
2078  * This is the second step of region initialization. Regions exist within an
2079  * address space which is mapped by a @cxlrd.
2080  *
2081  * Return: 0 if the region was added to the @cxlrd, else returns negative error
2082  * code. The region will be named "regionZ" where Z is the unique region number.
2083  */
2084 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2085                                               int id,
2086                                               enum cxl_decoder_mode mode,
2087                                               enum cxl_decoder_type type)
2088 {
2089         struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2090         struct cxl_region *cxlr;
2091         struct device *dev;
2092         int rc;
2093
2094         switch (mode) {
2095         case CXL_DECODER_RAM:
2096         case CXL_DECODER_PMEM:
2097                 break;
2098         default:
2099                 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2100                 return ERR_PTR(-EINVAL);
2101         }
2102
2103         cxlr = cxl_region_alloc(cxlrd, id);
2104         if (IS_ERR(cxlr))
2105                 return cxlr;
2106         cxlr->mode = mode;
2107         cxlr->type = type;
2108
2109         dev = &cxlr->dev;
2110         rc = dev_set_name(dev, "region%d", id);
2111         if (rc)
2112                 goto err;
2113
2114         rc = device_add(dev);
2115         if (rc)
2116                 goto err;
2117
2118         rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2119         if (rc)
2120                 return ERR_PTR(rc);
2121
2122         dev_dbg(port->uport_dev, "%s: created %s\n",
2123                 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2124         return cxlr;
2125
2126 err:
2127         put_device(dev);
2128         return ERR_PTR(rc);
2129 }
2130
2131 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2132 {
2133         return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2134 }
2135
2136 static ssize_t create_pmem_region_show(struct device *dev,
2137                                        struct device_attribute *attr, char *buf)
2138 {
2139         return __create_region_show(to_cxl_root_decoder(dev), buf);
2140 }
2141
2142 static ssize_t create_ram_region_show(struct device *dev,
2143                                       struct device_attribute *attr, char *buf)
2144 {
2145         return __create_region_show(to_cxl_root_decoder(dev), buf);
2146 }
2147
2148 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2149                                           enum cxl_decoder_mode mode, int id)
2150 {
2151         int rc;
2152
2153         rc = memregion_alloc(GFP_KERNEL);
2154         if (rc < 0)
2155                 return ERR_PTR(rc);
2156
2157         if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2158                 memregion_free(rc);
2159                 return ERR_PTR(-EBUSY);
2160         }
2161
2162         return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2163 }
2164
2165 static ssize_t create_pmem_region_store(struct device *dev,
2166                                         struct device_attribute *attr,
2167                                         const char *buf, size_t len)
2168 {
2169         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2170         struct cxl_region *cxlr;
2171         int rc, id;
2172
2173         rc = sscanf(buf, "region%d\n", &id);
2174         if (rc != 1)
2175                 return -EINVAL;
2176
2177         cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2178         if (IS_ERR(cxlr))
2179                 return PTR_ERR(cxlr);
2180
2181         return len;
2182 }
2183 DEVICE_ATTR_RW(create_pmem_region);
2184
2185 static ssize_t create_ram_region_store(struct device *dev,
2186                                        struct device_attribute *attr,
2187                                        const char *buf, size_t len)
2188 {
2189         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2190         struct cxl_region *cxlr;
2191         int rc, id;
2192
2193         rc = sscanf(buf, "region%d\n", &id);
2194         if (rc != 1)
2195                 return -EINVAL;
2196
2197         cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2198         if (IS_ERR(cxlr))
2199                 return PTR_ERR(cxlr);
2200
2201         return len;
2202 }
2203 DEVICE_ATTR_RW(create_ram_region);
2204
2205 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2206                            char *buf)
2207 {
2208         struct cxl_decoder *cxld = to_cxl_decoder(dev);
2209         ssize_t rc;
2210
2211         rc = down_read_interruptible(&cxl_region_rwsem);
2212         if (rc)
2213                 return rc;
2214
2215         if (cxld->region)
2216                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2217         else
2218                 rc = sysfs_emit(buf, "\n");
2219         up_read(&cxl_region_rwsem);
2220
2221         return rc;
2222 }
2223 DEVICE_ATTR_RO(region);
2224
2225 static struct cxl_region *
2226 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2227 {
2228         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2229         struct device *region_dev;
2230
2231         region_dev = device_find_child_by_name(&cxld->dev, name);
2232         if (!region_dev)
2233                 return ERR_PTR(-ENODEV);
2234
2235         return to_cxl_region(region_dev);
2236 }
2237
2238 static ssize_t delete_region_store(struct device *dev,
2239                                    struct device_attribute *attr,
2240                                    const char *buf, size_t len)
2241 {
2242         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2243         struct cxl_port *port = to_cxl_port(dev->parent);
2244         struct cxl_region *cxlr;
2245
2246         cxlr = cxl_find_region_by_name(cxlrd, buf);
2247         if (IS_ERR(cxlr))
2248                 return PTR_ERR(cxlr);
2249
2250         devm_release_action(port->uport_dev, unregister_region, cxlr);
2251         put_device(&cxlr->dev);
2252
2253         return len;
2254 }
2255 DEVICE_ATTR_WO(delete_region);
2256
2257 static void cxl_pmem_region_release(struct device *dev)
2258 {
2259         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2260         int i;
2261
2262         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2263                 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2264
2265                 put_device(&cxlmd->dev);
2266         }
2267
2268         kfree(cxlr_pmem);
2269 }
2270
2271 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2272         &cxl_base_attribute_group,
2273         NULL,
2274 };
2275
2276 const struct device_type cxl_pmem_region_type = {
2277         .name = "cxl_pmem_region",
2278         .release = cxl_pmem_region_release,
2279         .groups = cxl_pmem_region_attribute_groups,
2280 };
2281
2282 bool is_cxl_pmem_region(struct device *dev)
2283 {
2284         return dev->type == &cxl_pmem_region_type;
2285 }
2286 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2287
2288 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2289 {
2290         if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2291                           "not a cxl_pmem_region device\n"))
2292                 return NULL;
2293         return container_of(dev, struct cxl_pmem_region, dev);
2294 }
2295 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2296
2297 struct cxl_poison_context {
2298         struct cxl_port *port;
2299         enum cxl_decoder_mode mode;
2300         u64 offset;
2301 };
2302
2303 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2304                                    struct cxl_poison_context *ctx)
2305 {
2306         struct cxl_dev_state *cxlds = cxlmd->cxlds;
2307         u64 offset, length;
2308         int rc = 0;
2309
2310         /*
2311          * Collect poison for the remaining unmapped resources
2312          * after poison is collected by committed endpoints.
2313          *
2314          * Knowing that PMEM must always follow RAM, get poison
2315          * for unmapped resources based on the last decoder's mode:
2316          *      ram: scan remains of ram range, then any pmem range
2317          *      pmem: scan remains of pmem range
2318          */
2319
2320         if (ctx->mode == CXL_DECODER_RAM) {
2321                 offset = ctx->offset;
2322                 length = resource_size(&cxlds->ram_res) - offset;
2323                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2324                 if (rc == -EFAULT)
2325                         rc = 0;
2326                 if (rc)
2327                         return rc;
2328         }
2329         if (ctx->mode == CXL_DECODER_PMEM) {
2330                 offset = ctx->offset;
2331                 length = resource_size(&cxlds->dpa_res) - offset;
2332                 if (!length)
2333                         return 0;
2334         } else if (resource_size(&cxlds->pmem_res)) {
2335                 offset = cxlds->pmem_res.start;
2336                 length = resource_size(&cxlds->pmem_res);
2337         } else {
2338                 return 0;
2339         }
2340
2341         return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2342 }
2343
2344 static int poison_by_decoder(struct device *dev, void *arg)
2345 {
2346         struct cxl_poison_context *ctx = arg;
2347         struct cxl_endpoint_decoder *cxled;
2348         struct cxl_memdev *cxlmd;
2349         u64 offset, length;
2350         int rc = 0;
2351
2352         if (!is_endpoint_decoder(dev))
2353                 return rc;
2354
2355         cxled = to_cxl_endpoint_decoder(dev);
2356         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2357                 return rc;
2358
2359         /*
2360          * Regions are only created with single mode decoders: pmem or ram.
2361          * Linux does not support mixed mode decoders. This means that
2362          * reading poison per endpoint decoder adheres to the requirement
2363          * that poison reads of pmem and ram must be separated.
2364          * CXL 3.0 Spec 8.2.9.8.4.1
2365          */
2366         if (cxled->mode == CXL_DECODER_MIXED) {
2367                 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2368                 return rc;
2369         }
2370
2371         cxlmd = cxled_to_memdev(cxled);
2372         if (cxled->skip) {
2373                 offset = cxled->dpa_res->start - cxled->skip;
2374                 length = cxled->skip;
2375                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2376                 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2377                         rc = 0;
2378                 if (rc)
2379                         return rc;
2380         }
2381
2382         offset = cxled->dpa_res->start;
2383         length = cxled->dpa_res->end - offset + 1;
2384         rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2385         if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2386                 rc = 0;
2387         if (rc)
2388                 return rc;
2389
2390         /* Iterate until commit_end is reached */
2391         if (cxled->cxld.id == ctx->port->commit_end) {
2392                 ctx->offset = cxled->dpa_res->end + 1;
2393                 ctx->mode = cxled->mode;
2394                 return 1;
2395         }
2396
2397         return 0;
2398 }
2399
2400 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2401 {
2402         struct cxl_poison_context ctx;
2403         int rc = 0;
2404
2405         rc = down_read_interruptible(&cxl_region_rwsem);
2406         if (rc)
2407                 return rc;
2408
2409         ctx = (struct cxl_poison_context) {
2410                 .port = port
2411         };
2412
2413         rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2414         if (rc == 1)
2415                 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2416                                              &ctx);
2417
2418         up_read(&cxl_region_rwsem);
2419         return rc;
2420 }
2421
2422 static struct lock_class_key cxl_pmem_region_key;
2423
2424 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2425 {
2426         struct cxl_region_params *p = &cxlr->params;
2427         struct cxl_nvdimm_bridge *cxl_nvb;
2428         struct cxl_pmem_region *cxlr_pmem;
2429         struct device *dev;
2430         int i;
2431
2432         down_read(&cxl_region_rwsem);
2433         if (p->state != CXL_CONFIG_COMMIT) {
2434                 cxlr_pmem = ERR_PTR(-ENXIO);
2435                 goto out;
2436         }
2437
2438         cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2439                             GFP_KERNEL);
2440         if (!cxlr_pmem) {
2441                 cxlr_pmem = ERR_PTR(-ENOMEM);
2442                 goto out;
2443         }
2444
2445         cxlr_pmem->hpa_range.start = p->res->start;
2446         cxlr_pmem->hpa_range.end = p->res->end;
2447
2448         /* Snapshot the region configuration underneath the cxl_region_rwsem */
2449         cxlr_pmem->nr_mappings = p->nr_targets;
2450         for (i = 0; i < p->nr_targets; i++) {
2451                 struct cxl_endpoint_decoder *cxled = p->targets[i];
2452                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2453                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2454
2455                 /*
2456                  * Regions never span CXL root devices, so by definition the
2457                  * bridge for one device is the same for all.
2458                  */
2459                 if (i == 0) {
2460                         cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2461                         if (!cxl_nvb) {
2462                                 cxlr_pmem = ERR_PTR(-ENODEV);
2463                                 goto out;
2464                         }
2465                         cxlr->cxl_nvb = cxl_nvb;
2466                 }
2467                 m->cxlmd = cxlmd;
2468                 get_device(&cxlmd->dev);
2469                 m->start = cxled->dpa_res->start;
2470                 m->size = resource_size(cxled->dpa_res);
2471                 m->position = i;
2472         }
2473
2474         dev = &cxlr_pmem->dev;
2475         cxlr_pmem->cxlr = cxlr;
2476         cxlr->cxlr_pmem = cxlr_pmem;
2477         device_initialize(dev);
2478         lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2479         device_set_pm_not_required(dev);
2480         dev->parent = &cxlr->dev;
2481         dev->bus = &cxl_bus_type;
2482         dev->type = &cxl_pmem_region_type;
2483 out:
2484         up_read(&cxl_region_rwsem);
2485
2486         return cxlr_pmem;
2487 }
2488
2489 static void cxl_dax_region_release(struct device *dev)
2490 {
2491         struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2492
2493         kfree(cxlr_dax);
2494 }
2495
2496 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2497         &cxl_base_attribute_group,
2498         NULL,
2499 };
2500
2501 const struct device_type cxl_dax_region_type = {
2502         .name = "cxl_dax_region",
2503         .release = cxl_dax_region_release,
2504         .groups = cxl_dax_region_attribute_groups,
2505 };
2506
2507 static bool is_cxl_dax_region(struct device *dev)
2508 {
2509         return dev->type == &cxl_dax_region_type;
2510 }
2511
2512 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2513 {
2514         if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2515                           "not a cxl_dax_region device\n"))
2516                 return NULL;
2517         return container_of(dev, struct cxl_dax_region, dev);
2518 }
2519 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2520
2521 static struct lock_class_key cxl_dax_region_key;
2522
2523 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2524 {
2525         struct cxl_region_params *p = &cxlr->params;
2526         struct cxl_dax_region *cxlr_dax;
2527         struct device *dev;
2528
2529         down_read(&cxl_region_rwsem);
2530         if (p->state != CXL_CONFIG_COMMIT) {
2531                 cxlr_dax = ERR_PTR(-ENXIO);
2532                 goto out;
2533         }
2534
2535         cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2536         if (!cxlr_dax) {
2537                 cxlr_dax = ERR_PTR(-ENOMEM);
2538                 goto out;
2539         }
2540
2541         cxlr_dax->hpa_range.start = p->res->start;
2542         cxlr_dax->hpa_range.end = p->res->end;
2543
2544         dev = &cxlr_dax->dev;
2545         cxlr_dax->cxlr = cxlr;
2546         device_initialize(dev);
2547         lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2548         device_set_pm_not_required(dev);
2549         dev->parent = &cxlr->dev;
2550         dev->bus = &cxl_bus_type;
2551         dev->type = &cxl_dax_region_type;
2552 out:
2553         up_read(&cxl_region_rwsem);
2554
2555         return cxlr_dax;
2556 }
2557
2558 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2559 {
2560         struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2561         struct cxl_region *cxlr = cxlr_pmem->cxlr;
2562         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2563
2564         /*
2565          * Either the bridge is in ->remove() context under the device_lock(),
2566          * or cxlr_release_nvdimm() is cancelling the bridge's release action
2567          * for @cxlr_pmem and doing it itself (while manually holding the bridge
2568          * lock).
2569          */
2570         device_lock_assert(&cxl_nvb->dev);
2571         cxlr->cxlr_pmem = NULL;
2572         cxlr_pmem->cxlr = NULL;
2573         device_unregister(&cxlr_pmem->dev);
2574 }
2575
2576 static void cxlr_release_nvdimm(void *_cxlr)
2577 {
2578         struct cxl_region *cxlr = _cxlr;
2579         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2580
2581         device_lock(&cxl_nvb->dev);
2582         if (cxlr->cxlr_pmem)
2583                 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2584                                     cxlr->cxlr_pmem);
2585         device_unlock(&cxl_nvb->dev);
2586         cxlr->cxl_nvb = NULL;
2587         put_device(&cxl_nvb->dev);
2588 }
2589
2590 /**
2591  * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2592  * @cxlr: parent CXL region for this pmem region bridge device
2593  *
2594  * Return: 0 on success negative error code on failure.
2595  */
2596 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2597 {
2598         struct cxl_pmem_region *cxlr_pmem;
2599         struct cxl_nvdimm_bridge *cxl_nvb;
2600         struct device *dev;
2601         int rc;
2602
2603         cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2604         if (IS_ERR(cxlr_pmem))
2605                 return PTR_ERR(cxlr_pmem);
2606         cxl_nvb = cxlr->cxl_nvb;
2607
2608         dev = &cxlr_pmem->dev;
2609         rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2610         if (rc)
2611                 goto err;
2612
2613         rc = device_add(dev);
2614         if (rc)
2615                 goto err;
2616
2617         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2618                 dev_name(dev));
2619
2620         device_lock(&cxl_nvb->dev);
2621         if (cxl_nvb->dev.driver)
2622                 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2623                                               cxlr_pmem_unregister, cxlr_pmem);
2624         else
2625                 rc = -ENXIO;
2626         device_unlock(&cxl_nvb->dev);
2627
2628         if (rc)
2629                 goto err_bridge;
2630
2631         /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2632         return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2633
2634 err:
2635         put_device(dev);
2636 err_bridge:
2637         put_device(&cxl_nvb->dev);
2638         cxlr->cxl_nvb = NULL;
2639         return rc;
2640 }
2641
2642 static void cxlr_dax_unregister(void *_cxlr_dax)
2643 {
2644         struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2645
2646         device_unregister(&cxlr_dax->dev);
2647 }
2648
2649 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2650 {
2651         struct cxl_dax_region *cxlr_dax;
2652         struct device *dev;
2653         int rc;
2654
2655         cxlr_dax = cxl_dax_region_alloc(cxlr);
2656         if (IS_ERR(cxlr_dax))
2657                 return PTR_ERR(cxlr_dax);
2658
2659         dev = &cxlr_dax->dev;
2660         rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2661         if (rc)
2662                 goto err;
2663
2664         rc = device_add(dev);
2665         if (rc)
2666                 goto err;
2667
2668         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2669                 dev_name(dev));
2670
2671         return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2672                                         cxlr_dax);
2673 err:
2674         put_device(dev);
2675         return rc;
2676 }
2677
2678 static int match_decoder_by_range(struct device *dev, void *data)
2679 {
2680         struct range *r1, *r2 = data;
2681         struct cxl_root_decoder *cxlrd;
2682
2683         if (!is_root_decoder(dev))
2684                 return 0;
2685
2686         cxlrd = to_cxl_root_decoder(dev);
2687         r1 = &cxlrd->cxlsd.cxld.hpa_range;
2688         return range_contains(r1, r2);
2689 }
2690
2691 static int match_region_by_range(struct device *dev, void *data)
2692 {
2693         struct cxl_region_params *p;
2694         struct cxl_region *cxlr;
2695         struct range *r = data;
2696         int rc = 0;
2697
2698         if (!is_cxl_region(dev))
2699                 return 0;
2700
2701         cxlr = to_cxl_region(dev);
2702         p = &cxlr->params;
2703
2704         down_read(&cxl_region_rwsem);
2705         if (p->res && p->res->start == r->start && p->res->end == r->end)
2706                 rc = 1;
2707         up_read(&cxl_region_rwsem);
2708
2709         return rc;
2710 }
2711
2712 /* Establish an empty region covering the given HPA range */
2713 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2714                                            struct cxl_endpoint_decoder *cxled)
2715 {
2716         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2717         struct cxl_port *port = cxlrd_to_port(cxlrd);
2718         struct range *hpa = &cxled->cxld.hpa_range;
2719         struct cxl_region_params *p;
2720         struct cxl_region *cxlr;
2721         struct resource *res;
2722         int rc;
2723
2724         do {
2725                 cxlr = __create_region(cxlrd, cxled->mode,
2726                                        atomic_read(&cxlrd->region_id));
2727         } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2728
2729         if (IS_ERR(cxlr)) {
2730                 dev_err(cxlmd->dev.parent,
2731                         "%s:%s: %s failed assign region: %ld\n",
2732                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2733                         __func__, PTR_ERR(cxlr));
2734                 return cxlr;
2735         }
2736
2737         down_write(&cxl_region_rwsem);
2738         p = &cxlr->params;
2739         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2740                 dev_err(cxlmd->dev.parent,
2741                         "%s:%s: %s autodiscovery interrupted\n",
2742                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2743                         __func__);
2744                 rc = -EBUSY;
2745                 goto err;
2746         }
2747
2748         set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2749
2750         res = kmalloc(sizeof(*res), GFP_KERNEL);
2751         if (!res) {
2752                 rc = -ENOMEM;
2753                 goto err;
2754         }
2755
2756         *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2757                                     dev_name(&cxlr->dev));
2758         rc = insert_resource(cxlrd->res, res);
2759         if (rc) {
2760                 /*
2761                  * Platform-firmware may not have split resources like "System
2762                  * RAM" on CXL window boundaries see cxl_region_iomem_release()
2763                  */
2764                 dev_warn(cxlmd->dev.parent,
2765                          "%s:%s: %s %s cannot insert resource\n",
2766                          dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2767                          __func__, dev_name(&cxlr->dev));
2768         }
2769
2770         p->res = res;
2771         p->interleave_ways = cxled->cxld.interleave_ways;
2772         p->interleave_granularity = cxled->cxld.interleave_granularity;
2773         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2774
2775         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2776         if (rc)
2777                 goto err;
2778
2779         dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2780                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2781                 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2782                 p->interleave_granularity);
2783
2784         /* ...to match put_device() in cxl_add_to_region() */
2785         get_device(&cxlr->dev);
2786         up_write(&cxl_region_rwsem);
2787
2788         return cxlr;
2789
2790 err:
2791         up_write(&cxl_region_rwsem);
2792         devm_release_action(port->uport_dev, unregister_region, cxlr);
2793         return ERR_PTR(rc);
2794 }
2795
2796 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2797 {
2798         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2799         struct range *hpa = &cxled->cxld.hpa_range;
2800         struct cxl_decoder *cxld = &cxled->cxld;
2801         struct device *cxlrd_dev, *region_dev;
2802         struct cxl_root_decoder *cxlrd;
2803         struct cxl_region_params *p;
2804         struct cxl_region *cxlr;
2805         bool attach = false;
2806         int rc;
2807
2808         cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2809                                       match_decoder_by_range);
2810         if (!cxlrd_dev) {
2811                 dev_err(cxlmd->dev.parent,
2812                         "%s:%s no CXL window for range %#llx:%#llx\n",
2813                         dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2814                         cxld->hpa_range.start, cxld->hpa_range.end);
2815                 return -ENXIO;
2816         }
2817
2818         cxlrd = to_cxl_root_decoder(cxlrd_dev);
2819
2820         /*
2821          * Ensure that if multiple threads race to construct_region() for @hpa
2822          * one does the construction and the others add to that.
2823          */
2824         mutex_lock(&cxlrd->range_lock);
2825         region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2826                                        match_region_by_range);
2827         if (!region_dev) {
2828                 cxlr = construct_region(cxlrd, cxled);
2829                 region_dev = &cxlr->dev;
2830         } else
2831                 cxlr = to_cxl_region(region_dev);
2832         mutex_unlock(&cxlrd->range_lock);
2833
2834         rc = PTR_ERR_OR_ZERO(cxlr);
2835         if (rc)
2836                 goto out;
2837
2838         attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2839
2840         down_read(&cxl_region_rwsem);
2841         p = &cxlr->params;
2842         attach = p->state == CXL_CONFIG_COMMIT;
2843         up_read(&cxl_region_rwsem);
2844
2845         if (attach) {
2846                 /*
2847                  * If device_attach() fails the range may still be active via
2848                  * the platform-firmware memory map, otherwise the driver for
2849                  * regions is local to this file, so driver matching can't fail.
2850                  */
2851                 if (device_attach(&cxlr->dev) < 0)
2852                         dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2853                                 p->res);
2854         }
2855
2856         put_device(region_dev);
2857 out:
2858         put_device(cxlrd_dev);
2859         return rc;
2860 }
2861 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2862
2863 static int is_system_ram(struct resource *res, void *arg)
2864 {
2865         struct cxl_region *cxlr = arg;
2866         struct cxl_region_params *p = &cxlr->params;
2867
2868         dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2869         return 1;
2870 }
2871
2872 static int cxl_region_probe(struct device *dev)
2873 {
2874         struct cxl_region *cxlr = to_cxl_region(dev);
2875         struct cxl_region_params *p = &cxlr->params;
2876         int rc;
2877
2878         rc = down_read_interruptible(&cxl_region_rwsem);
2879         if (rc) {
2880                 dev_dbg(&cxlr->dev, "probe interrupted\n");
2881                 return rc;
2882         }
2883
2884         if (p->state < CXL_CONFIG_COMMIT) {
2885                 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2886                 rc = -ENXIO;
2887                 goto out;
2888         }
2889
2890         if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
2891                 dev_err(&cxlr->dev,
2892                         "failed to activate, re-commit region and retry\n");
2893                 rc = -ENXIO;
2894                 goto out;
2895         }
2896
2897         /*
2898          * From this point on any path that changes the region's state away from
2899          * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2900          */
2901 out:
2902         up_read(&cxl_region_rwsem);
2903
2904         if (rc)
2905                 return rc;
2906
2907         switch (cxlr->mode) {
2908         case CXL_DECODER_PMEM:
2909                 return devm_cxl_add_pmem_region(cxlr);
2910         case CXL_DECODER_RAM:
2911                 /*
2912                  * The region can not be manged by CXL if any portion of
2913                  * it is already online as 'System RAM'
2914                  */
2915                 if (walk_iomem_res_desc(IORES_DESC_NONE,
2916                                         IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2917                                         p->res->start, p->res->end, cxlr,
2918                                         is_system_ram) > 0)
2919                         return 0;
2920                 return devm_cxl_add_dax_region(cxlr);
2921         default:
2922                 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2923                         cxlr->mode);
2924                 return -ENXIO;
2925         }
2926 }
2927
2928 static struct cxl_driver cxl_region_driver = {
2929         .name = "cxl_region",
2930         .probe = cxl_region_probe,
2931         .id = CXL_DEVICE_REGION,
2932 };
2933
2934 int cxl_region_init(void)
2935 {
2936         return cxl_driver_register(&cxl_region_driver);
2937 }
2938
2939 void cxl_region_exit(void)
2940 {
2941         cxl_driver_unregister(&cxl_region_driver);
2942 }
2943
2944 MODULE_IMPORT_NS(CXL);
2945 MODULE_IMPORT_NS(DEVMEM);
2946 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);