projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cxl: Rename 'uport' to 'uport_dev'
[linux-block.git] / drivers / cxl / core / region.c
CommitLineData
779dd20c
BW		 1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/memregion.h>
4#include <linux/genalloc.h>
5#include <linux/device.h>
6#include <linux/module.h>
7#include <linux/slab.h>
dd5ba0eb 8#include <linux/uuid.h>
a32320b7 9#include <linux/sort.h>
779dd20c 10#include <linux/idr.h>
80d10a6c 11#include <cxlmem.h>
779dd20c
BW		 12#include <cxl.h>
13#include "core.h"
14
15/**
16 * DOC: cxl core region
17 *
18 * CXL Regions represent mapped memory capacity in system physical address
19 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
20 * Memory ranges, Regions represent the active mapped capacity by the HDM
21 * Decoder Capability structures throughout the Host Bridges, Switches, and
22 * Endpoints in the topology.
dd5ba0eb
BW		 23 *
24 * Region configuration has ordering constraints. UUID may be set at any time
25 * but is only visible for persistent regions.
80d10a6c
BW		 26 * 1. Interleave granularity
27 * 2. Interleave size
b9686e8c 28 * 3. Decoder targets
dd5ba0eb
BW		 29 */
30
31/*
32 * All changes to the interleave configuration occur with this lock held
33 * for write.
779dd20c 34 */
dd5ba0eb 35static DECLARE_RWSEM(cxl_region_rwsem);
779dd20c
BW		 36
37static struct cxl_region *to_cxl_region(struct device *dev);
38
dd5ba0eb
BW		 39static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
40 char *buf)
41{
42 struct cxl_region *cxlr = to_cxl_region(dev);
43 struct cxl_region_params *p = &cxlr->params;
44 ssize_t rc;
45
46 rc = down_read_interruptible(&cxl_region_rwsem);
47 if (rc)
48 return rc;
a8e7d558
DW		 49 if (cxlr->mode != CXL_DECODER_PMEM)
50 rc = sysfs_emit(buf, "\n");
51 else
52 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
dd5ba0eb
BW		 53 up_read(&cxl_region_rwsem);
54
55 return rc;
56}
57
58static int is_dup(struct device *match, void *data)
59{
60 struct cxl_region_params *p;
61 struct cxl_region *cxlr;
62 uuid_t *uuid = data;
63
64 if (!is_cxl_region(match))
65 return 0;
66
67 lockdep_assert_held(&cxl_region_rwsem);
68 cxlr = to_cxl_region(match);
69 p = &cxlr->params;
70
71 if (uuid_equal(&p->uuid, uuid)) {
72 dev_dbg(match, "already has uuid: %pUb\n", uuid);
73 return -EBUSY;
74 }
75
76 return 0;
77}
78
79static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
80 const char *buf, size_t len)
81{
82 struct cxl_region *cxlr = to_cxl_region(dev);
83 struct cxl_region_params *p = &cxlr->params;
84 uuid_t temp;
85 ssize_t rc;
86
87 if (len != UUID_STRING_LEN + 1)
88 return -EINVAL;
89
90 rc = uuid_parse(buf, &temp);
91 if (rc)
92 return rc;
93
94 if (uuid_is_null(&temp))
95 return -EINVAL;
96
97 rc = down_write_killable(&cxl_region_rwsem);
98 if (rc)
99 return rc;
100
101 if (uuid_equal(&p->uuid, &temp))
102 goto out;
103
104 rc = -EBUSY;
105 if (p->state >= CXL_CONFIG_ACTIVE)
106 goto out;
107
108 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
109 if (rc < 0)
110 goto out;
111
112 uuid_copy(&p->uuid, &temp);
113out:
114 up_write(&cxl_region_rwsem);
115
116 if (rc)
117 return rc;
118 return len;
119}
120static DEVICE_ATTR_RW(uuid);
121
176baefb
DW		 122static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
123 struct cxl_region *cxlr)
124{
125 return xa_load(&port->regions, (unsigned long)cxlr);
126}
127
128static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
129{
130 struct cxl_region_params *p = &cxlr->params;
131 int i;
132
133 for (i = count - 1; i >= 0; i--) {
134 struct cxl_endpoint_decoder *cxled = p->targets[i];
135 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
136 struct cxl_port *iter = cxled_to_port(cxled);
030f8803 137 struct cxl_dev_state *cxlds = cxlmd->cxlds;
176baefb 138 struct cxl_ep *ep;
4fa4302d 139 int rc = 0;
176baefb 140
030f8803
DW		 141 if (cxlds->rcd)
142 goto endpoint_reset;
143
176baefb
DW		 144 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
145 iter = to_cxl_port(iter->dev.parent);
146
147 for (ep = cxl_ep_load(iter, cxlmd); iter;
148 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
149 struct cxl_region_ref *cxl_rr;
150 struct cxl_decoder *cxld;
151
152 cxl_rr = cxl_rr_load(iter, cxlr);
153 cxld = cxl_rr->decoder;
4fa4302d
FN		 154 if (cxld->reset)
155 rc = cxld->reset(cxld);
176baefb
DW		 156 if (rc)
157 return rc;
158 }
159
030f8803 160endpoint_reset:
176baefb
DW		 161 rc = cxled->cxld.reset(&cxled->cxld);
162 if (rc)
163 return rc;
164 }
165
166 return 0;
167}
168
af3ea9ab
DW		 169static int commit_decoder(struct cxl_decoder *cxld)
170{
171 struct cxl_switch_decoder *cxlsd = NULL;
172
173 if (cxld->commit)
174 return cxld->commit(cxld);
175
176 if (is_switch_decoder(&cxld->dev))
177 cxlsd = to_cxl_switch_decoder(&cxld->dev);
178
179 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
180 "->commit() is required\n"))
181 return -ENXIO;
182 return 0;
183}
184
176baefb
DW		 185static int cxl_region_decode_commit(struct cxl_region *cxlr)
186{
187 struct cxl_region_params *p = &cxlr->params;
69c99613 188 int i, rc = 0;
176baefb
DW		 189
190 for (i = 0; i < p->nr_targets; i++) {
191 struct cxl_endpoint_decoder *cxled = p->targets[i];
192 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
193 struct cxl_region_ref *cxl_rr;
194 struct cxl_decoder *cxld;
195 struct cxl_port *iter;
196 struct cxl_ep *ep;
197
198 /* commit bottom up */
199 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
200 iter = to_cxl_port(iter->dev.parent)) {
201 cxl_rr = cxl_rr_load(iter, cxlr);
202 cxld = cxl_rr->decoder;
af3ea9ab 203 rc = commit_decoder(cxld);
176baefb
DW		 204 if (rc)
205 break;
206 }
207
69c99613
DW		 208 if (rc) {
209 /* programming @iter failed, teardown */
210 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
211 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
212 cxl_rr = cxl_rr_load(iter, cxlr);
213 cxld = cxl_rr->decoder;
4fa4302d
FN		 214 if (cxld->reset)
215 cxld->reset(cxld);
69c99613
DW		 216 }
217
218 cxled->cxld.reset(&cxled->cxld);
219 goto err;
176baefb 220 }
176baefb
DW		 221 }
222
69c99613 223 return 0;
176baefb 224
69c99613 225err:
176baefb
DW		 226 /* undo the targets that were successfully committed */
227 cxl_region_decode_reset(cxlr, i);
228 return rc;
229}
230
231static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
232 const char *buf, size_t len)
233{
234 struct cxl_region *cxlr = to_cxl_region(dev);
235 struct cxl_region_params *p = &cxlr->params;
236 bool commit;
237 ssize_t rc;
238
239 rc = kstrtobool(buf, &commit);
240 if (rc)
241 return rc;
242
243 rc = down_write_killable(&cxl_region_rwsem);
244 if (rc)
245 return rc;
246
247 /* Already in the requested state? */
248 if (commit && p->state >= CXL_CONFIG_COMMIT)
249 goto out;
250 if (!commit && p->state < CXL_CONFIG_COMMIT)
251 goto out;
252
253 /* Not ready to commit? */
254 if (commit && p->state < CXL_CONFIG_ACTIVE) {
255 rc = -ENXIO;
256 goto out;
257 }
258
259 if (commit)
260 rc = cxl_region_decode_commit(cxlr);
261 else {
262 p->state = CXL_CONFIG_RESET_PENDING;
263 up_write(&cxl_region_rwsem);
264 device_release_driver(&cxlr->dev);
265 down_write(&cxl_region_rwsem);
266
267 /*
268 * The lock was dropped, so need to revalidate that the reset is
269 * still pending.
270 */
271 if (p->state == CXL_CONFIG_RESET_PENDING)
272 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
273 }
274
275 if (rc)
276 goto out;
277
278 if (commit)
279 p->state = CXL_CONFIG_COMMIT;
280 else if (p->state == CXL_CONFIG_RESET_PENDING)
281 p->state = CXL_CONFIG_ACTIVE;
282
283out:
284 up_write(&cxl_region_rwsem);
285
286 if (rc)
287 return rc;
288 return len;
289}
290
291static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
292 char *buf)
293{
294 struct cxl_region *cxlr = to_cxl_region(dev);
295 struct cxl_region_params *p = &cxlr->params;
296 ssize_t rc;
297
298 rc = down_read_interruptible(&cxl_region_rwsem);
299 if (rc)
300 return rc;
301 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
302 up_read(&cxl_region_rwsem);
303
304 return rc;
305}
306static DEVICE_ATTR_RW(commit);
307
dd5ba0eb
BW		 308static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
309 int n)
310{
311 struct device *dev = kobj_to_dev(kobj);
312 struct cxl_region *cxlr = to_cxl_region(dev);
313
a8e7d558
DW		 314 /*
315 * Support tooling that expects to find a 'uuid' attribute for all
316 * regions regardless of mode.
317 */
dd5ba0eb 318 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
a8e7d558 319 return 0444;
dd5ba0eb
BW		 320 return a->mode;
321}
322
80d10a6c
BW		 323static ssize_t interleave_ways_show(struct device *dev,
324 struct device_attribute *attr, char *buf)
325{
326 struct cxl_region *cxlr = to_cxl_region(dev);
327 struct cxl_region_params *p = &cxlr->params;
328 ssize_t rc;
329
330 rc = down_read_interruptible(&cxl_region_rwsem);
331 if (rc)
332 return rc;
333 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
334 up_read(&cxl_region_rwsem);
335
336 return rc;
337}
338
b9686e8c
DW		 339static const struct attribute_group *get_cxl_region_target_group(void);
340
80d10a6c
BW		 341static ssize_t interleave_ways_store(struct device *dev,
342 struct device_attribute *attr,
343 const char *buf, size_t len)
344{
345 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
346 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
347 struct cxl_region *cxlr = to_cxl_region(dev);
348 struct cxl_region_params *p = &cxlr->params;
c7e3548c
DC		 349 unsigned int val, save;
350 int rc;
80d10a6c
BW		 351 u8 iw;
352
c7e3548c 353 rc = kstrtouint(buf, 0, &val);
80d10a6c
BW		 354 if (rc)
355 return rc;
356
c99b2e8c 357 rc = ways_to_eiw(val, &iw);
80d10a6c
BW		 358 if (rc)
359 return rc;
360
361 /*
362 * Even for x3, x9, and x12 interleaves the region interleave must be a
363 * power of 2 multiple of the host bridge interleave.
364 */
365 if (!is_power_of_2(val / cxld->interleave_ways) ||
366 (val % cxld->interleave_ways)) {
367 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
368 return -EINVAL;
369 }
370
371 rc = down_write_killable(&cxl_region_rwsem);
372 if (rc)
373 return rc;
374 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
375 rc = -EBUSY;
376 goto out;
377 }
378
b9686e8c 379 save = p->interleave_ways;
80d10a6c 380 p->interleave_ways = val;
b9686e8c
DW		 381 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
382 if (rc)
383 p->interleave_ways = save;
80d10a6c
BW		 384out:
385 up_write(&cxl_region_rwsem);
386 if (rc)
387 return rc;
388 return len;
389}
390static DEVICE_ATTR_RW(interleave_ways);
391
392static ssize_t interleave_granularity_show(struct device *dev,
393 struct device_attribute *attr,
394 char *buf)
395{
396 struct cxl_region *cxlr = to_cxl_region(dev);
397 struct cxl_region_params *p = &cxlr->params;
398 ssize_t rc;
399
400 rc = down_read_interruptible(&cxl_region_rwsem);
401 if (rc)
402 return rc;
403 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
404 up_read(&cxl_region_rwsem);
405
406 return rc;
407}
408
409static ssize_t interleave_granularity_store(struct device *dev,
410 struct device_attribute *attr,
411 const char *buf, size_t len)
412{
413 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
414 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
415 struct cxl_region *cxlr = to_cxl_region(dev);
416 struct cxl_region_params *p = &cxlr->params;
417 int rc, val;
418 u16 ig;
419
420 rc = kstrtoint(buf, 0, &val);
421 if (rc)
422 return rc;
423
83351ddb 424 rc = granularity_to_eig(val, &ig);
80d10a6c
BW		 425 if (rc)
426 return rc;
427
428 /*
4d8e4ea5
DW		 429 * When the host-bridge is interleaved, disallow region granularity !=
430 * root granularity. Regions with a granularity less than the root
431 * interleave result in needing multiple endpoints to support a single
cbbd05d0 432 * slot in the interleave (possible to support in the future). Regions
4d8e4ea5
DW		 433 * with a granularity greater than the root interleave result in invalid
434 * DPA translations (invalid to support).
80d10a6c 435 */
4d8e4ea5 436 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
80d10a6c
BW		 437 return -EINVAL;
438
439 rc = down_write_killable(&cxl_region_rwsem);
440 if (rc)
441 return rc;
442 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
443 rc = -EBUSY;
444 goto out;
445 }
446
447 p->interleave_granularity = val;
448out:
449 up_write(&cxl_region_rwsem);
450 if (rc)
451 return rc;
452 return len;
453}
454static DEVICE_ATTR_RW(interleave_granularity);
455
23a22cd1
DW		 456static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
457 char *buf)
458{
459 struct cxl_region *cxlr = to_cxl_region(dev);
460 struct cxl_region_params *p = &cxlr->params;
461 u64 resource = -1ULL;
462 ssize_t rc;
463
464 rc = down_read_interruptible(&cxl_region_rwsem);
465 if (rc)
466 return rc;
467 if (p->res)
468 resource = p->res->start;
469 rc = sysfs_emit(buf, "%#llx\n", resource);
470 up_read(&cxl_region_rwsem);
471
472 return rc;
473}
474static DEVICE_ATTR_RO(resource);
475
7d505f98
DW		 476static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
477 char *buf)
478{
479 struct cxl_region *cxlr = to_cxl_region(dev);
480
481 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
482}
483static DEVICE_ATTR_RO(mode);
484
23a22cd1
DW		 485static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
486{
487 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
488 struct cxl_region_params *p = &cxlr->params;
489 struct resource *res;
490 u32 remainder = 0;
491
492 lockdep_assert_held_write(&cxl_region_rwsem);
493
494 /* Nothing to do... */
88ab1dde 495 if (p->res && resource_size(p->res) == size)
23a22cd1
DW		 496 return 0;
497
498 /* To change size the old size must be freed first */
499 if (p->res)
500 return -EBUSY;
501
502 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
503 return -EBUSY;
504
505 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
506 if (!p->interleave_ways || !p->interleave_granularity ||
507 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
508 return -ENXIO;
509
510 div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
511 if (remainder)
512 return -EINVAL;
513
514 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
515 dev_name(&cxlr->dev));
516 if (IS_ERR(res)) {
517 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
518 PTR_ERR(res));
519 return PTR_ERR(res);
520 }
521
522 p->res = res;
523 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
524
525 return 0;
526}
527
528static void cxl_region_iomem_release(struct cxl_region *cxlr)
529{
530 struct cxl_region_params *p = &cxlr->params;
531
532 if (device_is_registered(&cxlr->dev))
533 lockdep_assert_held_write(&cxl_region_rwsem);
534 if (p->res) {
a32320b7
DW		 535 /*
536 * Autodiscovered regions may not have been able to insert their
537 * resource.
538 */
539 if (p->res->parent)
540 remove_resource(p->res);
23a22cd1
DW		 541 kfree(p->res);
542 p->res = NULL;
543 }
544}
545
546static int free_hpa(struct cxl_region *cxlr)
547{
548 struct cxl_region_params *p = &cxlr->params;
549
550 lockdep_assert_held_write(&cxl_region_rwsem);
551
552 if (!p->res)
553 return 0;
554
555 if (p->state >= CXL_CONFIG_ACTIVE)
556 return -EBUSY;
557
558 cxl_region_iomem_release(cxlr);
559 p->state = CXL_CONFIG_IDLE;
560 return 0;
561}
562
563static ssize_t size_store(struct device *dev, struct device_attribute *attr,
564 const char *buf, size_t len)
565{
566 struct cxl_region *cxlr = to_cxl_region(dev);
567 u64 val;
568 int rc;
569
570 rc = kstrtou64(buf, 0, &val);
571 if (rc)
572 return rc;
573
574 rc = down_write_killable(&cxl_region_rwsem);
575 if (rc)
576 return rc;
577
578 if (val)
579 rc = alloc_hpa(cxlr, val);
580 else
581 rc = free_hpa(cxlr);
582 up_write(&cxl_region_rwsem);
583
584 if (rc)
585 return rc;
586
587 return len;
588}
589
590static ssize_t size_show(struct device *dev, struct device_attribute *attr,
591 char *buf)
592{
593 struct cxl_region *cxlr = to_cxl_region(dev);
594 struct cxl_region_params *p = &cxlr->params;
595 u64 size = 0;
596 ssize_t rc;
597
598 rc = down_read_interruptible(&cxl_region_rwsem);
599 if (rc)
600 return rc;
601 if (p->res)
602 size = resource_size(p->res);
603 rc = sysfs_emit(buf, "%#llx\n", size);
604 up_read(&cxl_region_rwsem);
605
606 return rc;
607}
608static DEVICE_ATTR_RW(size);
609
dd5ba0eb
BW		 610static struct attribute *cxl_region_attrs[] = {
611 &dev_attr_uuid.attr,
176baefb 612 &dev_attr_commit.attr,
80d10a6c
BW		 613 &dev_attr_interleave_ways.attr,
614 &dev_attr_interleave_granularity.attr,
23a22cd1
DW		 615 &dev_attr_resource.attr,
616 &dev_attr_size.attr,
7d505f98 617 &dev_attr_mode.attr,
dd5ba0eb
BW		 618 NULL,
619};
620
621static const struct attribute_group cxl_region_group = {
622 .attrs = cxl_region_attrs,
623 .is_visible = cxl_region_visible,
624};
625
b9686e8c
DW		 626static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
627{
628 struct cxl_region_params *p = &cxlr->params;
629 struct cxl_endpoint_decoder *cxled;
630 int rc;
631
632 rc = down_read_interruptible(&cxl_region_rwsem);
633 if (rc)
634 return rc;
635
636 if (pos >= p->interleave_ways) {
637 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
638 p->interleave_ways);
639 rc = -ENXIO;
640 goto out;
641 }
642
643 cxled = p->targets[pos];
644 if (!cxled)
645 rc = sysfs_emit(buf, "\n");
646 else
647 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
648out:
649 up_read(&cxl_region_rwsem);
650
651 return rc;
652}
653
384e624b
DW		 654static int match_free_decoder(struct device *dev, void *data)
655{
656 struct cxl_decoder *cxld;
657 int *id = data;
658
659 if (!is_switch_decoder(dev))
660 return 0;
661
662 cxld = to_cxl_decoder(dev);
663
664 /* enforce ordered allocation */
665 if (cxld->id != *id)
666 return 0;
667
668 if (!cxld->region)
669 return 1;
670
671 (*id)++;
672
673 return 0;
674}
675
676static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
677 struct cxl_region *cxlr)
678{
679 struct device *dev;
680 int id = 0;
681
682 dev = device_find_child(&port->dev, &id, match_free_decoder);
683 if (!dev)
684 return NULL;
685 /*
686 * This decoder is pinned registered as long as the endpoint decoder is
687 * registered, and endpoint decoder unregistration holds the
688 * cxl_region_rwsem over unregister events, so no need to hold on to
689 * this extra reference.
690 */
691 put_device(dev);
692 return to_cxl_decoder(dev);
693}
694
695static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
696 struct cxl_region *cxlr)
697{
e29a8995
DW		 698 struct cxl_region_params *p = &cxlr->params;
699 struct cxl_region_ref *cxl_rr, *iter;
700 unsigned long index;
384e624b
DW		 701 int rc;
702
e29a8995
DW		 703 xa_for_each(&port->regions, index, iter) {
704 struct cxl_region_params *ip = &iter->region->params;
705
a90accb3
DW		 706 if (!ip->res)
707 continue;
708
e29a8995
DW		 709 if (ip->res->start > p->res->start) {
710 dev_dbg(&cxlr->dev,
711 "%s: HPA order violation %s:%pr vs %pr\n",
712 dev_name(&port->dev),
713 dev_name(&iter->region->dev), ip->res, p->res);
714 return ERR_PTR(-EBUSY);
715 }
716 }
717
384e624b
DW		 718 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
719 if (!cxl_rr)
e29a8995 720 return ERR_PTR(-ENOMEM);
384e624b
DW		 721 cxl_rr->port = port;
722 cxl_rr->region = cxlr;
27b3f8d1 723 cxl_rr->nr_targets = 1;
384e624b
DW		 724 xa_init(&cxl_rr->endpoints);
725
726 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
727 if (rc) {
728 dev_dbg(&cxlr->dev,
729 "%s: failed to track region reference: %d\n",
730 dev_name(&port->dev), rc);
731 kfree(cxl_rr);
e29a8995 732 return ERR_PTR(rc);
384e624b
DW		 733 }
734
735 return cxl_rr;
736}
737
71ee71d7 738static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
384e624b 739{
384e624b
DW		 740 struct cxl_region *cxlr = cxl_rr->region;
741 struct cxl_decoder *cxld = cxl_rr->decoder;
742
71ee71d7
VV		 743 if (!cxld)
744 return;
745
384e624b
DW		 746 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
747 if (cxld->region == cxlr) {
748 cxld->region = NULL;
749 put_device(&cxlr->dev);
750 }
71ee71d7
VV		 751}
752
753static void free_region_ref(struct cxl_region_ref *cxl_rr)
754{
755 struct cxl_port *port = cxl_rr->port;
756 struct cxl_region *cxlr = cxl_rr->region;
384e624b 757
71ee71d7 758 cxl_rr_free_decoder(cxl_rr);
384e624b
DW		 759 xa_erase(&port->regions, (unsigned long)cxlr);
760 xa_destroy(&cxl_rr->endpoints);
761 kfree(cxl_rr);
762}
763
764static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
765 struct cxl_endpoint_decoder *cxled)
766{
767 int rc;
768 struct cxl_port *port = cxl_rr->port;
769 struct cxl_region *cxlr = cxl_rr->region;
770 struct cxl_decoder *cxld = cxl_rr->decoder;
771 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
772
27b3f8d1
DW		 773 if (ep) {
774 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
775 GFP_KERNEL);
776 if (rc)
777 return rc;
778 }
384e624b
DW		 779 cxl_rr->nr_eps++;
780
781 if (!cxld->region) {
782 cxld->region = cxlr;
783 get_device(&cxlr->dev);
784 }
785
786 return 0;
787}
788
71ee71d7
VV		 789static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
790 struct cxl_endpoint_decoder *cxled,
791 struct cxl_region_ref *cxl_rr)
792{
793 struct cxl_decoder *cxld;
794
795 if (port == cxled_to_port(cxled))
796 cxld = &cxled->cxld;
797 else
798 cxld = cxl_region_find_decoder(port, cxlr);
799 if (!cxld) {
800 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
801 dev_name(&port->dev));
802 return -EBUSY;
803 }
804
805 if (cxld->region) {
806 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
807 dev_name(&port->dev), dev_name(&cxld->dev),
808 dev_name(&cxld->region->dev));
809 return -EBUSY;
810 }
811
812 cxl_rr->decoder = cxld;
813 return 0;
814}
815
384e624b
DW		 816/**
817 * cxl_port_attach_region() - track a region's interest in a port by endpoint
818 * @port: port to add a new region reference 'struct cxl_region_ref'
819 * @cxlr: region to attach to @port
820 * @cxled: endpoint decoder used to create or further pin a region reference
821 * @pos: interleave position of @cxled in @cxlr
822 *
823 * The attach event is an opportunity to validate CXL decode setup
824 * constraints and record metadata needed for programming HDM decoders,
825 * in particular decoder target lists.
826 *
827 * The steps are:
f13da0d9 828 *
384e624b
DW		 829 * - validate that there are no other regions with a higher HPA already
830 * associated with @port
831 * - establish a region reference if one is not already present
f13da0d9 832 *
384e624b
DW		 833 * - additionally allocate a decoder instance that will host @cxlr on
834 * @port
f13da0d9 835 *
384e624b
DW		 836 * - pin the region reference by the endpoint
837 * - account for how many entries in @port's target list are needed to
838 * cover all of the added endpoints.
b9686e8c 839 */
384e624b
DW		 840static int cxl_port_attach_region(struct cxl_port *port,
841 struct cxl_region *cxlr,
842 struct cxl_endpoint_decoder *cxled, int pos)
843{
844 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
845 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
e29a8995
DW		 846 struct cxl_region_ref *cxl_rr;
847 bool nr_targets_inc = false;
848 struct cxl_decoder *cxld;
384e624b
DW		 849 unsigned long index;
850 int rc = -EBUSY;
851
852 lockdep_assert_held_write(&cxl_region_rwsem);
853
e29a8995 854 cxl_rr = cxl_rr_load(port, cxlr);
384e624b
DW		 855 if (cxl_rr) {
856 struct cxl_ep *ep_iter;
857 int found = 0;
858
e29a8995
DW		 859 /*
860 * Walk the existing endpoints that have been attached to
861 * @cxlr at @port and see if they share the same 'next' port
862 * in the downstream direction. I.e. endpoints that share common
863 * upstream switch.
864 */
384e624b
DW		 865 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
866 if (ep_iter == ep)
867 continue;
868 if (ep_iter->next == ep->next) {
869 found++;
870 break;
871 }
872 }
873
874 /*
e29a8995
DW		 875 * New target port, or @port is an endpoint port that always
876 * accounts its own local decode as a target.
384e624b 877 */
e29a8995 878 if (!found || !ep->next) {
384e624b 879 cxl_rr->nr_targets++;
e29a8995
DW		 880 nr_targets_inc = true;
881 }
384e624b
DW		 882 } else {
883 cxl_rr = alloc_region_ref(port, cxlr);
e29a8995 884 if (IS_ERR(cxl_rr)) {
384e624b
DW		 885 dev_dbg(&cxlr->dev,
886 "%s: failed to allocate region reference\n",
887 dev_name(&port->dev));
e29a8995 888 return PTR_ERR(cxl_rr);
384e624b 889 }
e29a8995 890 nr_targets_inc = true;
384e624b 891
71ee71d7
VV		 892 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
893 if (rc)
384e624b 894 goto out_erase;
384e624b 895 }
71ee71d7 896 cxld = cxl_rr->decoder;
384e624b
DW		 897
898 rc = cxl_rr_ep_add(cxl_rr, cxled);
899 if (rc) {
900 dev_dbg(&cxlr->dev,
901 "%s: failed to track endpoint %s:%s reference\n",
902 dev_name(&port->dev), dev_name(&cxlmd->dev),
903 dev_name(&cxld->dev));
904 goto out_erase;
905 }
906
27b3f8d1
DW		 907 dev_dbg(&cxlr->dev,
908 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
7481653d 909 dev_name(port->uport_dev), dev_name(&port->dev),
27b3f8d1
DW		 910 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
911 dev_name(&cxled->cxld.dev), pos,
7481653d 912 ep ? ep->next ? dev_name(ep->next->uport_dev) :
27b3f8d1
DW		 913 dev_name(&cxlmd->dev) :
914 "none",
915 cxl_rr->nr_eps, cxl_rr->nr_targets);
916
384e624b
DW		 917 return 0;
918out_erase:
e29a8995
DW		 919 if (nr_targets_inc)
920 cxl_rr->nr_targets--;
384e624b
DW		 921 if (cxl_rr->nr_eps == 0)
922 free_region_ref(cxl_rr);
923 return rc;
924}
925
384e624b
DW		 926static void cxl_port_detach_region(struct cxl_port *port,
927 struct cxl_region *cxlr,
928 struct cxl_endpoint_decoder *cxled)
929{
930 struct cxl_region_ref *cxl_rr;
27b3f8d1 931 struct cxl_ep *ep = NULL;
384e624b
DW		 932
933 lockdep_assert_held_write(&cxl_region_rwsem);
934
935 cxl_rr = cxl_rr_load(port, cxlr);
936 if (!cxl_rr)
937 return;
938
27b3f8d1
DW		 939 /*
940 * Endpoint ports do not carry cxl_ep references, and they
941 * never target more than one endpoint by definition
942 */
943 if (cxl_rr->decoder == &cxled->cxld)
944 cxl_rr->nr_eps--;
945 else
946 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
384e624b
DW		 947 if (ep) {
948 struct cxl_ep *ep_iter;
949 unsigned long index;
950 int found = 0;
951
952 cxl_rr->nr_eps--;
953 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
954 if (ep_iter->next == ep->next) {
955 found++;
956 break;
957 }
958 }
959 if (!found)
960 cxl_rr->nr_targets--;
961 }
962
963 if (cxl_rr->nr_eps == 0)
964 free_region_ref(cxl_rr);
965}
966
27b3f8d1
DW		 967static int check_last_peer(struct cxl_endpoint_decoder *cxled,
968 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
969 int distance)
970{
971 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
972 struct cxl_region *cxlr = cxl_rr->region;
973 struct cxl_region_params *p = &cxlr->params;
974 struct cxl_endpoint_decoder *cxled_peer;
975 struct cxl_port *port = cxl_rr->port;
976 struct cxl_memdev *cxlmd_peer;
977 struct cxl_ep *ep_peer;
978 int pos = cxled->pos;
979
980 /*
981 * If this position wants to share a dport with the last endpoint mapped
982 * then that endpoint, at index 'position - distance', must also be
983 * mapped by this dport.
984 */
985 if (pos < distance) {
986 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
7481653d 987 dev_name(port->uport_dev), dev_name(&port->dev),
27b3f8d1
DW		 988 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
989 return -ENXIO;
990 }
991 cxled_peer = p->targets[pos - distance];
992 cxlmd_peer = cxled_to_memdev(cxled_peer);
993 ep_peer = cxl_ep_load(port, cxlmd_peer);
994 if (ep->dport != ep_peer->dport) {
995 dev_dbg(&cxlr->dev,
996 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
7481653d 997 dev_name(port->uport_dev), dev_name(&port->dev),
27b3f8d1
DW		 998 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
999 dev_name(&cxlmd_peer->dev),
1000 dev_name(&cxled_peer->cxld.dev));
1001 return -ENXIO;
1002 }
1003
1004 return 0;
1005}
1006
1007static int cxl_port_setup_targets(struct cxl_port *port,
1008 struct cxl_region *cxlr,
1009 struct cxl_endpoint_decoder *cxled)
1010{
1011 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1012 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1013 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1014 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1015 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1016 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1017 struct cxl_region_params *p = &cxlr->params;
1018 struct cxl_decoder *cxld = cxl_rr->decoder;
1019 struct cxl_switch_decoder *cxlsd;
1020 u16 eig, peig;
1021 u8 eiw, peiw;
1022
1023 /*
1024 * While root level decoders support x3, x6, x12, switch level
1025 * decoders only support powers of 2 up to x16.
1026 */
1027 if (!is_power_of_2(cxl_rr->nr_targets)) {
1028 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
7481653d 1029 dev_name(port->uport_dev), dev_name(&port->dev),
27b3f8d1
DW		 1030 cxl_rr->nr_targets);
1031 return -EINVAL;
1032 }
1033
1034 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1035 if (cxl_rr->nr_targets_set) {
1036 int i, distance;
1037
e4f6dfa9 1038 /*
711442e2 1039 * Passthrough decoders impose no distance requirements between
e4f6dfa9
DW		 1040 * peers
1041 */
711442e2 1042 if (cxl_rr->nr_targets == 1)
e4f6dfa9
DW		 1043 distance = 0;
1044 else
1045 distance = p->nr_targets / cxl_rr->nr_targets;
27b3f8d1
DW		 1046 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1047 if (ep->dport == cxlsd->target[i]) {
1048 rc = check_last_peer(cxled, ep, cxl_rr,
1049 distance);
1050 if (rc)
1051 return rc;
1052 goto out_target_set;
1053 }
1054 goto add_target;
1055 }
1056
1057 if (is_cxl_root(parent_port)) {
1058 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1059 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1060 /*
1061 * For purposes of address bit routing, use power-of-2 math for
1062 * switch ports.
1063 */
1064 if (!is_power_of_2(parent_iw))
1065 parent_iw /= 3;
1066 } else {
1067 struct cxl_region_ref *parent_rr;
1068 struct cxl_decoder *parent_cxld;
1069
1070 parent_rr = cxl_rr_load(parent_port, cxlr);
1071 parent_cxld = parent_rr->decoder;
1072 parent_ig = parent_cxld->interleave_granularity;
1073 parent_iw = parent_cxld->interleave_ways;
1074 }
1075
83351ddb 1076 rc = granularity_to_eig(parent_ig, &peig);
8d428542
DW		 1077 if (rc) {
1078 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
7481653d 1079 dev_name(parent_port->uport_dev),
8d428542
DW		 1080 dev_name(&parent_port->dev), parent_ig);
1081 return rc;
1082 }
1083
c99b2e8c 1084 rc = ways_to_eiw(parent_iw, &peiw);
8d428542
DW		 1085 if (rc) {
1086 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
7481653d 1087 dev_name(parent_port->uport_dev),
8d428542
DW		 1088 dev_name(&parent_port->dev), parent_iw);
1089 return rc;
1090 }
27b3f8d1
DW		 1091
1092 iw = cxl_rr->nr_targets;
c99b2e8c 1093 rc = ways_to_eiw(iw, &eiw);
8d428542
DW		 1094 if (rc) {
1095 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
7481653d 1096 dev_name(port->uport_dev), dev_name(&port->dev), iw);
8d428542
DW		 1097 return rc;
1098 }
1099
298d44d0
DW		 1100 /*
1101 * If @parent_port is masking address bits, pick the next unused address
1102 * bit to route @port's targets.
1103 */
1104 if (parent_iw > 1 && cxl_rr->nr_targets > 1) {
27b3f8d1
DW		 1105 u32 address_bit = max(peig + peiw, eiw + peig);
1106
1107 eig = address_bit - eiw + 1;
1108 } else {
1109 eiw = peiw;
1110 eig = peig;
1111 }
1112
83351ddb 1113 rc = eig_to_granularity(eig, &ig);
27b3f8d1
DW		 1114 if (rc) {
1115 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
7481653d 1116 dev_name(port->uport_dev), dev_name(&port->dev),
27b3f8d1
DW		 1117 256 << eig);
1118 return rc;
1119 }
1120
a32320b7
DW		 1121 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1122 if (cxld->interleave_ways != iw ||
1123 cxld->interleave_granularity != ig ||
1124 cxld->hpa_range.start != p->res->start ||
1125 cxld->hpa_range.end != p->res->end ||
1126 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1127 dev_err(&cxlr->dev,
1128 "%s:%s %s expected iw: %d ig: %d %pr\n",
7481653d 1129 dev_name(port->uport_dev), dev_name(&port->dev),
a32320b7
DW		 1130 __func__, iw, ig, p->res);
1131 dev_err(&cxlr->dev,
1132 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
7481653d 1133 dev_name(port->uport_dev), dev_name(&port->dev),
a32320b7
DW		 1134 __func__, cxld->interleave_ways,
1135 cxld->interleave_granularity,
1136 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1137 "enabled" :
1138 "disabled",
1139 cxld->hpa_range.start, cxld->hpa_range.end);
1140 return -ENXIO;
1141 }
1142 } else {
1143 cxld->interleave_ways = iw;
1144 cxld->interleave_granularity = ig;
1145 cxld->hpa_range = (struct range) {
1146 .start = p->res->start,
1147 .end = p->res->end,
1148 };
1149 }
7481653d 1150 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
27b3f8d1
DW		 1151 dev_name(&port->dev), iw, ig);
1152add_target:
1153 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1154 dev_dbg(&cxlr->dev,
1155 "%s:%s: targets full trying to add %s:%s at %d\n",
7481653d 1156 dev_name(port->uport_dev), dev_name(&port->dev),
27b3f8d1
DW		 1157 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1158 return -ENXIO;
1159 }
a32320b7
DW		 1160 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1161 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1162 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
7481653d 1163 dev_name(port->uport_dev), dev_name(&port->dev),
a32320b7 1164 dev_name(&cxlsd->cxld.dev),
227db574 1165 dev_name(ep->dport->dport_dev),
a32320b7
DW		 1166 cxl_rr->nr_targets_set);
1167 return -ENXIO;
1168 }
1169 } else
1170 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
27b3f8d1
DW		 1171 inc = 1;
1172out_target_set:
1173 cxl_rr->nr_targets_set += inc;
1174 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
7481653d 1175 dev_name(port->uport_dev), dev_name(&port->dev),
227db574 1176 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
27b3f8d1
DW		 1177 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1178
1179 return 0;
1180}
1181
1182static void cxl_port_reset_targets(struct cxl_port *port,
1183 struct cxl_region *cxlr)
1184{
1185 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
910bc55d 1186 struct cxl_decoder *cxld;
27b3f8d1
DW		 1187
1188 /*
1189 * After the last endpoint has been detached the entire cxl_rr may now
1190 * be gone.
1191 */
910bc55d
DW		 1192 if (!cxl_rr)
1193 return;
1194 cxl_rr->nr_targets_set = 0;
1195
1196 cxld = cxl_rr->decoder;
1197 cxld->hpa_range = (struct range) {
1198 .start = 0,
1199 .end = -1,
1200 };
27b3f8d1
DW		 1201}
1202
1203static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1204{
1205 struct cxl_region_params *p = &cxlr->params;
1206 struct cxl_endpoint_decoder *cxled;
030f8803 1207 struct cxl_dev_state *cxlds;
27b3f8d1
DW		 1208 struct cxl_memdev *cxlmd;
1209 struct cxl_port *iter;
1210 struct cxl_ep *ep;
1211 int i;
1212
a32320b7
DW		 1213 /*
1214 * In the auto-discovery case skip automatic teardown since the
1215 * address space is already active
1216 */
1217 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1218 return;
1219
27b3f8d1
DW		 1220 for (i = 0; i < p->nr_targets; i++) {
1221 cxled = p->targets[i];
1222 cxlmd = cxled_to_memdev(cxled);
030f8803
DW		 1223 cxlds = cxlmd->cxlds;
1224
1225 if (cxlds->rcd)
1226 continue;
27b3f8d1
DW		 1227
1228 iter = cxled_to_port(cxled);
1229 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1230 iter = to_cxl_port(iter->dev.parent);
1231
1232 for (ep = cxl_ep_load(iter, cxlmd); iter;
1233 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1234 cxl_port_reset_targets(iter, cxlr);
1235 }
1236}
1237
1238static int cxl_region_setup_targets(struct cxl_region *cxlr)
1239{
1240 struct cxl_region_params *p = &cxlr->params;
1241 struct cxl_endpoint_decoder *cxled;
030f8803
DW		 1242 struct cxl_dev_state *cxlds;
1243 int i, rc, rch = 0, vh = 0;
27b3f8d1
DW		 1244 struct cxl_memdev *cxlmd;
1245 struct cxl_port *iter;
1246 struct cxl_ep *ep;
27b3f8d1
DW		 1247
1248 for (i = 0; i < p->nr_targets; i++) {
1249 cxled = p->targets[i];
1250 cxlmd = cxled_to_memdev(cxled);
030f8803
DW		 1251 cxlds = cxlmd->cxlds;
1252
1253 /* validate that all targets agree on topology */
1254 if (!cxlds->rcd) {
1255 vh++;
1256 } else {
1257 rch++;
1258 continue;
1259 }
27b3f8d1
DW		 1260
1261 iter = cxled_to_port(cxled);
1262 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1263 iter = to_cxl_port(iter->dev.parent);
1264
1265 /*
a32320b7
DW		 1266 * Descend the topology tree programming / validating
1267 * targets while looking for conflicts.
27b3f8d1
DW		 1268 */
1269 for (ep = cxl_ep_load(iter, cxlmd); iter;
1270 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1271 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1272 if (rc) {
1273 cxl_region_teardown_targets(cxlr);
1274 return rc;
1275 }
1276 }
1277 }
1278
030f8803
DW		 1279 if (rch && vh) {
1280 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1281 cxl_region_teardown_targets(cxlr);
1282 return -ENXIO;
1283 }
1284
27b3f8d1
DW		 1285 return 0;
1286}
1287
9995576c
DW		 1288static int cxl_region_validate_position(struct cxl_region *cxlr,
1289 struct cxl_endpoint_decoder *cxled,
1290 int pos)
b9686e8c 1291{
384e624b 1292 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
b9686e8c 1293 struct cxl_region_params *p = &cxlr->params;
9995576c 1294 int i;
384e624b
DW		 1295
1296 if (pos < 0 || pos >= p->interleave_ways) {
b9686e8c
DW		 1297 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1298 p->interleave_ways);
1299 return -ENXIO;
1300 }
1301
1302 if (p->targets[pos] == cxled)
1303 return 0;
1304
1305 if (p->targets[pos]) {
1306 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1307 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1308
1309 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1310 pos, dev_name(&cxlmd_target->dev),
1311 dev_name(&cxled_target->cxld.dev));
1312 return -EBUSY;
1313 }
1314
384e624b
DW		 1315 for (i = 0; i < p->interleave_ways; i++) {
1316 struct cxl_endpoint_decoder *cxled_target;
1317 struct cxl_memdev *cxlmd_target;
1318
f04facfb 1319 cxled_target = p->targets[i];
384e624b
DW		 1320 if (!cxled_target)
1321 continue;
1322
1323 cxlmd_target = cxled_to_memdev(cxled_target);
1324 if (cxlmd_target == cxlmd) {
1325 dev_dbg(&cxlr->dev,
1326 "%s already specified at position %d via: %s\n",
1327 dev_name(&cxlmd->dev), pos,
1328 dev_name(&cxled_target->cxld.dev));
1329 return -EBUSY;
1330 }
1331 }
1332
9995576c
DW		 1333 return 0;
1334}
1335
1336static int cxl_region_attach_position(struct cxl_region *cxlr,
1337 struct cxl_root_decoder *cxlrd,
1338 struct cxl_endpoint_decoder *cxled,
1339 const struct cxl_dport *dport, int pos)
1340{
1341 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1342 struct cxl_port *iter;
1343 int rc;
1344
1345 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1346 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1347 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1348 dev_name(&cxlrd->cxlsd.cxld.dev));
1349 return -ENXIO;
1350 }
1351
1352 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1353 iter = to_cxl_port(iter->dev.parent)) {
1354 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1355 if (rc)
1356 goto err;
1357 }
1358
1359 return 0;
1360
1361err:
1362 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1363 iter = to_cxl_port(iter->dev.parent))
1364 cxl_port_detach_region(iter, cxlr, cxled);
1365 return rc;
1366}
1367
a32320b7
DW		 1368static int cxl_region_attach_auto(struct cxl_region *cxlr,
1369 struct cxl_endpoint_decoder *cxled, int pos)
1370{
1371 struct cxl_region_params *p = &cxlr->params;
1372
1373 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1374 dev_err(&cxlr->dev,
1375 "%s: unable to add decoder to autodetected region\n",
1376 dev_name(&cxled->cxld.dev));
1377 return -EINVAL;
1378 }
1379
1380 if (pos >= 0) {
1381 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1382 dev_name(&cxled->cxld.dev), pos);
1383 return -EINVAL;
1384 }
1385
1386 if (p->nr_targets >= p->interleave_ways) {
1387 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1388 dev_name(&cxled->cxld.dev));
1389 return -ENXIO;
1390 }
1391
1392 /*
1393 * Temporarily record the endpoint decoder into the target array. Yes,
1394 * this means that userspace can view devices in the wrong position
1395 * before the region activates, and must be careful to understand when
1396 * it might be racing region autodiscovery.
1397 */
1398 pos = p->nr_targets;
1399 p->targets[pos] = cxled;
1400 cxled->pos = pos;
1401 p->nr_targets++;
1402
1403 return 0;
1404}
1405
1406static struct cxl_port *next_port(struct cxl_port *port)
1407{
1408 if (!port->parent_dport)
1409 return NULL;
1410 return port->parent_dport->port;
1411}
1412
1413static int decoder_match_range(struct device *dev, void *data)
1414{
1415 struct cxl_endpoint_decoder *cxled = data;
1416 struct cxl_switch_decoder *cxlsd;
1417
1418 if (!is_switch_decoder(dev))
1419 return 0;
1420
1421 cxlsd = to_cxl_switch_decoder(dev);
1422 return range_contains(&cxlsd->cxld.hpa_range, &cxled->cxld.hpa_range);
1423}
1424
1425static void find_positions(const struct cxl_switch_decoder *cxlsd,
1426 const struct cxl_port *iter_a,
1427 const struct cxl_port *iter_b, int *a_pos,
1428 int *b_pos)
1429{
1430 int i;
1431
1432 for (i = 0, *a_pos = -1, *b_pos = -1; i < cxlsd->nr_targets; i++) {
1433 if (cxlsd->target[i] == iter_a->parent_dport)
1434 *a_pos = i;
1435 else if (cxlsd->target[i] == iter_b->parent_dport)
1436 *b_pos = i;
1437 if (*a_pos >= 0 && *b_pos >= 0)
1438 break;
1439 }
1440}
1441
1442static int cmp_decode_pos(const void *a, const void *b)
1443{
1444 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1445 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1446 struct cxl_memdev *cxlmd_a = cxled_to_memdev(cxled_a);
1447 struct cxl_memdev *cxlmd_b = cxled_to_memdev(cxled_b);
1448 struct cxl_port *port_a = cxled_to_port(cxled_a);
1449 struct cxl_port *port_b = cxled_to_port(cxled_b);
1450 struct cxl_port *iter_a, *iter_b, *port = NULL;
1451 struct cxl_switch_decoder *cxlsd;
1452 struct device *dev;
1453 int a_pos, b_pos;
1454 unsigned int seq;
1455
1456 /* Exit early if any prior sorting failed */
1457 if (cxled_a->pos < 0 || cxled_b->pos < 0)
1458 return 0;
1459
1460 /*
1461 * Walk up the hierarchy to find a shared port, find the decoder that
1462 * maps the range, compare the relative position of those dport
1463 * mappings.
1464 */
1465 for (iter_a = port_a; iter_a; iter_a = next_port(iter_a)) {
1466 struct cxl_port *next_a, *next_b;
1467
1468 next_a = next_port(iter_a);
1469 if (!next_a)
1470 break;
1471
1472 for (iter_b = port_b; iter_b; iter_b = next_port(iter_b)) {
1473 next_b = next_port(iter_b);
1474 if (next_a != next_b)
1475 continue;
1476 port = next_a;
1477 break;
1478 }
1479
1480 if (port)
1481 break;
1482 }
1483
1484 if (!port) {
1485 dev_err(cxlmd_a->dev.parent,
1486 "failed to find shared port with %s\n",
1487 dev_name(cxlmd_b->dev.parent));
1488 goto err;
1489 }
1490
1491 dev = device_find_child(&port->dev, cxled_a, decoder_match_range);
1492 if (!dev) {
1493 struct range *range = &cxled_a->cxld.hpa_range;
1494
7481653d 1495 dev_err(port->uport_dev,
a32320b7
DW		 1496 "failed to find decoder that maps %#llx-%#llx\n",
1497 range->start, range->end);
1498 goto err;
1499 }
1500
1501 cxlsd = to_cxl_switch_decoder(dev);
1502 do {
1503 seq = read_seqbegin(&cxlsd->target_lock);
1504 find_positions(cxlsd, iter_a, iter_b, &a_pos, &b_pos);
1505 } while (read_seqretry(&cxlsd->target_lock, seq));
1506
1507 put_device(dev);
1508
1509 if (a_pos < 0 || b_pos < 0) {
7481653d 1510 dev_err(port->uport_dev,
a32320b7
DW		 1511 "failed to find shared decoder for %s and %s\n",
1512 dev_name(cxlmd_a->dev.parent),
1513 dev_name(cxlmd_b->dev.parent));
1514 goto err;
1515 }
1516
7481653d
DW		 1517 dev_dbg(port->uport_dev, "%s comes %s %s\n",
1518 dev_name(cxlmd_a->dev.parent),
a32320b7
DW		 1519 a_pos - b_pos < 0 ? "before" : "after",
1520 dev_name(cxlmd_b->dev.parent));
1521
1522 return a_pos - b_pos;
1523err:
1524 cxled_a->pos = -1;
1525 return 0;
1526}
1527
1528static int cxl_region_sort_targets(struct cxl_region *cxlr)
1529{
1530 struct cxl_region_params *p = &cxlr->params;
1531 int i, rc = 0;
1532
1533 sort(p->targets, p->nr_targets, sizeof(p->targets[0]), cmp_decode_pos,
1534 NULL);
1535
1536 for (i = 0; i < p->nr_targets; i++) {
1537 struct cxl_endpoint_decoder *cxled = p->targets[i];
1538
1539 /*
1540 * Record that sorting failed, but still continue to restore
1541 * cxled->pos with its ->targets[] position so that follow-on
1542 * code paths can reliably do p->targets[cxled->pos] to
1543 * self-reference their entry.
1544 */
1545 if (cxled->pos < 0)
1546 rc = -ENXIO;
1547 cxled->pos = i;
1548 }
1549
1550 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1551 return rc;
1552}
1553
9995576c
DW		 1554static int cxl_region_attach(struct cxl_region *cxlr,
1555 struct cxl_endpoint_decoder *cxled, int pos)
1556{
1557 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1558 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1559 struct cxl_region_params *p = &cxlr->params;
1560 struct cxl_port *ep_port, *root_port;
1561 struct cxl_dport *dport;
1562 int rc = -ENXIO;
1563
1564 if (cxled->mode != cxlr->mode) {
1565 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1566 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1567 return -EINVAL;
1568 }
1569
1570 if (cxled->mode == CXL_DECODER_DEAD) {
1571 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1572 return -ENODEV;
1573 }
1574
1575 /* all full of members, or interleave config not established? */
1576 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1577 dev_dbg(&cxlr->dev, "region already active\n");
1578 return -EBUSY;
1579 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1580 dev_dbg(&cxlr->dev, "interleave config missing\n");
1581 return -ENXIO;
1582 }
1583
384e624b
DW		 1584 ep_port = cxled_to_port(cxled);
1585 root_port = cxlrd_to_port(cxlrd);
1586 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1587 if (!dport) {
1588 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1589 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1590 dev_name(cxlr->dev.parent));
1591 return -ENXIO;
1592 }
1593
384e624b
DW		 1594 if (cxled->cxld.target_type != cxlr->type) {
1595 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1596 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1597 cxled->cxld.target_type, cxlr->type);
1598 return -ENXIO;
1599 }
1600
1601 if (!cxled->dpa_res) {
1602 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1603 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1604 return -ENXIO;
1605 }
1606
1607 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1608 resource_size(p->res)) {
1609 dev_dbg(&cxlr->dev,
1610 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1611 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1612 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1613 (u64)resource_size(p->res));
1614 return -EINVAL;
1615 }
1616
a32320b7
DW		 1617 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1618 int i;
1619
1620 rc = cxl_region_attach_auto(cxlr, cxled, pos);
384e624b 1621 if (rc)
a32320b7
DW		 1622 return rc;
1623
1624 /* await more targets to arrive... */
1625 if (p->nr_targets < p->interleave_ways)
1626 return 0;
1627
1628 /*
1629 * All targets are here, which implies all PCI enumeration that
1630 * affects this region has been completed. Walk the topology to
1631 * sort the devices into their relative region decode position.
1632 */
1633 rc = cxl_region_sort_targets(cxlr);
1634 if (rc)
1635 return rc;
1636
1637 for (i = 0; i < p->nr_targets; i++) {
1638 cxled = p->targets[i];
1639 ep_port = cxled_to_port(cxled);
1640 dport = cxl_find_dport_by_dev(root_port,
1641 ep_port->host_bridge);
1642 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1643 dport, i);
1644 if (rc)
1645 return rc;
1646 }
1647
1648 rc = cxl_region_setup_targets(cxlr);
1649 if (rc)
1650 return rc;
1651
1652 /*
1653 * If target setup succeeds in the autodiscovery case
1654 * then the region is already committed.
1655 */
1656 p->state = CXL_CONFIG_COMMIT;
1657
1658 return 0;
384e624b
DW		 1659 }
1660
9995576c
DW		 1661 rc = cxl_region_validate_position(cxlr, cxled, pos);
1662 if (rc)
1663 return rc;
1664
1665 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1666 if (rc)
1667 return rc;
384e624b 1668
b9686e8c
DW		 1669 p->targets[pos] = cxled;
1670 cxled->pos = pos;
1671 p->nr_targets++;
1672
27b3f8d1
DW		 1673 if (p->nr_targets == p->interleave_ways) {
1674 rc = cxl_region_setup_targets(cxlr);
1675 if (rc)
5e42bcbc 1676 goto err_decrement;
384e624b 1677 p->state = CXL_CONFIG_ACTIVE;
9ff3eec9 1678 set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
27b3f8d1 1679 }
384e624b 1680
2901c8bd
DW		 1681 cxled->cxld.interleave_ways = p->interleave_ways;
1682 cxled->cxld.interleave_granularity = p->interleave_granularity;
910bc55d
DW		 1683 cxled->cxld.hpa_range = (struct range) {
1684 .start = p->res->start,
1685 .end = p->res->end,
1686 };
2901c8bd 1687
b9686e8c 1688 return 0;
384e624b 1689
5e42bcbc
DC		 1690err_decrement:
1691 p->nr_targets--;
86987c76
DW		 1692 cxled->pos = -1;
1693 p->targets[pos] = NULL;
384e624b 1694 return rc;
b9686e8c
DW		 1695}
1696
176baefb 1697static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
b9686e8c 1698{
384e624b 1699 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
b9686e8c
DW		 1700 struct cxl_region *cxlr = cxled->cxld.region;
1701 struct cxl_region_params *p;
176baefb 1702 int rc = 0;
b9686e8c
DW		 1703
1704 lockdep_assert_held_write(&cxl_region_rwsem);
1705
1706 if (!cxlr)
176baefb 1707 return 0;
b9686e8c
DW		 1708
1709 p = &cxlr->params;
1710 get_device(&cxlr->dev);
1711
176baefb
DW		 1712 if (p->state > CXL_CONFIG_ACTIVE) {
1713 /*
1714 * TODO: tear down all impacted regions if a device is
1715 * removed out of order
1716 */
1717 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1718 if (rc)
1719 goto out;
1720 p->state = CXL_CONFIG_ACTIVE;
1721 }
1722
384e624b
DW		 1723 for (iter = ep_port; !is_cxl_root(iter);
1724 iter = to_cxl_port(iter->dev.parent))
1725 cxl_port_detach_region(iter, cxlr, cxled);
1726
b9686e8c
DW		 1727 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1728 p->targets[cxled->pos] != cxled) {
1729 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1730
1731 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1732 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1733 cxled->pos);
1734 goto out;
1735 }
1736
27b3f8d1 1737 if (p->state == CXL_CONFIG_ACTIVE) {
384e624b 1738 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
27b3f8d1
DW		 1739 cxl_region_teardown_targets(cxlr);
1740 }
b9686e8c
DW		 1741 p->targets[cxled->pos] = NULL;
1742 p->nr_targets--;
910bc55d
DW		 1743 cxled->cxld.hpa_range = (struct range) {
1744 .start = 0,
1745 .end = -1,
1746 };
b9686e8c 1747
384e624b 1748 /* notify the region driver that one of its targets has departed */
b9686e8c
DW		 1749 up_write(&cxl_region_rwsem);
1750 device_release_driver(&cxlr->dev);
1751 down_write(&cxl_region_rwsem);
1752out:
1753 put_device(&cxlr->dev);
176baefb 1754 return rc;
b9686e8c
DW		 1755}
1756
1757void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1758{
1759 down_write(&cxl_region_rwsem);
1760 cxled->mode = CXL_DECODER_DEAD;
1761 cxl_region_detach(cxled);
1762 up_write(&cxl_region_rwsem);
1763}
1764
3528b1e1
DW		 1765static int attach_target(struct cxl_region *cxlr,
1766 struct cxl_endpoint_decoder *cxled, int pos,
1767 unsigned int state)
b9686e8c 1768{
3528b1e1 1769 int rc = 0;
b9686e8c 1770
3528b1e1
DW		 1771 if (state == TASK_INTERRUPTIBLE)
1772 rc = down_write_killable(&cxl_region_rwsem);
1773 else
1774 down_write(&cxl_region_rwsem);
b9686e8c 1775 if (rc)
3528b1e1
DW		 1776 return rc;
1777
b9686e8c 1778 down_read(&cxl_dpa_rwsem);
3528b1e1 1779 rc = cxl_region_attach(cxlr, cxled, pos);
b9686e8c
DW		 1780 up_read(&cxl_dpa_rwsem);
1781 up_write(&cxl_region_rwsem);
b9686e8c
DW		 1782 return rc;
1783}
1784
1785static int detach_target(struct cxl_region *cxlr, int pos)
1786{
1787 struct cxl_region_params *p = &cxlr->params;
1788 int rc;
1789
1790 rc = down_write_killable(&cxl_region_rwsem);
1791 if (rc)
1792 return rc;
1793
1794 if (pos >= p->interleave_ways) {
1795 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1796 p->interleave_ways);
1797 rc = -ENXIO;
1798 goto out;
1799 }
1800
1801 if (!p->targets[pos]) {
1802 rc = 0;
1803 goto out;
1804 }
1805
176baefb 1806 rc = cxl_region_detach(p->targets[pos]);
b9686e8c
DW		 1807out:
1808 up_write(&cxl_region_rwsem);
1809 return rc;
1810}
1811
1812static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1813 size_t len)
1814{
1815 int rc;
1816
1817 if (sysfs_streq(buf, "\n"))
1818 rc = detach_target(cxlr, pos);
3528b1e1
DW		 1819 else {
1820 struct device *dev;
1821
1822 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1823 if (!dev)
1824 return -ENODEV;
1825
1826 if (!is_endpoint_decoder(dev)) {
1827 rc = -EINVAL;
1828 goto out;
1829 }
1830
1831 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1832 TASK_INTERRUPTIBLE);
1833out:
1834 put_device(dev);
1835 }
b9686e8c
DW		 1836
1837 if (rc < 0)
1838 return rc;
1839 return len;
1840}
1841
1842#define TARGET_ATTR_RW(n) \
1843static ssize_t target##n##_show( \
1844 struct device *dev, struct device_attribute *attr, char *buf) \
1845{ \
1846 return show_targetN(to_cxl_region(dev), buf, (n)); \
1847} \
1848static ssize_t target##n##_store(struct device *dev, \
1849 struct device_attribute *attr, \
1850 const char *buf, size_t len) \
1851{ \
1852 return store_targetN(to_cxl_region(dev), buf, (n), len); \
1853} \
1854static DEVICE_ATTR_RW(target##n)
1855
1856TARGET_ATTR_RW(0);
1857TARGET_ATTR_RW(1);
1858TARGET_ATTR_RW(2);
1859TARGET_ATTR_RW(3);
1860TARGET_ATTR_RW(4);
1861TARGET_ATTR_RW(5);
1862TARGET_ATTR_RW(6);
1863TARGET_ATTR_RW(7);
1864TARGET_ATTR_RW(8);
1865TARGET_ATTR_RW(9);
1866TARGET_ATTR_RW(10);
1867TARGET_ATTR_RW(11);
1868TARGET_ATTR_RW(12);
1869TARGET_ATTR_RW(13);
1870TARGET_ATTR_RW(14);
1871TARGET_ATTR_RW(15);
1872
1873static struct attribute *target_attrs[] = {
1874 &dev_attr_target0.attr,
1875 &dev_attr_target1.attr,
1876 &dev_attr_target2.attr,
1877 &dev_attr_target3.attr,
1878 &dev_attr_target4.attr,
1879 &dev_attr_target5.attr,
1880 &dev_attr_target6.attr,
1881 &dev_attr_target7.attr,
1882 &dev_attr_target8.attr,
1883 &dev_attr_target9.attr,
1884 &dev_attr_target10.attr,
1885 &dev_attr_target11.attr,
1886 &dev_attr_target12.attr,
1887 &dev_attr_target13.attr,
1888 &dev_attr_target14.attr,
1889 &dev_attr_target15.attr,
1890 NULL,
1891};
1892
1893static umode_t cxl_region_target_visible(struct kobject *kobj,
1894 struct attribute *a, int n)
1895{
1896 struct device *dev = kobj_to_dev(kobj);
1897 struct cxl_region *cxlr = to_cxl_region(dev);
1898 struct cxl_region_params *p = &cxlr->params;
1899
1900 if (n < p->interleave_ways)
1901 return a->mode;
1902 return 0;
1903}
1904
1905static const struct attribute_group cxl_region_target_group = {
1906 .attrs = target_attrs,
1907 .is_visible = cxl_region_target_visible,
1908};
1909
1910static const struct attribute_group *get_cxl_region_target_group(void)
1911{
1912 return &cxl_region_target_group;
1913}
1914
dd5ba0eb
BW		 1915static const struct attribute_group *region_groups[] = {
1916 &cxl_base_attribute_group,
1917 &cxl_region_group,
b9686e8c 1918 &cxl_region_target_group,
dd5ba0eb
BW		 1919 NULL,
1920};
1921
779dd20c
BW		 1922static void cxl_region_release(struct device *dev)
1923{
8f401ec1 1924 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
779dd20c 1925 struct cxl_region *cxlr = to_cxl_region(dev);
8f401ec1
DW		 1926 int id = atomic_read(&cxlrd->region_id);
1927
1928 /*
1929 * Try to reuse the recently idled id rather than the cached
1930 * next id to prevent the region id space from increasing
1931 * unnecessarily.
1932 */
1933 if (cxlr->id < id)
1934 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
1935 memregion_free(id);
1936 goto out;
1937 }
779dd20c
BW		 1938
1939 memregion_free(cxlr->id);
8f401ec1
DW		 1940out:
1941 put_device(dev->parent);
779dd20c
BW		 1942 kfree(cxlr);
1943}
1944
8d48817d 1945const struct device_type cxl_region_type = {
779dd20c
BW		 1946 .name = "cxl_region",
1947 .release = cxl_region_release,
dd5ba0eb 1948 .groups = region_groups
779dd20c
BW		 1949};
1950
1951bool is_cxl_region(struct device *dev)
1952{
1953 return dev->type == &cxl_region_type;
1954}
1955EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
1956
1957static struct cxl_region *to_cxl_region(struct device *dev)
1958{
1959 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
1960 "not a cxl_region device\n"))
1961 return NULL;
1962
1963 return container_of(dev, struct cxl_region, dev);
1964}
1965
1966static void unregister_region(void *dev)
1967{
23a22cd1 1968 struct cxl_region *cxlr = to_cxl_region(dev);
0d9e7340
DW		 1969 struct cxl_region_params *p = &cxlr->params;
1970 int i;
23a22cd1
DW		 1971
1972 device_del(dev);
0d9e7340
DW		 1973
1974 /*
1975 * Now that region sysfs is shutdown, the parameter block is now
1976 * read-only, so no need to hold the region rwsem to access the
1977 * region parameters.
1978 */
1979 for (i = 0; i < p->interleave_ways; i++)
1980 detach_target(cxlr, i);
1981
23a22cd1
DW		 1982 cxl_region_iomem_release(cxlr);
1983 put_device(dev);
779dd20c
BW		 1984}
1985
1986static struct lock_class_key cxl_region_key;
1987
1988static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
1989{
1990 struct cxl_region *cxlr;
1991 struct device *dev;
1992
1993 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
1994 if (!cxlr) {
1995 memregion_free(id);
1996 return ERR_PTR(-ENOMEM);
1997 }
1998
1999 dev = &cxlr->dev;
2000 device_initialize(dev);
2001 lockdep_set_class(&dev->mutex, &cxl_region_key);
2002 dev->parent = &cxlrd->cxlsd.cxld.dev;
8f401ec1
DW		 2003 /*
2004 * Keep root decoder pinned through cxl_region_release to fixup
2005 * region id allocations
2006 */
2007 get_device(dev->parent);
779dd20c
BW		 2008 device_set_pm_not_required(dev);
2009 dev->bus = &cxl_bus_type;
2010 dev->type = &cxl_region_type;
2011 cxlr->id = id;
2012
2013 return cxlr;
2014}
2015
2016/**
2017 * devm_cxl_add_region - Adds a region to a decoder
2018 * @cxlrd: root decoder
2019 * @id: memregion id to create, or memregion_free() on failure
2020 * @mode: mode for the endpoint decoders of this region
2021 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2022 *
2023 * This is the second step of region initialization. Regions exist within an
2024 * address space which is mapped by a @cxlrd.
2025 *
2026 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2027 * code. The region will be named "regionZ" where Z is the unique region number.
2028 */
2029static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2030 int id,
2031 enum cxl_decoder_mode mode,
2032 enum cxl_decoder_type type)
2033{
2034 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2035 struct cxl_region *cxlr;
2036 struct device *dev;
2037 int rc;
2038
6e099264
DW		 2039 switch (mode) {
2040 case CXL_DECODER_RAM:
2041 case CXL_DECODER_PMEM:
2042 break;
2043 default:
2044 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2045 return ERR_PTR(-EINVAL);
2046 }
2047
779dd20c
BW		 2048 cxlr = cxl_region_alloc(cxlrd, id);
2049 if (IS_ERR(cxlr))
2050 return cxlr;
2051 cxlr->mode = mode;
2052 cxlr->type = type;
2053
2054 dev = &cxlr->dev;
2055 rc = dev_set_name(dev, "region%d", id);
2056 if (rc)
2057 goto err;
2058
2059 rc = device_add(dev);
2060 if (rc)
2061 goto err;
2062
7481653d 2063 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
779dd20c
BW		 2064 if (rc)
2065 return ERR_PTR(rc);
2066
7481653d 2067 dev_dbg(port->uport_dev, "%s: created %s\n",
779dd20c
BW		 2068 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2069 return cxlr;
2070
2071err:
2072 put_device(dev);
2073 return ERR_PTR(rc);
2074}
2075
6e099264
DW		 2076static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2077{
2078 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2079}
2080
779dd20c
BW		 2081static ssize_t create_pmem_region_show(struct device *dev,
2082 struct device_attribute *attr, char *buf)
2083{
6e099264
DW		 2084 return __create_region_show(to_cxl_root_decoder(dev), buf);
2085}
779dd20c 2086
6e099264
DW		 2087static ssize_t create_ram_region_show(struct device *dev,
2088 struct device_attribute *attr, char *buf)
2089{
2090 return __create_region_show(to_cxl_root_decoder(dev), buf);
2091}
2092
2093static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2094 enum cxl_decoder_mode mode, int id)
2095{
2096 int rc;
2097
2098 rc = memregion_alloc(GFP_KERNEL);
2099 if (rc < 0)
2100 return ERR_PTR(rc);
2101
2102 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2103 memregion_free(rc);
2104 return ERR_PTR(-EBUSY);
2105 }
2106
2107 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_EXPANDER);
779dd20c
BW		 2108}
2109
2110static ssize_t create_pmem_region_store(struct device *dev,
2111 struct device_attribute *attr,
2112 const char *buf, size_t len)
2113{
2114 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2115 struct cxl_region *cxlr;
6e099264 2116 int rc, id;
779dd20c
BW		 2117
2118 rc = sscanf(buf, "region%d\n", &id);
2119 if (rc != 1)
2120 return -EINVAL;
2121
6e099264
DW		 2122 cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2123 if (IS_ERR(cxlr))
2124 return PTR_ERR(cxlr);
779dd20c 2125
6e099264
DW		 2126 return len;
2127}
2128DEVICE_ATTR_RW(create_pmem_region);
779dd20c 2129
6e099264
DW		 2130static ssize_t create_ram_region_store(struct device *dev,
2131 struct device_attribute *attr,
2132 const char *buf, size_t len)
2133{
2134 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2135 struct cxl_region *cxlr;
2136 int rc, id;
779dd20c 2137
6e099264
DW		 2138 rc = sscanf(buf, "region%d\n", &id);
2139 if (rc != 1)
2140 return -EINVAL;
2141
2142 cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
779dd20c
BW		 2143 if (IS_ERR(cxlr))
2144 return PTR_ERR(cxlr);
2145
2146 return len;
2147}
6e099264 2148DEVICE_ATTR_RW(create_ram_region);
779dd20c 2149
b9686e8c
DW		 2150static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2151 char *buf)
2152{
2153 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2154 ssize_t rc;
2155
2156 rc = down_read_interruptible(&cxl_region_rwsem);
2157 if (rc)
2158 return rc;
2159
2160 if (cxld->region)
2161 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2162 else
2163 rc = sysfs_emit(buf, "\n");
2164 up_read(&cxl_region_rwsem);
2165
2166 return rc;
2167}
2168DEVICE_ATTR_RO(region);
2169
779dd20c
BW		 2170static struct cxl_region *
2171cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2172{
2173 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2174 struct device *region_dev;
2175
2176 region_dev = device_find_child_by_name(&cxld->dev, name);
2177 if (!region_dev)
2178 return ERR_PTR(-ENODEV);
2179
2180 return to_cxl_region(region_dev);
2181}
2182
2183static ssize_t delete_region_store(struct device *dev,
2184 struct device_attribute *attr,
2185 const char *buf, size_t len)
2186{
2187 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2188 struct cxl_port *port = to_cxl_port(dev->parent);
2189 struct cxl_region *cxlr;
2190
2191 cxlr = cxl_find_region_by_name(cxlrd, buf);
2192 if (IS_ERR(cxlr))
2193 return PTR_ERR(cxlr);
2194
7481653d 2195 devm_release_action(port->uport_dev, unregister_region, cxlr);
779dd20c
BW		 2196 put_device(&cxlr->dev);
2197
2198 return len;
2199}
2200DEVICE_ATTR_WO(delete_region);
23a22cd1 2201
04ad63f0
DW		 2202static void cxl_pmem_region_release(struct device *dev)
2203{
2204 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2205 int i;
2206
2207 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2208 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2209
2210 put_device(&cxlmd->dev);
2211 }
2212
2213 kfree(cxlr_pmem);
2214}
2215
2216static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2217 &cxl_base_attribute_group,
2218 NULL,
2219};
2220
2221const struct device_type cxl_pmem_region_type = {
2222 .name = "cxl_pmem_region",
2223 .release = cxl_pmem_region_release,
2224 .groups = cxl_pmem_region_attribute_groups,
2225};
2226
2227bool is_cxl_pmem_region(struct device *dev)
2228{
2229 return dev->type == &cxl_pmem_region_type;
2230}
2231EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2232
2233struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2234{
2235 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2236 "not a cxl_pmem_region device\n"))
2237 return NULL;
2238 return container_of(dev, struct cxl_pmem_region, dev);
2239}
2240EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2241
f0832a58
AS		 2242struct cxl_poison_context {
2243 struct cxl_port *port;
2244 enum cxl_decoder_mode mode;
2245 u64 offset;
2246};
2247
2248static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2249 struct cxl_poison_context *ctx)
2250{
2251 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2252 u64 offset, length;
2253 int rc = 0;
2254
2255 /*
2256 * Collect poison for the remaining unmapped resources
2257 * after poison is collected by committed endpoints.
2258 *
2259 * Knowing that PMEM must always follow RAM, get poison
2260 * for unmapped resources based on the last decoder's mode:
2261 * ram: scan remains of ram range, then any pmem range
2262 * pmem: scan remains of pmem range
2263 */
2264
2265 if (ctx->mode == CXL_DECODER_RAM) {
2266 offset = ctx->offset;
2267 length = resource_size(&cxlds->ram_res) - offset;
2268 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2269 if (rc == -EFAULT)
2270 rc = 0;
2271 if (rc)
2272 return rc;
2273 }
2274 if (ctx->mode == CXL_DECODER_PMEM) {
2275 offset = ctx->offset;
2276 length = resource_size(&cxlds->dpa_res) - offset;
2277 if (!length)
2278 return 0;
2279 } else if (resource_size(&cxlds->pmem_res)) {
2280 offset = cxlds->pmem_res.start;
2281 length = resource_size(&cxlds->pmem_res);
2282 } else {
2283 return 0;
2284 }
2285
2286 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2287}
2288
2289static int poison_by_decoder(struct device *dev, void *arg)
2290{
2291 struct cxl_poison_context *ctx = arg;
2292 struct cxl_endpoint_decoder *cxled;
2293 struct cxl_memdev *cxlmd;
2294 u64 offset, length;
2295 int rc = 0;
2296
2297 if (!is_endpoint_decoder(dev))
2298 return rc;
2299
2300 cxled = to_cxl_endpoint_decoder(dev);
2301 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2302 return rc;
2303
2304 /*
2305 * Regions are only created with single mode decoders: pmem or ram.
2306 * Linux does not support mixed mode decoders. This means that
2307 * reading poison per endpoint decoder adheres to the requirement
2308 * that poison reads of pmem and ram must be separated.
2309 * CXL 3.0 Spec 8.2.9.8.4.1
2310 */
2311 if (cxled->mode == CXL_DECODER_MIXED) {
2312 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2313 return rc;
2314 }
2315
2316 cxlmd = cxled_to_memdev(cxled);
2317 if (cxled->skip) {
2318 offset = cxled->dpa_res->start - cxled->skip;
2319 length = cxled->skip;
2320 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2321 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2322 rc = 0;
2323 if (rc)
2324 return rc;
2325 }
2326
2327 offset = cxled->dpa_res->start;
2328 length = cxled->dpa_res->end - offset + 1;
2329 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2330 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2331 rc = 0;
2332 if (rc)
2333 return rc;
2334
2335 /* Iterate until commit_end is reached */
2336 if (cxled->cxld.id == ctx->port->commit_end) {
2337 ctx->offset = cxled->dpa_res->end + 1;
2338 ctx->mode = cxled->mode;
2339 return 1;
2340 }
2341
2342 return 0;
2343}
2344
2345int cxl_get_poison_by_endpoint(struct cxl_port *port)
2346{
2347 struct cxl_poison_context ctx;
2348 int rc = 0;
2349
2350 rc = down_read_interruptible(&cxl_region_rwsem);
2351 if (rc)
2352 return rc;
2353
2354 ctx = (struct cxl_poison_context) {
2355 .port = port
2356 };
2357
2358 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2359 if (rc == 1)
7481653d
DW		 2360 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2361 &ctx);
f0832a58
AS		 2362
2363 up_read(&cxl_region_rwsem);
2364 return rc;
2365}
2366
04ad63f0
DW		 2367static struct lock_class_key cxl_pmem_region_key;
2368
2369static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2370{
2371 struct cxl_region_params *p = &cxlr->params;
f17b558d 2372 struct cxl_nvdimm_bridge *cxl_nvb;
04ad63f0
DW		 2373 struct cxl_pmem_region *cxlr_pmem;
2374 struct device *dev;
2375 int i;
2376
2377 down_read(&cxl_region_rwsem);
2378 if (p->state != CXL_CONFIG_COMMIT) {
2379 cxlr_pmem = ERR_PTR(-ENXIO);
2380 goto out;
2381 }
2382
2383 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2384 GFP_KERNEL);
2385 if (!cxlr_pmem) {
2386 cxlr_pmem = ERR_PTR(-ENOMEM);
2387 goto out;
2388 }
2389
2390 cxlr_pmem->hpa_range.start = p->res->start;
2391 cxlr_pmem->hpa_range.end = p->res->end;
2392
2393 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2394 cxlr_pmem->nr_mappings = p->nr_targets;
2395 for (i = 0; i < p->nr_targets; i++) {
2396 struct cxl_endpoint_decoder *cxled = p->targets[i];
2397 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2398 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2399
f17b558d
DW		 2400 /*
2401 * Regions never span CXL root devices, so by definition the
2402 * bridge for one device is the same for all.
2403 */
2404 if (i == 0) {
d35b495d 2405 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
f17b558d
DW		 2406 if (!cxl_nvb) {
2407 cxlr_pmem = ERR_PTR(-ENODEV);
2408 goto out;
2409 }
2410 cxlr->cxl_nvb = cxl_nvb;
2411 }
04ad63f0
DW		 2412 m->cxlmd = cxlmd;
2413 get_device(&cxlmd->dev);
2414 m->start = cxled->dpa_res->start;
2415 m->size = resource_size(cxled->dpa_res);
2416 m->position = i;
2417 }
2418
2419 dev = &cxlr_pmem->dev;
2420 cxlr_pmem->cxlr = cxlr;
f17b558d 2421 cxlr->cxlr_pmem = cxlr_pmem;
04ad63f0
DW		 2422 device_initialize(dev);
2423 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2424 device_set_pm_not_required(dev);
2425 dev->parent = &cxlr->dev;
2426 dev->bus = &cxl_bus_type;
2427 dev->type = &cxl_pmem_region_type;
2428out:
2429 up_read(&cxl_region_rwsem);
2430
2431 return cxlr_pmem;
2432}
2433
09d09e04
DW		 2434static void cxl_dax_region_release(struct device *dev)
2435{
2436 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2437
2438 kfree(cxlr_dax);
2439}
2440
2441static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2442 &cxl_base_attribute_group,
2443 NULL,
2444};
2445
2446const struct device_type cxl_dax_region_type = {
2447 .name = "cxl_dax_region",
2448 .release = cxl_dax_region_release,
2449 .groups = cxl_dax_region_attribute_groups,
2450};
2451
2452static bool is_cxl_dax_region(struct device *dev)
2453{
2454 return dev->type == &cxl_dax_region_type;
2455}
2456
2457struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2458{
2459 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2460 "not a cxl_dax_region device\n"))
2461 return NULL;
2462 return container_of(dev, struct cxl_dax_region, dev);
2463}
2464EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2465
2466static struct lock_class_key cxl_dax_region_key;
2467
2468static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2469{
2470 struct cxl_region_params *p = &cxlr->params;
2471 struct cxl_dax_region *cxlr_dax;
2472 struct device *dev;
2473
2474 down_read(&cxl_region_rwsem);
2475 if (p->state != CXL_CONFIG_COMMIT) {
2476 cxlr_dax = ERR_PTR(-ENXIO);
2477 goto out;
2478 }
2479
2480 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2481 if (!cxlr_dax) {
2482 cxlr_dax = ERR_PTR(-ENOMEM);
2483 goto out;
2484 }
2485
2486 cxlr_dax->hpa_range.start = p->res->start;
2487 cxlr_dax->hpa_range.end = p->res->end;
2488
2489 dev = &cxlr_dax->dev;
2490 cxlr_dax->cxlr = cxlr;
2491 device_initialize(dev);
2492 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2493 device_set_pm_not_required(dev);
2494 dev->parent = &cxlr->dev;
2495 dev->bus = &cxl_bus_type;
2496 dev->type = &cxl_dax_region_type;
2497out:
2498 up_read(&cxl_region_rwsem);
2499
2500 return cxlr_dax;
2501}
2502
f17b558d 2503static void cxlr_pmem_unregister(void *_cxlr_pmem)
04ad63f0 2504{
f17b558d
DW		 2505 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2506 struct cxl_region *cxlr = cxlr_pmem->cxlr;
2507 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2508
2509 /*
2510 * Either the bridge is in ->remove() context under the device_lock(),
2511 * or cxlr_release_nvdimm() is cancelling the bridge's release action
2512 * for @cxlr_pmem and doing it itself (while manually holding the bridge
2513 * lock).
2514 */
2515 device_lock_assert(&cxl_nvb->dev);
2516 cxlr->cxlr_pmem = NULL;
2517 cxlr_pmem->cxlr = NULL;
2518 device_unregister(&cxlr_pmem->dev);
2519}
2520
2521static void cxlr_release_nvdimm(void *_cxlr)
2522{
2523 struct cxl_region *cxlr = _cxlr;
2524 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2525
2526 device_lock(&cxl_nvb->dev);
2527 if (cxlr->cxlr_pmem)
2528 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2529 cxlr->cxlr_pmem);
2530 device_unlock(&cxl_nvb->dev);
2531 cxlr->cxl_nvb = NULL;
2532 put_device(&cxl_nvb->dev);
04ad63f0
DW		 2533}
2534
2535/**
2536 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2537 * @cxlr: parent CXL region for this pmem region bridge device
2538 *
2539 * Return: 0 on success negative error code on failure.
2540 */
2541static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2542{
2543 struct cxl_pmem_region *cxlr_pmem;
f17b558d 2544 struct cxl_nvdimm_bridge *cxl_nvb;
04ad63f0
DW		 2545 struct device *dev;
2546 int rc;
2547
2548 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2549 if (IS_ERR(cxlr_pmem))
2550 return PTR_ERR(cxlr_pmem);
f17b558d 2551 cxl_nvb = cxlr->cxl_nvb;
04ad63f0
DW		 2552
2553 dev = &cxlr_pmem->dev;
2554 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2555 if (rc)
2556 goto err;
2557
2558 rc = device_add(dev);
2559 if (rc)
2560 goto err;
2561
2562 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2563 dev_name(dev));
2564
f17b558d
DW		 2565 device_lock(&cxl_nvb->dev);
2566 if (cxl_nvb->dev.driver)
2567 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2568 cxlr_pmem_unregister, cxlr_pmem);
2569 else
2570 rc = -ENXIO;
2571 device_unlock(&cxl_nvb->dev);
2572
2573 if (rc)
2574 goto err_bridge;
2575
2576 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2577 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
04ad63f0
DW		 2578
2579err:
2580 put_device(dev);
f17b558d
DW		 2581err_bridge:
2582 put_device(&cxl_nvb->dev);
2583 cxlr->cxl_nvb = NULL;
04ad63f0
DW		 2584 return rc;
2585}
2586
09d09e04
DW		 2587static void cxlr_dax_unregister(void *_cxlr_dax)
2588{
2589 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2590
2591 device_unregister(&cxlr_dax->dev);
2592}
2593
2594static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2595{
2596 struct cxl_dax_region *cxlr_dax;
2597 struct device *dev;
2598 int rc;
2599
2600 cxlr_dax = cxl_dax_region_alloc(cxlr);
2601 if (IS_ERR(cxlr_dax))
2602 return PTR_ERR(cxlr_dax);
2603
2604 dev = &cxlr_dax->dev;
2605 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2606 if (rc)
2607 goto err;
2608
2609 rc = device_add(dev);
2610 if (rc)
2611 goto err;
2612
2613 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2614 dev_name(dev));
2615
2616 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2617 cxlr_dax);
2618err:
2619 put_device(dev);
2620 return rc;
2621}
2622
a32320b7
DW		 2623static int match_decoder_by_range(struct device *dev, void *data)
2624{
2625 struct range *r1, *r2 = data;
2626 struct cxl_root_decoder *cxlrd;
2627
2628 if (!is_root_decoder(dev))
2629 return 0;
2630
2631 cxlrd = to_cxl_root_decoder(dev);
2632 r1 = &cxlrd->cxlsd.cxld.hpa_range;
2633 return range_contains(r1, r2);
2634}
2635
2636static int match_region_by_range(struct device *dev, void *data)
2637{
2638 struct cxl_region_params *p;
2639 struct cxl_region *cxlr;
2640 struct range *r = data;
2641 int rc = 0;
2642
2643 if (!is_cxl_region(dev))
2644 return 0;
2645
2646 cxlr = to_cxl_region(dev);
2647 p = &cxlr->params;
2648
2649 down_read(&cxl_region_rwsem);
2650 if (p->res && p->res->start == r->start && p->res->end == r->end)
2651 rc = 1;
2652 up_read(&cxl_region_rwsem);
2653
2654 return rc;
2655}
2656
2657/* Establish an empty region covering the given HPA range */
2658static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2659 struct cxl_endpoint_decoder *cxled)
2660{
2661 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2662 struct cxl_port *port = cxlrd_to_port(cxlrd);
2663 struct range *hpa = &cxled->cxld.hpa_range;
2664 struct cxl_region_params *p;
2665 struct cxl_region *cxlr;
2666 struct resource *res;
2667 int rc;
2668
2669 do {
2670 cxlr = __create_region(cxlrd, cxled->mode,
2671 atomic_read(&cxlrd->region_id));
2672 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2673
2674 if (IS_ERR(cxlr)) {
2675 dev_err(cxlmd->dev.parent,
2676 "%s:%s: %s failed assign region: %ld\n",
2677 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2678 __func__, PTR_ERR(cxlr));
2679 return cxlr;
2680 }
2681
2682 down_write(&cxl_region_rwsem);
2683 p = &cxlr->params;
2684 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2685 dev_err(cxlmd->dev.parent,
2686 "%s:%s: %s autodiscovery interrupted\n",
2687 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2688 __func__);
2689 rc = -EBUSY;
2690 goto err;
2691 }
2692
2693 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2694
2695 res = kmalloc(sizeof(*res), GFP_KERNEL);
2696 if (!res) {
2697 rc = -ENOMEM;
2698 goto err;
2699 }
2700
2701 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2702 dev_name(&cxlr->dev));
2703 rc = insert_resource(cxlrd->res, res);
2704 if (rc) {
2705 /*
2706 * Platform-firmware may not have split resources like "System
2707 * RAM" on CXL window boundaries see cxl_region_iomem_release()
2708 */
2709 dev_warn(cxlmd->dev.parent,
2710 "%s:%s: %s %s cannot insert resource\n",
2711 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2712 __func__, dev_name(&cxlr->dev));
2713 }
2714
2715 p->res = res;
2716 p->interleave_ways = cxled->cxld.interleave_ways;
2717 p->interleave_granularity = cxled->cxld.interleave_granularity;
2718 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2719
2720 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2721 if (rc)
2722 goto err;
2723
2724 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2725 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2726 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2727 p->interleave_granularity);
2728
2729 /* ...to match put_device() in cxl_add_to_region() */
2730 get_device(&cxlr->dev);
2731 up_write(&cxl_region_rwsem);
2732
2733 return cxlr;
2734
2735err:
2736 up_write(&cxl_region_rwsem);
7481653d 2737 devm_release_action(port->uport_dev, unregister_region, cxlr);
a32320b7
DW		 2738 return ERR_PTR(rc);
2739}
2740
2741int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2742{
2743 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2744 struct range *hpa = &cxled->cxld.hpa_range;
2745 struct cxl_decoder *cxld = &cxled->cxld;
2746 struct device *cxlrd_dev, *region_dev;
2747 struct cxl_root_decoder *cxlrd;
2748 struct cxl_region_params *p;
2749 struct cxl_region *cxlr;
2750 bool attach = false;
2751 int rc;
2752
2753 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2754 match_decoder_by_range);
2755 if (!cxlrd_dev) {
2756 dev_err(cxlmd->dev.parent,
2757 "%s:%s no CXL window for range %#llx:%#llx\n",
2758 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2759 cxld->hpa_range.start, cxld->hpa_range.end);
2760 return -ENXIO;
2761 }
2762
2763 cxlrd = to_cxl_root_decoder(cxlrd_dev);
2764
2765 /*
2766 * Ensure that if multiple threads race to construct_region() for @hpa
2767 * one does the construction and the others add to that.
2768 */
2769 mutex_lock(&cxlrd->range_lock);
2770 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2771 match_region_by_range);
2772 if (!region_dev) {
2773 cxlr = construct_region(cxlrd, cxled);
2774 region_dev = &cxlr->dev;
2775 } else
2776 cxlr = to_cxl_region(region_dev);
2777 mutex_unlock(&cxlrd->range_lock);
2778
7abcb0b1
AB		 2779 rc = PTR_ERR_OR_ZERO(cxlr);
2780 if (rc)
a32320b7 2781 goto out;
a32320b7
DW		 2782
2783 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2784
2785 down_read(&cxl_region_rwsem);
2786 p = &cxlr->params;
2787 attach = p->state == CXL_CONFIG_COMMIT;
2788 up_read(&cxl_region_rwsem);
2789
2790 if (attach) {
2791 /*
2792 * If device_attach() fails the range may still be active via
2793 * the platform-firmware memory map, otherwise the driver for
2794 * regions is local to this file, so driver matching can't fail.
2795 */
2796 if (device_attach(&cxlr->dev) < 0)
2797 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2798 p->res);
2799 }
2800
2801 put_device(region_dev);
2802out:
2803 put_device(cxlrd_dev);
2804 return rc;
2805}
2806EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2807
d18bc74a
DW		 2808static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
2809{
2810 if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags))
2811 return 0;
2812
2813 if (!cpu_cache_has_invalidate_memregion()) {
2814 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
e520d52d 2815 dev_warn_once(
d18bc74a 2816 &cxlr->dev,
cb4cdf74 2817 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
d18bc74a
DW		 2818 clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
2819 return 0;
2820 } else {
2821 dev_err(&cxlr->dev,
2822 "Failed to synchronize CPU cache state\n");
2823 return -ENXIO;
2824 }
2825 }
2826
2827 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
2828 clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags);
2829 return 0;
2830}
2831
a32320b7
DW		 2832static int is_system_ram(struct resource *res, void *arg)
2833{
2834 struct cxl_region *cxlr = arg;
2835 struct cxl_region_params *p = &cxlr->params;
2836
2837 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2838 return 1;
2839}
2840
8d48817d
DW		 2841static int cxl_region_probe(struct device *dev)
2842{
2843 struct cxl_region *cxlr = to_cxl_region(dev);
2844 struct cxl_region_params *p = &cxlr->params;
2845 int rc;
2846
2847 rc = down_read_interruptible(&cxl_region_rwsem);
2848 if (rc) {
2849 dev_dbg(&cxlr->dev, "probe interrupted\n");
2850 return rc;
2851 }
2852
2853 if (p->state < CXL_CONFIG_COMMIT) {
2854 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2855 rc = -ENXIO;
d18bc74a 2856 goto out;
8d48817d
DW		 2857 }
2858
d18bc74a
DW		 2859 rc = cxl_region_invalidate_memregion(cxlr);
2860
8d48817d
DW		 2861 /*
2862 * From this point on any path that changes the region's state away from
2863 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2864 */
d18bc74a 2865out:
8d48817d
DW		 2866 up_read(&cxl_region_rwsem);
2867
bf3e5da8
DW		 2868 if (rc)
2869 return rc;
2870
04ad63f0
DW		 2871 switch (cxlr->mode) {
2872 case CXL_DECODER_PMEM:
2873 return devm_cxl_add_pmem_region(cxlr);
a32320b7
DW		 2874 case CXL_DECODER_RAM:
2875 /*
2876 * The region can not be manged by CXL if any portion of
2877 * it is already online as 'System RAM'
2878 */
2879 if (walk_iomem_res_desc(IORES_DESC_NONE,
2880 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2881 p->res->start, p->res->end, cxlr,
2882 is_system_ram) > 0)
2883 return 0;
09d09e04 2884 return devm_cxl_add_dax_region(cxlr);
04ad63f0
DW		 2885 default:
2886 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2887 cxlr->mode);
2888 return -ENXIO;
2889 }
8d48817d
DW		 2890}
2891
2892static struct cxl_driver cxl_region_driver = {
2893 .name = "cxl_region",
2894 .probe = cxl_region_probe,
2895 .id = CXL_DEVICE_REGION,
2896};
2897
2898int cxl_region_init(void)
2899{
2900 return cxl_driver_register(&cxl_region_driver);
2901}
2902
2903void cxl_region_exit(void)
2904{
2905 cxl_driver_unregister(&cxl_region_driver);
2906}
2907
23a22cd1 2908MODULE_IMPORT_NS(CXL);
d18bc74a 2909MODULE_IMPORT_NS(DEVMEM);
8d48817d 2910MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
Linux 6.x block layer and io_uring tree(s)