projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'sched_ext-for-6.12-rc1-fixes-1' of git://git.kernel.org/pub/scm/linux...
[linux-2.6-block.git] / drivers / s390 / cio / css.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * driver for channel subsystem
4 *
5 * Copyright IBM Corp. 2002, 2010
6 *
7 * Author(s): Arnd Bergmann (arndb@de.ibm.com)
8 * Cornelia Huck (cornelia.huck@de.ibm.com)
9 */
10
11#define KMSG_COMPONENT "cio"
12#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13
14#include <linux/export.h>
15#include <linux/init.h>
16#include <linux/device.h>
17#include <linux/slab.h>
18#include <linux/errno.h>
19#include <linux/list.h>
20#include <linux/reboot.h>
21#include <linux/proc_fs.h>
22#include <linux/genalloc.h>
23#include <linux/dma-mapping.h>
24#include <asm/isc.h>
25#include <asm/crw.h>
26
27#include "css.h"
28#include "cio.h"
29#include "blacklist.h"
30#include "cio_debug.h"
31#include "ioasm.h"
32#include "chsc.h"
33#include "device.h"
34#include "idset.h"
35#include "chp.h"
36
37int css_init_done = 0;
38int max_ssid;
39
40#define MAX_CSS_IDX 0
41struct channel_subsystem *channel_subsystems[MAX_CSS_IDX + 1];
42static const struct bus_type css_bus_type;
43
44int
45for_each_subchannel(int(*fn)(struct subchannel_id, void *), void *data)
46{
47 struct subchannel_id schid;
48 int ret;
49
50 init_subchannel_id(&schid);
51 do {
52 do {
53 ret = fn(schid, data);
54 if (ret)
55 break;
56 } while (schid.sch_no++ < __MAX_SUBCHANNEL);
57 schid.sch_no = 0;
58 } while (schid.ssid++ < max_ssid);
59 return ret;
60}
61
62struct cb_data {
63 void *data;
64 struct idset *set;
65 int (*fn_known_sch)(struct subchannel *, void *);
66 int (*fn_unknown_sch)(struct subchannel_id, void *);
67};
68
69static int call_fn_known_sch(struct device *dev, void *data)
70{
71 struct subchannel *sch = to_subchannel(dev);
72 struct cb_data *cb = data;
73 int rc = 0;
74
75 if (cb->set)
76 idset_sch_del(cb->set, sch->schid);
77 if (cb->fn_known_sch)
78 rc = cb->fn_known_sch(sch, cb->data);
79 return rc;
80}
81
82static int call_fn_unknown_sch(struct subchannel_id schid, void *data)
83{
84 struct cb_data *cb = data;
85 int rc = 0;
86
87 if (idset_sch_contains(cb->set, schid))
88 rc = cb->fn_unknown_sch(schid, cb->data);
89 return rc;
90}
91
92static int call_fn_all_sch(struct subchannel_id schid, void *data)
93{
94 struct cb_data *cb = data;
95 struct subchannel *sch;
96 int rc = 0;
97
98 sch = get_subchannel_by_schid(schid);
99 if (sch) {
100 if (cb->fn_known_sch)
101 rc = cb->fn_known_sch(sch, cb->data);
102 put_device(&sch->dev);
103 } else {
104 if (cb->fn_unknown_sch)
105 rc = cb->fn_unknown_sch(schid, cb->data);
106 }
107
108 return rc;
109}
110
111int for_each_subchannel_staged(int (*fn_known)(struct subchannel *, void *),
112 int (*fn_unknown)(struct subchannel_id,
113 void *), void *data)
114{
115 struct cb_data cb;
116 int rc;
117
118 cb.data = data;
119 cb.fn_known_sch = fn_known;
120 cb.fn_unknown_sch = fn_unknown;
121
122 if (fn_known && !fn_unknown) {
123 /* Skip idset allocation in case of known-only loop. */
124 cb.set = NULL;
125 return bus_for_each_dev(&css_bus_type, NULL, &cb,
126 call_fn_known_sch);
127 }
128
129 cb.set = idset_sch_new();
130 if (!cb.set)
131 /* fall back to brute force scanning in case of oom */
132 return for_each_subchannel(call_fn_all_sch, &cb);
133
134 idset_fill(cb.set);
135
136 /* Process registered subchannels. */
137 rc = bus_for_each_dev(&css_bus_type, NULL, &cb, call_fn_known_sch);
138 if (rc)
139 goto out;
140 /* Process unregistered subchannels. */
141 if (fn_unknown)
142 rc = for_each_subchannel(call_fn_unknown_sch, &cb);
143out:
144 idset_free(cb.set);
145
146 return rc;
147}
148
149static void css_sch_todo(struct work_struct *work);
150
151static void css_sch_create_locks(struct subchannel *sch)
152{
153 spin_lock_init(&sch->lock);
154 mutex_init(&sch->reg_mutex);
155}
156
157static void css_subchannel_release(struct device *dev)
158{
159 struct subchannel *sch = to_subchannel(dev);
160
161 sch->config.intparm = 0;
162 cio_commit_config(sch);
163 kfree(sch->driver_override);
164 kfree(sch);
165}
166
167static int css_validate_subchannel(struct subchannel_id schid,
168 struct schib *schib)
169{
170 int err;
171
172 switch (schib->pmcw.st) {
173 case SUBCHANNEL_TYPE_IO:
174 case SUBCHANNEL_TYPE_MSG:
175 if (!css_sch_is_valid(schib))
176 err = -ENODEV;
177 else if (is_blacklisted(schid.ssid, schib->pmcw.dev)) {
178 CIO_MSG_EVENT(6, "Blacklisted device detected "
179 "at devno %04X, subchannel set %x\n",
180 schib->pmcw.dev, schid.ssid);
181 err = -ENODEV;
182 } else
183 err = 0;
184 break;
185 default:
186 err = 0;
187 }
188 if (err)
189 goto out;
190
191 CIO_MSG_EVENT(4, "Subchannel 0.%x.%04x reports subchannel type %04X\n",
192 schid.ssid, schid.sch_no, schib->pmcw.st);
193out:
194 return err;
195}
196
197struct subchannel *css_alloc_subchannel(struct subchannel_id schid,
198 struct schib *schib)
199{
200 struct subchannel *sch;
201 int ret;
202
203 ret = css_validate_subchannel(schid, schib);
204 if (ret < 0)
205 return ERR_PTR(ret);
206
207 sch = kzalloc(sizeof(*sch), GFP_KERNEL | GFP_DMA);
208 if (!sch)
209 return ERR_PTR(-ENOMEM);
210
211 sch->schid = schid;
212 sch->schib = *schib;
213 sch->st = schib->pmcw.st;
214
215 css_sch_create_locks(sch);
216
217 INIT_WORK(&sch->todo_work, css_sch_todo);
218 sch->dev.release = &css_subchannel_release;
219 sch->dev.dma_mask = &sch->dma_mask;
220 device_initialize(&sch->dev);
221 /*
222 * The physical addresses for some of the dma structures that can
223 * belong to a subchannel need to fit 31 bit width (e.g. ccw).
224 */
225 ret = dma_set_coherent_mask(&sch->dev, DMA_BIT_MASK(31));
226 if (ret)
227 goto err;
228 /*
229 * But we don't have such restrictions imposed on the stuff that
230 * is handled by the streaming API.
231 */
232 ret = dma_set_mask(&sch->dev, DMA_BIT_MASK(64));
233 if (ret)
234 goto err;
235
236 return sch;
237
238err:
239 kfree(sch);
240 return ERR_PTR(ret);
241}
242
243static int css_sch_device_register(struct subchannel *sch)
244{
245 int ret;
246
247 mutex_lock(&sch->reg_mutex);
248 dev_set_name(&sch->dev, "0.%x.%04x", sch->schid.ssid,
249 sch->schid.sch_no);
250 ret = device_add(&sch->dev);
251 mutex_unlock(&sch->reg_mutex);
252 return ret;
253}
254
255/**
256 * css_sch_device_unregister - unregister a subchannel
257 * @sch: subchannel to be unregistered
258 */
259void css_sch_device_unregister(struct subchannel *sch)
260{
261 mutex_lock(&sch->reg_mutex);
262 if (device_is_registered(&sch->dev))
263 device_unregister(&sch->dev);
264 mutex_unlock(&sch->reg_mutex);
265}
266EXPORT_SYMBOL_GPL(css_sch_device_unregister);
267
268static void ssd_from_pmcw(struct chsc_ssd_info *ssd, struct pmcw *pmcw)
269{
270 int i;
271 int mask;
272
273 memset(ssd, 0, sizeof(struct chsc_ssd_info));
274 ssd->path_mask = pmcw->pim;
275 for (i = 0; i < 8; i++) {
276 mask = 0x80 >> i;
277 if (pmcw->pim & mask) {
278 chp_id_init(&ssd->chpid[i]);
279 ssd->chpid[i].id = pmcw->chpid[i];
280 }
281 }
282}
283
284static void ssd_register_chpids(struct chsc_ssd_info *ssd)
285{
286 int i;
287 int mask;
288
289 for (i = 0; i < 8; i++) {
290 mask = 0x80 >> i;
291 if (ssd->path_mask & mask)
292 chp_new(ssd->chpid[i]);
293 }
294}
295
296void css_update_ssd_info(struct subchannel *sch)
297{
298 int ret;
299
300 ret = chsc_get_ssd_info(sch->schid, &sch->ssd_info);
301 if (ret)
302 ssd_from_pmcw(&sch->ssd_info, &sch->schib.pmcw);
303
304 ssd_register_chpids(&sch->ssd_info);
305}
306
307static ssize_t type_show(struct device *dev, struct device_attribute *attr,
308 char *buf)
309{
310 struct subchannel *sch = to_subchannel(dev);
311
312 return sysfs_emit(buf, "%01x\n", sch->st);
313}
314
315static DEVICE_ATTR_RO(type);
316
317static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
318 char *buf)
319{
320 struct subchannel *sch = to_subchannel(dev);
321
322 return sysfs_emit(buf, "css:t%01X\n", sch->st);
323}
324
325static DEVICE_ATTR_RO(modalias);
326
327static ssize_t driver_override_store(struct device *dev,
328 struct device_attribute *attr,
329 const char *buf, size_t count)
330{
331 struct subchannel *sch = to_subchannel(dev);
332 int ret;
333
334 ret = driver_set_override(dev, &sch->driver_override, buf, count);
335 if (ret)
336 return ret;
337
338 return count;
339}
340
341static ssize_t driver_override_show(struct device *dev,
342 struct device_attribute *attr, char *buf)
343{
344 struct subchannel *sch = to_subchannel(dev);
345 ssize_t len;
346
347 device_lock(dev);
348 len = sysfs_emit(buf, "%s\n", sch->driver_override);
349 device_unlock(dev);
350 return len;
351}
352static DEVICE_ATTR_RW(driver_override);
353
354static struct attribute *subch_attrs[] = {
355 &dev_attr_type.attr,
356 &dev_attr_modalias.attr,
357 &dev_attr_driver_override.attr,
358 NULL,
359};
360
361static struct attribute_group subch_attr_group = {
362 .attrs = subch_attrs,
363};
364
365static const struct attribute_group *default_subch_attr_groups[] = {
366 &subch_attr_group,
367 NULL,
368};
369
370static ssize_t chpids_show(struct device *dev,
371 struct device_attribute *attr,
372 char *buf)
373{
374 struct subchannel *sch = to_subchannel(dev);
375 struct chsc_ssd_info *ssd = &sch->ssd_info;
376 ssize_t ret = 0;
377 int mask;
378 int chp;
379
380 for (chp = 0; chp < 8; chp++) {
381 mask = 0x80 >> chp;
382 if (ssd->path_mask & mask)
383 ret += sprintf(buf + ret, "%02x ", ssd->chpid[chp].id);
384 else
385 ret += sprintf(buf + ret, "00 ");
386 }
387 ret += sprintf(buf + ret, "\n");
388 return ret;
389}
390static DEVICE_ATTR_RO(chpids);
391
392static ssize_t pimpampom_show(struct device *dev,
393 struct device_attribute *attr,
394 char *buf)
395{
396 struct subchannel *sch = to_subchannel(dev);
397 struct pmcw *pmcw = &sch->schib.pmcw;
398
399 return sysfs_emit(buf, "%02x %02x %02x\n",
400 pmcw->pim, pmcw->pam, pmcw->pom);
401}
402static DEVICE_ATTR_RO(pimpampom);
403
404static ssize_t dev_busid_show(struct device *dev,
405 struct device_attribute *attr,
406 char *buf)
407{
408 struct subchannel *sch = to_subchannel(dev);
409 struct pmcw *pmcw = &sch->schib.pmcw;
410
411 if ((pmcw->st == SUBCHANNEL_TYPE_IO && pmcw->dnv) ||
412 (pmcw->st == SUBCHANNEL_TYPE_MSG && pmcw->w))
413 return sysfs_emit(buf, "0.%x.%04x\n", sch->schid.ssid,
414 pmcw->dev);
415 else
416 return sysfs_emit(buf, "none\n");
417}
418static DEVICE_ATTR_RO(dev_busid);
419
420static struct attribute *io_subchannel_type_attrs[] = {
421 &dev_attr_chpids.attr,
422 &dev_attr_pimpampom.attr,
423 &dev_attr_dev_busid.attr,
424 NULL,
425};
426ATTRIBUTE_GROUPS(io_subchannel_type);
427
428static const struct device_type io_subchannel_type = {
429 .groups = io_subchannel_type_groups,
430};
431
432int css_register_subchannel(struct subchannel *sch)
433{
434 int ret;
435
436 /* Initialize the subchannel structure */
437 sch->dev.parent = &channel_subsystems[0]->device;
438 sch->dev.bus = &css_bus_type;
439 sch->dev.groups = default_subch_attr_groups;
440
441 if (sch->st == SUBCHANNEL_TYPE_IO)
442 sch->dev.type = &io_subchannel_type;
443
444 css_update_ssd_info(sch);
445 /* make it known to the system */
446 ret = css_sch_device_register(sch);
447 if (ret) {
448 CIO_MSG_EVENT(0, "Could not register sch 0.%x.%04x: %d\n",
449 sch->schid.ssid, sch->schid.sch_no, ret);
450 return ret;
451 }
452 return ret;
453}
454
455static int css_probe_device(struct subchannel_id schid, struct schib *schib)
456{
457 struct subchannel *sch;
458 int ret;
459
460 sch = css_alloc_subchannel(schid, schib);
461 if (IS_ERR(sch))
462 return PTR_ERR(sch);
463
464 ret = css_register_subchannel(sch);
465 if (ret)
466 put_device(&sch->dev);
467
468 return ret;
469}
470
471static int
472check_subchannel(struct device *dev, const void *data)
473{
474 struct subchannel *sch;
475 struct subchannel_id *schid = (void *)data;
476
477 sch = to_subchannel(dev);
478 return schid_equal(&sch->schid, schid);
479}
480
481struct subchannel *
482get_subchannel_by_schid(struct subchannel_id schid)
483{
484 struct device *dev;
485
486 dev = bus_find_device(&css_bus_type, NULL,
487 &schid, check_subchannel);
488
489 return dev ? to_subchannel(dev) : NULL;
490}
491
492/**
493 * css_sch_is_valid() - check if a subchannel is valid
494 * @schib: subchannel information block for the subchannel
495 */
496int css_sch_is_valid(struct schib *schib)
497{
498 if ((schib->pmcw.st == SUBCHANNEL_TYPE_IO) && !schib->pmcw.dnv)
499 return 0;
500 if ((schib->pmcw.st == SUBCHANNEL_TYPE_MSG) && !schib->pmcw.w)
501 return 0;
502 return 1;
503}
504EXPORT_SYMBOL_GPL(css_sch_is_valid);
505
506static int css_evaluate_new_subchannel(struct subchannel_id schid, int slow)
507{
508 struct schib schib;
509 int ccode;
510
511 if (!slow) {
512 /* Will be done on the slow path. */
513 return -EAGAIN;
514 }
515 /*
516 * The first subchannel that is not-operational (ccode==3)
517 * indicates that there aren't any more devices available.
518 * If stsch gets an exception, it means the current subchannel set
519 * is not valid.
520 */
521 ccode = stsch(schid, &schib);
522 if (ccode)
523 return (ccode == 3) ? -ENXIO : ccode;
524
525 return css_probe_device(schid, &schib);
526}
527
528static int css_evaluate_known_subchannel(struct subchannel *sch, int slow)
529{
530 int ret = 0;
531
532 if (sch->driver) {
533 if (sch->driver->sch_event)
534 ret = sch->driver->sch_event(sch, slow);
535 else
536 dev_dbg(&sch->dev,
537 "Got subchannel machine check but "
538 "no sch_event handler provided.\n");
539 }
540 if (ret != 0 && ret != -EAGAIN) {
541 CIO_MSG_EVENT(2, "eval: sch 0.%x.%04x, rc=%d\n",
542 sch->schid.ssid, sch->schid.sch_no, ret);
543 }
544 return ret;
545}
546
547static void css_evaluate_subchannel(struct subchannel_id schid, int slow)
548{
549 struct subchannel *sch;
550 int ret;
551
552 sch = get_subchannel_by_schid(schid);
553 if (sch) {
554 ret = css_evaluate_known_subchannel(sch, slow);
555 put_device(&sch->dev);
556 } else
557 ret = css_evaluate_new_subchannel(schid, slow);
558 if (ret == -EAGAIN)
559 css_schedule_eval(schid);
560}
561
562/**
563 * css_sched_sch_todo - schedule a subchannel operation
564 * @sch: subchannel
565 * @todo: todo
566 *
567 * Schedule the operation identified by @todo to be performed on the slow path
568 * workqueue. Do nothing if another operation with higher priority is already
569 * scheduled. Needs to be called with subchannel lock held.
570 */
571void css_sched_sch_todo(struct subchannel *sch, enum sch_todo todo)
572{
573 CIO_MSG_EVENT(4, "sch_todo: sched sch=0.%x.%04x todo=%d\n",
574 sch->schid.ssid, sch->schid.sch_no, todo);
575 if (sch->todo >= todo)
576 return;
577 /* Get workqueue ref. */
578 if (!get_device(&sch->dev))
579 return;
580 sch->todo = todo;
581 if (!queue_work(cio_work_q, &sch->todo_work)) {
582 /* Already queued, release workqueue ref. */
583 put_device(&sch->dev);
584 }
585}
586EXPORT_SYMBOL_GPL(css_sched_sch_todo);
587
588static void css_sch_todo(struct work_struct *work)
589{
590 struct subchannel *sch;
591 enum sch_todo todo;
592 int ret;
593
594 sch = container_of(work, struct subchannel, todo_work);
595 /* Find out todo. */
596 spin_lock_irq(&sch->lock);
597 todo = sch->todo;
598 CIO_MSG_EVENT(4, "sch_todo: sch=0.%x.%04x, todo=%d\n", sch->schid.ssid,
599 sch->schid.sch_no, todo);
600 sch->todo = SCH_TODO_NOTHING;
601 spin_unlock_irq(&sch->lock);
602 /* Perform todo. */
603 switch (todo) {
604 case SCH_TODO_NOTHING:
605 break;
606 case SCH_TODO_EVAL:
607 ret = css_evaluate_known_subchannel(sch, 1);
608 if (ret == -EAGAIN) {
609 spin_lock_irq(&sch->lock);
610 css_sched_sch_todo(sch, todo);
611 spin_unlock_irq(&sch->lock);
612 }
613 break;
614 case SCH_TODO_UNREG:
615 css_sch_device_unregister(sch);
616 break;
617 }
618 /* Release workqueue ref. */
619 put_device(&sch->dev);
620}
621
622static struct idset *slow_subchannel_set;
623static DEFINE_SPINLOCK(slow_subchannel_lock);
624static DECLARE_WAIT_QUEUE_HEAD(css_eval_wq);
625static atomic_t css_eval_scheduled;
626
627static int __init slow_subchannel_init(void)
628{
629 atomic_set(&css_eval_scheduled, 0);
630 slow_subchannel_set = idset_sch_new();
631 if (!slow_subchannel_set) {
632 CIO_MSG_EVENT(0, "could not allocate slow subchannel set\n");
633 return -ENOMEM;
634 }
635 return 0;
636}
637
638static int slow_eval_known_fn(struct subchannel *sch, void *data)
639{
640 int eval;
641 int rc;
642
643 spin_lock_irq(&slow_subchannel_lock);
644 eval = idset_sch_contains(slow_subchannel_set, sch->schid);
645 idset_sch_del(slow_subchannel_set, sch->schid);
646 spin_unlock_irq(&slow_subchannel_lock);
647 if (eval) {
648 rc = css_evaluate_known_subchannel(sch, 1);
649 if (rc == -EAGAIN)
650 css_schedule_eval(sch->schid);
651 /*
652 * The loop might take long time for platforms with lots of
653 * known devices. Allow scheduling here.
654 */
655 cond_resched();
656 }
657 return 0;
658}
659
660static int slow_eval_unknown_fn(struct subchannel_id schid, void *data)
661{
662 int eval;
663 int rc = 0;
664
665 spin_lock_irq(&slow_subchannel_lock);
666 eval = idset_sch_contains(slow_subchannel_set, schid);
667 idset_sch_del(slow_subchannel_set, schid);
668 spin_unlock_irq(&slow_subchannel_lock);
669 if (eval) {
670 rc = css_evaluate_new_subchannel(schid, 1);
671 switch (rc) {
672 case -EAGAIN:
673 css_schedule_eval(schid);
674 rc = 0;
675 break;
676 case -ENXIO:
677 case -ENOMEM:
678 case -EIO:
679 /* These should abort looping */
680 spin_lock_irq(&slow_subchannel_lock);
681 idset_sch_del_subseq(slow_subchannel_set, schid);
682 spin_unlock_irq(&slow_subchannel_lock);
683 break;
684 default:
685 rc = 0;
686 }
687 /* Allow scheduling here since the containing loop might
688 * take a while. */
689 cond_resched();
690 }
691 return rc;
692}
693
694static void css_slow_path_func(struct work_struct *unused)
695{
696 unsigned long flags;
697
698 CIO_TRACE_EVENT(4, "slowpath");
699 for_each_subchannel_staged(slow_eval_known_fn, slow_eval_unknown_fn,
700 NULL);
701 spin_lock_irqsave(&slow_subchannel_lock, flags);
702 if (idset_is_empty(slow_subchannel_set)) {
703 atomic_set(&css_eval_scheduled, 0);
704 wake_up(&css_eval_wq);
705 }
706 spin_unlock_irqrestore(&slow_subchannel_lock, flags);
707}
708
709static DECLARE_DELAYED_WORK(slow_path_work, css_slow_path_func);
710struct workqueue_struct *cio_work_q;
711
712void css_schedule_eval(struct subchannel_id schid)
713{
714 unsigned long flags;
715
716 spin_lock_irqsave(&slow_subchannel_lock, flags);
717 idset_sch_add(slow_subchannel_set, schid);
718 atomic_set(&css_eval_scheduled, 1);
719 queue_delayed_work(cio_work_q, &slow_path_work, 0);
720 spin_unlock_irqrestore(&slow_subchannel_lock, flags);
721}
722
723void css_schedule_eval_all(void)
724{
725 unsigned long flags;
726
727 spin_lock_irqsave(&slow_subchannel_lock, flags);
728 idset_fill(slow_subchannel_set);
729 atomic_set(&css_eval_scheduled, 1);
730 queue_delayed_work(cio_work_q, &slow_path_work, 0);
731 spin_unlock_irqrestore(&slow_subchannel_lock, flags);
732}
733
734static int __unset_validpath(struct device *dev, void *data)
735{
736 struct idset *set = data;
737 struct subchannel *sch = to_subchannel(dev);
738 struct pmcw *pmcw = &sch->schib.pmcw;
739
740 /* Here we want to make sure that we are considering only those subchannels
741 * which do not have an operational device attached to it. This can be found
742 * with the help of PAM and POM values of pmcw. OPM provides the information
743 * about any path which is currently vary-off, so that we should not consider.
744 */
745 if (sch->st == SUBCHANNEL_TYPE_IO &&
746 (sch->opm & pmcw->pam & pmcw->pom))
747 idset_sch_del(set, sch->schid);
748
749 return 0;
750}
751
752static int __unset_online(struct device *dev, void *data)
753{
754 struct idset *set = data;
755 struct subchannel *sch = to_subchannel(dev);
756
757 if (sch->st == SUBCHANNEL_TYPE_IO && sch->config.ena)
758 idset_sch_del(set, sch->schid);
759
760 return 0;
761}
762
763void css_schedule_eval_cond(enum css_eval_cond cond, unsigned long delay)
764{
765 unsigned long flags;
766 struct idset *set;
767
768 /* Find unregistered subchannels. */
769 set = idset_sch_new();
770 if (!set) {
771 /* Fallback. */
772 css_schedule_eval_all();
773 return;
774 }
775 idset_fill(set);
776 switch (cond) {
777 case CSS_EVAL_NO_PATH:
778 bus_for_each_dev(&css_bus_type, NULL, set, __unset_validpath);
779 break;
780 case CSS_EVAL_NOT_ONLINE:
781 bus_for_each_dev(&css_bus_type, NULL, set, __unset_online);
782 break;
783 default:
784 break;
785 }
786
787 /* Apply to slow_subchannel_set. */
788 spin_lock_irqsave(&slow_subchannel_lock, flags);
789 idset_add_set(slow_subchannel_set, set);
790 atomic_set(&css_eval_scheduled, 1);
791 queue_delayed_work(cio_work_q, &slow_path_work, delay);
792 spin_unlock_irqrestore(&slow_subchannel_lock, flags);
793 idset_free(set);
794}
795
796void css_wait_for_slow_path(void)
797{
798 flush_workqueue(cio_work_q);
799}
800
801/* Schedule reprobing of all subchannels with no valid operational path. */
802void css_schedule_reprobe(void)
803{
804 /* Schedule with a delay to allow merging of subsequent calls. */
805 css_schedule_eval_cond(CSS_EVAL_NO_PATH, 1 * HZ);
806}
807EXPORT_SYMBOL_GPL(css_schedule_reprobe);
808
809/*
810 * Called from the machine check handler for subchannel report words.
811 */
812static void css_process_crw(struct crw *crw0, struct crw *crw1, int overflow)
813{
814 struct subchannel_id mchk_schid;
815 struct subchannel *sch;
816
817 if (overflow) {
818 css_schedule_eval_all();
819 return;
820 }
821 CIO_CRW_EVENT(2, "CRW0 reports slct=%d, oflw=%d, "
822 "chn=%d, rsc=%X, anc=%d, erc=%X, rsid=%X\n",
823 crw0->slct, crw0->oflw, crw0->chn, crw0->rsc, crw0->anc,
824 crw0->erc, crw0->rsid);
825 if (crw1)
826 CIO_CRW_EVENT(2, "CRW1 reports slct=%d, oflw=%d, "
827 "chn=%d, rsc=%X, anc=%d, erc=%X, rsid=%X\n",
828 crw1->slct, crw1->oflw, crw1->chn, crw1->rsc,
829 crw1->anc, crw1->erc, crw1->rsid);
830 init_subchannel_id(&mchk_schid);
831 mchk_schid.sch_no = crw0->rsid;
832 if (crw1)
833 mchk_schid.ssid = (crw1->rsid >> 4) & 3;
834
835 if (crw0->erc == CRW_ERC_PMOD) {
836 sch = get_subchannel_by_schid(mchk_schid);
837 if (sch) {
838 css_update_ssd_info(sch);
839 put_device(&sch->dev);
840 }
841 }
842 /*
843 * Since we are always presented with IPI in the CRW, we have to
844 * use stsch() to find out if the subchannel in question has come
845 * or gone.
846 */
847 css_evaluate_subchannel(mchk_schid, 0);
848}
849
850static void __init
851css_generate_pgid(struct channel_subsystem *css, u32 tod_high)
852{
853 struct cpuid cpu_id;
854
855 if (css_general_characteristics.mcss) {
856 css->global_pgid.pgid_high.ext_cssid.version = 0x80;
857 css->global_pgid.pgid_high.ext_cssid.cssid =
858 css->id_valid ? css->cssid : 0;
859 } else {
860 css->global_pgid.pgid_high.cpu_addr = stap();
861 }
862 get_cpu_id(&cpu_id);
863 css->global_pgid.cpu_id = cpu_id.ident;
864 css->global_pgid.cpu_model = cpu_id.machine;
865 css->global_pgid.tod_high = tod_high;
866}
867
868static void channel_subsystem_release(struct device *dev)
869{
870 struct channel_subsystem *css = to_css(dev);
871
872 mutex_destroy(&css->mutex);
873 kfree(css);
874}
875
876static ssize_t real_cssid_show(struct device *dev, struct device_attribute *a,
877 char *buf)
878{
879 struct channel_subsystem *css = to_css(dev);
880
881 if (!css->id_valid)
882 return -EINVAL;
883
884 return sysfs_emit(buf, "%x\n", css->cssid);
885}
886static DEVICE_ATTR_RO(real_cssid);
887
888static ssize_t rescan_store(struct device *dev, struct device_attribute *a,
889 const char *buf, size_t count)
890{
891 CIO_TRACE_EVENT(4, "usr-rescan");
892
893 css_schedule_eval_all();
894 css_complete_work();
895
896 return count;
897}
898static DEVICE_ATTR_WO(rescan);
899
900static ssize_t cm_enable_show(struct device *dev, struct device_attribute *a,
901 char *buf)
902{
903 struct channel_subsystem *css = to_css(dev);
904 int ret;
905
906 mutex_lock(&css->mutex);
907 ret = sysfs_emit(buf, "%x\n", css->cm_enabled);
908 mutex_unlock(&css->mutex);
909 return ret;
910}
911
912static ssize_t cm_enable_store(struct device *dev, struct device_attribute *a,
913 const char *buf, size_t count)
914{
915 struct channel_subsystem *css = to_css(dev);
916 unsigned long val;
917 int ret;
918
919 ret = kstrtoul(buf, 16, &val);
920 if (ret)
921 return ret;
922 mutex_lock(&css->mutex);
923 switch (val) {
924 case 0:
925 ret = css->cm_enabled ? chsc_secm(css, 0) : 0;
926 break;
927 case 1:
928 ret = css->cm_enabled ? 0 : chsc_secm(css, 1);
929 break;
930 default:
931 ret = -EINVAL;
932 }
933 mutex_unlock(&css->mutex);
934 return ret < 0 ? ret : count;
935}
936static DEVICE_ATTR_RW(cm_enable);
937
938static umode_t cm_enable_mode(struct kobject *kobj, struct attribute *attr,
939 int index)
940{
941 return css_chsc_characteristics.secm ? attr->mode : 0;
942}
943
944static struct attribute *cssdev_attrs[] = {
945 &dev_attr_real_cssid.attr,
946 &dev_attr_rescan.attr,
947 NULL,
948};
949
950static struct attribute_group cssdev_attr_group = {
951 .attrs = cssdev_attrs,
952};
953
954static struct attribute *cssdev_cm_attrs[] = {
955 &dev_attr_cm_enable.attr,
956 NULL,
957};
958
959static struct attribute_group cssdev_cm_attr_group = {
960 .attrs = cssdev_cm_attrs,
961 .is_visible = cm_enable_mode,
962};
963
964static const struct attribute_group *cssdev_attr_groups[] = {
965 &cssdev_attr_group,
966 &cssdev_cm_attr_group,
967 NULL,
968};
969
970static int __init setup_css(int nr)
971{
972 struct channel_subsystem *css;
973 int ret;
974
975 css = kzalloc(sizeof(*css), GFP_KERNEL);
976 if (!css)
977 return -ENOMEM;
978
979 channel_subsystems[nr] = css;
980 dev_set_name(&css->device, "css%x", nr);
981 css->device.groups = cssdev_attr_groups;
982 css->device.release = channel_subsystem_release;
983 /*
984 * We currently allocate notifier bits with this (using
985 * css->device as the device argument with the DMA API)
986 * and are fine with 64 bit addresses.
987 */
988 ret = dma_coerce_mask_and_coherent(&css->device, DMA_BIT_MASK(64));
989 if (ret) {
990 kfree(css);
991 goto out_err;
992 }
993
994 mutex_init(&css->mutex);
995 ret = chsc_get_cssid_iid(nr, &css->cssid, &css->iid);
996 if (!ret) {
997 css->id_valid = true;
998 pr_info("Partition identifier %01x.%01x\n", css->cssid,
999 css->iid);
1000 }
1001 css_generate_pgid(css, (u32) (get_tod_clock() >> 32));
1002
1003 ret = device_register(&css->device);
1004 if (ret) {
1005 put_device(&css->device);
1006 goto out_err;
1007 }
1008
1009 css->pseudo_subchannel = kzalloc(sizeof(*css->pseudo_subchannel),
1010 GFP_KERNEL);
1011 if (!css->pseudo_subchannel) {
1012 device_unregister(&css->device);
1013 ret = -ENOMEM;
1014 goto out_err;
1015 }
1016
1017 css->pseudo_subchannel->dev.parent = &css->device;
1018 css->pseudo_subchannel->dev.release = css_subchannel_release;
1019 mutex_init(&css->pseudo_subchannel->reg_mutex);
1020 css_sch_create_locks(css->pseudo_subchannel);
1021
1022 dev_set_name(&css->pseudo_subchannel->dev, "defunct");
1023 ret = device_register(&css->pseudo_subchannel->dev);
1024 if (ret) {
1025 put_device(&css->pseudo_subchannel->dev);
1026 device_unregister(&css->device);
1027 goto out_err;
1028 }
1029
1030 return ret;
1031out_err:
1032 channel_subsystems[nr] = NULL;
1033 return ret;
1034}
1035
1036static int css_reboot_event(struct notifier_block *this,
1037 unsigned long event,
1038 void *ptr)
1039{
1040 struct channel_subsystem *css;
1041 int ret;
1042
1043 ret = NOTIFY_DONE;
1044 for_each_css(css) {
1045 mutex_lock(&css->mutex);
1046 if (css->cm_enabled)
1047 if (chsc_secm(css, 0))
1048 ret = NOTIFY_BAD;
1049 mutex_unlock(&css->mutex);
1050 }
1051
1052 return ret;
1053}
1054
1055static struct notifier_block css_reboot_notifier = {
1056 .notifier_call = css_reboot_event,
1057};
1058
1059#define CIO_DMA_GFP (GFP_KERNEL | __GFP_ZERO)
1060static struct gen_pool *cio_dma_pool;
1061
1062/* Currently cio supports only a single css */
1063struct device *cio_get_dma_css_dev(void)
1064{
1065 return &channel_subsystems[0]->device;
1066}
1067
1068struct gen_pool *cio_gp_dma_create(struct device *dma_dev, int nr_pages)
1069{
1070 struct gen_pool *gp_dma;
1071 void *cpu_addr;
1072 dma_addr_t dma_addr;
1073 int i;
1074
1075 gp_dma = gen_pool_create(3, -1);
1076 if (!gp_dma)
1077 return NULL;
1078 for (i = 0; i < nr_pages; ++i) {
1079 cpu_addr = dma_alloc_coherent(dma_dev, PAGE_SIZE, &dma_addr,
1080 CIO_DMA_GFP);
1081 if (!cpu_addr)
1082 return gp_dma;
1083 gen_pool_add_virt(gp_dma, (unsigned long) cpu_addr,
1084 dma_addr, PAGE_SIZE, -1);
1085 }
1086 return gp_dma;
1087}
1088
1089static void __gp_dma_free_dma(struct gen_pool *pool,
1090 struct gen_pool_chunk *chunk, void *data)
1091{
1092 size_t chunk_size = chunk->end_addr - chunk->start_addr + 1;
1093
1094 dma_free_coherent((struct device *) data, chunk_size,
1095 (void *) chunk->start_addr,
1096 (dma_addr_t) chunk->phys_addr);
1097}
1098
1099void cio_gp_dma_destroy(struct gen_pool *gp_dma, struct device *dma_dev)
1100{
1101 if (!gp_dma)
1102 return;
1103 /* this is quite ugly but no better idea */
1104 gen_pool_for_each_chunk(gp_dma, __gp_dma_free_dma, dma_dev);
1105 gen_pool_destroy(gp_dma);
1106}
1107
1108static int cio_dma_pool_init(void)
1109{
1110 /* No need to free up the resources: compiled in */
1111 cio_dma_pool = cio_gp_dma_create(cio_get_dma_css_dev(), 1);
1112 if (!cio_dma_pool)
1113 return -ENOMEM;
1114 return 0;
1115}
1116
1117void *__cio_gp_dma_zalloc(struct gen_pool *gp_dma, struct device *dma_dev,
1118 size_t size, dma32_t *dma_handle)
1119{
1120 dma_addr_t dma_addr;
1121 size_t chunk_size;
1122 void *addr;
1123
1124 if (!gp_dma)
1125 return NULL;
1126 addr = gen_pool_dma_alloc(gp_dma, size, &dma_addr);
1127 while (!addr) {
1128 chunk_size = round_up(size, PAGE_SIZE);
1129 addr = dma_alloc_coherent(dma_dev, chunk_size, &dma_addr, CIO_DMA_GFP);
1130 if (!addr)
1131 return NULL;
1132 gen_pool_add_virt(gp_dma, (unsigned long)addr, dma_addr, chunk_size, -1);
1133 addr = gen_pool_dma_alloc(gp_dma, size, dma_handle ? &dma_addr : NULL);
1134 }
1135 if (dma_handle)
1136 *dma_handle = (__force dma32_t)dma_addr;
1137 return addr;
1138}
1139
1140void *cio_gp_dma_zalloc(struct gen_pool *gp_dma, struct device *dma_dev,
1141 size_t size)
1142{
1143 return __cio_gp_dma_zalloc(gp_dma, dma_dev, size, NULL);
1144}
1145
1146void cio_gp_dma_free(struct gen_pool *gp_dma, void *cpu_addr, size_t size)
1147{
1148 if (!cpu_addr)
1149 return;
1150 memset(cpu_addr, 0, size);
1151 gen_pool_free(gp_dma, (unsigned long) cpu_addr, size);
1152}
1153
1154/*
1155 * Allocate dma memory from the css global pool. Intended for memory not
1156 * specific to any single device within the css. The allocated memory
1157 * is not guaranteed to be 31-bit addressable.
1158 *
1159 * Caution: Not suitable for early stuff like console.
1160 */
1161void *cio_dma_zalloc(size_t size)
1162{
1163 return cio_gp_dma_zalloc(cio_dma_pool, cio_get_dma_css_dev(), size);
1164}
1165
1166void cio_dma_free(void *cpu_addr, size_t size)
1167{
1168 cio_gp_dma_free(cio_dma_pool, cpu_addr, size);
1169}
1170
1171/*
1172 * Now that the driver core is running, we can setup our channel subsystem.
1173 * The struct subchannel's are created during probing.
1174 */
1175static int __init css_bus_init(void)
1176{
1177 int ret, i;
1178
1179 ret = chsc_init();
1180 if (ret)
1181 return ret;
1182
1183 chsc_determine_css_characteristics();
1184 /* Try to enable MSS. */
1185 ret = chsc_enable_facility(CHSC_SDA_OC_MSS);
1186 if (ret)
1187 max_ssid = 0;
1188 else /* Success. */
1189 max_ssid = __MAX_SSID;
1190
1191 ret = slow_subchannel_init();
1192 if (ret)
1193 goto out;
1194
1195 ret = crw_register_handler(CRW_RSC_SCH, css_process_crw);
1196 if (ret)
1197 goto out;
1198
1199 if ((ret = bus_register(&css_bus_type)))
1200 goto out;
1201
1202 /* Setup css structure. */
1203 for (i = 0; i <= MAX_CSS_IDX; i++) {
1204 ret = setup_css(i);
1205 if (ret)
1206 goto out_unregister;
1207 }
1208 ret = register_reboot_notifier(&css_reboot_notifier);
1209 if (ret)
1210 goto out_unregister;
1211 ret = cio_dma_pool_init();
1212 if (ret)
1213 goto out_unregister_rn;
1214 airq_init();
1215 css_init_done = 1;
1216
1217 /* Enable default isc for I/O subchannels. */
1218 isc_register(IO_SCH_ISC);
1219
1220 return 0;
1221out_unregister_rn:
1222 unregister_reboot_notifier(&css_reboot_notifier);
1223out_unregister:
1224 while (i-- > 0) {
1225 struct channel_subsystem *css = channel_subsystems[i];
1226 device_unregister(&css->pseudo_subchannel->dev);
1227 device_unregister(&css->device);
1228 }
1229 bus_unregister(&css_bus_type);
1230out:
1231 crw_unregister_handler(CRW_RSC_SCH);
1232 idset_free(slow_subchannel_set);
1233 chsc_init_cleanup();
1234 pr_alert("The CSS device driver initialization failed with "
1235 "errno=%d\n", ret);
1236 return ret;
1237}
1238
1239static void __init css_bus_cleanup(void)
1240{
1241 struct channel_subsystem *css;
1242
1243 for_each_css(css) {
1244 device_unregister(&css->pseudo_subchannel->dev);
1245 device_unregister(&css->device);
1246 }
1247 bus_unregister(&css_bus_type);
1248 crw_unregister_handler(CRW_RSC_SCH);
1249 idset_free(slow_subchannel_set);
1250 chsc_init_cleanup();
1251 isc_unregister(IO_SCH_ISC);
1252}
1253
1254static int __init channel_subsystem_init(void)
1255{
1256 int ret;
1257
1258 ret = css_bus_init();
1259 if (ret)
1260 return ret;
1261 cio_work_q = create_singlethread_workqueue("cio");
1262 if (!cio_work_q) {
1263 ret = -ENOMEM;
1264 goto out_bus;
1265 }
1266 ret = io_subchannel_init();
1267 if (ret)
1268 goto out_wq;
1269
1270 /* Register subchannels which are already in use. */
1271 cio_register_early_subchannels();
1272 /* Start initial subchannel evaluation. */
1273 css_schedule_eval_all();
1274
1275 return ret;
1276out_wq:
1277 destroy_workqueue(cio_work_q);
1278out_bus:
1279 css_bus_cleanup();
1280 return ret;
1281}
1282subsys_initcall(channel_subsystem_init);
1283
1284static int css_settle(struct device_driver *drv, void *unused)
1285{
1286 struct css_driver *cssdrv = to_cssdriver(drv);
1287
1288 if (cssdrv->settle)
1289 return cssdrv->settle();
1290 return 0;
1291}
1292
1293int css_complete_work(void)
1294{
1295 int ret;
1296
1297 /* Wait for the evaluation of subchannels to finish. */
1298 ret = wait_event_interruptible(css_eval_wq,
1299 atomic_read(&css_eval_scheduled) == 0);
1300 if (ret)
1301 return -EINTR;
1302 flush_workqueue(cio_work_q);
1303 /* Wait for the subchannel type specific initialization to finish */
1304 return bus_for_each_drv(&css_bus_type, NULL, NULL, css_settle);
1305}
1306
1307
1308/*
1309 * Wait for the initialization of devices to finish, to make sure we are
1310 * done with our setup if the search for the root device starts.
1311 */
1312static int __init channel_subsystem_init_sync(void)
1313{
1314 css_complete_work();
1315 return 0;
1316}
1317subsys_initcall_sync(channel_subsystem_init_sync);
1318
1319#ifdef CONFIG_PROC_FS
1320static ssize_t cio_settle_write(struct file *file, const char __user *buf,
1321 size_t count, loff_t *ppos)
1322{
1323 int ret;
1324
1325 /* Handle pending CRW's. */
1326 crw_wait_for_channel_report();
1327 ret = css_complete_work();
1328
1329 return ret ? ret : count;
1330}
1331
1332static const struct proc_ops cio_settle_proc_ops = {
1333 .proc_open = nonseekable_open,
1334 .proc_write = cio_settle_write,
1335};
1336
1337static int __init cio_settle_init(void)
1338{
1339 struct proc_dir_entry *entry;
1340
1341 entry = proc_create("cio_settle", S_IWUSR, NULL, &cio_settle_proc_ops);
1342 if (!entry)
1343 return -ENOMEM;
1344 return 0;
1345}
1346device_initcall(cio_settle_init);
1347#endif /*CONFIG_PROC_FS*/
1348
1349int sch_is_pseudo_sch(struct subchannel *sch)
1350{
1351 if (!sch->dev.parent)
1352 return 0;
1353 return sch == to_css(sch->dev.parent)->pseudo_subchannel;
1354}
1355
1356static int css_bus_match(struct device *dev, const struct device_driver *drv)
1357{
1358 struct subchannel *sch = to_subchannel(dev);
1359 const struct css_driver *driver = to_cssdriver(drv);
1360 struct css_device_id *id;
1361
1362 /* When driver_override is set, only bind to the matching driver */
1363 if (sch->driver_override && strcmp(sch->driver_override, drv->name))
1364 return 0;
1365
1366 for (id = driver->subchannel_type; id->match_flags; id++) {
1367 if (sch->st == id->type)
1368 return 1;
1369 }
1370
1371 return 0;
1372}
1373
1374static int css_probe(struct device *dev)
1375{
1376 struct subchannel *sch;
1377 int ret;
1378
1379 sch = to_subchannel(dev);
1380 sch->driver = to_cssdriver(dev->driver);
1381 ret = sch->driver->probe ? sch->driver->probe(sch) : 0;
1382 if (ret)
1383 sch->driver = NULL;
1384 return ret;
1385}
1386
1387static void css_remove(struct device *dev)
1388{
1389 struct subchannel *sch;
1390
1391 sch = to_subchannel(dev);
1392 if (sch->driver->remove)
1393 sch->driver->remove(sch);
1394 sch->driver = NULL;
1395}
1396
1397static void css_shutdown(struct device *dev)
1398{
1399 struct subchannel *sch;
1400
1401 sch = to_subchannel(dev);
1402 if (sch->driver && sch->driver->shutdown)
1403 sch->driver->shutdown(sch);
1404}
1405
1406static int css_uevent(const struct device *dev, struct kobj_uevent_env *env)
1407{
1408 const struct subchannel *sch = to_subchannel(dev);
1409 int ret;
1410
1411 ret = add_uevent_var(env, "ST=%01X", sch->st);
1412 if (ret)
1413 return ret;
1414 ret = add_uevent_var(env, "MODALIAS=css:t%01X", sch->st);
1415 return ret;
1416}
1417
1418static const struct bus_type css_bus_type = {
1419 .name = "css",
1420 .match = css_bus_match,
1421 .probe = css_probe,
1422 .remove = css_remove,
1423 .shutdown = css_shutdown,
1424 .uevent = css_uevent,
1425};
1426
1427/**
1428 * css_driver_register - register a css driver
1429 * @cdrv: css driver to register
1430 *
1431 * This is mainly a wrapper around driver_register that sets name
1432 * and bus_type in the embedded struct device_driver correctly.
1433 */
1434int css_driver_register(struct css_driver *cdrv)
1435{
1436 cdrv->drv.bus = &css_bus_type;
1437 return driver_register(&cdrv->drv);
1438}
1439EXPORT_SYMBOL_GPL(css_driver_register);
1440
1441/**
1442 * css_driver_unregister - unregister a css driver
1443 * @cdrv: css driver to unregister
1444 *
1445 * This is a wrapper around driver_unregister.
1446 */
1447void css_driver_unregister(struct css_driver *cdrv)
1448{
1449 driver_unregister(&cdrv->drv);
1450}
1451EXPORT_SYMBOL_GPL(css_driver_unregister);
Linux 6.x block layer and io_uring tree(s)