Merge tag 'parisc-for-6.6-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/delle...
[linux-2.6-block.git] / drivers / nvdimm / bus.c
CommitLineData
5b497af4 1// SPDX-License-Identifier: GPL-2.0-only
45def22c
DW
2/*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
45def22c
DW
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
aa9ad44a 6#include <linux/libnvdimm.h>
0930a750 7#include <linux/sched/mm.h>
62232e45 8#include <linux/vmalloc.h>
45def22c 9#include <linux/uaccess.h>
3d88002e 10#include <linux/module.h>
8c2f7e86 11#include <linux/blkdev.h>
45def22c 12#include <linux/fcntl.h>
e6dfb2de 13#include <linux/async.h>
62232e45 14#include <linux/ndctl.h>
4d88a97a 15#include <linux/sched.h>
45def22c 16#include <linux/slab.h>
af87b9a7 17#include <linux/cpu.h>
45def22c
DW
18#include <linux/fs.h>
19#include <linux/io.h>
62232e45 20#include <linux/mm.h>
4d88a97a 21#include <linux/nd.h>
45def22c 22#include "nd-core.h"
4d88a97a 23#include "nd.h"
006358b3 24#include "pfn.h"
45def22c 25
62232e45 26int nvdimm_major;
45def22c 27static int nvdimm_bus_major;
191a9f3a 28static struct class *nd_class;
18515942 29static DEFINE_IDA(nd_ida);
45def22c 30
2a81ada3 31static int to_nd_device_type(const struct device *dev)
4d88a97a
DW
32{
33 if (is_nvdimm(dev))
34 return ND_DEVICE_DIMM;
c9e582aa 35 else if (is_memory(dev))
3d88002e 36 return ND_DEVICE_REGION_PMEM;
cd03412a
DW
37 else if (is_nd_dax(dev))
38 return ND_DEVICE_DAX_PMEM;
c9e582aa 39 else if (is_nd_region(dev->parent))
3d88002e 40 return nd_region_to_nstype(to_nd_region(dev->parent));
4d88a97a
DW
41
42 return 0;
43}
44
2a81ada3 45static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
4d88a97a
DW
46{
47 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
48 to_nd_device_type(dev));
49}
50
3d88002e
DW
51static struct module *to_bus_provider(struct device *dev)
52{
53 /* pin bus providers while regions are enabled */
c9e582aa 54 if (is_nd_region(dev)) {
3d88002e
DW
55 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
56
bc9775d8 57 return nvdimm_bus->nd_desc->module;
3d88002e
DW
58 }
59 return NULL;
60}
61
eaf96153
DW
62static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
63{
64 nvdimm_bus_lock(&nvdimm_bus->dev);
65 nvdimm_bus->probe_active++;
66 nvdimm_bus_unlock(&nvdimm_bus->dev);
67}
68
69static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
70{
71 nvdimm_bus_lock(&nvdimm_bus->dev);
72 if (--nvdimm_bus->probe_active == 0)
b70d31d0 73 wake_up(&nvdimm_bus->wait);
eaf96153
DW
74 nvdimm_bus_unlock(&nvdimm_bus->dev);
75}
76
4d88a97a
DW
77static int nvdimm_bus_probe(struct device *dev)
78{
79 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
3d88002e 80 struct module *provider = to_bus_provider(dev);
4d88a97a
DW
81 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
82 int rc;
83
3d88002e
DW
84 if (!try_module_get(provider))
85 return -ENXIO;
86
3f46833d
DW
87 dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
88 dev->driver->name, dev_name(dev));
89
eaf96153 90 nvdimm_bus_probe_start(nvdimm_bus);
4d88a97a 91 rc = nd_drv->probe(dev);
1c97afa7
AK
92 if ((rc == 0 || rc == -EOPNOTSUPP) &&
93 dev->parent && is_nd_region(dev->parent))
a2d1c7a6 94 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
eaf96153
DW
95 nvdimm_bus_probe_end(nvdimm_bus);
96
3f46833d 97 dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
4d88a97a 98 dev_name(dev), rc);
8c2f7e86 99
3d88002e
DW
100 if (rc != 0)
101 module_put(provider);
4d88a97a
DW
102 return rc;
103}
104
fc7a6209 105static void nvdimm_bus_remove(struct device *dev)
4d88a97a
DW
106{
107 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
3d88002e 108 struct module *provider = to_bus_provider(dev);
4d88a97a 109 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
4d88a97a 110
81beea55 111 if (nd_drv->remove)
1f975074 112 nd_drv->remove(dev);
eaf96153 113
1f975074
UKK
114 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
115 dev_name(dev));
3d88002e 116 module_put(provider);
4d88a97a
DW
117}
118
476f848a
DW
119static void nvdimm_bus_shutdown(struct device *dev)
120{
121 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
122 struct nd_device_driver *nd_drv = NULL;
123
124 if (dev->driver)
125 nd_drv = to_nd_device_driver(dev->driver);
126
127 if (nd_drv && nd_drv->shutdown) {
128 nd_drv->shutdown(dev);
129 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
130 dev->driver->name, dev_name(dev));
131 }
132}
133
71999466
DW
134void nd_device_notify(struct device *dev, enum nvdimm_event event)
135{
81beea55 136 device_lock(dev);
71999466
DW
137 if (dev->driver) {
138 struct nd_device_driver *nd_drv;
139
140 nd_drv = to_nd_device_driver(dev->driver);
141 if (nd_drv->notify)
142 nd_drv->notify(dev, event);
143 }
81beea55 144 device_unlock(dev);
71999466
DW
145}
146EXPORT_SYMBOL(nd_device_notify);
147
148void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
149{
150 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
151
152 if (!nvdimm_bus)
153 return;
154
155 /* caller is responsible for holding a reference on the device */
156 nd_device_notify(&nd_region->dev, event);
157}
158EXPORT_SYMBOL_GPL(nvdimm_region_notify);
159
23f49844
DW
160struct clear_badblocks_context {
161 resource_size_t phys, cleared;
162};
163
164static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
165{
166 struct clear_badblocks_context *ctx = data;
167 struct nd_region *nd_region;
168 resource_size_t ndr_end;
169 sector_t sector;
170
171 /* make sure device is a region */
c42adf87 172 if (!is_memory(dev))
23f49844
DW
173 return 0;
174
175 nd_region = to_nd_region(dev);
176 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
177
178 /* make sure we are in the region */
ef910200
CY
179 if (ctx->phys < nd_region->ndr_start ||
180 (ctx->phys + ctx->cleared - 1) > ndr_end)
23f49844
DW
181 return 0;
182
183 sector = (ctx->phys - nd_region->ndr_start) / 512;
184 badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
185
975750a9
TK
186 if (nd_region->bb_state)
187 sysfs_notify_dirent(nd_region->bb_state);
188
23f49844
DW
189 return 0;
190}
191
192static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
193 phys_addr_t phys, u64 cleared)
194{
195 struct clear_badblocks_context ctx = {
196 .phys = phys,
197 .cleared = cleared,
198 };
199
200 device_for_each_child(&nvdimm_bus->dev, &ctx,
201 nvdimm_clear_badblocks_region);
202}
203
204static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
205 phys_addr_t phys, u64 cleared)
206{
207 if (cleared > 0)
aa9ad44a 208 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
23f49844
DW
209
210 if (cleared > 0 && cleared / 512)
211 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
212}
213
59e64739
DW
214long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
215 unsigned int len)
216{
217 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
218 struct nvdimm_bus_descriptor *nd_desc;
219 struct nd_cmd_clear_error clear_err;
220 struct nd_cmd_ars_cap ars_cap;
221 u32 clear_err_unit, mask;
0930a750 222 unsigned int noio_flag;
59e64739
DW
223 int cmd_rc, rc;
224
225 if (!nvdimm_bus)
226 return -ENXIO;
227
228 nd_desc = nvdimm_bus->nd_desc;
1e8b8d96
DJ
229 /*
230 * if ndctl does not exist, it's PMEM_LEGACY and
231 * we want to just pretend everything is handled.
232 */
59e64739 233 if (!nd_desc->ndctl)
1e8b8d96 234 return len;
59e64739
DW
235
236 memset(&ars_cap, 0, sizeof(ars_cap));
237 ars_cap.address = phys;
238 ars_cap.length = len;
0930a750 239 noio_flag = memalloc_noio_save();
59e64739
DW
240 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
241 sizeof(ars_cap), &cmd_rc);
0930a750 242 memalloc_noio_restore(noio_flag);
59e64739
DW
243 if (rc < 0)
244 return rc;
245 if (cmd_rc < 0)
246 return cmd_rc;
247 clear_err_unit = ars_cap.clear_err_unit;
248 if (!clear_err_unit || !is_power_of_2(clear_err_unit))
249 return -ENXIO;
250
251 mask = clear_err_unit - 1;
252 if ((phys | len) & mask)
253 return -ENXIO;
254 memset(&clear_err, 0, sizeof(clear_err));
255 clear_err.address = phys;
256 clear_err.length = len;
0930a750 257 noio_flag = memalloc_noio_save();
59e64739
DW
258 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
259 sizeof(clear_err), &cmd_rc);
0930a750 260 memalloc_noio_restore(noio_flag);
59e64739
DW
261 if (rc < 0)
262 return rc;
263 if (cmd_rc < 0)
264 return cmd_rc;
e046114a 265
23f49844 266 nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
8d13c029 267
59e64739
DW
268 return clear_err.cleared;
269}
270EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
271
18515942
DW
272static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
273
4d88a97a 274static struct bus_type nvdimm_bus_type = {
e6dfb2de 275 .name = "nd",
4d88a97a
DW
276 .uevent = nvdimm_bus_uevent,
277 .match = nvdimm_bus_match,
278 .probe = nvdimm_bus_probe,
279 .remove = nvdimm_bus_remove,
476f848a 280 .shutdown = nvdimm_bus_shutdown,
4d88a97a
DW
281};
282
18515942
DW
283static void nvdimm_bus_release(struct device *dev)
284{
285 struct nvdimm_bus *nvdimm_bus;
286
287 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
288 ida_simple_remove(&nd_ida, nvdimm_bus->id);
289 kfree(nvdimm_bus);
290}
291
e755799a
DW
292static const struct device_type nvdimm_bus_dev_type = {
293 .release = nvdimm_bus_release,
294 .groups = nvdimm_bus_attribute_groups,
295};
296
87a30e1f 297bool is_nvdimm_bus(struct device *dev)
18515942 298{
e755799a 299 return dev->type == &nvdimm_bus_dev_type;
18515942
DW
300}
301
302struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
303{
304 struct device *dev;
305
306 for (dev = nd_dev; dev; dev = dev->parent)
307 if (is_nvdimm_bus(dev))
308 break;
309 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
310 if (dev)
311 return to_nvdimm_bus(dev);
312 return NULL;
313}
314
315struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
316{
317 struct nvdimm_bus *nvdimm_bus;
318
319 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
320 WARN_ON(!is_nvdimm_bus(dev));
321 return nvdimm_bus;
322}
323EXPORT_SYMBOL_GPL(to_nvdimm_bus);
324
f2989396
DJ
325struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
326{
327 return to_nvdimm_bus(nvdimm->dev.parent);
328}
329EXPORT_SYMBOL_GPL(nvdimm_to_bus);
330
4a0079bc
DW
331static struct lock_class_key nvdimm_bus_key;
332
18515942
DW
333struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
334 struct nvdimm_bus_descriptor *nd_desc)
335{
336 struct nvdimm_bus *nvdimm_bus;
337 int rc;
338
339 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
340 if (!nvdimm_bus)
341 return NULL;
342 INIT_LIST_HEAD(&nvdimm_bus->list);
343 INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
b70d31d0 344 init_waitqueue_head(&nvdimm_bus->wait);
18515942 345 nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
18515942
DW
346 if (nvdimm_bus->id < 0) {
347 kfree(nvdimm_bus);
348 return NULL;
349 }
9bf3aa44
OH
350 mutex_init(&nvdimm_bus->reconfig_mutex);
351 badrange_init(&nvdimm_bus->badrange);
18515942
DW
352 nvdimm_bus->nd_desc = nd_desc;
353 nvdimm_bus->dev.parent = parent;
e755799a 354 nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
18515942
DW
355 nvdimm_bus->dev.groups = nd_desc->attr_groups;
356 nvdimm_bus->dev.bus = &nvdimm_bus_type;
1ff19f48 357 nvdimm_bus->dev.of_node = nd_desc->of_node;
2bbafda4 358 device_initialize(&nvdimm_bus->dev);
4a0079bc 359 lockdep_set_class(&nvdimm_bus->dev.mutex, &nvdimm_bus_key);
2bbafda4
DW
360 device_set_pm_not_required(&nvdimm_bus->dev);
361 rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
362 if (rc)
363 goto err;
364
365 rc = device_add(&nvdimm_bus->dev);
18515942
DW
366 if (rc) {
367 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
368 goto err;
369 }
370
371 return nvdimm_bus;
372 err:
373 put_device(&nvdimm_bus->dev);
374 return NULL;
375}
376EXPORT_SYMBOL_GPL(nvdimm_bus_register);
377
378void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
379{
380 if (!nvdimm_bus)
381 return;
382 device_unregister(&nvdimm_bus->dev);
383}
384EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
385
386static int child_unregister(struct device *dev, void *data)
387{
388 /*
389 * the singular ndctl class device per bus needs to be
390 * "device_destroy"ed, so skip it here
391 *
392 * i.e. remove classless children
393 */
394 if (dev->class)
7d988097
DJ
395 return 0;
396
fd14602d
DW
397 if (is_nvdimm(dev))
398 nvdimm_delete(to_nvdimm(dev));
399 else
400 nd_device_unregister(dev, ND_SYNC);
7d988097 401
18515942
DW
402 return 0;
403}
404
aa9ad44a 405static void free_badrange_list(struct list_head *badrange_list)
18515942 406{
aa9ad44a 407 struct badrange_entry *bre, *next;
18515942 408
aa9ad44a
DJ
409 list_for_each_entry_safe(bre, next, badrange_list, list) {
410 list_del(&bre->list);
411 kfree(bre);
18515942 412 }
aa9ad44a 413 list_del_init(badrange_list);
18515942
DW
414}
415
1f975074 416static void nd_bus_remove(struct device *dev)
18515942
DW
417{
418 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
419
420 mutex_lock(&nvdimm_bus_list_mutex);
421 list_del_init(&nvdimm_bus->list);
422 mutex_unlock(&nvdimm_bus_list_mutex);
423
b70d31d0
DW
424 wait_event(nvdimm_bus->wait,
425 atomic_read(&nvdimm_bus->ioctl_active) == 0);
426
18515942
DW
427 nd_synchronize();
428 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
429
aa9ad44a
DJ
430 spin_lock(&nvdimm_bus->badrange.lock);
431 free_badrange_list(&nvdimm_bus->badrange.list);
432 spin_unlock(&nvdimm_bus->badrange.lock);
18515942
DW
433
434 nvdimm_bus_destroy_ndctl(nvdimm_bus);
18515942
DW
435}
436
437static int nd_bus_probe(struct device *dev)
438{
439 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
440 int rc;
441
442 rc = nvdimm_bus_create_ndctl(nvdimm_bus);
443 if (rc)
444 return rc;
445
446 mutex_lock(&nvdimm_bus_list_mutex);
447 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
448 mutex_unlock(&nvdimm_bus_list_mutex);
449
450 /* enable bus provider attributes to look up their local context */
451 dev_set_drvdata(dev, nvdimm_bus->nd_desc);
452
453 return 0;
454}
455
456static struct nd_device_driver nd_bus_driver = {
457 .probe = nd_bus_probe,
458 .remove = nd_bus_remove,
459 .drv = {
460 .name = "nd_bus",
461 .suppress_bind_attrs = true,
462 .bus = &nvdimm_bus_type,
463 .owner = THIS_MODULE,
464 .mod_name = KBUILD_MODNAME,
465 },
4d88a97a
DW
466};
467
18515942
DW
468static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
469{
470 struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
471
472 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
473 return true;
474
475 return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
476}
477
4d88a97a
DW
478static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
479
480void nd_synchronize(void)
481{
482 async_synchronize_full_domain(&nd_async_domain);
483}
484EXPORT_SYMBOL_GPL(nd_synchronize);
485
486static void nd_async_device_register(void *d, async_cookie_t cookie)
487{
488 struct device *dev = d;
489
490 if (device_add(dev) != 0) {
491 dev_err(dev, "%s: failed\n", __func__);
492 put_device(dev);
493 }
494 put_device(dev);
b6eae0f6
AD
495 if (dev->parent)
496 put_device(dev->parent);
4d88a97a
DW
497}
498
499static void nd_async_device_unregister(void *d, async_cookie_t cookie)
500{
501 struct device *dev = d;
502
0ba1c634
DW
503 /* flush bus operations before delete */
504 nvdimm_bus_lock(dev);
505 nvdimm_bus_unlock(dev);
506
4d88a97a
DW
507 device_unregister(dev);
508 put_device(dev);
509}
510
f57aec44 511static void __nd_device_register(struct device *dev, bool sync)
4d88a97a 512{
cd03412a
DW
513 if (!dev)
514 return;
1a091d16
AD
515
516 /*
517 * Ensure that region devices always have their NUMA node set as
518 * early as possible. This way we are able to make certain that
519 * any memory associated with the creation and the creation
520 * itself of the region is associated with the correct node.
521 */
522 if (is_nd_region(dev))
523 set_dev_node(dev, to_nd_region(dev)->numa_node);
524
4d88a97a 525 dev->bus = &nvdimm_bus_type;
2bbafda4 526 device_set_pm_not_required(dev);
af87b9a7 527 if (dev->parent) {
b6eae0f6 528 get_device(dev->parent);
af87b9a7
AD
529 if (dev_to_node(dev) == NUMA_NO_NODE)
530 set_dev_node(dev, dev_to_node(dev->parent));
531 }
4d88a97a 532 get_device(dev);
af87b9a7 533
f57aec44
DW
534 if (sync)
535 nd_async_device_register(dev, 0);
536 else
537 async_schedule_dev_domain(nd_async_device_register, dev,
538 &nd_async_domain);
539}
540
541void nd_device_register(struct device *dev)
542{
543 __nd_device_register(dev, false);
4d88a97a
DW
544}
545EXPORT_SYMBOL(nd_device_register);
546
f57aec44
DW
547void nd_device_register_sync(struct device *dev)
548{
549 __nd_device_register(dev, true);
550}
551
4d88a97a
DW
552void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
553{
8aac0e23
DW
554 bool killed;
555
4d88a97a
DW
556 switch (mode) {
557 case ND_ASYNC:
8aac0e23
DW
558 /*
559 * In the async case this is being triggered with the
560 * device lock held and the unregistration work needs to
561 * be moved out of line iff this is thread has won the
562 * race to schedule the deletion.
563 */
564 if (!kill_device(dev))
565 return;
566
4d88a97a
DW
567 get_device(dev);
568 async_schedule_domain(nd_async_device_unregister, dev,
569 &nd_async_domain);
570 break;
571 case ND_SYNC:
8aac0e23
DW
572 /*
573 * In the sync case the device is being unregistered due
574 * to a state change of the parent. Claim the kill state
575 * to synchronize against other unregistration requests,
576 * or otherwise let the async path handle it if the
577 * unregistration was already queued.
578 */
81beea55 579 device_lock(dev);
8aac0e23 580 killed = kill_device(dev);
81beea55 581 device_unlock(dev);
8aac0e23
DW
582
583 if (!killed)
584 return;
585
4d88a97a
DW
586 nd_synchronize();
587 device_unregister(dev);
588 break;
589 }
590}
591EXPORT_SYMBOL(nd_device_unregister);
592
593/**
594 * __nd_driver_register() - register a region or a namespace driver
595 * @nd_drv: driver to register
596 * @owner: automatically set by nd_driver_register() macro
597 * @mod_name: automatically set by nd_driver_register() macro
598 */
599int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
600 const char *mod_name)
601{
602 struct device_driver *drv = &nd_drv->drv;
603
604 if (!nd_drv->type) {
d75f773c 605 pr_debug("driver type bitmask not set (%ps)\n",
4d88a97a
DW
606 __builtin_return_address(0));
607 return -EINVAL;
608 }
609
6cf9c5ba
DW
610 if (!nd_drv->probe) {
611 pr_debug("%s ->probe() must be specified\n", mod_name);
4d88a97a
DW
612 return -EINVAL;
613 }
614
615 drv->bus = &nvdimm_bus_type;
616 drv->owner = owner;
617 drv->mod_name = mod_name;
618
619 return driver_register(drv);
620}
621EXPORT_SYMBOL(__nd_driver_register);
622
32f61d67 623void nvdimm_check_and_set_ro(struct gendisk *disk)
58138820 624{
52c44d93 625 struct device *dev = disk_to_dev(disk)->parent;
58138820 626 struct nd_region *nd_region = to_nd_region(dev->parent);
254a4cd5 627 int disk_ro = get_disk_ro(disk);
58138820 628
2361db89
DW
629 /* catch the disk up with the region ro state */
630 if (disk_ro == nd_region->ro)
32f61d67 631 return;
58138820 632
2361db89
DW
633 dev_info(dev, "%s read-%s, marking %s read-%s\n",
634 dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
635 disk->disk_name, nd_region->ro ? "only" : "write");
636 set_disk_ro(disk, nd_region->ro);
58138820 637}
32f61d67 638EXPORT_SYMBOL(nvdimm_check_and_set_ro);
58138820 639
4d88a97a
DW
640static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
641 char *buf)
642{
643 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
644 to_nd_device_type(dev));
645}
646static DEVICE_ATTR_RO(modalias);
647
648static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
649 char *buf)
650{
651 return sprintf(buf, "%s\n", dev->type->name);
652}
653static DEVICE_ATTR_RO(devtype);
654
655static struct attribute *nd_device_attributes[] = {
656 &dev_attr_modalias.attr,
657 &dev_attr_devtype.attr,
658 NULL,
659};
660
c01dafad 661/*
4d88a97a
DW
662 * nd_device_attribute_group - generic attributes for all devices on an nd bus
663 */
adbb6829 664const struct attribute_group nd_device_attribute_group = {
4d88a97a 665 .attrs = nd_device_attributes,
e6dfb2de
DW
666};
667
74ae66c3
TK
668static ssize_t numa_node_show(struct device *dev,
669 struct device_attribute *attr, char *buf)
670{
671 return sprintf(buf, "%d\n", dev_to_node(dev));
672}
673static DEVICE_ATTR_RO(numa_node);
674
bcba0c45
DW
675static int nvdimm_dev_to_target_node(struct device *dev)
676{
677 struct device *parent = dev->parent;
678 struct nd_region *nd_region = NULL;
679
680 if (is_nd_region(dev))
681 nd_region = to_nd_region(dev);
682 else if (parent && is_nd_region(parent))
683 nd_region = to_nd_region(parent);
684
685 if (!nd_region)
686 return NUMA_NO_NODE;
687 return nd_region->target_node;
688}
689
690static ssize_t target_node_show(struct device *dev,
691 struct device_attribute *attr, char *buf)
692{
693 return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
694}
695static DEVICE_ATTR_RO(target_node);
696
74ae66c3
TK
697static struct attribute *nd_numa_attributes[] = {
698 &dev_attr_numa_node.attr,
bcba0c45 699 &dev_attr_target_node.attr,
74ae66c3
TK
700 NULL,
701};
702
703static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
704 int n)
705{
bcba0c45
DW
706 struct device *dev = container_of(kobj, typeof(*dev), kobj);
707
74ae66c3
TK
708 if (!IS_ENABLED(CONFIG_NUMA))
709 return 0;
710
bcba0c45
DW
711 if (a == &dev_attr_target_node.attr &&
712 nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
713 return 0;
714
74ae66c3
TK
715 return a->mode;
716}
717
c01dafad 718/*
74ae66c3
TK
719 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
720 */
e2f6a0e3 721const struct attribute_group nd_numa_attribute_group = {
74ae66c3
TK
722 .attrs = nd_numa_attributes,
723 .is_visible = nd_numa_attr_visible,
724};
74ae66c3 725
2bbafda4
DW
726static void ndctl_release(struct device *dev)
727{
728 kfree(dev);
729}
730
4a0079bc
DW
731static struct lock_class_key nvdimm_ndctl_key;
732
45def22c
DW
733int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
734{
735 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
736 struct device *dev;
2bbafda4
DW
737 int rc;
738
739 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
740 if (!dev)
741 return -ENOMEM;
742 device_initialize(dev);
4a0079bc 743 lockdep_set_class(&dev->mutex, &nvdimm_ndctl_key);
2bbafda4
DW
744 device_set_pm_not_required(dev);
745 dev->class = nd_class;
746 dev->parent = &nvdimm_bus->dev;
747 dev->devt = devt;
748 dev->release = ndctl_release;
749 rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id);
750 if (rc)
751 goto err;
45def22c 752
2bbafda4
DW
753 rc = device_add(dev);
754 if (rc) {
755 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
756 nvdimm_bus->id, rc);
757 goto err;
758 }
759 return 0;
45def22c 760
2bbafda4
DW
761err:
762 put_device(dev);
763 return rc;
45def22c
DW
764}
765
766void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
767{
768 device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
769}
770
62232e45
DW
771static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
772 [ND_CMD_IMPLEMENTED] = { },
773 [ND_CMD_SMART] = {
774 .out_num = 2,
21129112 775 .out_sizes = { 4, 128, },
62232e45
DW
776 },
777 [ND_CMD_SMART_THRESHOLD] = {
778 .out_num = 2,
779 .out_sizes = { 4, 8, },
780 },
781 [ND_CMD_DIMM_FLAGS] = {
782 .out_num = 2,
783 .out_sizes = { 4, 4 },
784 },
785 [ND_CMD_GET_CONFIG_SIZE] = {
786 .out_num = 3,
787 .out_sizes = { 4, 4, 4, },
788 },
789 [ND_CMD_GET_CONFIG_DATA] = {
790 .in_num = 2,
791 .in_sizes = { 4, 4, },
792 .out_num = 2,
793 .out_sizes = { 4, UINT_MAX, },
794 },
795 [ND_CMD_SET_CONFIG_DATA] = {
796 .in_num = 3,
797 .in_sizes = { 4, 4, UINT_MAX, },
798 .out_num = 1,
799 .out_sizes = { 4, },
800 },
801 [ND_CMD_VENDOR] = {
802 .in_num = 3,
803 .in_sizes = { 4, 4, UINT_MAX, },
804 .out_num = 3,
805 .out_sizes = { 4, 4, UINT_MAX, },
806 },
31eca76b
DW
807 [ND_CMD_CALL] = {
808 .in_num = 2,
809 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
810 .out_num = 1,
811 .out_sizes = { UINT_MAX, },
812 },
62232e45
DW
813};
814
815const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
816{
817 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
818 return &__nd_cmd_dimm_descs[cmd];
819 return NULL;
820}
821EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
822
823static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
824 [ND_CMD_IMPLEMENTED] = { },
825 [ND_CMD_ARS_CAP] = {
826 .in_num = 2,
827 .in_sizes = { 8, 8, },
4577b066
DW
828 .out_num = 4,
829 .out_sizes = { 4, 4, 4, 4, },
62232e45
DW
830 },
831 [ND_CMD_ARS_START] = {
4577b066
DW
832 .in_num = 5,
833 .in_sizes = { 8, 8, 2, 1, 5, },
834 .out_num = 2,
835 .out_sizes = { 4, 4, },
62232e45
DW
836 },
837 [ND_CMD_ARS_STATUS] = {
747ffe11
DW
838 .out_num = 3,
839 .out_sizes = { 4, 4, UINT_MAX, },
62232e45 840 },
d4f32367
DW
841 [ND_CMD_CLEAR_ERROR] = {
842 .in_num = 2,
843 .in_sizes = { 8, 8, },
844 .out_num = 3,
845 .out_sizes = { 4, 4, 8, },
846 },
31eca76b
DW
847 [ND_CMD_CALL] = {
848 .in_num = 2,
849 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
850 .out_num = 1,
851 .out_sizes = { UINT_MAX, },
852 },
62232e45
DW
853};
854
855const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
856{
857 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
858 return &__nd_cmd_bus_descs[cmd];
859 return NULL;
860}
861EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
862
863u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
864 const struct nd_cmd_desc *desc, int idx, void *buf)
865{
866 if (idx >= desc->in_num)
867 return UINT_MAX;
868
869 if (desc->in_sizes[idx] < UINT_MAX)
870 return desc->in_sizes[idx];
871
872 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
873 struct nd_cmd_set_config_hdr *hdr = buf;
874
875 return hdr->in_length;
876 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
877 struct nd_cmd_vendor_hdr *hdr = buf;
878
879 return hdr->in_length;
31eca76b
DW
880 } else if (cmd == ND_CMD_CALL) {
881 struct nd_cmd_pkg *pkg = buf;
882
883 return pkg->nd_size_in;
62232e45
DW
884 }
885
886 return UINT_MAX;
887}
888EXPORT_SYMBOL_GPL(nd_cmd_in_size);
889
890u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
891 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
efda1b5d 892 const u32 *out_field, unsigned long remainder)
62232e45
DW
893{
894 if (idx >= desc->out_num)
895 return UINT_MAX;
896
897 if (desc->out_sizes[idx] < UINT_MAX)
898 return desc->out_sizes[idx];
899
900 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
901 return in_field[1];
902 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
903 return out_field[1];
efda1b5d
DW
904 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
905 /*
906 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
907 * "Size of Output Buffer in bytes, including this
908 * field."
909 */
910 if (out_field[1] < 4)
911 return 0;
912 /*
913 * ACPI 6.1 is ambiguous if 'status' is included in the
914 * output size. If we encounter an output size that
915 * overshoots the remainder by 4 bytes, assume it was
916 * including 'status'.
917 */
286e8771 918 if (out_field[1] - 4 == remainder)
efda1b5d 919 return remainder;
286e8771 920 return out_field[1] - 8;
efda1b5d 921 } else if (cmd == ND_CMD_CALL) {
31eca76b
DW
922 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
923
924 return pkg->nd_size_out;
925 }
926
62232e45
DW
927
928 return UINT_MAX;
929}
930EXPORT_SYMBOL_GPL(nd_cmd_out_size);
931
bf9bccc1 932void wait_nvdimm_bus_probe_idle(struct device *dev)
eaf96153 933{
bf9bccc1
DW
934 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
935
eaf96153
DW
936 do {
937 if (nvdimm_bus->probe_active == 0)
938 break;
ca6bf264 939 nvdimm_bus_unlock(dev);
81beea55 940 device_unlock(dev);
b70d31d0 941 wait_event(nvdimm_bus->wait,
eaf96153 942 nvdimm_bus->probe_active == 0);
81beea55 943 device_lock(dev);
ca6bf264 944 nvdimm_bus_lock(dev);
eaf96153
DW
945 } while (true);
946}
947
006358b3 948static int nd_pmem_forget_poison_check(struct device *dev, void *data)
d4f32367 949{
006358b3
DJ
950 struct nd_cmd_clear_error *clear_err =
951 (struct nd_cmd_clear_error *)data;
952 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
953 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
954 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
955 struct nd_namespace_common *ndns = NULL;
956 struct nd_namespace_io *nsio;
957 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
958
959 if (nd_dax || !dev->driver)
960 return 0;
961
962 start = clear_err->address;
963 end = clear_err->address + clear_err->cleared - 1;
964
965 if (nd_btt || nd_pfn || nd_dax) {
966 if (nd_btt)
967 ndns = nd_btt->ndns;
968 else if (nd_pfn)
969 ndns = nd_pfn->ndns;
970 else if (nd_dax)
971 ndns = nd_dax->nd_pfn.ndns;
972
973 if (!ndns)
974 return 0;
975 } else
976 ndns = to_ndns(dev);
977
978 nsio = to_nd_namespace_io(&ndns->dev);
979 pstart = nsio->res.start + offset;
980 pend = nsio->res.end - end_trunc;
981
982 if ((pstart >= start) && (pend <= end))
d4f32367 983 return -EBUSY;
006358b3 984
d4f32367 985 return 0;
006358b3
DJ
986
987}
988
989static int nd_ns_forget_poison_check(struct device *dev, void *data)
990{
991 return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
d4f32367
DW
992}
993
eaf96153 994/* set_config requires an idle interleave set */
87bf572e 995static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
006358b3 996 struct nvdimm *nvdimm, unsigned int cmd, void *data)
eaf96153 997{
87bf572e
DW
998 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
999
1000 /* ask the bus provider if it would like to block this request */
1001 if (nd_desc->clear_to_send) {
b3ed2ce0 1002 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
87bf572e
DW
1003
1004 if (rc)
1005 return rc;
1006 }
eaf96153 1007
d4f32367
DW
1008 /* require clear error to go through the pmem driver */
1009 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
006358b3
DJ
1010 return device_for_each_child(&nvdimm_bus->dev, data,
1011 nd_ns_forget_poison_check);
d4f32367 1012
eaf96153
DW
1013 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1014 return 0;
1015
87bf572e 1016 /* prevent label manipulation while the kernel owns label updates */
bf9bccc1 1017 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
eaf96153
DW
1018 if (atomic_read(&nvdimm->busy))
1019 return -EBUSY;
1020 return 0;
1021}
1022
62232e45
DW
1023static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1024 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1025{
1026 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
62232e45
DW
1027 const struct nd_cmd_desc *desc = NULL;
1028 unsigned int cmd = _IOC_NR(ioctl_cmd);
62232e45 1029 struct device *dev = &nvdimm_bus->dev;
58738c49 1030 void __user *p = (void __user *) arg;
6de5d06e 1031 char *out_env = NULL, *in_env = NULL;
62232e45 1032 const char *cmd_name, *dimm_name;
58738c49
DW
1033 u32 in_len = 0, out_len = 0;
1034 unsigned int func = cmd;
e3654eca 1035 unsigned long cmd_mask;
58738c49 1036 struct nd_cmd_pkg pkg;
006358b3 1037 int rc, i, cmd_rc;
6de5d06e 1038 void *buf = NULL;
58738c49 1039 u64 buf_len = 0;
62232e45
DW
1040
1041 if (nvdimm) {
1042 desc = nd_cmd_dimm_desc(cmd);
1043 cmd_name = nvdimm_cmd_name(cmd);
e3654eca 1044 cmd_mask = nvdimm->cmd_mask;
62232e45
DW
1045 dimm_name = dev_name(&nvdimm->dev);
1046 } else {
1047 desc = nd_cmd_bus_desc(cmd);
1048 cmd_name = nvdimm_bus_cmd_name(cmd);
e3654eca 1049 cmd_mask = nd_desc->cmd_mask;
62232e45
DW
1050 dimm_name = "bus";
1051 }
1052
92fe2aa8 1053 /* Validate command family support against bus declared support */
31eca76b 1054 if (cmd == ND_CMD_CALL) {
92fe2aa8
DW
1055 unsigned long *mask;
1056
31eca76b
DW
1057 if (copy_from_user(&pkg, p, sizeof(pkg)))
1058 return -EFAULT;
92fe2aa8
DW
1059
1060 if (nvdimm) {
1061 if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1062 return -EINVAL;
1063 mask = &nd_desc->dimm_family_mask;
1064 } else {
1065 if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1066 return -EINVAL;
1067 mask = &nd_desc->bus_family_mask;
1068 }
1069
1070 if (!test_bit(pkg.nd_family, mask))
1071 return -EINVAL;
31eca76b
DW
1072 }
1073
f84afbdd
DC
1074 if (!desc ||
1075 (desc->out_num + desc->in_num == 0) ||
1076 cmd > ND_CMD_CALL ||
1077 !test_bit(cmd, &cmd_mask))
62232e45
DW
1078 return -ENOTTY;
1079
1080 /* fail write commands (when read-only) */
1081 if (read_only)
07accfa9
JH
1082 switch (cmd) {
1083 case ND_CMD_VENDOR:
1084 case ND_CMD_SET_CONFIG_DATA:
1085 case ND_CMD_ARS_START:
d4f32367 1086 case ND_CMD_CLEAR_ERROR:
31eca76b 1087 case ND_CMD_CALL:
ca6bf264 1088 dev_dbg(dev, "'%s' command while read-only.\n",
62232e45
DW
1089 nvdimm ? nvdimm_cmd_name(cmd)
1090 : nvdimm_bus_cmd_name(cmd));
1091 return -EPERM;
1092 default:
1093 break;
1094 }
1095
1096 /* process an input envelope */
6de5d06e
DW
1097 in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1098 if (!in_env)
1099 return -ENOMEM;
62232e45
DW
1100 for (i = 0; i < desc->in_num; i++) {
1101 u32 in_size, copy;
1102
1103 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1104 if (in_size == UINT_MAX) {
1105 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1106 __func__, dimm_name, cmd_name, i);
6de5d06e
DW
1107 rc = -ENXIO;
1108 goto out;
62232e45 1109 }
6de5d06e
DW
1110 if (in_len < ND_CMD_MAX_ENVELOPE)
1111 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
62232e45
DW
1112 else
1113 copy = 0;
6de5d06e
DW
1114 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1115 rc = -EFAULT;
1116 goto out;
1117 }
62232e45
DW
1118 in_len += in_size;
1119 }
1120
31eca76b 1121 if (cmd == ND_CMD_CALL) {
53b85a44 1122 func = pkg.nd_command;
426824d6
DW
1123 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1124 dimm_name, pkg.nd_command,
31eca76b 1125 in_len, out_len, buf_len);
31eca76b
DW
1126 }
1127
62232e45 1128 /* process an output envelope */
6de5d06e
DW
1129 out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1130 if (!out_env) {
1131 rc = -ENOMEM;
1132 goto out;
1133 }
1134
62232e45
DW
1135 for (i = 0; i < desc->out_num; i++) {
1136 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
efda1b5d 1137 (u32 *) in_env, (u32 *) out_env, 0);
62232e45
DW
1138 u32 copy;
1139
1140 if (out_size == UINT_MAX) {
426824d6
DW
1141 dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1142 dimm_name, cmd_name, i);
6de5d06e
DW
1143 rc = -EFAULT;
1144 goto out;
62232e45 1145 }
6de5d06e
DW
1146 if (out_len < ND_CMD_MAX_ENVELOPE)
1147 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
62232e45
DW
1148 else
1149 copy = 0;
1150 if (copy && copy_from_user(&out_env[out_len],
6de5d06e
DW
1151 p + in_len + out_len, copy)) {
1152 rc = -EFAULT;
1153 goto out;
1154 }
62232e45
DW
1155 out_len += out_size;
1156 }
1157
58738c49 1158 buf_len = (u64) out_len + (u64) in_len;
62232e45 1159 if (buf_len > ND_IOCTL_MAX_BUFLEN) {
426824d6
DW
1160 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1161 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
6de5d06e
DW
1162 rc = -EINVAL;
1163 goto out;
62232e45
DW
1164 }
1165
1166 buf = vmalloc(buf_len);
6de5d06e
DW
1167 if (!buf) {
1168 rc = -ENOMEM;
1169 goto out;
1170 }
62232e45
DW
1171
1172 if (copy_from_user(buf, p, buf_len)) {
1173 rc = -EFAULT;
1174 goto out;
1175 }
1176
81beea55 1177 device_lock(dev);
ca6bf264 1178 nvdimm_bus_lock(dev);
53b85a44 1179 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
eaf96153
DW
1180 if (rc)
1181 goto out_unlock;
1182
006358b3 1183 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
62232e45 1184 if (rc < 0)
eaf96153 1185 goto out_unlock;
006358b3
DJ
1186
1187 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1188 struct nd_cmd_clear_error *clear_err = buf;
006358b3 1189
23f49844
DW
1190 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1191 clear_err->cleared);
006358b3 1192 }
0beb2012 1193
62232e45
DW
1194 if (copy_to_user(p, buf, buf_len))
1195 rc = -EFAULT;
0beb2012 1196
6de5d06e 1197out_unlock:
ca6bf264 1198 nvdimm_bus_unlock(dev);
81beea55 1199 device_unlock(dev);
6de5d06e
DW
1200out:
1201 kfree(in_env);
1202 kfree(out_env);
62232e45
DW
1203 vfree(buf);
1204 return rc;
1205}
1206
b70d31d0
DW
1207enum nd_ioctl_mode {
1208 BUS_IOCTL,
1209 DIMM_IOCTL,
1210};
62232e45
DW
1211
1212static int match_dimm(struct device *dev, void *data)
1213{
1214 long id = (long) data;
1215
1216 if (is_nvdimm(dev)) {
1217 struct nvdimm *nvdimm = to_nvdimm(dev);
1218
1219 return nvdimm->id == id;
1220 }
1221
1222 return 0;
1223}
1224
b70d31d0
DW
1225static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1226 enum nd_ioctl_mode mode)
1227
62232e45 1228{
b70d31d0
DW
1229 struct nvdimm_bus *nvdimm_bus, *found = NULL;
1230 long id = (long) file->private_data;
1231 struct nvdimm *nvdimm = NULL;
1232 int rc, ro;
62232e45 1233
4dc0e7be 1234 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
62232e45
DW
1235 mutex_lock(&nvdimm_bus_list_mutex);
1236 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
b70d31d0
DW
1237 if (mode == DIMM_IOCTL) {
1238 struct device *dev;
1239
1240 dev = device_find_child(&nvdimm_bus->dev,
1241 file->private_data, match_dimm);
1242 if (!dev)
1243 continue;
1244 nvdimm = to_nvdimm(dev);
1245 found = nvdimm_bus;
1246 } else if (nvdimm_bus->id == id) {
1247 found = nvdimm_bus;
1248 }
62232e45 1249
b70d31d0
DW
1250 if (found) {
1251 atomic_inc(&nvdimm_bus->ioctl_active);
1252 break;
1253 }
62232e45
DW
1254 }
1255 mutex_unlock(&nvdimm_bus_list_mutex);
1256
b70d31d0
DW
1257 if (!found)
1258 return -ENXIO;
1259
1260 nvdimm_bus = found;
1261 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1262
1263 if (nvdimm)
1264 put_device(&nvdimm->dev);
1265 if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1266 wake_up(&nvdimm_bus->wait);
1267
62232e45
DW
1268 return rc;
1269}
1270
b70d31d0
DW
1271static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1272{
1273 return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1274}
1275
1276static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1277{
1278 return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1279}
1280
62232e45
DW
1281static int nd_open(struct inode *inode, struct file *file)
1282{
1283 long minor = iminor(inode);
1284
1285 file->private_data = (void *) minor;
1286 return 0;
45def22c
DW
1287}
1288
1289static const struct file_operations nvdimm_bus_fops = {
1290 .owner = THIS_MODULE,
62232e45 1291 .open = nd_open,
b70d31d0 1292 .unlocked_ioctl = bus_ioctl,
1832f2d8 1293 .compat_ioctl = compat_ptr_ioctl,
45def22c
DW
1294 .llseek = noop_llseek,
1295};
1296
62232e45
DW
1297static const struct file_operations nvdimm_fops = {
1298 .owner = THIS_MODULE,
1299 .open = nd_open,
b70d31d0 1300 .unlocked_ioctl = dimm_ioctl,
1832f2d8 1301 .compat_ioctl = compat_ptr_ioctl,
62232e45
DW
1302 .llseek = noop_llseek,
1303};
1304
45def22c
DW
1305int __init nvdimm_bus_init(void)
1306{
1307 int rc;
1308
e6dfb2de
DW
1309 rc = bus_register(&nvdimm_bus_type);
1310 if (rc)
1311 return rc;
1312
45def22c
DW
1313 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1314 if (rc < 0)
62232e45 1315 goto err_bus_chrdev;
45def22c
DW
1316 nvdimm_bus_major = rc;
1317
62232e45
DW
1318 rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1319 if (rc < 0)
1320 goto err_dimm_chrdev;
1321 nvdimm_major = rc;
1322
1aaba11d 1323 nd_class = class_create("nd");
daa1dee4
AL
1324 if (IS_ERR(nd_class)) {
1325 rc = PTR_ERR(nd_class);
45def22c 1326 goto err_class;
daa1dee4 1327 }
45def22c 1328
18515942
DW
1329 rc = driver_register(&nd_bus_driver.drv);
1330 if (rc)
1331 goto err_nd_bus;
1332
45def22c
DW
1333 return 0;
1334
18515942
DW
1335 err_nd_bus:
1336 class_destroy(nd_class);
45def22c 1337 err_class:
62232e45
DW
1338 unregister_chrdev(nvdimm_major, "dimmctl");
1339 err_dimm_chrdev:
45def22c 1340 unregister_chrdev(nvdimm_bus_major, "ndctl");
62232e45 1341 err_bus_chrdev:
e6dfb2de 1342 bus_unregister(&nvdimm_bus_type);
45def22c
DW
1343
1344 return rc;
1345}
1346
4d88a97a 1347void nvdimm_bus_exit(void)
45def22c 1348{
18515942 1349 driver_unregister(&nd_bus_driver.drv);
45def22c
DW
1350 class_destroy(nd_class);
1351 unregister_chrdev(nvdimm_bus_major, "ndctl");
62232e45 1352 unregister_chrdev(nvdimm_major, "dimmctl");
e6dfb2de 1353 bus_unregister(&nvdimm_bus_type);
18515942 1354 ida_destroy(&nd_ida);
45def22c 1355}