projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
lightnvm: add fpg_size and pfpg_size to struct nvm_dev
[linux-2.6-block.git] / drivers / lightnvm / core.c
CommitLineData
cd9e9808
MB		 1/*
2 * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
3 * Initial release: Matias Bjorling <m@bjorling.me>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; see the file COPYING. If not, write to
16 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
17 * USA.
18 *
19 */
20
21#include <linux/blkdev.h>
22#include <linux/blk-mq.h>
23#include <linux/list.h>
24#include <linux/types.h>
25#include <linux/sem.h>
26#include <linux/bitmap.h>
27#include <linux/module.h>
28#include <linux/miscdevice.h>
29#include <linux/lightnvm.h>
91276162 30#include <linux/sched/sysctl.h>
cd9e9808
MB		 31#include <uapi/linux/lightnvm.h>
32
33static LIST_HEAD(nvm_targets);
34static LIST_HEAD(nvm_mgrs);
35static LIST_HEAD(nvm_devices);
36static DECLARE_RWSEM(nvm_lock);
37
38static struct nvm_tgt_type *nvm_find_target_type(const char *name)
39{
40 struct nvm_tgt_type *tt;
41
42 list_for_each_entry(tt, &nvm_targets, list)
43 if (!strcmp(name, tt->name))
44 return tt;
45
46 return NULL;
47}
48
49int nvm_register_target(struct nvm_tgt_type *tt)
50{
51 int ret = 0;
52
53 down_write(&nvm_lock);
54 if (nvm_find_target_type(tt->name))
55 ret = -EEXIST;
56 else
57 list_add(&tt->list, &nvm_targets);
58 up_write(&nvm_lock);
59
60 return ret;
61}
62EXPORT_SYMBOL(nvm_register_target);
63
64void nvm_unregister_target(struct nvm_tgt_type *tt)
65{
66 if (!tt)
67 return;
68
69 down_write(&nvm_lock);
70 list_del(&tt->list);
71 up_write(&nvm_lock);
72}
73EXPORT_SYMBOL(nvm_unregister_target);
74
75void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
76 dma_addr_t *dma_handler)
77{
16f26c3a 78 return dev->ops->dev_dma_alloc(dev, dev->ppalist_pool, mem_flags,
cd9e9808
MB		 79 dma_handler);
80}
81EXPORT_SYMBOL(nvm_dev_dma_alloc);
82
83void nvm_dev_dma_free(struct nvm_dev *dev, void *ppa_list,
84 dma_addr_t dma_handler)
85{
86 dev->ops->dev_dma_free(dev->ppalist_pool, ppa_list, dma_handler);
87}
88EXPORT_SYMBOL(nvm_dev_dma_free);
89
90static struct nvmm_type *nvm_find_mgr_type(const char *name)
91{
92 struct nvmm_type *mt;
93
94 list_for_each_entry(mt, &nvm_mgrs, list)
95 if (!strcmp(name, mt->name))
96 return mt;
97
98 return NULL;
99}
100
762796bc
MB		 101struct nvmm_type *nvm_init_mgr(struct nvm_dev *dev)
102{
103 struct nvmm_type *mt;
104 int ret;
105
106 lockdep_assert_held(&nvm_lock);
107
108 list_for_each_entry(mt, &nvm_mgrs, list) {
b7692076
MB		 109 if (strncmp(dev->sb.mmtype, mt->name, NVM_MMTYPE_LEN))
110 continue;
111
762796bc
MB		 112 ret = mt->register_mgr(dev);
113 if (ret < 0) {
114 pr_err("nvm: media mgr failed to init (%d) on dev %s\n",
115 ret, dev->name);
116 return NULL; /* initialization failed */
117 } else if (ret > 0)
118 return mt;
119 }
120
121 return NULL;
122}
123
cd9e9808
MB		 124int nvm_register_mgr(struct nvmm_type *mt)
125{
762796bc 126 struct nvm_dev *dev;
cd9e9808
MB		 127 int ret = 0;
128
129 down_write(&nvm_lock);
762796bc 130 if (nvm_find_mgr_type(mt->name)) {
cd9e9808 131 ret = -EEXIST;
762796bc
MB		 132 goto finish;
133 } else {
cd9e9808 134 list_add(&mt->list, &nvm_mgrs);
762796bc
MB		 135 }
136
137 /* try to register media mgr if any device have none configured */
138 list_for_each_entry(dev, &nvm_devices, devices) {
139 if (dev->mt)
140 continue;
141
142 dev->mt = nvm_init_mgr(dev);
143 }
144finish:
cd9e9808
MB		 145 up_write(&nvm_lock);
146
147 return ret;
148}
149EXPORT_SYMBOL(nvm_register_mgr);
150
151void nvm_unregister_mgr(struct nvmm_type *mt)
152{
153 if (!mt)
154 return;
155
156 down_write(&nvm_lock);
157 list_del(&mt->list);
158 up_write(&nvm_lock);
159}
160EXPORT_SYMBOL(nvm_unregister_mgr);
161
162static struct nvm_dev *nvm_find_nvm_dev(const char *name)
163{
164 struct nvm_dev *dev;
165
166 list_for_each_entry(dev, &nvm_devices, devices)
167 if (!strcmp(name, dev->name))
168 return dev;
169
170 return NULL;
171}
172
ff0e498b
JG		 173struct nvm_block *nvm_get_blk_unlocked(struct nvm_dev *dev, struct nvm_lun *lun,
174 unsigned long flags)
175{
176 return dev->mt->get_blk_unlocked(dev, lun, flags);
177}
178EXPORT_SYMBOL(nvm_get_blk_unlocked);
179
180/* Assumes that all valid pages have already been moved on release to bm */
181void nvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk)
182{
183 return dev->mt->put_blk_unlocked(dev, blk);
184}
185EXPORT_SYMBOL(nvm_put_blk_unlocked);
186
cd9e9808
MB		 187struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun,
188 unsigned long flags)
189{
190 return dev->mt->get_blk(dev, lun, flags);
191}
192EXPORT_SYMBOL(nvm_get_blk);
193
194/* Assumes that all valid pages have already been moved on release to bm */
195void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
196{
197 return dev->mt->put_blk(dev, blk);
198}
199EXPORT_SYMBOL(nvm_put_blk);
200
201int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
202{
203 return dev->mt->submit_io(dev, rqd);
204}
205EXPORT_SYMBOL(nvm_submit_io);
206
207int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk)
208{
209 return dev->mt->erase_blk(dev, blk, 0);
210}
211EXPORT_SYMBOL(nvm_erase_blk);
212
069368e9
MB		 213void nvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
214{
215 int i;
216
217 if (rqd->nr_pages > 1) {
218 for (i = 0; i < rqd->nr_pages; i++)
219 rqd->ppa_list[i] = dev_to_generic_addr(dev,
220 rqd->ppa_list[i]);
221 } else {
222 rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
223 }
224}
225EXPORT_SYMBOL(nvm_addr_to_generic_mode);
226
227void nvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
228{
229 int i;
230
231 if (rqd->nr_pages > 1) {
232 for (i = 0; i < rqd->nr_pages; i++)
233 rqd->ppa_list[i] = generic_to_dev_addr(dev,
234 rqd->ppa_list[i]);
235 } else {
236 rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
237 }
238}
239EXPORT_SYMBOL(nvm_generic_to_addr_mode);
240
abd805ec
MB		 241int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
242 struct ppa_addr *ppas, int nr_ppas)
069368e9 243{
abd805ec
MB		 244 int i, plane_cnt, pl_idx;
245
246 if (dev->plane_mode == NVM_PLANE_SINGLE && nr_ppas == 1) {
247 rqd->nr_pages = 1;
248 rqd->ppa_addr = ppas[0];
069368e9 249
069368e9 250 return 0;
abd805ec 251 }
069368e9 252
d5bdec8d 253 plane_cnt = dev->plane_mode;
abd805ec 254 rqd->nr_pages = plane_cnt * nr_ppas;
069368e9 255
abd805ec
MB		 256 if (dev->ops->max_phys_sect < rqd->nr_pages)
257 return -EINVAL;
258
259 rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
260 if (!rqd->ppa_list) {
261 pr_err("nvm: failed to allocate dma memory\n");
262 return -ENOMEM;
263 }
264
556755e9
MB		 265 for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
266 for (i = 0; i < nr_ppas; i++) {
abd805ec 267 ppas[i].g.pl = pl_idx;
556755e9 268 rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppas[i];
069368e9
MB		 269 }
270 }
271
abd805ec
MB		 272 return 0;
273}
274EXPORT_SYMBOL(nvm_set_rqd_ppalist);
275
276void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd)
277{
278 if (!rqd->ppa_list)
279 return;
280
281 nvm_dev_dma_free(dev, rqd->ppa_list, rqd->dma_ppa_list);
282}
283EXPORT_SYMBOL(nvm_free_rqd_ppalist);
284
81e681d3 285int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas)
abd805ec
MB		 286{
287 struct nvm_rq rqd;
288 int ret;
289
290 if (!dev->ops->erase_block)
291 return 0;
292
293 memset(&rqd, 0, sizeof(struct nvm_rq));
294
81e681d3 295 ret = nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas);
abd805ec
MB		 296 if (ret)
297 return ret;
298
069368e9
MB		 299 nvm_generic_to_addr_mode(dev, &rqd);
300
301 ret = dev->ops->erase_block(dev, &rqd);
302
abd805ec 303 nvm_free_rqd_ppalist(dev, &rqd);
069368e9
MB		 304
305 return ret;
306}
307EXPORT_SYMBOL(nvm_erase_ppa);
308
91276162
MB		 309void nvm_end_io(struct nvm_rq *rqd, int error)
310{
72d256ec
MB		 311 rqd->error = error;
312 rqd->end_io(rqd);
91276162
MB		 313}
314EXPORT_SYMBOL(nvm_end_io);
315
09719b62 316static void nvm_end_io_sync(struct nvm_rq *rqd)
91276162
MB		 317{
318 struct completion *waiting = rqd->wait;
319
320 rqd->wait = NULL;
321
322 complete(waiting);
323}
324
1145e635
MB		 325int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode,
326 int flags, void *buf, int len)
09719b62
MB		 327{
328 DECLARE_COMPLETION_ONSTACK(wait);
09719b62
MB		 329 struct bio *bio;
330 int ret;
331 unsigned long hang_check;
332
333 bio = bio_map_kern(dev->q, buf, len, GFP_KERNEL);
334 if (IS_ERR_OR_NULL(bio))
335 return -ENOMEM;
336
1145e635 337 nvm_generic_to_addr_mode(dev, rqd);
09719b62 338
1145e635
MB		 339 rqd->dev = dev;
340 rqd->opcode = opcode;
341 rqd->flags = flags;
342 rqd->bio = bio;
343 rqd->wait = &wait;
344 rqd->end_io = nvm_end_io_sync;
09719b62 345
1145e635 346 ret = dev->ops->submit_io(dev, rqd);
ecfb40c6 347 if (ret) {
ecfb40c6
MB		 348 bio_put(bio);
349 return ret;
350 }
09719b62
MB		 351
352 /* Prevent hang_check timer from firing at us during very long I/O */
353 hang_check = sysctl_hung_task_timeout_secs;
354 if (hang_check)
355 while (!wait_for_completion_io_timeout(&wait, hang_check * (HZ/2)));
356 else
357 wait_for_completion_io(&wait);
358
1145e635
MB		 359 return rqd->error;
360}
361
362/**
363 * nvm_submit_ppa_list - submit user-defined ppa list to device. The user must
364 * take to free ppa list if necessary.
365 * @dev: device
366 * @ppa_list: user created ppa_list
367 * @nr_ppas: length of ppa_list
368 * @opcode: device opcode
369 * @flags: device flags
370 * @buf: data buffer
371 * @len: data buffer length
372 */
373int nvm_submit_ppa_list(struct nvm_dev *dev, struct ppa_addr *ppa_list,
374 int nr_ppas, int opcode, int flags, void *buf, int len)
375{
376 struct nvm_rq rqd;
377
378 if (dev->ops->max_phys_sect < nr_ppas)
379 return -EINVAL;
380
381 memset(&rqd, 0, sizeof(struct nvm_rq));
382
383 rqd.nr_pages = nr_ppas;
384 if (nr_ppas > 1)
385 rqd.ppa_list = ppa_list;
386 else
387 rqd.ppa_addr = ppa_list[0];
388
389 return __nvm_submit_ppa(dev, &rqd, opcode, flags, buf, len);
390}
391EXPORT_SYMBOL(nvm_submit_ppa_list);
392
393/**
394 * nvm_submit_ppa - submit PPAs to device. PPAs will automatically be unfolded
395 * as single, dual, quad plane PPAs depending on device type.
396 * @dev: device
397 * @ppa: user created ppa_list
398 * @nr_ppas: length of ppa_list
399 * @opcode: device opcode
400 * @flags: device flags
401 * @buf: data buffer
402 * @len: data buffer length
403 */
404int nvm_submit_ppa(struct nvm_dev *dev, struct ppa_addr *ppa, int nr_ppas,
405 int opcode, int flags, void *buf, int len)
406{
407 struct nvm_rq rqd;
408 int ret;
409
410 memset(&rqd, 0, sizeof(struct nvm_rq));
411 ret = nvm_set_rqd_ppalist(dev, &rqd, ppa, nr_ppas);
412 if (ret)
413 return ret;
414
415 ret = __nvm_submit_ppa(dev, &rqd, opcode, flags, buf, len);
416
09719b62
MB		 417 nvm_free_rqd_ppalist(dev, &rqd);
418
1145e635 419 return ret;
09719b62
MB		 420}
421EXPORT_SYMBOL(nvm_submit_ppa);
422
ca5927e7
MB		 423static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
424{
425 int i;
426
427 dev->lps_per_blk = dev->pgs_per_blk;
428 dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
429 if (!dev->lptbl)
430 return -ENOMEM;
431
432 /* Just a linear array */
433 for (i = 0; i < dev->lps_per_blk; i++)
434 dev->lptbl[i] = i;
435
436 return 0;
437}
438
439static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
440{
441 int i, p;
442 struct nvm_id_lp_mlc *mlc = &grp->lptbl.mlc;
443
444 if (!mlc->num_pairs)
445 return 0;
446
447 dev->lps_per_blk = mlc->num_pairs;
448 dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
449 if (!dev->lptbl)
450 return -ENOMEM;
451
452 /* The lower page table encoding consists of a list of bytes, where each
453 * has a lower and an upper half. The first half byte maintains the
454 * increment value and every value after is an offset added to the
455 * previous incrementation value */
456 dev->lptbl[0] = mlc->pairs[0] & 0xF;
457 for (i = 1; i < dev->lps_per_blk; i++) {
458 p = mlc->pairs[i >> 1];
459 if (i & 0x1) /* upper */
460 dev->lptbl[i] = dev->lptbl[i - 1] + ((p & 0xF0) >> 4);
461 else /* lower */
462 dev->lptbl[i] = dev->lptbl[i - 1] + (p & 0xF);
463 }
464
465 return 0;
466}
467
cd9e9808
MB		 468static int nvm_core_init(struct nvm_dev *dev)
469{
470 struct nvm_id *id = &dev->identity;
471 struct nvm_id_group *grp = &id->groups[0];
472
473 /* device values */
474 dev->nr_chnls = grp->num_ch;
475 dev->luns_per_chnl = grp->num_lun;
476 dev->pgs_per_blk = grp->num_pg;
477 dev->blks_per_lun = grp->num_blk;
478 dev->nr_planes = grp->num_pln;
4891d120
MB		 479 dev->fpg_size = grp->fpg_sz;
480 dev->pfpg_size = grp->fpg_sz * grp->num_pln;
cd9e9808
MB		 481 dev->sec_size = grp->csecs;
482 dev->oob_size = grp->sos;
483 dev->sec_per_pg = grp->fpg_sz / grp->csecs;
f9a99950 484 dev->mccap = grp->mccap;
7386af27 485 memcpy(&dev->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
cd9e9808
MB		 486
487 dev->plane_mode = NVM_PLANE_SINGLE;
488 dev->max_rq_size = dev->ops->max_phys_sect * dev->sec_size;
489
4264c980
MB		 490 if (grp->mtype != 0) {
491 pr_err("nvm: memory type not supported\n");
492 return -EINVAL;
493 }
494
ca5927e7
MB		 495 switch (grp->fmtype) {
496 case NVM_ID_FMTYPE_SLC:
497 if (nvm_init_slc_tbl(dev, grp))
498 return -ENOMEM;
499 break;
500 case NVM_ID_FMTYPE_MLC:
501 if (nvm_init_mlc_tbl(dev, grp))
502 return -ENOMEM;
503 break;
504 default:
4264c980
MB		 505 pr_err("nvm: flash type not supported\n");
506 return -EINVAL;
507 }
508
ca5927e7
MB		 509 if (!dev->lps_per_blk)
510 pr_info("nvm: lower page programming table missing\n");
511
cd9e9808
MB		 512 if (grp->mpos & 0x020202)
513 dev->plane_mode = NVM_PLANE_DOUBLE;
514 if (grp->mpos & 0x040404)
515 dev->plane_mode = NVM_PLANE_QUAD;
516
517 /* calculated values */
518 dev->sec_per_pl = dev->sec_per_pg * dev->nr_planes;
519 dev->sec_per_blk = dev->sec_per_pl * dev->pgs_per_blk;
520 dev->sec_per_lun = dev->sec_per_blk * dev->blks_per_lun;
521 dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls;
522
ed2a92a6 523 dev->total_secs = dev->nr_luns * dev->sec_per_lun;
da1e2849
WT		 524 dev->lun_map = kcalloc(BITS_TO_LONGS(dev->nr_luns),
525 sizeof(unsigned long), GFP_KERNEL);
526 if (!dev->lun_map)
527 return -ENOMEM;
cd9e9808 528 INIT_LIST_HEAD(&dev->online_targets);
e3eb3799 529 mutex_init(&dev->mlock);
4c9dacb8 530 spin_lock_init(&dev->lock);
cd9e9808
MB		 531
532 return 0;
533}
534
535static void nvm_free(struct nvm_dev *dev)
536{
537 if (!dev)
538 return;
539
540 if (dev->mt)
541 dev->mt->unregister_mgr(dev);
ca5927e7
MB		 542
543 kfree(dev->lptbl);
cd9e9808
MB		 544}
545
546static int nvm_init(struct nvm_dev *dev)
547{
480fc0db 548 int ret = -EINVAL;
cd9e9808
MB		 549
550 if (!dev->q || !dev->ops)
480fc0db 551 return ret;
cd9e9808 552
16f26c3a 553 if (dev->ops->identity(dev, &dev->identity)) {
cd9e9808 554 pr_err("nvm: device could not be identified\n");
cd9e9808
MB		 555 goto err;
556 }
557
558 pr_debug("nvm: ver:%x nvm_vendor:%x groups:%u\n",
559 dev->identity.ver_id, dev->identity.vmnt,
560 dev->identity.cgrps);
561
562 if (dev->identity.ver_id != 1) {
563 pr_err("nvm: device not supported by kernel.");
564 goto err;
565 }
566
567 if (dev->identity.cgrps != 1) {
568 pr_err("nvm: only one group configuration supported.");
569 goto err;
570 }
571
572 ret = nvm_core_init(dev);
573 if (ret) {
574 pr_err("nvm: could not initialize core structures.\n");
575 goto err;
576 }
577
cd9e9808
MB		 578 pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
579 dev->name, dev->sec_per_pg, dev->nr_planes,
580 dev->pgs_per_blk, dev->blks_per_lun, dev->nr_luns,
581 dev->nr_chnls);
582 return 0;
583err:
cd9e9808
MB		 584 pr_err("nvm: failed to initialize nvm\n");
585 return ret;
586}
587
588static void nvm_exit(struct nvm_dev *dev)
589{
590 if (dev->ppalist_pool)
591 dev->ops->destroy_dma_pool(dev->ppalist_pool);
592 nvm_free(dev);
593
594 pr_info("nvm: successfully unloaded\n");
595}
596
597int nvm_register(struct request_queue *q, char *disk_name,
598 struct nvm_dev_ops *ops)
599{
600 struct nvm_dev *dev;
601 int ret;
602
603 if (!ops->identity)
604 return -EINVAL;
605
606 dev = kzalloc(sizeof(struct nvm_dev), GFP_KERNEL);
607 if (!dev)
608 return -ENOMEM;
609
610 dev->q = q;
611 dev->ops = ops;
612 strncpy(dev->name, disk_name, DISK_NAME_LEN);
613
614 ret = nvm_init(dev);
615 if (ret)
616 goto err_init;
617
d160147b
WT		 618 if (dev->ops->max_phys_sect > 256) {
619 pr_info("nvm: max sectors supported is 256.\n");
620 ret = -EINVAL;
621 goto err_init;
622 }
623
cd9e9808 624 if (dev->ops->max_phys_sect > 1) {
16f26c3a 625 dev->ppalist_pool = dev->ops->create_dma_pool(dev, "ppalist");
cd9e9808
MB		 626 if (!dev->ppalist_pool) {
627 pr_err("nvm: could not create ppa pool\n");
93e70c1f
MB		 628 ret = -ENOMEM;
629 goto err_init;
cd9e9808 630 }
cd9e9808
MB		 631 }
632
bf643185
MB		 633 if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
634 ret = nvm_get_sysblock(dev, &dev->sb);
635 if (!ret)
636 pr_err("nvm: device not initialized.\n");
637 else if (ret < 0)
638 pr_err("nvm: err (%d) on device initialization\n", ret);
639 }
b7692076 640
762796bc 641 /* register device with a supported media manager */
edad2e66 642 down_write(&nvm_lock);
b7692076
MB		 643 if (ret > 0)
644 dev->mt = nvm_init_mgr(dev);
edad2e66
MB		 645 list_add(&dev->devices, &nvm_devices);
646 up_write(&nvm_lock);
647
cd9e9808
MB		 648 return 0;
649err_init:
da1e2849 650 kfree(dev->lun_map);
cd9e9808
MB		 651 kfree(dev);
652 return ret;
653}
654EXPORT_SYMBOL(nvm_register);
655
656void nvm_unregister(char *disk_name)
657{
d0a712ce 658 struct nvm_dev *dev;
cd9e9808 659
d0a712ce
WT		 660 down_write(&nvm_lock);
661 dev = nvm_find_nvm_dev(disk_name);
cd9e9808
MB		 662 if (!dev) {
663 pr_err("nvm: could not find device %s to unregister\n",
664 disk_name);
d0a712ce 665 up_write(&nvm_lock);
cd9e9808
MB		 666 return;
667 }
668
cd9e9808
MB		 669 list_del(&dev->devices);
670 up_write(&nvm_lock);
c1480ad5
MB		 671
672 nvm_exit(dev);
da1e2849 673 kfree(dev->lun_map);
c1480ad5 674 kfree(dev);
cd9e9808
MB		 675}
676EXPORT_SYMBOL(nvm_unregister);
677
678static const struct block_device_operations nvm_fops = {
679 .owner = THIS_MODULE,
680};
681
682static int nvm_create_target(struct nvm_dev *dev,
683 struct nvm_ioctl_create *create)
684{
685 struct nvm_ioctl_create_simple *s = &create->conf.s;
686 struct request_queue *tqueue;
cd9e9808
MB		 687 struct gendisk *tdisk;
688 struct nvm_tgt_type *tt;
689 struct nvm_target *t;
690 void *targetdata;
cd9e9808
MB		 691
692 if (!dev->mt) {
762796bc
MB		 693 pr_info("nvm: device has no media manager registered.\n");
694 return -ENODEV;
cd9e9808
MB		 695 }
696
762796bc 697 down_write(&nvm_lock);
cd9e9808
MB		 698 tt = nvm_find_target_type(create->tgttype);
699 if (!tt) {
700 pr_err("nvm: target type %s not found\n", create->tgttype);
d0a712ce 701 up_write(&nvm_lock);
cd9e9808
MB		 702 return -EINVAL;
703 }
704
cd9e9808
MB		 705 list_for_each_entry(t, &dev->online_targets, list) {
706 if (!strcmp(create->tgtname, t->disk->disk_name)) {
707 pr_err("nvm: target name already exists.\n");
708 up_write(&nvm_lock);
709 return -EINVAL;
710 }
711 }
712 up_write(&nvm_lock);
713
714 t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL);
715 if (!t)
716 return -ENOMEM;
717
718 tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
719 if (!tqueue)
720 goto err_t;
721 blk_queue_make_request(tqueue, tt->make_rq);
722
723 tdisk = alloc_disk(0);
724 if (!tdisk)
725 goto err_queue;
726
727 sprintf(tdisk->disk_name, "%s", create->tgtname);
728 tdisk->flags = GENHD_FL_EXT_DEVT;
729 tdisk->major = 0;
730 tdisk->first_minor = 0;
731 tdisk->fops = &nvm_fops;
732 tdisk->queue = tqueue;
733
734 targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end);
735 if (IS_ERR(targetdata))
736 goto err_init;
737
738 tdisk->private_data = targetdata;
739 tqueue->queuedata = targetdata;
740
741 blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect);
742
743 set_capacity(tdisk, tt->capacity(targetdata));
744 add_disk(tdisk);
745
746 t->type = tt;
747 t->disk = tdisk;
748
749 down_write(&nvm_lock);
750 list_add_tail(&t->list, &dev->online_targets);
751 up_write(&nvm_lock);
752
753 return 0;
754err_init:
755 put_disk(tdisk);
756err_queue:
757 blk_cleanup_queue(tqueue);
758err_t:
759 kfree(t);
760 return -ENOMEM;
761}
762
763static void nvm_remove_target(struct nvm_target *t)
764{
765 struct nvm_tgt_type *tt = t->type;
766 struct gendisk *tdisk = t->disk;
767 struct request_queue *q = tdisk->queue;
768
769 lockdep_assert_held(&nvm_lock);
770
771 del_gendisk(tdisk);
d09f9581
JG		 772 blk_cleanup_queue(q);
773
cd9e9808
MB		 774 if (tt->exit)
775 tt->exit(tdisk->private_data);
776
cd9e9808
MB		 777 put_disk(tdisk);
778
779 list_del(&t->list);
780 kfree(t);
781}
782
783static int __nvm_configure_create(struct nvm_ioctl_create *create)
784{
785 struct nvm_dev *dev;
786 struct nvm_ioctl_create_simple *s;
787
d0a712ce 788 down_write(&nvm_lock);
cd9e9808 789 dev = nvm_find_nvm_dev(create->dev);
d0a712ce 790 up_write(&nvm_lock);
cd9e9808
MB		 791 if (!dev) {
792 pr_err("nvm: device not found\n");
793 return -EINVAL;
794 }
795
796 if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
797 pr_err("nvm: config type not valid\n");
798 return -EINVAL;
799 }
800 s = &create->conf.s;
801
802 if (s->lun_begin > s->lun_end || s->lun_end > dev->nr_luns) {
803 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
804 s->lun_begin, s->lun_end, dev->nr_luns);
805 return -EINVAL;
806 }
807
808 return nvm_create_target(dev, create);
809}
810
811static int __nvm_configure_remove(struct nvm_ioctl_remove *remove)
812{
813 struct nvm_target *t = NULL;
814 struct nvm_dev *dev;
815 int ret = -1;
816
817 down_write(&nvm_lock);
818 list_for_each_entry(dev, &nvm_devices, devices)
819 list_for_each_entry(t, &dev->online_targets, list) {
820 if (!strcmp(remove->tgtname, t->disk->disk_name)) {
821 nvm_remove_target(t);
822 ret = 0;
823 break;
824 }
825 }
826 up_write(&nvm_lock);
827
828 if (ret) {
829 pr_err("nvm: target \"%s\" doesn't exist.\n", remove->tgtname);
830 return -EINVAL;
831 }
832
833 return 0;
834}
835
836#ifdef CONFIG_NVM_DEBUG
837static int nvm_configure_show(const char *val)
838{
839 struct nvm_dev *dev;
840 char opcode, devname[DISK_NAME_LEN];
841 int ret;
842
843 ret = sscanf(val, "%c %32s", &opcode, devname);
844 if (ret != 2) {
845 pr_err("nvm: invalid command. Use \"opcode devicename\".\n");
846 return -EINVAL;
847 }
848
d0a712ce 849 down_write(&nvm_lock);
cd9e9808 850 dev = nvm_find_nvm_dev(devname);
d0a712ce 851 up_write(&nvm_lock);
cd9e9808
MB		 852 if (!dev) {
853 pr_err("nvm: device not found\n");
854 return -EINVAL;
855 }
856
857 if (!dev->mt)
858 return 0;
859
2fde0e48 860 dev->mt->lun_info_print(dev);
cd9e9808
MB		 861
862 return 0;
863}
864
865static int nvm_configure_remove(const char *val)
866{
867 struct nvm_ioctl_remove remove;
868 char opcode;
869 int ret;
870
871 ret = sscanf(val, "%c %256s", &opcode, remove.tgtname);
872 if (ret != 2) {
873 pr_err("nvm: invalid command. Use \"d targetname\".\n");
874 return -EINVAL;
875 }
876
877 remove.flags = 0;
878
879 return __nvm_configure_remove(&remove);
880}
881
882static int nvm_configure_create(const char *val)
883{
884 struct nvm_ioctl_create create;
885 char opcode;
886 int lun_begin, lun_end, ret;
887
888 ret = sscanf(val, "%c %256s %256s %48s %u:%u", &opcode, create.dev,
889 create.tgtname, create.tgttype,
890 &lun_begin, &lun_end);
891 if (ret != 6) {
892 pr_err("nvm: invalid command. Use \"opcode device name tgttype lun_begin:lun_end\".\n");
893 return -EINVAL;
894 }
895
896 create.flags = 0;
897 create.conf.type = NVM_CONFIG_TYPE_SIMPLE;
898 create.conf.s.lun_begin = lun_begin;
899 create.conf.s.lun_end = lun_end;
900
901 return __nvm_configure_create(&create);
902}
903
904
905/* Exposes administrative interface through /sys/module/lnvm/configure_by_str */
906static int nvm_configure_by_str_event(const char *val,
907 const struct kernel_param *kp)
908{
909 char opcode;
910 int ret;
911
912 ret = sscanf(val, "%c", &opcode);
913 if (ret != 1) {
914 pr_err("nvm: string must have the format of \"cmd ...\"\n");
915 return -EINVAL;
916 }
917
918 switch (opcode) {
919 case 'a':
920 return nvm_configure_create(val);
921 case 'd':
922 return nvm_configure_remove(val);
923 case 's':
924 return nvm_configure_show(val);
925 default:
926 pr_err("nvm: invalid command\n");
927 return -EINVAL;
928 }
929
930 return 0;
931}
932
933static int nvm_configure_get(char *buf, const struct kernel_param *kp)
934{
5e422cff 935 int sz;
cd9e9808
MB		 936 struct nvm_dev *dev;
937
5e422cff 938 sz = sprintf(buf, "available devices:\n");
cd9e9808
MB		 939 down_write(&nvm_lock);
940 list_for_each_entry(dev, &nvm_devices, devices) {
5e422cff 941 if (sz > 4095 - DISK_NAME_LEN - 2)
cd9e9808 942 break;
5e422cff 943 sz += sprintf(buf + sz, " %32s\n", dev->name);
cd9e9808
MB		 944 }
945 up_write(&nvm_lock);
946
5e422cff 947 return sz;
cd9e9808
MB		 948}
949
950static const struct kernel_param_ops nvm_configure_by_str_event_param_ops = {
951 .set = nvm_configure_by_str_event,
952 .get = nvm_configure_get,
953};
954
955#undef MODULE_PARAM_PREFIX
956#define MODULE_PARAM_PREFIX "lnvm."
957
958module_param_cb(configure_debug, &nvm_configure_by_str_event_param_ops, NULL,
959 0644);
960
961#endif /* CONFIG_NVM_DEBUG */
962
963static long nvm_ioctl_info(struct file *file, void __user *arg)
964{
965 struct nvm_ioctl_info *info;
966 struct nvm_tgt_type *tt;
967 int tgt_iter = 0;
968
969 if (!capable(CAP_SYS_ADMIN))
970 return -EPERM;
971
972 info = memdup_user(arg, sizeof(struct nvm_ioctl_info));
973 if (IS_ERR(info))
974 return -EFAULT;
975
976 info->version[0] = NVM_VERSION_MAJOR;
977 info->version[1] = NVM_VERSION_MINOR;
978 info->version[2] = NVM_VERSION_PATCH;
979
980 down_write(&nvm_lock);
981 list_for_each_entry(tt, &nvm_targets, list) {
982 struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter];
983
984 tgt->version[0] = tt->version[0];
985 tgt->version[1] = tt->version[1];
986 tgt->version[2] = tt->version[2];
987 strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX);
988
989 tgt_iter++;
990 }
991
992 info->tgtsize = tgt_iter;
993 up_write(&nvm_lock);
994
76e25081
SM		 995 if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) {
996 kfree(info);
cd9e9808 997 return -EFAULT;
76e25081 998 }
cd9e9808
MB		 999
1000 kfree(info);
1001 return 0;
1002}
1003
1004static long nvm_ioctl_get_devices(struct file *file, void __user *arg)
1005{
1006 struct nvm_ioctl_get_devices *devices;
1007 struct nvm_dev *dev;
1008 int i = 0;
1009
1010 if (!capable(CAP_SYS_ADMIN))
1011 return -EPERM;
1012
1013 devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL);
1014 if (!devices)
1015 return -ENOMEM;
1016
1017 down_write(&nvm_lock);
1018 list_for_each_entry(dev, &nvm_devices, devices) {
1019 struct nvm_ioctl_device_info *info = &devices->info[i];
1020
1021 sprintf(info->devname, "%s", dev->name);
1022 if (dev->mt) {
1023 info->bmversion[0] = dev->mt->version[0];
1024 info->bmversion[1] = dev->mt->version[1];
1025 info->bmversion[2] = dev->mt->version[2];
1026 sprintf(info->bmname, "%s", dev->mt->name);
1027 } else {
1028 sprintf(info->bmname, "none");
1029 }
1030
1031 i++;
1032 if (i > 31) {
1033 pr_err("nvm: max 31 devices can be reported.\n");
1034 break;
1035 }
1036 }
1037 up_write(&nvm_lock);
1038
1039 devices->nr_devices = i;
1040
76e25081
SM		 1041 if (copy_to_user(arg, devices,
1042 sizeof(struct nvm_ioctl_get_devices))) {
1043 kfree(devices);
cd9e9808 1044 return -EFAULT;
76e25081 1045 }
cd9e9808
MB		 1046
1047 kfree(devices);
1048 return 0;
1049}
1050
1051static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
1052{
1053 struct nvm_ioctl_create create;
1054
1055 if (!capable(CAP_SYS_ADMIN))
1056 return -EPERM;
1057
1058 if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
1059 return -EFAULT;
1060
1061 create.dev[DISK_NAME_LEN - 1] = '\0';
1062 create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
1063 create.tgtname[DISK_NAME_LEN - 1] = '\0';
1064
1065 if (create.flags != 0) {
1066 pr_err("nvm: no flags supported\n");
1067 return -EINVAL;
1068 }
1069
1070 return __nvm_configure_create(&create);
1071}
1072
1073static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
1074{
1075 struct nvm_ioctl_remove remove;
1076
1077 if (!capable(CAP_SYS_ADMIN))
1078 return -EPERM;
1079
1080 if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove)))
1081 return -EFAULT;
1082
1083 remove.tgtname[DISK_NAME_LEN - 1] = '\0';
1084
1085 if (remove.flags != 0) {
1086 pr_err("nvm: no flags supported\n");
1087 return -EINVAL;
1088 }
1089
1090 return __nvm_configure_remove(&remove);
1091}
1092
55696154
MB		 1093static void nvm_setup_nvm_sb_info(struct nvm_sb_info *info)
1094{
1095 info->seqnr = 1;
1096 info->erase_cnt = 0;
1097 info->version = 1;
1098}
1099
1100static long __nvm_ioctl_dev_init(struct nvm_ioctl_dev_init *init)
1101{
1102 struct nvm_dev *dev;
1103 struct nvm_sb_info info;
b7692076 1104 int ret;
55696154
MB		 1105
1106 down_write(&nvm_lock);
1107 dev = nvm_find_nvm_dev(init->dev);
1108 up_write(&nvm_lock);
1109 if (!dev) {
1110 pr_err("nvm: device not found\n");
1111 return -EINVAL;
1112 }
1113
1114 nvm_setup_nvm_sb_info(&info);
1115
1116 strncpy(info.mmtype, init->mmtype, NVM_MMTYPE_LEN);
1117 info.fs_ppa.ppa = -1;
1118
bf643185
MB		 1119 if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
1120 ret = nvm_init_sysblock(dev, &info);
1121 if (ret)
1122 return ret;
1123 }
b7692076
MB		 1124
1125 memcpy(&dev->sb, &info, sizeof(struct nvm_sb_info));
1126
1127 down_write(&nvm_lock);
1128 dev->mt = nvm_init_mgr(dev);
1129 up_write(&nvm_lock);
1130
1131 return 0;
55696154
MB		 1132}
1133
1134static long nvm_ioctl_dev_init(struct file *file, void __user *arg)
1135{
1136 struct nvm_ioctl_dev_init init;
1137
1138 if (!capable(CAP_SYS_ADMIN))
1139 return -EPERM;
1140
1141 if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init)))
1142 return -EFAULT;
1143
1144 if (init.flags != 0) {
1145 pr_err("nvm: no flags supported\n");
1146 return -EINVAL;
1147 }
1148
1149 init.dev[DISK_NAME_LEN - 1] = '\0';
1150
1151 return __nvm_ioctl_dev_init(&init);
1152}
1153
8b4970c4
MB		 1154static long nvm_ioctl_dev_factory(struct file *file, void __user *arg)
1155{
1156 struct nvm_ioctl_dev_factory fact;
1157 struct nvm_dev *dev;
1158
1159 if (!capable(CAP_SYS_ADMIN))
1160 return -EPERM;
1161
1162 if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory)))
1163 return -EFAULT;
1164
1165 fact.dev[DISK_NAME_LEN - 1] = '\0';
1166
1167 if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1))
1168 return -EINVAL;
1169
1170 down_write(&nvm_lock);
1171 dev = nvm_find_nvm_dev(fact.dev);
1172 up_write(&nvm_lock);
1173 if (!dev) {
1174 pr_err("nvm: device not found\n");
1175 return -EINVAL;
1176 }
1177
1178 if (dev->mt) {
1179 dev->mt->unregister_mgr(dev);
1180 dev->mt = NULL;
1181 }
1182
bf643185
MB		 1183 if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT)
1184 return nvm_dev_factory(dev, fact.flags);
1185
1186 return 0;
8b4970c4
MB		 1187}
1188
cd9e9808
MB		 1189static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
1190{
1191 void __user *argp = (void __user *)arg;
1192
1193 switch (cmd) {
1194 case NVM_INFO:
1195 return nvm_ioctl_info(file, argp);
1196 case NVM_GET_DEVICES:
1197 return nvm_ioctl_get_devices(file, argp);
1198 case NVM_DEV_CREATE:
1199 return nvm_ioctl_dev_create(file, argp);
1200 case NVM_DEV_REMOVE:
1201 return nvm_ioctl_dev_remove(file, argp);
55696154
MB		 1202 case NVM_DEV_INIT:
1203 return nvm_ioctl_dev_init(file, argp);
8b4970c4
MB		 1204 case NVM_DEV_FACTORY:
1205 return nvm_ioctl_dev_factory(file, argp);
cd9e9808
MB		 1206 }
1207 return 0;
1208}
1209
1210static const struct file_operations _ctl_fops = {
1211 .open = nonseekable_open,
1212 .unlocked_ioctl = nvm_ctl_ioctl,
1213 .owner = THIS_MODULE,
1214 .llseek = noop_llseek,
1215};
1216
1217static struct miscdevice _nvm_misc = {
1218 .minor = MISC_DYNAMIC_MINOR,
1219 .name = "lightnvm",
1220 .nodename = "lightnvm/control",
1221 .fops = &_ctl_fops,
1222};
1223
1224MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR);
1225
1226static int __init nvm_mod_init(void)
1227{
1228 int ret;
1229
1230 ret = misc_register(&_nvm_misc);
1231 if (ret)
1232 pr_err("nvm: misc_register failed for control device");
1233
1234 return ret;
1235}
1236
1237static void __exit nvm_mod_exit(void)
1238{
1239 misc_deregister(&_nvm_misc);
1240}
1241
1242MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>");
1243MODULE_LICENSE("GPL v2");
1244MODULE_VERSION("0.1");
1245module_init(nvm_mod_init);
1246module_exit(nvm_mod_exit);
Linux 6.x block layer and io_uring tree(s)