projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
lightnvm: rename nvm_targets to nvm_tgt_type
[linux-2.6-block.git] / drivers / lightnvm / core.c
CommitLineData
cd9e9808
MB		 1/*
2 * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
3 * Initial release: Matias Bjorling <m@bjorling.me>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; see the file COPYING. If not, write to
16 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
17 * USA.
18 *
19 */
20
21#include <linux/blkdev.h>
22#include <linux/blk-mq.h>
23#include <linux/list.h>
24#include <linux/types.h>
25#include <linux/sem.h>
26#include <linux/bitmap.h>
27#include <linux/module.h>
28#include <linux/miscdevice.h>
29#include <linux/lightnvm.h>
91276162 30#include <linux/sched/sysctl.h>
cd9e9808
MB		 31#include <uapi/linux/lightnvm.h>
32
6063fe39 33static LIST_HEAD(nvm_tgt_types);
cd9e9808
MB		 34static LIST_HEAD(nvm_mgrs);
35static LIST_HEAD(nvm_devices);
36static DECLARE_RWSEM(nvm_lock);
37
38static struct nvm_tgt_type *nvm_find_target_type(const char *name)
39{
40 struct nvm_tgt_type *tt;
41
6063fe39 42 list_for_each_entry(tt, &nvm_tgt_types, list)
cd9e9808
MB		 43 if (!strcmp(name, tt->name))
44 return tt;
45
46 return NULL;
47}
48
6063fe39 49int nvm_register_tgt_type(struct nvm_tgt_type *tt)
cd9e9808
MB		 50{
51 int ret = 0;
52
53 down_write(&nvm_lock);
54 if (nvm_find_target_type(tt->name))
55 ret = -EEXIST;
56 else
6063fe39 57 list_add(&tt->list, &nvm_tgt_types);
cd9e9808
MB		 58 up_write(&nvm_lock);
59
60 return ret;
61}
6063fe39 62EXPORT_SYMBOL(nvm_register_tgt_type);
cd9e9808 63
6063fe39 64void nvm_unregister_tgt_type(struct nvm_tgt_type *tt)
cd9e9808
MB		 65{
66 if (!tt)
67 return;
68
69 down_write(&nvm_lock);
70 list_del(&tt->list);
71 up_write(&nvm_lock);
72}
6063fe39 73EXPORT_SYMBOL(nvm_unregister_tgt_type);
cd9e9808
MB		 74
75void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
76 dma_addr_t *dma_handler)
77{
16f26c3a 78 return dev->ops->dev_dma_alloc(dev, dev->ppalist_pool, mem_flags,
cd9e9808
MB		 79 dma_handler);
80}
81EXPORT_SYMBOL(nvm_dev_dma_alloc);
82
83void nvm_dev_dma_free(struct nvm_dev *dev, void *ppa_list,
84 dma_addr_t dma_handler)
85{
86 dev->ops->dev_dma_free(dev->ppalist_pool, ppa_list, dma_handler);
87}
88EXPORT_SYMBOL(nvm_dev_dma_free);
89
90static struct nvmm_type *nvm_find_mgr_type(const char *name)
91{
92 struct nvmm_type *mt;
93
94 list_for_each_entry(mt, &nvm_mgrs, list)
95 if (!strcmp(name, mt->name))
96 return mt;
97
98 return NULL;
99}
100
762796bc
MB		 101struct nvmm_type *nvm_init_mgr(struct nvm_dev *dev)
102{
103 struct nvmm_type *mt;
104 int ret;
105
106 lockdep_assert_held(&nvm_lock);
107
108 list_for_each_entry(mt, &nvm_mgrs, list) {
b7692076
MB		 109 if (strncmp(dev->sb.mmtype, mt->name, NVM_MMTYPE_LEN))
110 continue;
111
762796bc
MB		 112 ret = mt->register_mgr(dev);
113 if (ret < 0) {
114 pr_err("nvm: media mgr failed to init (%d) on dev %s\n",
115 ret, dev->name);
116 return NULL; /* initialization failed */
117 } else if (ret > 0)
118 return mt;
119 }
120
121 return NULL;
122}
123
cd9e9808
MB		 124int nvm_register_mgr(struct nvmm_type *mt)
125{
762796bc 126 struct nvm_dev *dev;
cd9e9808
MB		 127 int ret = 0;
128
129 down_write(&nvm_lock);
762796bc 130 if (nvm_find_mgr_type(mt->name)) {
cd9e9808 131 ret = -EEXIST;
762796bc
MB		 132 goto finish;
133 } else {
cd9e9808 134 list_add(&mt->list, &nvm_mgrs);
762796bc
MB		 135 }
136
137 /* try to register media mgr if any device have none configured */
138 list_for_each_entry(dev, &nvm_devices, devices) {
139 if (dev->mt)
140 continue;
141
142 dev->mt = nvm_init_mgr(dev);
143 }
144finish:
cd9e9808
MB		 145 up_write(&nvm_lock);
146
147 return ret;
148}
149EXPORT_SYMBOL(nvm_register_mgr);
150
151void nvm_unregister_mgr(struct nvmm_type *mt)
152{
153 if (!mt)
154 return;
155
156 down_write(&nvm_lock);
157 list_del(&mt->list);
158 up_write(&nvm_lock);
159}
160EXPORT_SYMBOL(nvm_unregister_mgr);
161
162static struct nvm_dev *nvm_find_nvm_dev(const char *name)
163{
164 struct nvm_dev *dev;
165
166 list_for_each_entry(dev, &nvm_devices, devices)
167 if (!strcmp(name, dev->name))
168 return dev;
169
170 return NULL;
171}
172
ff0e498b
JG		 173struct nvm_block *nvm_get_blk_unlocked(struct nvm_dev *dev, struct nvm_lun *lun,
174 unsigned long flags)
175{
176 return dev->mt->get_blk_unlocked(dev, lun, flags);
177}
178EXPORT_SYMBOL(nvm_get_blk_unlocked);
179
180/* Assumes that all valid pages have already been moved on release to bm */
181void nvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk)
182{
183 return dev->mt->put_blk_unlocked(dev, blk);
184}
185EXPORT_SYMBOL(nvm_put_blk_unlocked);
186
cd9e9808
MB		 187struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun,
188 unsigned long flags)
189{
190 return dev->mt->get_blk(dev, lun, flags);
191}
192EXPORT_SYMBOL(nvm_get_blk);
193
194/* Assumes that all valid pages have already been moved on release to bm */
195void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
196{
197 return dev->mt->put_blk(dev, blk);
198}
199EXPORT_SYMBOL(nvm_put_blk);
200
201int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
202{
203 return dev->mt->submit_io(dev, rqd);
204}
205EXPORT_SYMBOL(nvm_submit_io);
206
207int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk)
208{
209 return dev->mt->erase_blk(dev, blk, 0);
210}
211EXPORT_SYMBOL(nvm_erase_blk);
212
069368e9
MB		 213void nvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
214{
215 int i;
216
217 if (rqd->nr_pages > 1) {
218 for (i = 0; i < rqd->nr_pages; i++)
219 rqd->ppa_list[i] = dev_to_generic_addr(dev,
220 rqd->ppa_list[i]);
221 } else {
222 rqd->ppa_addr = dev_to_generic_addr(dev, rqd->ppa_addr);
223 }
224}
225EXPORT_SYMBOL(nvm_addr_to_generic_mode);
226
227void nvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
228{
229 int i;
230
231 if (rqd->nr_pages > 1) {
232 for (i = 0; i < rqd->nr_pages; i++)
233 rqd->ppa_list[i] = generic_to_dev_addr(dev,
234 rqd->ppa_list[i]);
235 } else {
236 rqd->ppa_addr = generic_to_dev_addr(dev, rqd->ppa_addr);
237 }
238}
239EXPORT_SYMBOL(nvm_generic_to_addr_mode);
240
abd805ec
MB		 241int nvm_set_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd,
242 struct ppa_addr *ppas, int nr_ppas)
069368e9 243{
abd805ec
MB		 244 int i, plane_cnt, pl_idx;
245
246 if (dev->plane_mode == NVM_PLANE_SINGLE && nr_ppas == 1) {
247 rqd->nr_pages = 1;
248 rqd->ppa_addr = ppas[0];
069368e9 249
069368e9 250 return 0;
abd805ec 251 }
069368e9 252
d5bdec8d 253 plane_cnt = dev->plane_mode;
abd805ec 254 rqd->nr_pages = plane_cnt * nr_ppas;
069368e9 255
abd805ec
MB		 256 if (dev->ops->max_phys_sect < rqd->nr_pages)
257 return -EINVAL;
258
259 rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list);
260 if (!rqd->ppa_list) {
261 pr_err("nvm: failed to allocate dma memory\n");
262 return -ENOMEM;
263 }
264
556755e9
MB		 265 for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
266 for (i = 0; i < nr_ppas; i++) {
abd805ec 267 ppas[i].g.pl = pl_idx;
556755e9 268 rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppas[i];
069368e9
MB		 269 }
270 }
271
abd805ec
MB		 272 return 0;
273}
274EXPORT_SYMBOL(nvm_set_rqd_ppalist);
275
276void nvm_free_rqd_ppalist(struct nvm_dev *dev, struct nvm_rq *rqd)
277{
278 if (!rqd->ppa_list)
279 return;
280
281 nvm_dev_dma_free(dev, rqd->ppa_list, rqd->dma_ppa_list);
282}
283EXPORT_SYMBOL(nvm_free_rqd_ppalist);
284
81e681d3 285int nvm_erase_ppa(struct nvm_dev *dev, struct ppa_addr *ppas, int nr_ppas)
abd805ec
MB		 286{
287 struct nvm_rq rqd;
288 int ret;
289
290 if (!dev->ops->erase_block)
291 return 0;
292
293 memset(&rqd, 0, sizeof(struct nvm_rq));
294
81e681d3 295 ret = nvm_set_rqd_ppalist(dev, &rqd, ppas, nr_ppas);
abd805ec
MB		 296 if (ret)
297 return ret;
298
069368e9
MB		 299 nvm_generic_to_addr_mode(dev, &rqd);
300
301 ret = dev->ops->erase_block(dev, &rqd);
302
abd805ec 303 nvm_free_rqd_ppalist(dev, &rqd);
069368e9
MB		 304
305 return ret;
306}
307EXPORT_SYMBOL(nvm_erase_ppa);
308
91276162
MB		 309void nvm_end_io(struct nvm_rq *rqd, int error)
310{
72d256ec
MB		 311 rqd->error = error;
312 rqd->end_io(rqd);
91276162
MB		 313}
314EXPORT_SYMBOL(nvm_end_io);
315
09719b62 316static void nvm_end_io_sync(struct nvm_rq *rqd)
91276162
MB		 317{
318 struct completion *waiting = rqd->wait;
319
320 rqd->wait = NULL;
321
322 complete(waiting);
323}
324
1145e635
MB		 325int __nvm_submit_ppa(struct nvm_dev *dev, struct nvm_rq *rqd, int opcode,
326 int flags, void *buf, int len)
09719b62
MB		 327{
328 DECLARE_COMPLETION_ONSTACK(wait);
09719b62
MB		 329 struct bio *bio;
330 int ret;
331 unsigned long hang_check;
332
333 bio = bio_map_kern(dev->q, buf, len, GFP_KERNEL);
334 if (IS_ERR_OR_NULL(bio))
335 return -ENOMEM;
336
1145e635 337 nvm_generic_to_addr_mode(dev, rqd);
09719b62 338
1145e635
MB		 339 rqd->dev = dev;
340 rqd->opcode = opcode;
341 rqd->flags = flags;
342 rqd->bio = bio;
343 rqd->wait = &wait;
344 rqd->end_io = nvm_end_io_sync;
09719b62 345
1145e635 346 ret = dev->ops->submit_io(dev, rqd);
ecfb40c6 347 if (ret) {
ecfb40c6
MB		 348 bio_put(bio);
349 return ret;
350 }
09719b62
MB		 351
352 /* Prevent hang_check timer from firing at us during very long I/O */
353 hang_check = sysctl_hung_task_timeout_secs;
354 if (hang_check)
355 while (!wait_for_completion_io_timeout(&wait, hang_check * (HZ/2)));
356 else
357 wait_for_completion_io(&wait);
358
1145e635
MB		 359 return rqd->error;
360}
361
362/**
363 * nvm_submit_ppa_list - submit user-defined ppa list to device. The user must
364 * take to free ppa list if necessary.
365 * @dev: device
366 * @ppa_list: user created ppa_list
367 * @nr_ppas: length of ppa_list
368 * @opcode: device opcode
369 * @flags: device flags
370 * @buf: data buffer
371 * @len: data buffer length
372 */
373int nvm_submit_ppa_list(struct nvm_dev *dev, struct ppa_addr *ppa_list,
374 int nr_ppas, int opcode, int flags, void *buf, int len)
375{
376 struct nvm_rq rqd;
377
378 if (dev->ops->max_phys_sect < nr_ppas)
379 return -EINVAL;
380
381 memset(&rqd, 0, sizeof(struct nvm_rq));
382
383 rqd.nr_pages = nr_ppas;
384 if (nr_ppas > 1)
385 rqd.ppa_list = ppa_list;
386 else
387 rqd.ppa_addr = ppa_list[0];
388
389 return __nvm_submit_ppa(dev, &rqd, opcode, flags, buf, len);
390}
391EXPORT_SYMBOL(nvm_submit_ppa_list);
392
393/**
394 * nvm_submit_ppa - submit PPAs to device. PPAs will automatically be unfolded
395 * as single, dual, quad plane PPAs depending on device type.
396 * @dev: device
397 * @ppa: user created ppa_list
398 * @nr_ppas: length of ppa_list
399 * @opcode: device opcode
400 * @flags: device flags
401 * @buf: data buffer
402 * @len: data buffer length
403 */
404int nvm_submit_ppa(struct nvm_dev *dev, struct ppa_addr *ppa, int nr_ppas,
405 int opcode, int flags, void *buf, int len)
406{
407 struct nvm_rq rqd;
408 int ret;
409
410 memset(&rqd, 0, sizeof(struct nvm_rq));
411 ret = nvm_set_rqd_ppalist(dev, &rqd, ppa, nr_ppas);
412 if (ret)
413 return ret;
414
415 ret = __nvm_submit_ppa(dev, &rqd, opcode, flags, buf, len);
416
09719b62
MB		 417 nvm_free_rqd_ppalist(dev, &rqd);
418
1145e635 419 return ret;
09719b62
MB		 420}
421EXPORT_SYMBOL(nvm_submit_ppa);
422
22e8c976
MB		 423/*
424 * folds a bad block list from its plane representation to its virtual
425 * block representation. The fold is done in place and reduced size is
426 * returned.
427 *
428 * If any of the planes status are bad or grown bad block, the virtual block
429 * is marked bad. If not bad, the first plane state acts as the block state.
430 */
431int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
432{
433 int blk, offset, pl, blktype;
434
435 if (nr_blks != dev->blks_per_lun * dev->plane_mode)
436 return -EINVAL;
437
438 for (blk = 0; blk < dev->blks_per_lun; blk++) {
439 offset = blk * dev->plane_mode;
440 blktype = blks[offset];
441
442 /* Bad blocks on any planes take precedence over other types */
443 for (pl = 0; pl < dev->plane_mode; pl++) {
444 if (blks[offset + pl] &
445 (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) {
446 blktype = blks[offset + pl];
447 break;
448 }
449 }
450
451 blks[blk] = blktype;
452 }
453
454 return dev->blks_per_lun;
455}
456EXPORT_SYMBOL(nvm_bb_tbl_fold);
457
ca5927e7
MB		 458static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
459{
460 int i;
461
462 dev->lps_per_blk = dev->pgs_per_blk;
463 dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
464 if (!dev->lptbl)
465 return -ENOMEM;
466
467 /* Just a linear array */
468 for (i = 0; i < dev->lps_per_blk; i++)
469 dev->lptbl[i] = i;
470
471 return 0;
472}
473
474static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
475{
476 int i, p;
477 struct nvm_id_lp_mlc *mlc = &grp->lptbl.mlc;
478
479 if (!mlc->num_pairs)
480 return 0;
481
482 dev->lps_per_blk = mlc->num_pairs;
483 dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
484 if (!dev->lptbl)
485 return -ENOMEM;
486
487 /* The lower page table encoding consists of a list of bytes, where each
488 * has a lower and an upper half. The first half byte maintains the
489 * increment value and every value after is an offset added to the
490 * previous incrementation value */
491 dev->lptbl[0] = mlc->pairs[0] & 0xF;
492 for (i = 1; i < dev->lps_per_blk; i++) {
493 p = mlc->pairs[i >> 1];
494 if (i & 0x1) /* upper */
495 dev->lptbl[i] = dev->lptbl[i - 1] + ((p & 0xF0) >> 4);
496 else /* lower */
497 dev->lptbl[i] = dev->lptbl[i - 1] + (p & 0xF);
498 }
499
500 return 0;
501}
502
cd9e9808
MB		 503static int nvm_core_init(struct nvm_dev *dev)
504{
505 struct nvm_id *id = &dev->identity;
506 struct nvm_id_group *grp = &id->groups[0];
7f7c5d03 507 int ret;
cd9e9808
MB		 508
509 /* device values */
510 dev->nr_chnls = grp->num_ch;
511 dev->luns_per_chnl = grp->num_lun;
512 dev->pgs_per_blk = grp->num_pg;
513 dev->blks_per_lun = grp->num_blk;
514 dev->nr_planes = grp->num_pln;
4891d120
MB		 515 dev->fpg_size = grp->fpg_sz;
516 dev->pfpg_size = grp->fpg_sz * grp->num_pln;
cd9e9808
MB		 517 dev->sec_size = grp->csecs;
518 dev->oob_size = grp->sos;
519 dev->sec_per_pg = grp->fpg_sz / grp->csecs;
f9a99950 520 dev->mccap = grp->mccap;
7386af27 521 memcpy(&dev->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
cd9e9808
MB		 522
523 dev->plane_mode = NVM_PLANE_SINGLE;
524 dev->max_rq_size = dev->ops->max_phys_sect * dev->sec_size;
525
526 if (grp->mpos & 0x020202)
527 dev->plane_mode = NVM_PLANE_DOUBLE;
528 if (grp->mpos & 0x040404)
529 dev->plane_mode = NVM_PLANE_QUAD;
530
7f7c5d03
MB		 531 if (grp->mtype != 0) {
532 pr_err("nvm: memory type not supported\n");
533 return -EINVAL;
534 }
535
cd9e9808
MB		 536 /* calculated values */
537 dev->sec_per_pl = dev->sec_per_pg * dev->nr_planes;
538 dev->sec_per_blk = dev->sec_per_pl * dev->pgs_per_blk;
539 dev->sec_per_lun = dev->sec_per_blk * dev->blks_per_lun;
540 dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls;
541
ed2a92a6 542 dev->total_secs = dev->nr_luns * dev->sec_per_lun;
da1e2849
WT		 543 dev->lun_map = kcalloc(BITS_TO_LONGS(dev->nr_luns),
544 sizeof(unsigned long), GFP_KERNEL);
545 if (!dev->lun_map)
546 return -ENOMEM;
7f7c5d03
MB		 547
548 switch (grp->fmtype) {
549 case NVM_ID_FMTYPE_SLC:
550 if (nvm_init_slc_tbl(dev, grp)) {
551 ret = -ENOMEM;
552 goto err_fmtype;
553 }
554 break;
555 case NVM_ID_FMTYPE_MLC:
556 if (nvm_init_mlc_tbl(dev, grp)) {
557 ret = -ENOMEM;
558 goto err_fmtype;
559 }
560 break;
561 default:
562 pr_err("nvm: flash type not supported\n");
563 ret = -EINVAL;
564 goto err_fmtype;
565 }
566
cd9e9808 567 INIT_LIST_HEAD(&dev->online_targets);
e3eb3799 568 mutex_init(&dev->mlock);
4c9dacb8 569 spin_lock_init(&dev->lock);
cd9e9808
MB		 570
571 return 0;
7f7c5d03
MB		 572err_fmtype:
573 kfree(dev->lun_map);
574 return ret;
cd9e9808
MB		 575}
576
577static void nvm_free(struct nvm_dev *dev)
578{
579 if (!dev)
580 return;
581
582 if (dev->mt)
583 dev->mt->unregister_mgr(dev);
ca5927e7
MB		 584
585 kfree(dev->lptbl);
7f7c5d03 586 kfree(dev->lun_map);
cd9e9808
MB		 587}
588
589static int nvm_init(struct nvm_dev *dev)
590{
480fc0db 591 int ret = -EINVAL;
cd9e9808
MB		 592
593 if (!dev->q || !dev->ops)
480fc0db 594 return ret;
cd9e9808 595
16f26c3a 596 if (dev->ops->identity(dev, &dev->identity)) {
cd9e9808 597 pr_err("nvm: device could not be identified\n");
cd9e9808
MB		 598 goto err;
599 }
600
601 pr_debug("nvm: ver:%x nvm_vendor:%x groups:%u\n",
602 dev->identity.ver_id, dev->identity.vmnt,
603 dev->identity.cgrps);
604
605 if (dev->identity.ver_id != 1) {
606 pr_err("nvm: device not supported by kernel.");
607 goto err;
608 }
609
610 if (dev->identity.cgrps != 1) {
611 pr_err("nvm: only one group configuration supported.");
612 goto err;
613 }
614
615 ret = nvm_core_init(dev);
616 if (ret) {
617 pr_err("nvm: could not initialize core structures.\n");
618 goto err;
619 }
620
cd9e9808
MB		 621 pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
622 dev->name, dev->sec_per_pg, dev->nr_planes,
623 dev->pgs_per_blk, dev->blks_per_lun, dev->nr_luns,
624 dev->nr_chnls);
625 return 0;
626err:
cd9e9808
MB		 627 pr_err("nvm: failed to initialize nvm\n");
628 return ret;
629}
630
631static void nvm_exit(struct nvm_dev *dev)
632{
633 if (dev->ppalist_pool)
634 dev->ops->destroy_dma_pool(dev->ppalist_pool);
635 nvm_free(dev);
636
637 pr_info("nvm: successfully unloaded\n");
638}
639
640int nvm_register(struct request_queue *q, char *disk_name,
641 struct nvm_dev_ops *ops)
642{
643 struct nvm_dev *dev;
644 int ret;
645
646 if (!ops->identity)
647 return -EINVAL;
648
649 dev = kzalloc(sizeof(struct nvm_dev), GFP_KERNEL);
650 if (!dev)
651 return -ENOMEM;
652
653 dev->q = q;
654 dev->ops = ops;
655 strncpy(dev->name, disk_name, DISK_NAME_LEN);
656
657 ret = nvm_init(dev);
658 if (ret)
659 goto err_init;
660
d160147b
WT		 661 if (dev->ops->max_phys_sect > 256) {
662 pr_info("nvm: max sectors supported is 256.\n");
663 ret = -EINVAL;
664 goto err_init;
665 }
666
cd9e9808 667 if (dev->ops->max_phys_sect > 1) {
16f26c3a 668 dev->ppalist_pool = dev->ops->create_dma_pool(dev, "ppalist");
cd9e9808
MB		 669 if (!dev->ppalist_pool) {
670 pr_err("nvm: could not create ppa pool\n");
93e70c1f
MB		 671 ret = -ENOMEM;
672 goto err_init;
cd9e9808 673 }
cd9e9808
MB		 674 }
675
bf643185
MB		 676 if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
677 ret = nvm_get_sysblock(dev, &dev->sb);
678 if (!ret)
679 pr_err("nvm: device not initialized.\n");
680 else if (ret < 0)
681 pr_err("nvm: err (%d) on device initialization\n", ret);
682 }
b7692076 683
762796bc 684 /* register device with a supported media manager */
edad2e66 685 down_write(&nvm_lock);
b7692076
MB		 686 if (ret > 0)
687 dev->mt = nvm_init_mgr(dev);
edad2e66
MB		 688 list_add(&dev->devices, &nvm_devices);
689 up_write(&nvm_lock);
690
cd9e9808
MB		 691 return 0;
692err_init:
da1e2849 693 kfree(dev->lun_map);
cd9e9808
MB		 694 kfree(dev);
695 return ret;
696}
697EXPORT_SYMBOL(nvm_register);
698
699void nvm_unregister(char *disk_name)
700{
d0a712ce 701 struct nvm_dev *dev;
cd9e9808 702
d0a712ce
WT		 703 down_write(&nvm_lock);
704 dev = nvm_find_nvm_dev(disk_name);
cd9e9808
MB		 705 if (!dev) {
706 pr_err("nvm: could not find device %s to unregister\n",
707 disk_name);
d0a712ce 708 up_write(&nvm_lock);
cd9e9808
MB		 709 return;
710 }
711
cd9e9808
MB		 712 list_del(&dev->devices);
713 up_write(&nvm_lock);
c1480ad5
MB		 714
715 nvm_exit(dev);
716 kfree(dev);
cd9e9808
MB		 717}
718EXPORT_SYMBOL(nvm_unregister);
719
720static const struct block_device_operations nvm_fops = {
721 .owner = THIS_MODULE,
722};
723
724static int nvm_create_target(struct nvm_dev *dev,
725 struct nvm_ioctl_create *create)
726{
727 struct nvm_ioctl_create_simple *s = &create->conf.s;
728 struct request_queue *tqueue;
cd9e9808
MB		 729 struct gendisk *tdisk;
730 struct nvm_tgt_type *tt;
731 struct nvm_target *t;
732 void *targetdata;
cd9e9808
MB		 733
734 if (!dev->mt) {
762796bc
MB		 735 pr_info("nvm: device has no media manager registered.\n");
736 return -ENODEV;
cd9e9808
MB		 737 }
738
762796bc 739 down_write(&nvm_lock);
cd9e9808
MB		 740 tt = nvm_find_target_type(create->tgttype);
741 if (!tt) {
742 pr_err("nvm: target type %s not found\n", create->tgttype);
d0a712ce 743 up_write(&nvm_lock);
cd9e9808
MB		 744 return -EINVAL;
745 }
746
cd9e9808
MB		 747 list_for_each_entry(t, &dev->online_targets, list) {
748 if (!strcmp(create->tgtname, t->disk->disk_name)) {
749 pr_err("nvm: target name already exists.\n");
750 up_write(&nvm_lock);
751 return -EINVAL;
752 }
753 }
754 up_write(&nvm_lock);
755
756 t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL);
757 if (!t)
758 return -ENOMEM;
759
760 tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
761 if (!tqueue)
762 goto err_t;
763 blk_queue_make_request(tqueue, tt->make_rq);
764
765 tdisk = alloc_disk(0);
766 if (!tdisk)
767 goto err_queue;
768
769 sprintf(tdisk->disk_name, "%s", create->tgtname);
770 tdisk->flags = GENHD_FL_EXT_DEVT;
771 tdisk->major = 0;
772 tdisk->first_minor = 0;
773 tdisk->fops = &nvm_fops;
774 tdisk->queue = tqueue;
775
776 targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end);
777 if (IS_ERR(targetdata))
778 goto err_init;
779
780 tdisk->private_data = targetdata;
781 tqueue->queuedata = targetdata;
782
783 blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect);
784
785 set_capacity(tdisk, tt->capacity(targetdata));
786 add_disk(tdisk);
787
788 t->type = tt;
789 t->disk = tdisk;
790
791 down_write(&nvm_lock);
792 list_add_tail(&t->list, &dev->online_targets);
793 up_write(&nvm_lock);
794
795 return 0;
796err_init:
797 put_disk(tdisk);
798err_queue:
799 blk_cleanup_queue(tqueue);
800err_t:
801 kfree(t);
802 return -ENOMEM;
803}
804
805static void nvm_remove_target(struct nvm_target *t)
806{
807 struct nvm_tgt_type *tt = t->type;
808 struct gendisk *tdisk = t->disk;
809 struct request_queue *q = tdisk->queue;
810
811 lockdep_assert_held(&nvm_lock);
812
813 del_gendisk(tdisk);
d09f9581
JG		 814 blk_cleanup_queue(q);
815
cd9e9808
MB		 816 if (tt->exit)
817 tt->exit(tdisk->private_data);
818
cd9e9808
MB		 819 put_disk(tdisk);
820
821 list_del(&t->list);
822 kfree(t);
823}
824
825static int __nvm_configure_create(struct nvm_ioctl_create *create)
826{
827 struct nvm_dev *dev;
828 struct nvm_ioctl_create_simple *s;
829
d0a712ce 830 down_write(&nvm_lock);
cd9e9808 831 dev = nvm_find_nvm_dev(create->dev);
d0a712ce 832 up_write(&nvm_lock);
cd9e9808
MB		 833 if (!dev) {
834 pr_err("nvm: device not found\n");
835 return -EINVAL;
836 }
837
838 if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
839 pr_err("nvm: config type not valid\n");
840 return -EINVAL;
841 }
842 s = &create->conf.s;
843
844 if (s->lun_begin > s->lun_end || s->lun_end > dev->nr_luns) {
845 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
846 s->lun_begin, s->lun_end, dev->nr_luns);
847 return -EINVAL;
848 }
849
850 return nvm_create_target(dev, create);
851}
852
853static int __nvm_configure_remove(struct nvm_ioctl_remove *remove)
854{
855 struct nvm_target *t = NULL;
856 struct nvm_dev *dev;
857 int ret = -1;
858
859 down_write(&nvm_lock);
860 list_for_each_entry(dev, &nvm_devices, devices)
861 list_for_each_entry(t, &dev->online_targets, list) {
862 if (!strcmp(remove->tgtname, t->disk->disk_name)) {
863 nvm_remove_target(t);
864 ret = 0;
865 break;
866 }
867 }
868 up_write(&nvm_lock);
869
870 if (ret) {
871 pr_err("nvm: target \"%s\" doesn't exist.\n", remove->tgtname);
872 return -EINVAL;
873 }
874
875 return 0;
876}
877
878#ifdef CONFIG_NVM_DEBUG
879static int nvm_configure_show(const char *val)
880{
881 struct nvm_dev *dev;
882 char opcode, devname[DISK_NAME_LEN];
883 int ret;
884
885 ret = sscanf(val, "%c %32s", &opcode, devname);
886 if (ret != 2) {
887 pr_err("nvm: invalid command. Use \"opcode devicename\".\n");
888 return -EINVAL;
889 }
890
d0a712ce 891 down_write(&nvm_lock);
cd9e9808 892 dev = nvm_find_nvm_dev(devname);
d0a712ce 893 up_write(&nvm_lock);
cd9e9808
MB		 894 if (!dev) {
895 pr_err("nvm: device not found\n");
896 return -EINVAL;
897 }
898
899 if (!dev->mt)
900 return 0;
901
2fde0e48 902 dev->mt->lun_info_print(dev);
cd9e9808
MB		 903
904 return 0;
905}
906
907static int nvm_configure_remove(const char *val)
908{
909 struct nvm_ioctl_remove remove;
910 char opcode;
911 int ret;
912
913 ret = sscanf(val, "%c %256s", &opcode, remove.tgtname);
914 if (ret != 2) {
915 pr_err("nvm: invalid command. Use \"d targetname\".\n");
916 return -EINVAL;
917 }
918
919 remove.flags = 0;
920
921 return __nvm_configure_remove(&remove);
922}
923
924static int nvm_configure_create(const char *val)
925{
926 struct nvm_ioctl_create create;
927 char opcode;
928 int lun_begin, lun_end, ret;
929
930 ret = sscanf(val, "%c %256s %256s %48s %u:%u", &opcode, create.dev,
931 create.tgtname, create.tgttype,
932 &lun_begin, &lun_end);
933 if (ret != 6) {
934 pr_err("nvm: invalid command. Use \"opcode device name tgttype lun_begin:lun_end\".\n");
935 return -EINVAL;
936 }
937
938 create.flags = 0;
939 create.conf.type = NVM_CONFIG_TYPE_SIMPLE;
940 create.conf.s.lun_begin = lun_begin;
941 create.conf.s.lun_end = lun_end;
942
943 return __nvm_configure_create(&create);
944}
945
946
947/* Exposes administrative interface through /sys/module/lnvm/configure_by_str */
948static int nvm_configure_by_str_event(const char *val,
949 const struct kernel_param *kp)
950{
951 char opcode;
952 int ret;
953
954 ret = sscanf(val, "%c", &opcode);
955 if (ret != 1) {
956 pr_err("nvm: string must have the format of \"cmd ...\"\n");
957 return -EINVAL;
958 }
959
960 switch (opcode) {
961 case 'a':
962 return nvm_configure_create(val);
963 case 'd':
964 return nvm_configure_remove(val);
965 case 's':
966 return nvm_configure_show(val);
967 default:
968 pr_err("nvm: invalid command\n");
969 return -EINVAL;
970 }
971
972 return 0;
973}
974
975static int nvm_configure_get(char *buf, const struct kernel_param *kp)
976{
5e422cff 977 int sz;
cd9e9808
MB		 978 struct nvm_dev *dev;
979
5e422cff 980 sz = sprintf(buf, "available devices:\n");
cd9e9808
MB		 981 down_write(&nvm_lock);
982 list_for_each_entry(dev, &nvm_devices, devices) {
5e422cff 983 if (sz > 4095 - DISK_NAME_LEN - 2)
cd9e9808 984 break;
5e422cff 985 sz += sprintf(buf + sz, " %32s\n", dev->name);
cd9e9808
MB		 986 }
987 up_write(&nvm_lock);
988
5e422cff 989 return sz;
cd9e9808
MB		 990}
991
992static const struct kernel_param_ops nvm_configure_by_str_event_param_ops = {
993 .set = nvm_configure_by_str_event,
994 .get = nvm_configure_get,
995};
996
997#undef MODULE_PARAM_PREFIX
998#define MODULE_PARAM_PREFIX "lnvm."
999
1000module_param_cb(configure_debug, &nvm_configure_by_str_event_param_ops, NULL,
1001 0644);
1002
1003#endif /* CONFIG_NVM_DEBUG */
1004
1005static long nvm_ioctl_info(struct file *file, void __user *arg)
1006{
1007 struct nvm_ioctl_info *info;
1008 struct nvm_tgt_type *tt;
1009 int tgt_iter = 0;
1010
1011 if (!capable(CAP_SYS_ADMIN))
1012 return -EPERM;
1013
1014 info = memdup_user(arg, sizeof(struct nvm_ioctl_info));
1015 if (IS_ERR(info))
1016 return -EFAULT;
1017
1018 info->version[0] = NVM_VERSION_MAJOR;
1019 info->version[1] = NVM_VERSION_MINOR;
1020 info->version[2] = NVM_VERSION_PATCH;
1021
1022 down_write(&nvm_lock);
6063fe39 1023 list_for_each_entry(tt, &nvm_tgt_types, list) {
cd9e9808
MB		 1024 struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter];
1025
1026 tgt->version[0] = tt->version[0];
1027 tgt->version[1] = tt->version[1];
1028 tgt->version[2] = tt->version[2];
1029 strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX);
1030
1031 tgt_iter++;
1032 }
1033
1034 info->tgtsize = tgt_iter;
1035 up_write(&nvm_lock);
1036
76e25081
SM		 1037 if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) {
1038 kfree(info);
cd9e9808 1039 return -EFAULT;
76e25081 1040 }
cd9e9808
MB		 1041
1042 kfree(info);
1043 return 0;
1044}
1045
1046static long nvm_ioctl_get_devices(struct file *file, void __user *arg)
1047{
1048 struct nvm_ioctl_get_devices *devices;
1049 struct nvm_dev *dev;
1050 int i = 0;
1051
1052 if (!capable(CAP_SYS_ADMIN))
1053 return -EPERM;
1054
1055 devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL);
1056 if (!devices)
1057 return -ENOMEM;
1058
1059 down_write(&nvm_lock);
1060 list_for_each_entry(dev, &nvm_devices, devices) {
1061 struct nvm_ioctl_device_info *info = &devices->info[i];
1062
1063 sprintf(info->devname, "%s", dev->name);
1064 if (dev->mt) {
1065 info->bmversion[0] = dev->mt->version[0];
1066 info->bmversion[1] = dev->mt->version[1];
1067 info->bmversion[2] = dev->mt->version[2];
1068 sprintf(info->bmname, "%s", dev->mt->name);
1069 } else {
1070 sprintf(info->bmname, "none");
1071 }
1072
1073 i++;
1074 if (i > 31) {
1075 pr_err("nvm: max 31 devices can be reported.\n");
1076 break;
1077 }
1078 }
1079 up_write(&nvm_lock);
1080
1081 devices->nr_devices = i;
1082
76e25081
SM		 1083 if (copy_to_user(arg, devices,
1084 sizeof(struct nvm_ioctl_get_devices))) {
1085 kfree(devices);
cd9e9808 1086 return -EFAULT;
76e25081 1087 }
cd9e9808
MB		 1088
1089 kfree(devices);
1090 return 0;
1091}
1092
1093static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
1094{
1095 struct nvm_ioctl_create create;
1096
1097 if (!capable(CAP_SYS_ADMIN))
1098 return -EPERM;
1099
1100 if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
1101 return -EFAULT;
1102
1103 create.dev[DISK_NAME_LEN - 1] = '\0';
1104 create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
1105 create.tgtname[DISK_NAME_LEN - 1] = '\0';
1106
1107 if (create.flags != 0) {
1108 pr_err("nvm: no flags supported\n");
1109 return -EINVAL;
1110 }
1111
1112 return __nvm_configure_create(&create);
1113}
1114
1115static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
1116{
1117 struct nvm_ioctl_remove remove;
1118
1119 if (!capable(CAP_SYS_ADMIN))
1120 return -EPERM;
1121
1122 if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove)))
1123 return -EFAULT;
1124
1125 remove.tgtname[DISK_NAME_LEN - 1] = '\0';
1126
1127 if (remove.flags != 0) {
1128 pr_err("nvm: no flags supported\n");
1129 return -EINVAL;
1130 }
1131
1132 return __nvm_configure_remove(&remove);
1133}
1134
55696154
MB		 1135static void nvm_setup_nvm_sb_info(struct nvm_sb_info *info)
1136{
1137 info->seqnr = 1;
1138 info->erase_cnt = 0;
1139 info->version = 1;
1140}
1141
1142static long __nvm_ioctl_dev_init(struct nvm_ioctl_dev_init *init)
1143{
1144 struct nvm_dev *dev;
1145 struct nvm_sb_info info;
b7692076 1146 int ret;
55696154
MB		 1147
1148 down_write(&nvm_lock);
1149 dev = nvm_find_nvm_dev(init->dev);
1150 up_write(&nvm_lock);
1151 if (!dev) {
1152 pr_err("nvm: device not found\n");
1153 return -EINVAL;
1154 }
1155
1156 nvm_setup_nvm_sb_info(&info);
1157
1158 strncpy(info.mmtype, init->mmtype, NVM_MMTYPE_LEN);
1159 info.fs_ppa.ppa = -1;
1160
bf643185
MB		 1161 if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT) {
1162 ret = nvm_init_sysblock(dev, &info);
1163 if (ret)
1164 return ret;
1165 }
b7692076
MB		 1166
1167 memcpy(&dev->sb, &info, sizeof(struct nvm_sb_info));
1168
1169 down_write(&nvm_lock);
1170 dev->mt = nvm_init_mgr(dev);
1171 up_write(&nvm_lock);
1172
1173 return 0;
55696154
MB		 1174}
1175
1176static long nvm_ioctl_dev_init(struct file *file, void __user *arg)
1177{
1178 struct nvm_ioctl_dev_init init;
1179
1180 if (!capable(CAP_SYS_ADMIN))
1181 return -EPERM;
1182
1183 if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init)))
1184 return -EFAULT;
1185
1186 if (init.flags != 0) {
1187 pr_err("nvm: no flags supported\n");
1188 return -EINVAL;
1189 }
1190
1191 init.dev[DISK_NAME_LEN - 1] = '\0';
1192
1193 return __nvm_ioctl_dev_init(&init);
1194}
1195
8b4970c4
MB		 1196static long nvm_ioctl_dev_factory(struct file *file, void __user *arg)
1197{
1198 struct nvm_ioctl_dev_factory fact;
1199 struct nvm_dev *dev;
1200
1201 if (!capable(CAP_SYS_ADMIN))
1202 return -EPERM;
1203
1204 if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory)))
1205 return -EFAULT;
1206
1207 fact.dev[DISK_NAME_LEN - 1] = '\0';
1208
1209 if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1))
1210 return -EINVAL;
1211
1212 down_write(&nvm_lock);
1213 dev = nvm_find_nvm_dev(fact.dev);
1214 up_write(&nvm_lock);
1215 if (!dev) {
1216 pr_err("nvm: device not found\n");
1217 return -EINVAL;
1218 }
1219
1220 if (dev->mt) {
1221 dev->mt->unregister_mgr(dev);
1222 dev->mt = NULL;
1223 }
1224
bf643185
MB		 1225 if (dev->identity.cap & NVM_ID_DCAP_BBLKMGMT)
1226 return nvm_dev_factory(dev, fact.flags);
1227
1228 return 0;
8b4970c4
MB		 1229}
1230
cd9e9808
MB		 1231static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
1232{
1233 void __user *argp = (void __user *)arg;
1234
1235 switch (cmd) {
1236 case NVM_INFO:
1237 return nvm_ioctl_info(file, argp);
1238 case NVM_GET_DEVICES:
1239 return nvm_ioctl_get_devices(file, argp);
1240 case NVM_DEV_CREATE:
1241 return nvm_ioctl_dev_create(file, argp);
1242 case NVM_DEV_REMOVE:
1243 return nvm_ioctl_dev_remove(file, argp);
55696154
MB		 1244 case NVM_DEV_INIT:
1245 return nvm_ioctl_dev_init(file, argp);
8b4970c4
MB		 1246 case NVM_DEV_FACTORY:
1247 return nvm_ioctl_dev_factory(file, argp);
cd9e9808
MB		 1248 }
1249 return 0;
1250}
1251
1252static const struct file_operations _ctl_fops = {
1253 .open = nonseekable_open,
1254 .unlocked_ioctl = nvm_ctl_ioctl,
1255 .owner = THIS_MODULE,
1256 .llseek = noop_llseek,
1257};
1258
1259static struct miscdevice _nvm_misc = {
1260 .minor = MISC_DYNAMIC_MINOR,
1261 .name = "lightnvm",
1262 .nodename = "lightnvm/control",
1263 .fops = &_ctl_fops,
1264};
1265
1266MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR);
1267
1268static int __init nvm_mod_init(void)
1269{
1270 int ret;
1271
1272 ret = misc_register(&_nvm_misc);
1273 if (ret)
1274 pr_err("nvm: misc_register failed for control device");
1275
1276 return ret;
1277}
1278
1279static void __exit nvm_mod_exit(void)
1280{
1281 misc_deregister(&_nvm_misc);
1282}
1283
1284MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>");
1285MODULE_LICENSE("GPL v2");
1286MODULE_VERSION("0.1");
1287module_init(nvm_mod_init);
1288module_exit(nvm_mod_exit);
Linux 6.x block layer and io_uring tree(s)