Commit | Line | Data |
---|---|---|
48add0f5 MB |
1 | /* |
2 | * Copyright (C) 2015 Matias Bjorling <m@bjorling.me> | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License version | |
6 | * 2 as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * along with this program; see the file COPYING. If not, write to | |
15 | * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, | |
16 | * USA. | |
17 | * | |
18 | * Implementation of a generic nvm manager for Open-Channel SSDs. | |
19 | */ | |
20 | ||
21 | #include "gennvm.h" | |
22 | ||
23 | static void gennvm_blocks_free(struct nvm_dev *dev) | |
24 | { | |
25 | struct gen_nvm *gn = dev->mp; | |
26 | struct gen_lun *lun; | |
27 | int i; | |
28 | ||
29 | gennvm_for_each_lun(gn, lun, i) { | |
30 | if (!lun->vlun.blocks) | |
31 | break; | |
32 | vfree(lun->vlun.blocks); | |
33 | } | |
34 | } | |
35 | ||
36 | static void gennvm_luns_free(struct nvm_dev *dev) | |
37 | { | |
38 | struct gen_nvm *gn = dev->mp; | |
39 | ||
40 | kfree(gn->luns); | |
41 | } | |
42 | ||
43 | static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn) | |
44 | { | |
45 | struct gen_lun *lun; | |
46 | int i; | |
47 | ||
48 | gn->luns = kcalloc(dev->nr_luns, sizeof(struct gen_lun), GFP_KERNEL); | |
49 | if (!gn->luns) | |
50 | return -ENOMEM; | |
51 | ||
52 | gennvm_for_each_lun(gn, lun, i) { | |
53 | spin_lock_init(&lun->vlun.lock); | |
54 | INIT_LIST_HEAD(&lun->free_list); | |
55 | INIT_LIST_HEAD(&lun->used_list); | |
56 | INIT_LIST_HEAD(&lun->bb_list); | |
57 | ||
58 | lun->reserved_blocks = 2; /* for GC only */ | |
59 | lun->vlun.id = i; | |
60 | lun->vlun.lun_id = i % dev->luns_per_chnl; | |
61 | lun->vlun.chnl_id = i / dev->luns_per_chnl; | |
62 | lun->vlun.nr_free_blocks = dev->blks_per_lun; | |
ff0e498b JG |
63 | lun->vlun.nr_open_blocks = 0; |
64 | lun->vlun.nr_closed_blocks = 0; | |
0b59733b | 65 | lun->vlun.nr_bad_blocks = 0; |
48add0f5 MB |
66 | } |
67 | return 0; | |
68 | } | |
69 | ||
11450469 | 70 | static int gennvm_block_bb(struct ppa_addr ppa, int nr_blocks, u8 *blks, |
48add0f5 MB |
71 | void *private) |
72 | { | |
73 | struct gen_nvm *gn = private; | |
11450469 MB |
74 | struct nvm_dev *dev = gn->dev; |
75 | struct gen_lun *lun; | |
48add0f5 MB |
76 | struct nvm_block *blk; |
77 | int i; | |
78 | ||
c3293a9a | 79 | lun = &gn->luns[(dev->luns_per_chnl * ppa.g.ch) + ppa.g.lun]; |
11450469 MB |
80 | |
81 | for (i = 0; i < nr_blocks; i++) { | |
82 | if (blks[i] == 0) | |
83 | continue; | |
48add0f5 | 84 | |
48add0f5 MB |
85 | blk = &lun->vlun.blocks[i]; |
86 | if (!blk) { | |
87 | pr_err("gennvm: BB data is out of bounds.\n"); | |
88 | return -EINVAL; | |
89 | } | |
90 | ||
91 | list_move_tail(&blk->list, &lun->bb_list); | |
0b59733b | 92 | lun->vlun.nr_bad_blocks++; |
bdded155 | 93 | lun->vlun.nr_free_blocks--; |
48add0f5 MB |
94 | } |
95 | ||
96 | return 0; | |
97 | } | |
98 | ||
99 | static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) | |
100 | { | |
101 | struct nvm_dev *dev = private; | |
102 | struct gen_nvm *gn = dev->mp; | |
103 | sector_t max_pages = dev->total_pages * (dev->sec_size >> 9); | |
104 | u64 elba = slba + nlb; | |
105 | struct gen_lun *lun; | |
106 | struct nvm_block *blk; | |
107 | u64 i; | |
108 | int lun_id; | |
109 | ||
110 | if (unlikely(elba > dev->total_pages)) { | |
111 | pr_err("gennvm: L2P data from device is out of bounds!\n"); | |
112 | return -EINVAL; | |
113 | } | |
114 | ||
115 | for (i = 0; i < nlb; i++) { | |
116 | u64 pba = le64_to_cpu(entries[i]); | |
117 | ||
118 | if (unlikely(pba >= max_pages && pba != U64_MAX)) { | |
119 | pr_err("gennvm: L2P data entry is out of bounds!\n"); | |
120 | return -EINVAL; | |
121 | } | |
122 | ||
123 | /* Address zero is a special one. The first page on a disk is | |
124 | * protected. It often holds internal device boot | |
125 | * information. | |
126 | */ | |
127 | if (!pba) | |
128 | continue; | |
129 | ||
130 | /* resolve block from physical address */ | |
131 | lun_id = div_u64(pba, dev->sec_per_lun); | |
132 | lun = &gn->luns[lun_id]; | |
133 | ||
134 | /* Calculate block offset into lun */ | |
135 | pba = pba - (dev->sec_per_lun * lun_id); | |
136 | blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)]; | |
137 | ||
ff0e498b | 138 | if (!blk->state) { |
48add0f5 MB |
139 | /* at this point, we don't know anything about the |
140 | * block. It's up to the FTL on top to re-etablish the | |
ff0e498b | 141 | * block state. The block is assumed to be open. |
48add0f5 MB |
142 | */ |
143 | list_move_tail(&blk->list, &lun->used_list); | |
ff0e498b | 144 | blk->state = NVM_BLK_ST_OPEN; |
48add0f5 | 145 | lun->vlun.nr_free_blocks--; |
ff0e498b | 146 | lun->vlun.nr_open_blocks++; |
48add0f5 MB |
147 | } |
148 | } | |
149 | ||
150 | return 0; | |
151 | } | |
152 | ||
153 | static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn) | |
154 | { | |
155 | struct gen_lun *lun; | |
156 | struct nvm_block *block; | |
157 | sector_t lun_iter, blk_iter, cur_block_id = 0; | |
158 | int ret; | |
159 | ||
160 | gennvm_for_each_lun(gn, lun, lun_iter) { | |
161 | lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) * | |
162 | dev->blks_per_lun); | |
163 | if (!lun->vlun.blocks) | |
164 | return -ENOMEM; | |
165 | ||
166 | for (blk_iter = 0; blk_iter < dev->blks_per_lun; blk_iter++) { | |
167 | block = &lun->vlun.blocks[blk_iter]; | |
168 | ||
169 | INIT_LIST_HEAD(&block->list); | |
170 | ||
171 | block->lun = &lun->vlun; | |
172 | block->id = cur_block_id++; | |
173 | ||
174 | /* First block is reserved for device */ | |
0b59733b JG |
175 | if (unlikely(lun_iter == 0 && blk_iter == 0)) { |
176 | lun->vlun.nr_free_blocks--; | |
48add0f5 | 177 | continue; |
0b59733b | 178 | } |
48add0f5 MB |
179 | |
180 | list_add_tail(&block->list, &lun->free_list); | |
181 | } | |
182 | ||
183 | if (dev->ops->get_bb_tbl) { | |
11450469 MB |
184 | struct ppa_addr ppa; |
185 | ||
186 | ppa.ppa = 0; | |
187 | ppa.g.ch = lun->vlun.chnl_id; | |
188 | ppa.g.lun = lun->vlun.id; | |
7386af27 | 189 | ppa = generic_to_dev_addr(dev, ppa); |
11450469 | 190 | |
08236c6b | 191 | ret = dev->ops->get_bb_tbl(dev, ppa, |
11450469 MB |
192 | dev->blks_per_lun, |
193 | gennvm_block_bb, gn); | |
48add0f5 MB |
194 | if (ret) |
195 | pr_err("gennvm: could not read BB table\n"); | |
196 | } | |
197 | } | |
198 | ||
199 | if (dev->ops->get_l2p_tbl) { | |
16f26c3a | 200 | ret = dev->ops->get_l2p_tbl(dev, 0, dev->total_pages, |
48add0f5 MB |
201 | gennvm_block_map, dev); |
202 | if (ret) { | |
203 | pr_err("gennvm: could not read L2P table.\n"); | |
204 | pr_warn("gennvm: default block initialization"); | |
205 | } | |
206 | } | |
207 | ||
208 | return 0; | |
209 | } | |
210 | ||
8261bd48 WT |
211 | static void gennvm_free(struct nvm_dev *dev) |
212 | { | |
213 | gennvm_blocks_free(dev); | |
214 | gennvm_luns_free(dev); | |
215 | kfree(dev->mp); | |
216 | dev->mp = NULL; | |
217 | } | |
218 | ||
48add0f5 MB |
219 | static int gennvm_register(struct nvm_dev *dev) |
220 | { | |
221 | struct gen_nvm *gn; | |
222 | int ret; | |
223 | ||
008b7443 MB |
224 | if (!try_module_get(THIS_MODULE)) |
225 | return -ENODEV; | |
226 | ||
48add0f5 MB |
227 | gn = kzalloc(sizeof(struct gen_nvm), GFP_KERNEL); |
228 | if (!gn) | |
229 | return -ENOMEM; | |
230 | ||
11450469 | 231 | gn->dev = dev; |
48add0f5 MB |
232 | gn->nr_luns = dev->nr_luns; |
233 | dev->mp = gn; | |
234 | ||
235 | ret = gennvm_luns_init(dev, gn); | |
236 | if (ret) { | |
237 | pr_err("gennvm: could not initialize luns\n"); | |
238 | goto err; | |
239 | } | |
240 | ||
241 | ret = gennvm_blocks_init(dev, gn); | |
242 | if (ret) { | |
243 | pr_err("gennvm: could not initialize blocks\n"); | |
244 | goto err; | |
245 | } | |
246 | ||
247 | return 1; | |
248 | err: | |
8261bd48 | 249 | gennvm_free(dev); |
008b7443 | 250 | module_put(THIS_MODULE); |
48add0f5 MB |
251 | return ret; |
252 | } | |
253 | ||
254 | static void gennvm_unregister(struct nvm_dev *dev) | |
255 | { | |
8261bd48 | 256 | gennvm_free(dev); |
008b7443 | 257 | module_put(THIS_MODULE); |
48add0f5 MB |
258 | } |
259 | ||
ff0e498b | 260 | static struct nvm_block *gennvm_get_blk_unlocked(struct nvm_dev *dev, |
48add0f5 MB |
261 | struct nvm_lun *vlun, unsigned long flags) |
262 | { | |
263 | struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); | |
264 | struct nvm_block *blk = NULL; | |
265 | int is_gc = flags & NVM_IOTYPE_GC; | |
266 | ||
ff0e498b | 267 | assert_spin_locked(&vlun->lock); |
48add0f5 MB |
268 | |
269 | if (list_empty(&lun->free_list)) { | |
270 | pr_err_ratelimited("gennvm: lun %u have no free pages available", | |
271 | lun->vlun.id); | |
48add0f5 MB |
272 | goto out; |
273 | } | |
274 | ||
e9b76a80 | 275 | if (!is_gc && lun->vlun.nr_free_blocks < lun->reserved_blocks) |
48add0f5 | 276 | goto out; |
48add0f5 MB |
277 | |
278 | blk = list_first_entry(&lun->free_list, struct nvm_block, list); | |
279 | list_move_tail(&blk->list, &lun->used_list); | |
ff0e498b | 280 | blk->state = NVM_BLK_ST_OPEN; |
48add0f5 MB |
281 | |
282 | lun->vlun.nr_free_blocks--; | |
ff0e498b | 283 | lun->vlun.nr_open_blocks++; |
48add0f5 | 284 | |
48add0f5 | 285 | out: |
ff0e498b JG |
286 | return blk; |
287 | } | |
288 | ||
289 | static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev, | |
290 | struct nvm_lun *vlun, unsigned long flags) | |
291 | { | |
292 | struct nvm_block *blk; | |
293 | ||
294 | spin_lock(&vlun->lock); | |
295 | blk = gennvm_get_blk_unlocked(dev, vlun, flags); | |
e9b76a80 | 296 | spin_unlock(&vlun->lock); |
48add0f5 MB |
297 | return blk; |
298 | } | |
299 | ||
ff0e498b | 300 | static void gennvm_put_blk_unlocked(struct nvm_dev *dev, struct nvm_block *blk) |
48add0f5 MB |
301 | { |
302 | struct nvm_lun *vlun = blk->lun; | |
303 | struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); | |
304 | ||
ff0e498b | 305 | assert_spin_locked(&vlun->lock); |
48add0f5 | 306 | |
ff0e498b | 307 | if (blk->state & NVM_BLK_ST_OPEN) { |
48add0f5 | 308 | list_move_tail(&blk->list, &lun->free_list); |
ff0e498b | 309 | lun->vlun.nr_open_blocks--; |
48add0f5 | 310 | lun->vlun.nr_free_blocks++; |
ff0e498b JG |
311 | blk->state = NVM_BLK_ST_FREE; |
312 | } else if (blk->state & NVM_BLK_ST_CLOSED) { | |
313 | list_move_tail(&blk->list, &lun->free_list); | |
314 | lun->vlun.nr_closed_blocks--; | |
315 | lun->vlun.nr_free_blocks++; | |
316 | blk->state = NVM_BLK_ST_FREE; | |
317 | } else if (blk->state & NVM_BLK_ST_BAD) { | |
48add0f5 | 318 | list_move_tail(&blk->list, &lun->bb_list); |
0b59733b | 319 | lun->vlun.nr_bad_blocks++; |
ff0e498b JG |
320 | blk->state = NVM_BLK_ST_BAD; |
321 | } else { | |
48add0f5 MB |
322 | WARN_ON_ONCE(1); |
323 | pr_err("gennvm: erroneous block type (%lu -> %u)\n", | |
ff0e498b | 324 | blk->id, blk->state); |
48add0f5 | 325 | list_move_tail(&blk->list, &lun->bb_list); |
0b59733b | 326 | lun->vlun.nr_bad_blocks++; |
ff0e498b | 327 | blk->state = NVM_BLK_ST_BAD; |
48add0f5 | 328 | } |
ff0e498b | 329 | } |
48add0f5 | 330 | |
ff0e498b JG |
331 | static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) |
332 | { | |
333 | struct nvm_lun *vlun = blk->lun; | |
334 | ||
335 | spin_lock(&vlun->lock); | |
336 | gennvm_put_blk_unlocked(dev, blk); | |
48add0f5 MB |
337 | spin_unlock(&vlun->lock); |
338 | } | |
339 | ||
48add0f5 MB |
340 | static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa, |
341 | int type) | |
342 | { | |
343 | struct gen_nvm *gn = dev->mp; | |
344 | struct gen_lun *lun; | |
345 | struct nvm_block *blk; | |
346 | ||
347 | if (unlikely(ppa->g.ch > dev->nr_chnls || | |
348 | ppa->g.lun > dev->luns_per_chnl || | |
349 | ppa->g.blk > dev->blks_per_lun)) { | |
350 | WARN_ON_ONCE(1); | |
351 | pr_err("gennvm: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u", | |
352 | ppa->g.ch, dev->nr_chnls, | |
353 | ppa->g.lun, dev->luns_per_chnl, | |
354 | ppa->g.blk, dev->blks_per_lun); | |
355 | return; | |
356 | } | |
357 | ||
358 | lun = &gn->luns[ppa->g.lun * ppa->g.ch]; | |
359 | blk = &lun->vlun.blocks[ppa->g.blk]; | |
360 | ||
361 | /* will be moved to bb list on put_blk from target */ | |
ff0e498b | 362 | blk->state = type; |
48add0f5 MB |
363 | } |
364 | ||
365 | /* mark block bad. It is expected the target recover from the error. */ | |
366 | static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) | |
367 | { | |
368 | int i; | |
369 | ||
11450469 | 370 | if (!dev->ops->set_bb_tbl) |
48add0f5 MB |
371 | return; |
372 | ||
16f26c3a | 373 | if (dev->ops->set_bb_tbl(dev, rqd, 1)) |
48add0f5 MB |
374 | return; |
375 | ||
069368e9 | 376 | nvm_addr_to_generic_mode(dev, rqd); |
48add0f5 MB |
377 | |
378 | /* look up blocks and mark them as bad */ | |
379 | if (rqd->nr_pages > 1) | |
380 | for (i = 0; i < rqd->nr_pages; i++) | |
ff0e498b JG |
381 | gennvm_blk_set_type(dev, &rqd->ppa_list[i], |
382 | NVM_BLK_ST_BAD); | |
48add0f5 | 383 | else |
ff0e498b | 384 | gennvm_blk_set_type(dev, &rqd->ppa_addr, NVM_BLK_ST_BAD); |
48add0f5 MB |
385 | } |
386 | ||
72d256ec | 387 | static void gennvm_end_io(struct nvm_rq *rqd) |
48add0f5 MB |
388 | { |
389 | struct nvm_tgt_instance *ins = rqd->ins; | |
48add0f5 | 390 | |
72d256ec | 391 | switch (rqd->error) { |
48add0f5 | 392 | case NVM_RSP_SUCCESS: |
48add0f5 MB |
393 | case NVM_RSP_ERR_EMPTYPAGE: |
394 | break; | |
395 | case NVM_RSP_ERR_FAILWRITE: | |
396 | gennvm_mark_blk_bad(rqd->dev, rqd); | |
48add0f5 MB |
397 | } |
398 | ||
72d256ec | 399 | ins->tt->end_io(rqd); |
91276162 | 400 | } |
48add0f5 | 401 | |
91276162 MB |
402 | static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) |
403 | { | |
404 | if (!dev->ops->submit_io) | |
405 | return -ENODEV; | |
406 | ||
407 | /* Convert address space */ | |
408 | nvm_generic_to_addr_mode(dev, rqd); | |
409 | ||
410 | rqd->dev = dev; | |
411 | rqd->end_io = gennvm_end_io; | |
412 | return dev->ops->submit_io(dev, rqd); | |
48add0f5 MB |
413 | } |
414 | ||
415 | static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, | |
416 | unsigned long flags) | |
417 | { | |
069368e9 | 418 | struct ppa_addr addr = block_to_ppa(dev, blk); |
48add0f5 | 419 | |
81e681d3 | 420 | return nvm_erase_ppa(dev, &addr, 1); |
48add0f5 MB |
421 | } |
422 | ||
423 | static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid) | |
424 | { | |
425 | struct gen_nvm *gn = dev->mp; | |
426 | ||
427 | return &gn->luns[lunid].vlun; | |
428 | } | |
429 | ||
2fde0e48 | 430 | static void gennvm_lun_info_print(struct nvm_dev *dev) |
48add0f5 MB |
431 | { |
432 | struct gen_nvm *gn = dev->mp; | |
433 | struct gen_lun *lun; | |
434 | unsigned int i; | |
435 | ||
2fde0e48 JG |
436 | |
437 | gennvm_for_each_lun(gn, lun, i) { | |
438 | spin_lock(&lun->vlun.lock); | |
439 | ||
ff0e498b | 440 | pr_info("%s: lun%8u\t%u\t%u\t%u\t%u\n", |
2fde0e48 JG |
441 | dev->name, i, |
442 | lun->vlun.nr_free_blocks, | |
ff0e498b JG |
443 | lun->vlun.nr_open_blocks, |
444 | lun->vlun.nr_closed_blocks, | |
2fde0e48 JG |
445 | lun->vlun.nr_bad_blocks); |
446 | ||
447 | spin_unlock(&lun->vlun.lock); | |
448 | } | |
48add0f5 MB |
449 | } |
450 | ||
451 | static struct nvmm_type gennvm = { | |
ff0e498b JG |
452 | .name = "gennvm", |
453 | .version = {0, 1, 0}, | |
454 | ||
455 | .register_mgr = gennvm_register, | |
456 | .unregister_mgr = gennvm_unregister, | |
48add0f5 | 457 | |
ff0e498b JG |
458 | .get_blk_unlocked = gennvm_get_blk_unlocked, |
459 | .put_blk_unlocked = gennvm_put_blk_unlocked, | |
48add0f5 | 460 | |
ff0e498b JG |
461 | .get_blk = gennvm_get_blk, |
462 | .put_blk = gennvm_put_blk, | |
48add0f5 | 463 | |
ff0e498b JG |
464 | .submit_io = gennvm_submit_io, |
465 | .erase_blk = gennvm_erase_blk, | |
48add0f5 | 466 | |
ff0e498b JG |
467 | .get_lun = gennvm_get_lun, |
468 | .lun_info_print = gennvm_lun_info_print, | |
48add0f5 MB |
469 | }; |
470 | ||
471 | static int __init gennvm_module_init(void) | |
472 | { | |
473 | return nvm_register_mgr(&gennvm); | |
474 | } | |
475 | ||
476 | static void gennvm_module_exit(void) | |
477 | { | |
478 | nvm_unregister_mgr(&gennvm); | |
479 | } | |
480 | ||
481 | module_init(gennvm_module_init); | |
482 | module_exit(gennvm_module_exit); | |
483 | MODULE_LICENSE("GPL v2"); | |
484 | MODULE_DESCRIPTION("Generic media manager for Open-Channel SSDs"); |