Commit | Line | Data |
---|---|---|
02a1520d | 1 | // SPDX-License-Identifier: GPL-2.0 |
a4bd217b JG |
2 | /* |
3 | * Copyright (C) 2016 CNEX Labs | |
4 | * Initial: Javier Gonzalez <javier@cnexlabs.com> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License version | |
8 | * 2 as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License for more details. | |
14 | * | |
15 | * pblk-recovery.c - pblk's recovery path | |
361d889f JG |
16 | * |
17 | * The L2P recovery path is single threaded as the L2P table is updated in order | |
18 | * following the line sequence ID. | |
a4bd217b JG |
19 | */ |
20 | ||
21 | #include "pblk.h" | |
f2937232 | 22 | #include "pblk-trace.h" |
a4bd217b | 23 | |
06bc072b | 24 | int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta_buf) |
a4bd217b JG |
25 | { |
26 | u32 crc; | |
27 | ||
dd2a4343 JG |
28 | crc = pblk_calc_emeta_crc(pblk, emeta_buf); |
29 | if (le32_to_cpu(emeta_buf->crc) != crc) | |
06bc072b | 30 | return 1; |
a4bd217b | 31 | |
dd2a4343 | 32 | if (le32_to_cpu(emeta_buf->header.identifier) != PBLK_MAGIC) |
06bc072b | 33 | return 1; |
a4bd217b | 34 | |
06bc072b | 35 | return 0; |
a4bd217b JG |
36 | } |
37 | ||
38 | static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line) | |
39 | { | |
40 | struct nvm_tgt_dev *dev = pblk->dev; | |
41 | struct nvm_geo *geo = &dev->geo; | |
42 | struct pblk_line_meta *lm = &pblk->lm; | |
dd2a4343 JG |
43 | struct pblk_emeta *emeta = line->emeta; |
44 | struct line_emeta *emeta_buf = emeta->buf; | |
a4bd217b | 45 | __le64 *lba_list; |
75610cd9 HH |
46 | u64 data_start, data_end; |
47 | u64 nr_valid_lbas, nr_lbas = 0; | |
48 | u64 i; | |
a4bd217b | 49 | |
06bc072b | 50 | lba_list = emeta_to_lbas(pblk, emeta_buf); |
a4bd217b JG |
51 | if (!lba_list) |
52 | return 1; | |
53 | ||
54 | data_start = pblk_line_smeta_start(pblk, line) + lm->smeta_sec; | |
75610cd9 | 55 | data_end = line->emeta_ssec; |
dd2a4343 | 56 | nr_valid_lbas = le64_to_cpu(emeta_buf->nr_valid_lbas); |
a4bd217b | 57 | |
92957091 | 58 | for (i = data_start; i < data_end; i++) { |
a4bd217b JG |
59 | struct ppa_addr ppa; |
60 | int pos; | |
61 | ||
b1bcfda1 | 62 | ppa = addr_to_gen_ppa(pblk, i, line->id); |
a4bd217b JG |
63 | pos = pblk_ppa_to_pos(geo, ppa); |
64 | ||
65 | /* Do not update bad blocks */ | |
66 | if (test_bit(pos, line->blk_bitmap)) | |
67 | continue; | |
68 | ||
69 | if (le64_to_cpu(lba_list[i]) == ADDR_EMPTY) { | |
70 | spin_lock(&line->lock); | |
71 | if (test_and_set_bit(i, line->invalid_bitmap)) | |
2a79efd8 | 72 | WARN_ONCE(1, "pblk: rec. double invalidate:\n"); |
a4bd217b | 73 | else |
dd2a4343 | 74 | le32_add_cpu(line->vsc, -1); |
a4bd217b JG |
75 | spin_unlock(&line->lock); |
76 | ||
77 | continue; | |
78 | } | |
79 | ||
80 | pblk_update_map(pblk, le64_to_cpu(lba_list[i]), ppa); | |
81 | nr_lbas++; | |
82 | } | |
83 | ||
84 | if (nr_valid_lbas != nr_lbas) | |
4e495a46 | 85 | pblk_err(pblk, "line %d - inconsistent lba list(%llu/%llu)\n", |
75610cd9 | 86 | line->id, nr_valid_lbas, nr_lbas); |
a4bd217b JG |
87 | |
88 | line->left_msecs = 0; | |
89 | ||
90 | return 0; | |
91 | } | |
92 | ||
6ad2f619 JG |
93 | static void pblk_update_line_wp(struct pblk *pblk, struct pblk_line *line, |
94 | u64 written_secs) | |
95 | { | |
2b0ae81e | 96 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; |
6ad2f619 JG |
97 | int i; |
98 | ||
99 | for (i = 0; i < written_secs; i += pblk->min_write_pgs) | |
2b0ae81e IK |
100 | __pblk_alloc_page(pblk, line, pblk->min_write_pgs); |
101 | ||
102 | spin_lock(&l_mg->free_lock); | |
103 | if (written_secs > line->left_msecs) { | |
104 | /* | |
105 | * We have all data sectors written | |
106 | * and some emeta sectors written too. | |
107 | */ | |
108 | line->left_msecs = 0; | |
109 | } else { | |
110 | /* We have only some data sectors written. */ | |
111 | line->left_msecs -= written_secs; | |
112 | } | |
113 | spin_unlock(&l_mg->free_lock); | |
6ad2f619 JG |
114 | } |
115 | ||
116 | static u64 pblk_sec_in_open_line(struct pblk *pblk, struct pblk_line *line) | |
a4bd217b | 117 | { |
a4bd217b JG |
118 | struct pblk_line_meta *lm = &pblk->lm; |
119 | int nr_bb = bitmap_weight(line->blk_bitmap, lm->blk_per_line); | |
6ad2f619 JG |
120 | u64 written_secs = 0; |
121 | int valid_chunks = 0; | |
122 | int i; | |
123 | ||
124 | for (i = 0; i < lm->blk_per_line; i++) { | |
125 | struct nvm_chk_meta *chunk = &line->chks[i]; | |
a4bd217b | 126 | |
6ad2f619 JG |
127 | if (chunk->state & NVM_CHK_ST_OFFLINE) |
128 | continue; | |
129 | ||
130 | written_secs += chunk->wp; | |
131 | valid_chunks++; | |
132 | } | |
133 | ||
134 | if (lm->blk_per_line - nr_bb != valid_chunks) | |
135 | pblk_err(pblk, "recovery line %d is bad\n", line->id); | |
136 | ||
137 | pblk_update_line_wp(pblk, line, written_secs - lm->smeta_sec); | |
138 | ||
139 | return written_secs; | |
a4bd217b JG |
140 | } |
141 | ||
142 | struct pblk_recov_alloc { | |
143 | struct ppa_addr *ppa_list; | |
faa79f27 | 144 | void *meta_list; |
a4bd217b JG |
145 | struct nvm_rq *rqd; |
146 | void *data; | |
147 | dma_addr_t dma_ppa_list; | |
148 | dma_addr_t dma_meta_list; | |
149 | }; | |
150 | ||
ee8d5c1a JG |
151 | static void pblk_recov_complete(struct kref *ref) |
152 | { | |
153 | struct pblk_pad_rq *pad_rq = container_of(ref, struct pblk_pad_rq, ref); | |
154 | ||
155 | complete(&pad_rq->wait); | |
156 | } | |
157 | ||
158 | static void pblk_end_io_recov(struct nvm_rq *rqd) | |
159 | { | |
d68a9344 | 160 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
ee8d5c1a JG |
161 | struct pblk_pad_rq *pad_rq = rqd->private; |
162 | struct pblk *pblk = pad_rq->pblk; | |
ee8d5c1a | 163 | |
43241cfe | 164 | pblk_up_chunk(pblk, ppa_list[0]); |
3eaa11e2 | 165 | |
e2cddf20 | 166 | pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); |
3eaa11e2 JG |
167 | |
168 | atomic_dec(&pblk->inflight_io); | |
169 | kref_put(&pad_rq->ref, pblk_recov_complete); | |
ee8d5c1a JG |
170 | } |
171 | ||
6ad2f619 JG |
172 | /* pad line using line bitmap. */ |
173 | static int pblk_recov_pad_line(struct pblk *pblk, struct pblk_line *line, | |
174 | int left_ppas) | |
a4bd217b JG |
175 | { |
176 | struct nvm_tgt_dev *dev = pblk->dev; | |
177 | struct nvm_geo *geo = &dev->geo; | |
faa79f27 | 178 | void *meta_list; |
ee8d5c1a | 179 | struct pblk_pad_rq *pad_rq; |
a4bd217b JG |
180 | struct nvm_rq *rqd; |
181 | struct bio *bio; | |
45c5fcbb | 182 | struct ppa_addr *ppa_list; |
a4bd217b | 183 | void *data; |
dd2a4343 | 184 | __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf); |
a4bd217b | 185 | u64 w_ptr = line->cur_sec; |
ee8d5c1a | 186 | int left_line_ppas, rq_ppas, rq_len; |
a4bd217b JG |
187 | int i, j; |
188 | int ret = 0; | |
a4bd217b | 189 | |
ee8d5c1a JG |
190 | spin_lock(&line->lock); |
191 | left_line_ppas = line->left_msecs; | |
192 | spin_unlock(&line->lock); | |
193 | ||
194 | pad_rq = kmalloc(sizeof(struct pblk_pad_rq), GFP_KERNEL); | |
195 | if (!pad_rq) | |
196 | return -ENOMEM; | |
197 | ||
fad953ce | 198 | data = vzalloc(array_size(pblk->max_write_pgs, geo->csecs)); |
ee8d5c1a JG |
199 | if (!data) { |
200 | ret = -ENOMEM; | |
201 | goto free_rq; | |
202 | } | |
203 | ||
204 | pad_rq->pblk = pblk; | |
205 | init_completion(&pad_rq->wait); | |
206 | kref_init(&pad_rq->ref); | |
a4bd217b JG |
207 | |
208 | next_pad_rq: | |
55d8ec35 | 209 | rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); |
ee8d5c1a | 210 | if (rq_ppas < pblk->min_write_pgs) { |
4e495a46 | 211 | pblk_err(pblk, "corrupted pad line %d\n", line->id); |
1fc3b305 | 212 | goto fail_complete; |
ee8d5c1a JG |
213 | } |
214 | ||
e46f4e48 | 215 | rq_len = rq_ppas * geo->csecs; |
a4bd217b | 216 | |
3eaa11e2 JG |
217 | bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len, |
218 | PBLK_VMALLOC_META, GFP_KERNEL); | |
ee8d5c1a JG |
219 | if (IS_ERR(bio)) { |
220 | ret = PTR_ERR(bio); | |
1fc3b305 | 221 | goto fail_complete; |
ee8d5c1a | 222 | } |
a4bd217b JG |
223 | |
224 | bio->bi_iter.bi_sector = 0; /* internal bio */ | |
225 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); | |
226 | ||
e2cddf20 | 227 | rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT); |
2942f50f | 228 | |
45dcf29b | 229 | ret = pblk_alloc_rqd_meta(pblk, rqd); |
1fc3b305 IK |
230 | if (ret) { |
231 | pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); | |
232 | bio_put(bio); | |
233 | goto fail_complete; | |
234 | } | |
45dcf29b | 235 | |
a4bd217b JG |
236 | rqd->bio = bio; |
237 | rqd->opcode = NVM_OP_PWRITE; | |
d7b68016 | 238 | rqd->is_seq = 1; |
a4bd217b | 239 | rqd->nr_ppas = rq_ppas; |
ee8d5c1a JG |
240 | rqd->end_io = pblk_end_io_recov; |
241 | rqd->private = pad_rq; | |
a4bd217b | 242 | |
45c5fcbb | 243 | ppa_list = nvm_rq_to_ppa_list(rqd); |
45dcf29b JG |
244 | meta_list = rqd->meta_list; |
245 | ||
a4bd217b JG |
246 | for (i = 0; i < rqd->nr_ppas; ) { |
247 | struct ppa_addr ppa; | |
248 | int pos; | |
249 | ||
250 | w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs); | |
b1bcfda1 | 251 | ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); |
a4bd217b JG |
252 | pos = pblk_ppa_to_pos(geo, ppa); |
253 | ||
254 | while (test_bit(pos, line->blk_bitmap)) { | |
255 | w_ptr += pblk->min_write_pgs; | |
b1bcfda1 | 256 | ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); |
a4bd217b JG |
257 | pos = pblk_ppa_to_pos(geo, ppa); |
258 | } | |
259 | ||
260 | for (j = 0; j < pblk->min_write_pgs; j++, i++, w_ptr++) { | |
261 | struct ppa_addr dev_ppa; | |
faa79f27 | 262 | struct pblk_sec_meta *meta; |
f417aa0b | 263 | __le64 addr_empty = cpu_to_le64(ADDR_EMPTY); |
a4bd217b JG |
264 | |
265 | dev_ppa = addr_to_gen_ppa(pblk, w_ptr, line->id); | |
266 | ||
267 | pblk_map_invalidate(pblk, dev_ppa); | |
faa79f27 IK |
268 | lba_list[w_ptr] = addr_empty; |
269 | meta = pblk_get_meta(pblk, meta_list, i); | |
270 | meta->lba = addr_empty; | |
45c5fcbb | 271 | ppa_list[i] = dev_ppa; |
a4bd217b JG |
272 | } |
273 | } | |
274 | ||
ee8d5c1a | 275 | kref_get(&pad_rq->ref); |
45c5fcbb | 276 | pblk_down_chunk(pblk, ppa_list[0]); |
ee8d5c1a | 277 | |
a4bd217b JG |
278 | ret = pblk_submit_io(pblk, rqd); |
279 | if (ret) { | |
4e495a46 | 280 | pblk_err(pblk, "I/O submission failed: %d\n", ret); |
45c5fcbb | 281 | pblk_up_chunk(pblk, ppa_list[0]); |
1fc3b305 IK |
282 | kref_put(&pad_rq->ref, pblk_recov_complete); |
283 | pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); | |
284 | bio_put(bio); | |
285 | goto fail_complete; | |
a4bd217b JG |
286 | } |
287 | ||
a4bd217b JG |
288 | left_line_ppas -= rq_ppas; |
289 | left_ppas -= rq_ppas; | |
ee8d5c1a | 290 | if (left_ppas && left_line_ppas) |
a4bd217b JG |
291 | goto next_pad_rq; |
292 | ||
1fc3b305 | 293 | fail_complete: |
ee8d5c1a | 294 | kref_put(&pad_rq->ref, pblk_recov_complete); |
32ac0fa3 | 295 | wait_for_completion(&pad_rq->wait); |
ee8d5c1a | 296 | |
3eaa11e2 | 297 | if (!pblk_line_is_full(line)) |
4e495a46 | 298 | pblk_err(pblk, "corrupted padded line: %d\n", line->id); |
3eaa11e2 JG |
299 | |
300 | vfree(data); | |
ee8d5c1a JG |
301 | free_rq: |
302 | kfree(pad_rq); | |
ee8d5c1a | 303 | return ret; |
a4bd217b JG |
304 | } |
305 | ||
63dee3a6 JG |
306 | static int pblk_pad_distance(struct pblk *pblk, struct pblk_line *line) |
307 | { | |
308 | struct nvm_tgt_dev *dev = pblk->dev; | |
309 | struct nvm_geo *geo = &dev->geo; | |
310 | int distance = geo->mw_cunits * geo->all_luns * geo->ws_opt; | |
311 | ||
312 | return (distance > line->left_msecs) ? line->left_msecs : distance; | |
313 | } | |
314 | ||
aa8759d8 HH |
315 | /* Return a chunk belonging to a line by stripe(write order) index */ |
316 | static struct nvm_chk_meta *pblk_get_stripe_chunk(struct pblk *pblk, | |
317 | struct pblk_line *line, | |
318 | int index) | |
a4bd217b JG |
319 | { |
320 | struct nvm_tgt_dev *dev = pblk->dev; | |
321 | struct nvm_geo *geo = &dev->geo; | |
6ad2f619 | 322 | struct pblk_lun *rlun; |
6ad2f619 | 323 | struct ppa_addr ppa; |
aa8759d8 | 324 | int pos; |
a4bd217b | 325 | |
aa8759d8 | 326 | rlun = &pblk->luns[index]; |
6ad2f619 JG |
327 | ppa = rlun->bppa; |
328 | pos = pblk_ppa_to_pos(geo, ppa); | |
f9c10152 | 329 | |
aa8759d8 HH |
330 | return &line->chks[pos]; |
331 | } | |
a4bd217b | 332 | |
aa8759d8 HH |
333 | static int pblk_line_wps_are_unbalanced(struct pblk *pblk, |
334 | struct pblk_line *line) | |
335 | { | |
336 | struct pblk_line_meta *lm = &pblk->lm; | |
337 | int blk_in_line = lm->blk_per_line; | |
338 | struct nvm_chk_meta *chunk; | |
339 | u64 max_wp, min_wp; | |
340 | int i; | |
a4bd217b | 341 | |
aa8759d8 HH |
342 | i = find_first_zero_bit(line->blk_bitmap, blk_in_line); |
343 | ||
344 | /* If there is one or zero good chunks in the line, | |
345 | * the write pointers can't be unbalanced. | |
346 | */ | |
347 | if (i >= (blk_in_line - 1)) | |
348 | return 0; | |
349 | ||
350 | chunk = pblk_get_stripe_chunk(pblk, line, i); | |
351 | max_wp = chunk->wp; | |
352 | if (max_wp > pblk->max_write_pgs) | |
353 | min_wp = max_wp - pblk->max_write_pgs; | |
354 | else | |
355 | min_wp = 0; | |
356 | ||
357 | i = find_next_zero_bit(line->blk_bitmap, blk_in_line, i + 1); | |
358 | while (i < blk_in_line) { | |
359 | chunk = pblk_get_stripe_chunk(pblk, line, i); | |
360 | if (chunk->wp > max_wp || chunk->wp < min_wp) | |
6ad2f619 | 361 | return 1; |
aa8759d8 HH |
362 | |
363 | i = find_next_zero_bit(line->blk_bitmap, blk_in_line, i + 1); | |
a4bd217b JG |
364 | } |
365 | ||
6ad2f619 | 366 | return 0; |
a4bd217b JG |
367 | } |
368 | ||
369 | static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line, | |
6ad2f619 | 370 | struct pblk_recov_alloc p) |
a4bd217b JG |
371 | { |
372 | struct nvm_tgt_dev *dev = pblk->dev; | |
f40a62d2 | 373 | struct pblk_line_meta *lm = &pblk->lm; |
a4bd217b JG |
374 | struct nvm_geo *geo = &dev->geo; |
375 | struct ppa_addr *ppa_list; | |
faa79f27 | 376 | void *meta_list; |
a4bd217b JG |
377 | struct nvm_rq *rqd; |
378 | struct bio *bio; | |
379 | void *data; | |
380 | dma_addr_t dma_ppa_list, dma_meta_list; | |
6ad2f619 | 381 | __le64 *lba_list; |
f40a62d2 | 382 | u64 paddr = pblk_line_smeta_start(pblk, line) + lm->smeta_sec; |
6ad2f619 | 383 | bool padded = false; |
a4bd217b JG |
384 | int rq_ppas, rq_len; |
385 | int i, j; | |
6ad2f619 | 386 | int ret; |
f40a62d2 | 387 | u64 left_ppas = pblk_sec_in_open_line(pblk, line) - lm->smeta_sec; |
6ad2f619 | 388 | |
aa8759d8 | 389 | if (pblk_line_wps_are_unbalanced(pblk, line)) |
6ad2f619 | 390 | pblk_warn(pblk, "recovering unbalanced line (%d)\n", line->id); |
a4bd217b JG |
391 | |
392 | ppa_list = p.ppa_list; | |
393 | meta_list = p.meta_list; | |
394 | rqd = p.rqd; | |
395 | data = p.data; | |
396 | dma_ppa_list = p.dma_ppa_list; | |
397 | dma_meta_list = p.dma_meta_list; | |
398 | ||
6ad2f619 | 399 | lba_list = emeta_to_lbas(pblk, line->emeta->buf); |
a4bd217b JG |
400 | |
401 | next_rq: | |
084ec9ba | 402 | memset(rqd, 0, pblk_g_rq_size); |
a4bd217b | 403 | |
55d8ec35 | 404 | rq_ppas = pblk_calc_secs(pblk, left_ppas, 0, false); |
a4bd217b JG |
405 | if (!rq_ppas) |
406 | rq_ppas = pblk->min_write_pgs; | |
e46f4e48 | 407 | rq_len = rq_ppas * geo->csecs; |
a4bd217b | 408 | |
55d8ec35 | 409 | retry_rq: |
a4bd217b JG |
410 | bio = bio_map_kern(dev->q, data, rq_len, GFP_KERNEL); |
411 | if (IS_ERR(bio)) | |
412 | return PTR_ERR(bio); | |
413 | ||
414 | bio->bi_iter.bi_sector = 0; /* internal bio */ | |
415 | bio_set_op_attrs(bio, REQ_OP_READ, 0); | |
55d8ec35 | 416 | bio_get(bio); |
a4bd217b JG |
417 | |
418 | rqd->bio = bio; | |
419 | rqd->opcode = NVM_OP_PREAD; | |
a4bd217b JG |
420 | rqd->meta_list = meta_list; |
421 | rqd->nr_ppas = rq_ppas; | |
422 | rqd->ppa_list = ppa_list; | |
423 | rqd->dma_ppa_list = dma_ppa_list; | |
424 | rqd->dma_meta_list = dma_meta_list; | |
45c5fcbb | 425 | ppa_list = nvm_rq_to_ppa_list(rqd); |
a4bd217b | 426 | |
f9c10152 | 427 | if (pblk_io_aligned(pblk, rq_ppas)) |
d7b68016 | 428 | rqd->is_seq = 1; |
f9c10152 | 429 | |
a4bd217b JG |
430 | for (i = 0; i < rqd->nr_ppas; ) { |
431 | struct ppa_addr ppa; | |
432 | int pos; | |
433 | ||
a4bd217b | 434 | ppa = addr_to_gen_ppa(pblk, paddr, line->id); |
b1bcfda1 | 435 | pos = pblk_ppa_to_pos(geo, ppa); |
a4bd217b JG |
436 | |
437 | while (test_bit(pos, line->blk_bitmap)) { | |
438 | paddr += pblk->min_write_pgs; | |
439 | ppa = addr_to_gen_ppa(pblk, paddr, line->id); | |
b1bcfda1 | 440 | pos = pblk_ppa_to_pos(geo, ppa); |
a4bd217b JG |
441 | } |
442 | ||
6ad2f619 | 443 | for (j = 0; j < pblk->min_write_pgs; j++, i++) |
45c5fcbb | 444 | ppa_list[i] = |
6ad2f619 | 445 | addr_to_gen_ppa(pblk, paddr + j, line->id); |
a4bd217b JG |
446 | } |
447 | ||
1a94b2d4 | 448 | ret = pblk_submit_io_sync(pblk, rqd); |
a4bd217b | 449 | if (ret) { |
4e495a46 | 450 | pblk_err(pblk, "I/O submission failed: %d\n", ret); |
a4bd217b JG |
451 | bio_put(bio); |
452 | return ret; | |
453 | } | |
454 | ||
588726d3 | 455 | atomic_dec(&pblk->inflight_io); |
a4bd217b | 456 | |
6ad2f619 | 457 | /* If a read fails, do a best effort by padding the line and retrying */ |
d165a7a6 | 458 | if (rqd->error && rqd->error != NVM_RSP_WARN_HIGHECC) { |
6ad2f619 | 459 | int pad_distance, ret; |
a4bd217b | 460 | |
6ad2f619 JG |
461 | if (padded) { |
462 | pblk_log_read_err(pblk, rqd); | |
55d8ec35 | 463 | bio_put(bio); |
6ad2f619 JG |
464 | return -EINTR; |
465 | } | |
a4bd217b | 466 | |
6ad2f619 JG |
467 | pad_distance = pblk_pad_distance(pblk, line); |
468 | ret = pblk_recov_pad_line(pblk, line, pad_distance); | |
55d8ec35 IK |
469 | if (ret) { |
470 | bio_put(bio); | |
6ad2f619 | 471 | return ret; |
55d8ec35 | 472 | } |
a4bd217b | 473 | |
6ad2f619 | 474 | padded = true; |
55d8ec35 | 475 | bio_put(bio); |
6ad2f619 | 476 | goto retry_rq; |
a4bd217b JG |
477 | } |
478 | ||
55d8ec35 IK |
479 | pblk_get_packed_meta(pblk, rqd); |
480 | bio_put(bio); | |
481 | ||
a4bd217b | 482 | for (i = 0; i < rqd->nr_ppas; i++) { |
faa79f27 IK |
483 | struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); |
484 | u64 lba = le64_to_cpu(meta->lba); | |
a4bd217b | 485 | |
6ad2f619 JG |
486 | lba_list[paddr++] = cpu_to_le64(lba); |
487 | ||
847a3a27 | 488 | if (lba == ADDR_EMPTY || lba >= pblk->capacity) |
a4bd217b JG |
489 | continue; |
490 | ||
6ad2f619 | 491 | line->nr_valid_lbas++; |
45c5fcbb | 492 | pblk_update_map(pblk, lba, ppa_list[i]); |
a4bd217b JG |
493 | } |
494 | ||
495 | left_ppas -= rq_ppas; | |
496 | if (left_ppas > 0) | |
497 | goto next_rq; | |
498 | ||
6ad2f619 JG |
499 | #ifdef CONFIG_NVM_PBLK_DEBUG |
500 | WARN_ON(padded && !pblk_line_is_full(line)); | |
501 | #endif | |
502 | ||
503 | return 0; | |
a4bd217b JG |
504 | } |
505 | ||
506 | /* Scan line for lbas on out of bound area */ | |
507 | static int pblk_recov_l2p_from_oob(struct pblk *pblk, struct pblk_line *line) | |
508 | { | |
509 | struct nvm_tgt_dev *dev = pblk->dev; | |
510 | struct nvm_geo *geo = &dev->geo; | |
511 | struct nvm_rq *rqd; | |
512 | struct ppa_addr *ppa_list; | |
faa79f27 | 513 | void *meta_list; |
a4bd217b JG |
514 | struct pblk_recov_alloc p; |
515 | void *data; | |
516 | dma_addr_t dma_ppa_list, dma_meta_list; | |
6ad2f619 | 517 | int ret = 0; |
a4bd217b | 518 | |
a4bd217b | 519 | meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list); |
2942f50f JG |
520 | if (!meta_list) |
521 | return -ENOMEM; | |
a4bd217b | 522 | |
24828d05 IK |
523 | ppa_list = (void *)(meta_list) + pblk_dma_meta_size(pblk); |
524 | dma_ppa_list = dma_meta_list + pblk_dma_meta_size(pblk); | |
a4bd217b | 525 | |
e46f4e48 | 526 | data = kcalloc(pblk->max_write_pgs, geo->csecs, GFP_KERNEL); |
a4bd217b JG |
527 | if (!data) { |
528 | ret = -ENOMEM; | |
529 | goto free_meta_list; | |
530 | } | |
531 | ||
6ad2f619 JG |
532 | rqd = mempool_alloc(&pblk->r_rq_pool, GFP_KERNEL); |
533 | memset(rqd, 0, pblk_g_rq_size); | |
2942f50f | 534 | |
a4bd217b JG |
535 | p.ppa_list = ppa_list; |
536 | p.meta_list = meta_list; | |
537 | p.rqd = rqd; | |
538 | p.data = data; | |
539 | p.dma_ppa_list = dma_ppa_list; | |
540 | p.dma_meta_list = dma_meta_list; | |
541 | ||
6ad2f619 | 542 | ret = pblk_recov_scan_oob(pblk, line, p); |
a4bd217b | 543 | if (ret) { |
6ad2f619 | 544 | pblk_err(pblk, "could not recover L2P form OOB\n"); |
a4bd217b JG |
545 | goto out; |
546 | } | |
547 | ||
a4bd217b JG |
548 | if (pblk_line_is_full(line)) |
549 | pblk_line_recov_close(pblk, line); | |
550 | ||
551 | out: | |
6ad2f619 | 552 | mempool_free(rqd, &pblk->r_rq_pool); |
a4bd217b JG |
553 | kfree(data); |
554 | free_meta_list: | |
555 | nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list); | |
a4bd217b JG |
556 | |
557 | return ret; | |
558 | } | |
559 | ||
560 | /* Insert lines ordered by sequence number (seq_num) on list */ | |
561 | static void pblk_recov_line_add_ordered(struct list_head *head, | |
562 | struct pblk_line *line) | |
563 | { | |
564 | struct pblk_line *t = NULL; | |
565 | ||
566 | list_for_each_entry(t, head, list) | |
567 | if (t->seq_nr > line->seq_nr) | |
568 | break; | |
569 | ||
570 | __list_add(&line->list, t->list.prev, &t->list); | |
571 | } | |
572 | ||
75610cd9 | 573 | static u64 pblk_line_emeta_start(struct pblk *pblk, struct pblk_line *line) |
a4bd217b JG |
574 | { |
575 | struct nvm_tgt_dev *dev = pblk->dev; | |
576 | struct nvm_geo *geo = &dev->geo; | |
75610cd9 HH |
577 | struct pblk_line_meta *lm = &pblk->lm; |
578 | unsigned int emeta_secs; | |
579 | u64 emeta_start; | |
580 | struct ppa_addr ppa; | |
581 | int pos; | |
582 | ||
583 | emeta_secs = lm->emeta_sec[0]; | |
584 | emeta_start = lm->sec_per_line; | |
585 | ||
586 | while (emeta_secs) { | |
587 | emeta_start--; | |
b1bcfda1 | 588 | ppa = addr_to_gen_ppa(pblk, emeta_start, line->id); |
75610cd9 HH |
589 | pos = pblk_ppa_to_pos(geo, ppa); |
590 | if (!test_bit(pos, line->blk_bitmap)) | |
591 | emeta_secs--; | |
592 | } | |
593 | ||
594 | return emeta_start; | |
595 | } | |
596 | ||
d0ab0b1a HH |
597 | static int pblk_recov_check_line_version(struct pblk *pblk, |
598 | struct line_emeta *emeta) | |
599 | { | |
600 | struct line_header *header = &emeta->header; | |
601 | ||
602 | if (header->version_major != EMETA_VERSION_MAJOR) { | |
4e495a46 MB |
603 | pblk_err(pblk, "line major version mismatch: %d, expected: %d\n", |
604 | header->version_major, EMETA_VERSION_MAJOR); | |
d0ab0b1a HH |
605 | return 1; |
606 | } | |
607 | ||
99b8dad1 | 608 | #ifdef CONFIG_NVM_PBLK_DEBUG |
d0ab0b1a | 609 | if (header->version_minor > EMETA_VERSION_MINOR) |
4e495a46 | 610 | pblk_info(pblk, "newer line minor version found: %d\n", |
99b8dad1 | 611 | header->version_minor); |
d0ab0b1a HH |
612 | #endif |
613 | ||
614 | return 0; | |
615 | } | |
616 | ||
76758390 HH |
617 | static void pblk_recov_wa_counters(struct pblk *pblk, |
618 | struct line_emeta *emeta) | |
619 | { | |
620 | struct pblk_line_meta *lm = &pblk->lm; | |
621 | struct line_header *header = &emeta->header; | |
622 | struct wa_counters *wa = emeta_to_wa(lm, emeta); | |
623 | ||
624 | /* WA counters were introduced in emeta version 0.2 */ | |
625 | if (header->version_major > 0 || header->version_minor >= 2) { | |
626 | u64 user = le64_to_cpu(wa->user); | |
627 | u64 pad = le64_to_cpu(wa->pad); | |
628 | u64 gc = le64_to_cpu(wa->gc); | |
629 | ||
630 | atomic64_set(&pblk->user_wa, user); | |
631 | atomic64_set(&pblk->pad_wa, pad); | |
632 | atomic64_set(&pblk->gc_wa, gc); | |
633 | ||
634 | pblk->user_rst_wa = user; | |
635 | pblk->pad_rst_wa = pad; | |
636 | pblk->gc_rst_wa = gc; | |
637 | } | |
638 | } | |
639 | ||
9156f360 | 640 | static int pblk_line_was_written(struct pblk_line *line, |
6ad2f619 | 641 | struct pblk *pblk) |
9156f360 HH |
642 | { |
643 | ||
b06be287 HH |
644 | struct pblk_line_meta *lm = &pblk->lm; |
645 | struct nvm_tgt_dev *dev = pblk->dev; | |
646 | struct nvm_geo *geo = &dev->geo; | |
647 | struct nvm_chk_meta *chunk; | |
648 | struct ppa_addr bppa; | |
649 | int smeta_blk; | |
9156f360 | 650 | |
b06be287 HH |
651 | if (line->state == PBLK_LINESTATE_BAD) |
652 | return 0; | |
9156f360 | 653 | |
b06be287 HH |
654 | smeta_blk = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line); |
655 | if (smeta_blk >= lm->blk_per_line) | |
656 | return 0; | |
657 | ||
658 | bppa = pblk->luns[smeta_blk].bppa; | |
659 | chunk = &line->chks[pblk_ppa_to_pos(geo, bppa)]; | |
660 | ||
a24eab59 IK |
661 | if (chunk->state & NVM_CHK_ST_CLOSED || |
662 | (chunk->state & NVM_CHK_ST_OPEN | |
663 | && chunk->wp >= lm->smeta_sec)) | |
664 | return 1; | |
b06be287 | 665 | |
a24eab59 | 666 | return 0; |
9156f360 HH |
667 | } |
668 | ||
6ad2f619 JG |
669 | static bool pblk_line_is_open(struct pblk *pblk, struct pblk_line *line) |
670 | { | |
671 | struct pblk_line_meta *lm = &pblk->lm; | |
672 | int i; | |
673 | ||
674 | for (i = 0; i < lm->blk_per_line; i++) | |
675 | if (line->chks[i].state & NVM_CHK_ST_OPEN) | |
676 | return true; | |
677 | ||
678 | return false; | |
679 | } | |
680 | ||
75610cd9 HH |
681 | struct pblk_line *pblk_recov_l2p(struct pblk *pblk) |
682 | { | |
a4bd217b JG |
683 | struct pblk_line_meta *lm = &pblk->lm; |
684 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; | |
685 | struct pblk_line *line, *tline, *data_line = NULL; | |
dd2a4343 JG |
686 | struct pblk_smeta *smeta; |
687 | struct pblk_emeta *emeta; | |
688 | struct line_smeta *smeta_buf; | |
a4bd217b JG |
689 | int found_lines = 0, recovered_lines = 0, open_lines = 0; |
690 | int is_next = 0; | |
691 | int meta_line; | |
692 | int i, valid_uuid = 0; | |
693 | LIST_HEAD(recov_list); | |
694 | ||
695 | /* TODO: Implement FTL snapshot */ | |
696 | ||
697 | /* Scan recovery - takes place when FTL snapshot fails */ | |
698 | spin_lock(&l_mg->free_lock); | |
699 | meta_line = find_first_zero_bit(&l_mg->meta_bitmap, PBLK_DATA_LINES); | |
700 | set_bit(meta_line, &l_mg->meta_bitmap); | |
dd2a4343 JG |
701 | smeta = l_mg->sline_meta[meta_line]; |
702 | emeta = l_mg->eline_meta[meta_line]; | |
8224cbd8 | 703 | smeta_buf = (struct line_smeta *)smeta; |
a4bd217b JG |
704 | spin_unlock(&l_mg->free_lock); |
705 | ||
706 | /* Order data lines using their sequence number */ | |
707 | for (i = 0; i < l_mg->nr_lines; i++) { | |
708 | u32 crc; | |
709 | ||
710 | line = &pblk->lines[i]; | |
711 | ||
712 | memset(smeta, 0, lm->smeta_len); | |
713 | line->smeta = smeta; | |
dd2a4343 | 714 | line->lun_bitmap = ((void *)(smeta_buf)) + |
a4bd217b JG |
715 | sizeof(struct line_smeta); |
716 | ||
b06be287 | 717 | if (!pblk_line_was_written(line, pblk)) |
9156f360 HH |
718 | continue; |
719 | ||
a4bd217b | 720 | /* Lines that cannot be read are assumed as not written here */ |
af3fac16 | 721 | if (pblk_line_smeta_read(pblk, line)) |
a4bd217b JG |
722 | continue; |
723 | ||
dd2a4343 JG |
724 | crc = pblk_calc_smeta_crc(pblk, smeta_buf); |
725 | if (le32_to_cpu(smeta_buf->crc) != crc) | |
a4bd217b JG |
726 | continue; |
727 | ||
dd2a4343 | 728 | if (le32_to_cpu(smeta_buf->header.identifier) != PBLK_MAGIC) |
a4bd217b JG |
729 | continue; |
730 | ||
d0ab0b1a | 731 | if (smeta_buf->header.version_major != SMETA_VERSION_MAJOR) { |
4e495a46 | 732 | pblk_err(pblk, "found incompatible line version %u\n", |
d0ab0b1a | 733 | smeta_buf->header.version_major); |
a4bd217b JG |
734 | return ERR_PTR(-EINVAL); |
735 | } | |
736 | ||
737 | /* The first valid instance uuid is used for initialization */ | |
738 | if (!valid_uuid) { | |
7e0a0847 AS |
739 | guid_copy(&pblk->instance_uuid, |
740 | (guid_t *)&smeta_buf->header.uuid); | |
a4bd217b JG |
741 | valid_uuid = 1; |
742 | } | |
743 | ||
7e0a0847 AS |
744 | if (!guid_equal(&pblk->instance_uuid, |
745 | (guid_t *)&smeta_buf->header.uuid)) { | |
4e495a46 | 746 | pblk_debug(pblk, "ignore line %u due to uuid mismatch\n", |
a4bd217b JG |
747 | i); |
748 | continue; | |
749 | } | |
750 | ||
751 | /* Update line metadata */ | |
752 | spin_lock(&line->lock); | |
dd2a4343 JG |
753 | line->id = le32_to_cpu(smeta_buf->header.id); |
754 | line->type = le16_to_cpu(smeta_buf->header.type); | |
755 | line->seq_nr = le64_to_cpu(smeta_buf->seq_nr); | |
a4bd217b JG |
756 | spin_unlock(&line->lock); |
757 | ||
758 | /* Update general metadata */ | |
759 | spin_lock(&l_mg->free_lock); | |
760 | if (line->seq_nr >= l_mg->d_seq_nr) | |
761 | l_mg->d_seq_nr = line->seq_nr + 1; | |
762 | l_mg->nr_free_lines--; | |
763 | spin_unlock(&l_mg->free_lock); | |
764 | ||
765 | if (pblk_line_recov_alloc(pblk, line)) | |
766 | goto out; | |
767 | ||
768 | pblk_recov_line_add_ordered(&recov_list, line); | |
769 | found_lines++; | |
4e495a46 | 770 | pblk_debug(pblk, "recovering data line %d, seq:%llu\n", |
dd2a4343 | 771 | line->id, smeta_buf->seq_nr); |
a4bd217b JG |
772 | } |
773 | ||
774 | if (!found_lines) { | |
7e0a0847 | 775 | guid_gen(&pblk->instance_uuid); |
a4bd217b JG |
776 | |
777 | spin_lock(&l_mg->free_lock); | |
778 | WARN_ON_ONCE(!test_and_clear_bit(meta_line, | |
779 | &l_mg->meta_bitmap)); | |
780 | spin_unlock(&l_mg->free_lock); | |
781 | ||
782 | goto out; | |
783 | } | |
784 | ||
785 | /* Verify closed blocks and recover this portion of L2P table*/ | |
786 | list_for_each_entry_safe(line, tline, &recov_list, list) { | |
a4bd217b | 787 | recovered_lines++; |
a4bd217b | 788 | |
75610cd9 | 789 | line->emeta_ssec = pblk_line_emeta_start(pblk, line); |
8224cbd8 JG |
790 | line->emeta = emeta; |
791 | memset(line->emeta->buf, 0, lm->emeta_len[0]); | |
a4bd217b | 792 | |
6ad2f619 JG |
793 | if (pblk_line_is_open(pblk, line)) { |
794 | pblk_recov_l2p_from_oob(pblk, line); | |
795 | goto next; | |
796 | } | |
797 | ||
af3fac16 | 798 | if (pblk_line_emeta_read(pblk, line, line->emeta->buf)) { |
a4bd217b JG |
799 | pblk_recov_l2p_from_oob(pblk, line); |
800 | goto next; | |
801 | } | |
802 | ||
06bc072b HH |
803 | if (pblk_recov_check_emeta(pblk, line->emeta->buf)) { |
804 | pblk_recov_l2p_from_oob(pblk, line); | |
805 | goto next; | |
806 | } | |
807 | ||
d0ab0b1a HH |
808 | if (pblk_recov_check_line_version(pblk, line->emeta->buf)) |
809 | return ERR_PTR(-EINVAL); | |
810 | ||
76758390 HH |
811 | pblk_recov_wa_counters(pblk, line->emeta->buf); |
812 | ||
a4bd217b JG |
813 | if (pblk_recov_l2p_from_emeta(pblk, line)) |
814 | pblk_recov_l2p_from_oob(pblk, line); | |
815 | ||
816 | next: | |
817 | if (pblk_line_is_full(line)) { | |
818 | struct list_head *move_list; | |
819 | ||
820 | spin_lock(&line->lock); | |
821 | line->state = PBLK_LINESTATE_CLOSED; | |
f2937232 HH |
822 | trace_pblk_line_state(pblk_disk_name(pblk), line->id, |
823 | line->state); | |
a4bd217b JG |
824 | move_list = pblk_line_gc_list(pblk, line); |
825 | spin_unlock(&line->lock); | |
826 | ||
827 | spin_lock(&l_mg->gc_lock); | |
828 | list_move_tail(&line->list, move_list); | |
829 | spin_unlock(&l_mg->gc_lock); | |
830 | ||
53d82db6 | 831 | mempool_free(line->map_bitmap, l_mg->bitmap_pool); |
a4bd217b JG |
832 | line->map_bitmap = NULL; |
833 | line->smeta = NULL; | |
834 | line->emeta = NULL; | |
835 | } else { | |
6ad2f619 JG |
836 | spin_lock(&line->lock); |
837 | line->state = PBLK_LINESTATE_OPEN; | |
838 | spin_unlock(&line->lock); | |
839 | ||
840 | line->emeta->mem = 0; | |
841 | atomic_set(&line->emeta->sync, 0); | |
a4bd217b | 842 | |
f2937232 HH |
843 | trace_pblk_line_state(pblk_disk_name(pblk), line->id, |
844 | line->state); | |
845 | ||
a4bd217b | 846 | data_line = line; |
6ad2f619 JG |
847 | line->meta_line = meta_line; |
848 | ||
849 | open_lines++; | |
a4bd217b JG |
850 | } |
851 | } | |
852 | ||
a4bd217b | 853 | if (!open_lines) { |
7325b4bb | 854 | spin_lock(&l_mg->free_lock); |
a4bd217b JG |
855 | WARN_ON_ONCE(!test_and_clear_bit(meta_line, |
856 | &l_mg->meta_bitmap)); | |
7325b4bb | 857 | spin_unlock(&l_mg->free_lock); |
a4bd217b | 858 | } else { |
7325b4bb | 859 | spin_lock(&l_mg->free_lock); |
4bbae699 | 860 | l_mg->data_line = data_line; |
a4bd217b JG |
861 | /* Allocate next line for preparation */ |
862 | l_mg->data_next = pblk_line_get(pblk); | |
863 | if (l_mg->data_next) { | |
864 | l_mg->data_next->seq_nr = l_mg->d_seq_nr++; | |
865 | l_mg->data_next->type = PBLK_LINETYPE_DATA; | |
866 | is_next = 1; | |
867 | } | |
7325b4bb | 868 | spin_unlock(&l_mg->free_lock); |
a4bd217b | 869 | } |
a4bd217b | 870 | |
a7689938 | 871 | if (is_next) |
a4bd217b | 872 | pblk_line_erase(pblk, l_mg->data_next); |
a4bd217b JG |
873 | |
874 | out: | |
875 | if (found_lines != recovered_lines) | |
4e495a46 | 876 | pblk_err(pblk, "failed to recover all found lines %d/%d\n", |
a4bd217b JG |
877 | found_lines, recovered_lines); |
878 | ||
879 | return data_line; | |
880 | } | |
881 | ||
882 | /* | |
588726d3 | 883 | * Pad current line |
a4bd217b | 884 | */ |
588726d3 | 885 | int pblk_recov_pad(struct pblk *pblk) |
a4bd217b | 886 | { |
a4bd217b JG |
887 | struct pblk_line *line; |
888 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; | |
588726d3 JG |
889 | int left_msecs; |
890 | int ret = 0; | |
a4bd217b JG |
891 | |
892 | spin_lock(&l_mg->free_lock); | |
893 | line = l_mg->data_line; | |
588726d3 | 894 | left_msecs = line->left_msecs; |
a4bd217b JG |
895 | spin_unlock(&l_mg->free_lock); |
896 | ||
6ad2f619 | 897 | ret = pblk_recov_pad_line(pblk, line, left_msecs); |
588726d3 | 898 | if (ret) { |
4e495a46 | 899 | pblk_err(pblk, "tear down padding failed (%d)\n", ret); |
ee8d5c1a | 900 | return ret; |
a4bd217b JG |
901 | } |
902 | ||
588726d3 | 903 | pblk_line_close_meta(pblk, line); |
588726d3 | 904 | return ret; |
a4bd217b | 905 | } |