Commit | Line | Data |
---|---|---|
02a1520d | 1 | // SPDX-License-Identifier: GPL-2.0 |
a4bd217b JG |
2 | /* |
3 | * Copyright (C) 2016 CNEX Labs | |
4 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> | |
5 | * Matias Bjorling <matias@cnexlabs.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License version | |
9 | * 2 as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, but | |
12 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * General Public License for more details. | |
15 | * | |
16 | * pblk-read.c - pblk's read path | |
17 | */ | |
18 | ||
19 | #include "pblk.h" | |
20 | ||
21 | /* | |
22 | * There is no guarantee that the value read from cache has not been updated and | |
23 | * resides at another location in the cache. We guarantee though that if the | |
24 | * value is read from the cache, it belongs to the mapped lba. In order to | |
25 | * guarantee and order between writes and reads are ordered, a flush must be | |
26 | * issued. | |
27 | */ | |
28 | static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio, | |
29 | sector_t lba, struct ppa_addr ppa, | |
75cb8e93 | 30 | int bio_iter, bool advanced_bio) |
a4bd217b | 31 | { |
880eda54 | 32 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
33 | /* Callers must ensure that the ppa points to a cache address */ |
34 | BUG_ON(pblk_ppa_empty(ppa)); | |
35 | BUG_ON(!pblk_addr_in_cache(ppa)); | |
36 | #endif | |
37 | ||
75cb8e93 JG |
38 | return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa, |
39 | bio_iter, advanced_bio); | |
a4bd217b JG |
40 | } |
41 | ||
42 | static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd, | |
87cc40bb JG |
43 | struct bio *bio, sector_t blba, |
44 | unsigned long *read_bitmap) | |
a4bd217b | 45 | { |
a4809fee | 46 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
afdc23c9 | 47 | struct ppa_addr ppas[NVM_MAX_VLBA]; |
a4bd217b | 48 | int nr_secs = rqd->nr_ppas; |
75cb8e93 | 49 | bool advanced_bio = false; |
a4bd217b JG |
50 | int i, j = 0; |
51 | ||
a4bd217b JG |
52 | pblk_lookup_l2p_seq(pblk, ppas, blba, nr_secs); |
53 | ||
54 | for (i = 0; i < nr_secs; i++) { | |
55 | struct ppa_addr p = ppas[i]; | |
56 | sector_t lba = blba + i; | |
57 | ||
58 | retry: | |
59 | if (pblk_ppa_empty(p)) { | |
60 | WARN_ON(test_and_set_bit(i, read_bitmap)); | |
a4809fee | 61 | meta_list[i].lba = cpu_to_le64(ADDR_EMPTY); |
75cb8e93 JG |
62 | |
63 | if (unlikely(!advanced_bio)) { | |
64 | bio_advance(bio, (i) * PBLK_EXPOSED_PAGE_SIZE); | |
65 | advanced_bio = true; | |
66 | } | |
67 | ||
68 | goto next; | |
a4bd217b JG |
69 | } |
70 | ||
71 | /* Try to read from write buffer. The address is later checked | |
72 | * on the write buffer to prevent retrieving overwritten data. | |
73 | */ | |
74 | if (pblk_addr_in_cache(p)) { | |
75cb8e93 JG |
75 | if (!pblk_read_from_cache(pblk, bio, lba, p, i, |
76 | advanced_bio)) { | |
a4bd217b JG |
77 | pblk_lookup_l2p_seq(pblk, &p, lba, 1); |
78 | goto retry; | |
79 | } | |
80 | WARN_ON(test_and_set_bit(i, read_bitmap)); | |
a4809fee | 81 | meta_list[i].lba = cpu_to_le64(lba); |
75cb8e93 | 82 | advanced_bio = true; |
880eda54 | 83 | #ifdef CONFIG_NVM_PBLK_DEBUG |
db7ada33 JG |
84 | atomic_long_inc(&pblk->cache_reads); |
85 | #endif | |
a4bd217b JG |
86 | } else { |
87 | /* Read from media non-cached sectors */ | |
88 | rqd->ppa_list[j++] = p; | |
89 | } | |
90 | ||
75cb8e93 | 91 | next: |
a4bd217b JG |
92 | if (advanced_bio) |
93 | bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE); | |
94 | } | |
95 | ||
f9c10152 | 96 | if (pblk_io_aligned(pblk, nr_secs)) |
d7b68016 | 97 | rqd->is_seq = 1; |
f9c10152 | 98 | |
880eda54 | 99 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
100 | atomic_long_add(nr_secs, &pblk->inflight_reads); |
101 | #endif | |
102 | } | |
103 | ||
a4bd217b | 104 | |
03a34b2d JG |
105 | static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd, |
106 | sector_t blba) | |
a4809fee | 107 | { |
03a34b2d JG |
108 | struct pblk_sec_meta *meta_lba_list = rqd->meta_list; |
109 | int nr_lbas = rqd->nr_ppas; | |
a4809fee JG |
110 | int i; |
111 | ||
112 | for (i = 0; i < nr_lbas; i++) { | |
310df582 | 113 | u64 lba = le64_to_cpu(meta_lba_list[i].lba); |
a4809fee JG |
114 | |
115 | if (lba == ADDR_EMPTY) | |
116 | continue; | |
117 | ||
03a34b2d | 118 | if (lba != blba + i) { |
880eda54 | 119 | #ifdef CONFIG_NVM_PBLK_DEBUG |
d68a9344 | 120 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
03a34b2d | 121 | |
d68a9344 | 122 | print_ppa(pblk, &ppa_list[i], "seq", i); |
03a34b2d | 123 | #endif |
4e495a46 | 124 | pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", |
03a34b2d JG |
125 | lba, (u64)blba + i); |
126 | WARN_ON(1); | |
127 | } | |
a4809fee JG |
128 | } |
129 | } | |
130 | ||
310df582 JG |
131 | /* |
132 | * There can be holes in the lba list. | |
133 | */ | |
03a34b2d JG |
134 | static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, |
135 | u64 *lba_list, int nr_lbas) | |
310df582 | 136 | { |
03a34b2d | 137 | struct pblk_sec_meta *meta_lba_list = rqd->meta_list; |
310df582 JG |
138 | int i, j; |
139 | ||
140 | for (i = 0, j = 0; i < nr_lbas; i++) { | |
141 | u64 lba = lba_list[i]; | |
142 | u64 meta_lba; | |
143 | ||
144 | if (lba == ADDR_EMPTY) | |
145 | continue; | |
146 | ||
03a34b2d | 147 | meta_lba = le64_to_cpu(meta_lba_list[j].lba); |
310df582 JG |
148 | |
149 | if (lba != meta_lba) { | |
880eda54 | 150 | #ifdef CONFIG_NVM_PBLK_DEBUG |
d68a9344 | 151 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
03a34b2d | 152 | |
ea1d24bc | 153 | print_ppa(pblk, &ppa_list[j], "rnd", j); |
03a34b2d | 154 | #endif |
4e495a46 | 155 | pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n", |
ea1d24bc | 156 | meta_lba, lba); |
310df582 JG |
157 | WARN_ON(1); |
158 | } | |
03a34b2d JG |
159 | |
160 | j++; | |
310df582 | 161 | } |
03a34b2d JG |
162 | |
163 | WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n"); | |
310df582 JG |
164 | } |
165 | ||
7bd4d370 JG |
166 | static void pblk_end_user_read(struct bio *bio) |
167 | { | |
880eda54 | 168 | #ifdef CONFIG_NVM_PBLK_DEBUG |
7bd4d370 JG |
169 | WARN_ONCE(bio->bi_status, "pblk: corrupted read bio\n"); |
170 | #endif | |
171 | bio_endio(bio); | |
7bd4d370 JG |
172 | } |
173 | ||
174 | static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd, | |
175 | bool put_line) | |
a4bd217b | 176 | { |
998ba629 | 177 | struct nvm_tgt_dev *dev = pblk->dev; |
084ec9ba | 178 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
87cc40bb | 179 | struct bio *int_bio = rqd->bio; |
998ba629 JG |
180 | unsigned long start_time = r_ctx->start_time; |
181 | ||
ddcf35d3 | 182 | generic_end_io_acct(dev->q, REQ_OP_READ, &pblk->disk->part0, start_time); |
a4bd217b JG |
183 | |
184 | if (rqd->error) | |
185 | pblk_log_read_err(pblk, rqd); | |
a4bd217b | 186 | |
03a34b2d | 187 | pblk_read_check_seq(pblk, rqd, r_ctx->lba); |
a4809fee | 188 | |
87cc40bb JG |
189 | if (int_bio) |
190 | bio_put(int_bio); | |
084ec9ba | 191 | |
7bd4d370 | 192 | if (put_line) |
ae14cc04 | 193 | pblk_rq_to_line_put(pblk, rqd); |
a4bd217b | 194 | |
880eda54 | 195 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
196 | atomic_long_add(rqd->nr_ppas, &pblk->sync_reads); |
197 | atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads); | |
198 | #endif | |
199 | ||
e2cddf20 | 200 | pblk_free_rqd(pblk, rqd, PBLK_READ); |
588726d3 | 201 | atomic_dec(&pblk->inflight_io); |
a4bd217b JG |
202 | } |
203 | ||
7bd4d370 JG |
204 | static void pblk_end_io_read(struct nvm_rq *rqd) |
205 | { | |
206 | struct pblk *pblk = rqd->private; | |
87cc40bb JG |
207 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
208 | struct bio *bio = (struct bio *)r_ctx->private; | |
7bd4d370 | 209 | |
87cc40bb | 210 | pblk_end_user_read(bio); |
7bd4d370 JG |
211 | __pblk_end_io_read(pblk, rqd, true); |
212 | } | |
213 | ||
11f6ad69 | 214 | static void pblk_end_partial_read(struct nvm_rq *rqd) |
a4bd217b | 215 | { |
11f6ad69 HL |
216 | struct pblk *pblk = rqd->private; |
217 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); | |
218 | struct pblk_pr_ctx *pr_ctx = r_ctx->private; | |
219 | struct bio *new_bio = rqd->bio; | |
220 | struct bio *bio = pr_ctx->orig_bio; | |
a4bd217b | 221 | struct bio_vec src_bv, dst_bv; |
11f6ad69 HL |
222 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
223 | int bio_init_idx = pr_ctx->bio_init_idx; | |
224 | unsigned long *read_bitmap = pr_ctx->bitmap; | |
225 | int nr_secs = pr_ctx->orig_nr_secs; | |
a4bd217b | 226 | int nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs); |
11f6ad69 HL |
227 | __le64 *lba_list_mem, *lba_list_media; |
228 | void *src_p, *dst_p; | |
229 | int hole, i; | |
a4bd217b | 230 | |
0f9248cf | 231 | if (unlikely(nr_holes == 1)) { |
7bd4d370 JG |
232 | struct ppa_addr ppa; |
233 | ||
234 | ppa = rqd->ppa_addr; | |
11f6ad69 HL |
235 | rqd->ppa_list = pr_ctx->ppa_ptr; |
236 | rqd->dma_ppa_list = pr_ctx->dma_ppa_list; | |
7bd4d370 | 237 | rqd->ppa_list[0] = ppa; |
a4bd217b JG |
238 | } |
239 | ||
11f6ad69 HL |
240 | /* Re-use allocated memory for intermediate lbas */ |
241 | lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); | |
242 | lba_list_media = (((void *)rqd->ppa_list) + 2 * pblk_dma_ppa_size); | |
243 | ||
a4809fee JG |
244 | for (i = 0; i < nr_secs; i++) { |
245 | lba_list_media[i] = meta_list[i].lba; | |
246 | meta_list[i].lba = lba_list_mem[i]; | |
247 | } | |
248 | ||
a4bd217b JG |
249 | /* Fill the holes in the original bio */ |
250 | i = 0; | |
251 | hole = find_first_zero_bit(read_bitmap, nr_secs); | |
252 | do { | |
cb21665c | 253 | struct pblk_line *line; |
7bd4d370 | 254 | |
cb21665c | 255 | line = pblk_ppa_to_line(pblk, rqd->ppa_list[i]); |
7bd4d370 JG |
256 | kref_put(&line->ref, pblk_line_put); |
257 | ||
a4809fee JG |
258 | meta_list[hole].lba = lba_list_media[i]; |
259 | ||
a4bd217b | 260 | src_bv = new_bio->bi_io_vec[i++]; |
11f6ad69 | 261 | dst_bv = bio->bi_io_vec[bio_init_idx + hole]; |
a4bd217b JG |
262 | |
263 | src_p = kmap_atomic(src_bv.bv_page); | |
264 | dst_p = kmap_atomic(dst_bv.bv_page); | |
265 | ||
266 | memcpy(dst_p + dst_bv.bv_offset, | |
267 | src_p + src_bv.bv_offset, | |
268 | PBLK_EXPOSED_PAGE_SIZE); | |
269 | ||
270 | kunmap_atomic(src_p); | |
271 | kunmap_atomic(dst_p); | |
272 | ||
b906bbb6 | 273 | mempool_free(src_bv.bv_page, &pblk->page_bio_pool); |
a4bd217b JG |
274 | |
275 | hole = find_next_zero_bit(read_bitmap, nr_secs, hole + 1); | |
276 | } while (hole < nr_secs); | |
277 | ||
278 | bio_put(new_bio); | |
11f6ad69 | 279 | kfree(pr_ctx); |
a4bd217b | 280 | |
87cc40bb JG |
281 | /* restore original request */ |
282 | rqd->bio = NULL; | |
a4bd217b | 283 | rqd->nr_ppas = nr_secs; |
a4bd217b | 284 | |
11f6ad69 | 285 | bio_endio(bio); |
7bd4d370 | 286 | __pblk_end_io_read(pblk, rqd, false); |
11f6ad69 | 287 | } |
a4bd217b | 288 | |
11f6ad69 HL |
289 | static int pblk_setup_partial_read(struct pblk *pblk, struct nvm_rq *rqd, |
290 | unsigned int bio_init_idx, | |
291 | unsigned long *read_bitmap, | |
292 | int nr_holes) | |
293 | { | |
294 | struct pblk_sec_meta *meta_list = rqd->meta_list; | |
295 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); | |
296 | struct pblk_pr_ctx *pr_ctx; | |
297 | struct bio *new_bio, *bio = r_ctx->private; | |
298 | __le64 *lba_list_mem; | |
299 | int nr_secs = rqd->nr_ppas; | |
300 | int i; | |
301 | ||
302 | /* Re-use allocated memory for intermediate lbas */ | |
303 | lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); | |
304 | ||
305 | new_bio = bio_alloc(GFP_KERNEL, nr_holes); | |
306 | ||
307 | if (pblk_bio_add_pages(pblk, new_bio, GFP_KERNEL, nr_holes)) | |
308 | goto fail_bio_put; | |
309 | ||
310 | if (nr_holes != new_bio->bi_vcnt) { | |
311 | WARN_ONCE(1, "pblk: malformed bio\n"); | |
312 | goto fail_free_pages; | |
313 | } | |
314 | ||
315 | pr_ctx = kmalloc(sizeof(struct pblk_pr_ctx), GFP_KERNEL); | |
316 | if (!pr_ctx) | |
317 | goto fail_free_pages; | |
318 | ||
319 | for (i = 0; i < nr_secs; i++) | |
320 | lba_list_mem[i] = meta_list[i].lba; | |
321 | ||
322 | new_bio->bi_iter.bi_sector = 0; /* internal bio */ | |
323 | bio_set_op_attrs(new_bio, REQ_OP_READ, 0); | |
324 | ||
325 | rqd->bio = new_bio; | |
326 | rqd->nr_ppas = nr_holes; | |
11f6ad69 HL |
327 | |
328 | pr_ctx->ppa_ptr = NULL; | |
329 | pr_ctx->orig_bio = bio; | |
330 | bitmap_copy(pr_ctx->bitmap, read_bitmap, NVM_MAX_VLBA); | |
331 | pr_ctx->bio_init_idx = bio_init_idx; | |
332 | pr_ctx->orig_nr_secs = nr_secs; | |
333 | r_ctx->private = pr_ctx; | |
334 | ||
335 | if (unlikely(nr_holes == 1)) { | |
336 | pr_ctx->ppa_ptr = rqd->ppa_list; | |
337 | pr_ctx->dma_ppa_list = rqd->dma_ppa_list; | |
338 | rqd->ppa_addr = rqd->ppa_list[0]; | |
339 | } | |
340 | return 0; | |
341 | ||
342 | fail_free_pages: | |
fbadca73 | 343 | pblk_bio_free_pages(pblk, new_bio, 0, new_bio->bi_vcnt); |
11f6ad69 HL |
344 | fail_bio_put: |
345 | bio_put(new_bio); | |
346 | ||
347 | return -ENOMEM; | |
348 | } | |
349 | ||
350 | static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd, | |
351 | unsigned int bio_init_idx, | |
352 | unsigned long *read_bitmap, int nr_secs) | |
353 | { | |
354 | int nr_holes; | |
355 | int ret; | |
356 | ||
357 | nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs); | |
358 | ||
359 | if (pblk_setup_partial_read(pblk, rqd, bio_init_idx, read_bitmap, | |
360 | nr_holes)) | |
361 | return NVM_IO_ERR; | |
362 | ||
363 | rqd->end_io = pblk_end_partial_read; | |
364 | ||
365 | ret = pblk_submit_io(pblk, rqd); | |
366 | if (ret) { | |
367 | bio_put(rqd->bio); | |
368 | pblk_err(pblk, "partial read IO submission failed\n"); | |
369 | goto err; | |
370 | } | |
371 | ||
372 | return NVM_IO_OK; | |
373 | ||
374 | err: | |
4e495a46 | 375 | pblk_err(pblk, "failed to perform partial read\n"); |
11f6ad69 HL |
376 | |
377 | /* Free allocated pages in new bio */ | |
378 | pblk_bio_free_pages(pblk, rqd->bio, 0, rqd->bio->bi_vcnt); | |
7bd4d370 | 379 | __pblk_end_io_read(pblk, rqd, false); |
a4bd217b JG |
380 | return NVM_IO_ERR; |
381 | } | |
382 | ||
87cc40bb | 383 | static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio, |
84454e6d | 384 | sector_t lba, unsigned long *read_bitmap) |
a4bd217b | 385 | { |
a4809fee | 386 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
a4bd217b | 387 | struct ppa_addr ppa; |
a4bd217b JG |
388 | |
389 | pblk_lookup_l2p_seq(pblk, &ppa, lba, 1); | |
390 | ||
880eda54 | 391 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
392 | atomic_long_inc(&pblk->inflight_reads); |
393 | #endif | |
394 | ||
395 | retry: | |
396 | if (pblk_ppa_empty(ppa)) { | |
397 | WARN_ON(test_and_set_bit(0, read_bitmap)); | |
a4809fee | 398 | meta_list[0].lba = cpu_to_le64(ADDR_EMPTY); |
a4bd217b JG |
399 | return; |
400 | } | |
401 | ||
402 | /* Try to read from write buffer. The address is later checked on the | |
403 | * write buffer to prevent retrieving overwritten data. | |
404 | */ | |
405 | if (pblk_addr_in_cache(ppa)) { | |
75cb8e93 | 406 | if (!pblk_read_from_cache(pblk, bio, lba, ppa, 0, 1)) { |
a4bd217b JG |
407 | pblk_lookup_l2p_seq(pblk, &ppa, lba, 1); |
408 | goto retry; | |
409 | } | |
a4809fee | 410 | |
7bd4d370 | 411 | WARN_ON(test_and_set_bit(0, read_bitmap)); |
a4809fee JG |
412 | meta_list[0].lba = cpu_to_le64(lba); |
413 | ||
880eda54 | 414 | #ifdef CONFIG_NVM_PBLK_DEBUG |
7bd4d370 | 415 | atomic_long_inc(&pblk->cache_reads); |
db7ada33 | 416 | #endif |
a4bd217b JG |
417 | } else { |
418 | rqd->ppa_addr = ppa; | |
419 | } | |
420 | } | |
421 | ||
422 | int pblk_submit_read(struct pblk *pblk, struct bio *bio) | |
423 | { | |
424 | struct nvm_tgt_dev *dev = pblk->dev; | |
998ba629 | 425 | struct request_queue *q = dev->q; |
84454e6d | 426 | sector_t blba = pblk_get_lba(bio); |
5bf1e1ee | 427 | unsigned int nr_secs = pblk_get_secs(bio); |
a4809fee | 428 | struct pblk_g_ctx *r_ctx; |
a4bd217b | 429 | struct nvm_rq *rqd; |
a4bd217b | 430 | unsigned int bio_init_idx; |
921aebfa | 431 | DECLARE_BITMAP(read_bitmap, NVM_MAX_VLBA); |
a4bd217b JG |
432 | int ret = NVM_IO_ERR; |
433 | ||
ddcf35d3 MC |
434 | generic_start_io_acct(q, REQ_OP_READ, bio_sectors(bio), |
435 | &pblk->disk->part0); | |
998ba629 | 436 | |
921aebfa | 437 | bitmap_zero(read_bitmap, nr_secs); |
a4bd217b | 438 | |
e2cddf20 | 439 | rqd = pblk_alloc_rqd(pblk, PBLK_READ); |
a4bd217b JG |
440 | |
441 | rqd->opcode = NVM_OP_PREAD; | |
a4bd217b | 442 | rqd->nr_ppas = nr_secs; |
87cc40bb | 443 | rqd->bio = NULL; /* cloned bio if needed */ |
a4bd217b JG |
444 | rqd->private = pblk; |
445 | rqd->end_io = pblk_end_io_read; | |
446 | ||
a4809fee | 447 | r_ctx = nvm_rq_to_pdu(rqd); |
998ba629 | 448 | r_ctx->start_time = jiffies; |
a4809fee | 449 | r_ctx->lba = blba; |
87cc40bb | 450 | r_ctx->private = bio; /* original bio */ |
a4809fee | 451 | |
a4bd217b JG |
452 | /* Save the index for this bio's start. This is needed in case |
453 | * we need to fill a partial read. | |
454 | */ | |
455 | bio_init_idx = pblk_get_bi_idx(bio); | |
456 | ||
45dcf29b | 457 | if (pblk_alloc_rqd_meta(pblk, rqd)) |
63e3809c | 458 | goto fail_rqd_free; |
a4bd217b | 459 | |
45dcf29b | 460 | if (nr_secs > 1) |
921aebfa | 461 | pblk_read_ppalist_rq(pblk, rqd, bio, blba, read_bitmap); |
45dcf29b | 462 | else |
921aebfa | 463 | pblk_read_rq(pblk, rqd, bio, blba, read_bitmap); |
a4bd217b | 464 | |
921aebfa | 465 | if (bitmap_full(read_bitmap, nr_secs)) { |
588726d3 | 466 | atomic_inc(&pblk->inflight_io); |
7bd4d370 | 467 | __pblk_end_io_read(pblk, rqd, false); |
87cc40bb | 468 | return NVM_IO_DONE; |
a4bd217b JG |
469 | } |
470 | ||
471 | /* All sectors are to be read from the device */ | |
921aebfa | 472 | if (bitmap_empty(read_bitmap, rqd->nr_ppas)) { |
a4bd217b | 473 | struct bio *int_bio = NULL; |
a4bd217b JG |
474 | |
475 | /* Clone read bio to deal with read errors internally */ | |
b906bbb6 | 476 | int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set); |
a4bd217b | 477 | if (!int_bio) { |
4e495a46 | 478 | pblk_err(pblk, "could not clone read bio\n"); |
998ba629 | 479 | goto fail_end_io; |
a4bd217b JG |
480 | } |
481 | ||
482 | rqd->bio = int_bio; | |
a4bd217b | 483 | |
e13f421b | 484 | if (pblk_submit_io(pblk, rqd)) { |
4e495a46 | 485 | pblk_err(pblk, "read IO submission failed\n"); |
e13f421b | 486 | ret = NVM_IO_ERR; |
998ba629 | 487 | goto fail_end_io; |
a4bd217b JG |
488 | } |
489 | ||
490 | return NVM_IO_OK; | |
491 | } | |
492 | ||
493 | /* The read bio request could be partially filled by the write buffer, | |
494 | * but there are some holes that need to be read from the drive. | |
495 | */ | |
11f6ad69 HL |
496 | ret = pblk_partial_read_bio(pblk, rqd, bio_init_idx, read_bitmap, |
497 | nr_secs); | |
498 | if (ret) | |
499 | goto fail_meta_free; | |
500 | ||
501 | return NVM_IO_OK; | |
a4bd217b | 502 | |
11f6ad69 HL |
503 | fail_meta_free: |
504 | nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); | |
a4bd217b | 505 | fail_rqd_free: |
e2cddf20 | 506 | pblk_free_rqd(pblk, rqd, PBLK_READ); |
a4bd217b | 507 | return ret; |
998ba629 JG |
508 | fail_end_io: |
509 | __pblk_end_io_read(pblk, rqd, false); | |
510 | return ret; | |
a4bd217b JG |
511 | } |
512 | ||
513 | static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, | |
514 | struct pblk_line *line, u64 *lba_list, | |
d340121e | 515 | u64 *paddr_list_gc, unsigned int nr_secs) |
a4bd217b | 516 | { |
afdc23c9 | 517 | struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA]; |
d340121e | 518 | struct ppa_addr ppa_gc; |
a4bd217b JG |
519 | int valid_secs = 0; |
520 | int i; | |
521 | ||
d340121e | 522 | pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs); |
a4bd217b JG |
523 | |
524 | for (i = 0; i < nr_secs; i++) { | |
d340121e JG |
525 | if (lba_list[i] == ADDR_EMPTY) |
526 | continue; | |
527 | ||
528 | ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id); | |
529 | if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) { | |
530 | paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY; | |
a4bd217b JG |
531 | continue; |
532 | } | |
533 | ||
d340121e | 534 | rqd->ppa_list[valid_secs++] = ppa_list_l2p[i]; |
a4bd217b JG |
535 | } |
536 | ||
880eda54 | 537 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
538 | atomic_long_add(valid_secs, &pblk->inflight_reads); |
539 | #endif | |
d340121e | 540 | |
a4bd217b JG |
541 | return valid_secs; |
542 | } | |
543 | ||
544 | static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, | |
d340121e JG |
545 | struct pblk_line *line, sector_t lba, |
546 | u64 paddr_gc) | |
a4bd217b | 547 | { |
d340121e | 548 | struct ppa_addr ppa_l2p, ppa_gc; |
a4bd217b JG |
549 | int valid_secs = 0; |
550 | ||
659226eb DC |
551 | if (lba == ADDR_EMPTY) |
552 | goto out; | |
553 | ||
a4bd217b | 554 | /* logic error: lba out-of-bounds */ |
2a79efd8 DC |
555 | if (lba >= pblk->rl.nr_secs) { |
556 | WARN(1, "pblk: read lba out of bounds\n"); | |
a4bd217b JG |
557 | goto out; |
558 | } | |
559 | ||
a4bd217b | 560 | spin_lock(&pblk->trans_lock); |
d340121e | 561 | ppa_l2p = pblk_trans_map_get(pblk, lba); |
a4bd217b JG |
562 | spin_unlock(&pblk->trans_lock); |
563 | ||
d340121e JG |
564 | ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id); |
565 | if (!pblk_ppa_comp(ppa_l2p, ppa_gc)) | |
a4bd217b JG |
566 | goto out; |
567 | ||
d340121e | 568 | rqd->ppa_addr = ppa_l2p; |
a4bd217b JG |
569 | valid_secs = 1; |
570 | ||
880eda54 | 571 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
572 | atomic_long_inc(&pblk->inflight_reads); |
573 | #endif | |
574 | ||
575 | out: | |
576 | return valid_secs; | |
577 | } | |
578 | ||
d340121e | 579 | int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) |
a4bd217b JG |
580 | { |
581 | struct nvm_tgt_dev *dev = pblk->dev; | |
582 | struct nvm_geo *geo = &dev->geo; | |
a4bd217b JG |
583 | struct bio *bio; |
584 | struct nvm_rq rqd; | |
d340121e JG |
585 | int data_len; |
586 | int ret = NVM_IO_OK; | |
a4bd217b JG |
587 | |
588 | memset(&rqd, 0, sizeof(struct nvm_rq)); | |
589 | ||
45dcf29b JG |
590 | ret = pblk_alloc_rqd_meta(pblk, &rqd); |
591 | if (ret) | |
592 | return ret; | |
63e3809c | 593 | |
d340121e | 594 | if (gc_rq->nr_secs > 1) { |
d340121e JG |
595 | gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line, |
596 | gc_rq->lba_list, | |
597 | gc_rq->paddr_list, | |
598 | gc_rq->nr_secs); | |
599 | if (gc_rq->secs_to_gc == 1) | |
63e3809c | 600 | rqd.ppa_addr = rqd.ppa_list[0]; |
a4bd217b | 601 | } else { |
d340121e JG |
602 | gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line, |
603 | gc_rq->lba_list[0], | |
604 | gc_rq->paddr_list[0]); | |
a4bd217b JG |
605 | } |
606 | ||
d340121e | 607 | if (!(gc_rq->secs_to_gc)) |
a4bd217b JG |
608 | goto out; |
609 | ||
e46f4e48 | 610 | data_len = (gc_rq->secs_to_gc) * geo->csecs; |
d340121e | 611 | bio = pblk_bio_map_addr(pblk, gc_rq->data, gc_rq->secs_to_gc, data_len, |
7d327a9e | 612 | PBLK_VMALLOC_META, GFP_KERNEL); |
a4bd217b | 613 | if (IS_ERR(bio)) { |
4e495a46 | 614 | pblk_err(pblk, "could not allocate GC bio (%lu)\n", |
45dcf29b JG |
615 | PTR_ERR(bio)); |
616 | ret = PTR_ERR(bio); | |
a4bd217b JG |
617 | goto err_free_dma; |
618 | } | |
619 | ||
620 | bio->bi_iter.bi_sector = 0; /* internal bio */ | |
621 | bio_set_op_attrs(bio, REQ_OP_READ, 0); | |
622 | ||
623 | rqd.opcode = NVM_OP_PREAD; | |
d340121e | 624 | rqd.nr_ppas = gc_rq->secs_to_gc; |
a4bd217b JG |
625 | rqd.bio = bio; |
626 | ||
1a94b2d4 | 627 | if (pblk_submit_io_sync(pblk, &rqd)) { |
d340121e | 628 | ret = -EIO; |
4e495a46 | 629 | pblk_err(pblk, "GC read request failed\n"); |
7d327a9e | 630 | goto err_free_bio; |
a4bd217b JG |
631 | } |
632 | ||
03a34b2d | 633 | pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs); |
310df582 | 634 | |
588726d3 | 635 | atomic_dec(&pblk->inflight_io); |
a4bd217b JG |
636 | |
637 | if (rqd.error) { | |
638 | atomic_long_inc(&pblk->read_failed_gc); | |
880eda54 | 639 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
640 | pblk_print_failed_rqd(pblk, &rqd, rqd.error); |
641 | #endif | |
642 | } | |
643 | ||
880eda54 | 644 | #ifdef CONFIG_NVM_PBLK_DEBUG |
d340121e JG |
645 | atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads); |
646 | atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads); | |
647 | atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads); | |
a4bd217b JG |
648 | #endif |
649 | ||
650 | out: | |
45dcf29b | 651 | pblk_free_rqd_meta(pblk, &rqd); |
d340121e | 652 | return ret; |
a4bd217b | 653 | |
7d327a9e JG |
654 | err_free_bio: |
655 | bio_put(bio); | |
a4bd217b | 656 | err_free_dma: |
45dcf29b | 657 | pblk_free_rqd_meta(pblk, &rqd); |
d340121e | 658 | return ret; |
a4bd217b | 659 | } |