Commit | Line | Data |
---|---|---|
a4bd217b JG |
1 | /* |
2 | * Copyright (C) 2016 CNEX Labs | |
3 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> | |
4 | * Matias Bjorling <matias@cnexlabs.com> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License version | |
8 | * 2 as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License for more details. | |
14 | * | |
15 | * pblk-write.c - pblk's write path from write buffer to media | |
16 | */ | |
17 | ||
18 | #include "pblk.h" | |
4c44abf4 | 19 | #include "pblk-trace.h" |
a4bd217b | 20 | |
a4bd217b JG |
21 | static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd, |
22 | struct pblk_c_ctx *c_ctx) | |
23 | { | |
a4bd217b | 24 | struct bio *original_bio; |
533657c1 | 25 | struct pblk_rb *rwb = &pblk->rwb; |
a4bd217b JG |
26 | unsigned long ret; |
27 | int i; | |
28 | ||
29 | for (i = 0; i < c_ctx->nr_valid; i++) { | |
30 | struct pblk_w_ctx *w_ctx; | |
533657c1 HH |
31 | int pos = c_ctx->sentry + i; |
32 | int flags; | |
33 | ||
34 | w_ctx = pblk_rb_w_ctx(rwb, pos); | |
35 | flags = READ_ONCE(w_ctx->flags); | |
36 | ||
37 | if (flags & PBLK_FLUSH_ENTRY) { | |
38 | flags &= ~PBLK_FLUSH_ENTRY; | |
39 | /* Release flags on context. Protect from writes */ | |
40 | smp_store_release(&w_ctx->flags, flags); | |
41 | ||
880eda54 | 42 | #ifdef CONFIG_NVM_PBLK_DEBUG |
533657c1 HH |
43 | atomic_dec(&rwb->inflight_flush_point); |
44 | #endif | |
45 | } | |
a4bd217b | 46 | |
a4bd217b JG |
47 | while ((original_bio = bio_list_pop(&w_ctx->bios))) |
48 | bio_endio(original_bio); | |
49 | } | |
50 | ||
cd8ddbf7 JG |
51 | if (c_ctx->nr_padded) |
52 | pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid, | |
53 | c_ctx->nr_padded); | |
54 | ||
880eda54 | 55 | #ifdef CONFIG_NVM_PBLK_DEBUG |
e0e12a70 | 56 | atomic_long_add(rqd->nr_ppas, &pblk->sync_writes); |
a4bd217b JG |
57 | #endif |
58 | ||
59 | ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid); | |
60 | ||
a4bd217b | 61 | bio_put(rqd->bio); |
e2cddf20 | 62 | pblk_free_rqd(pblk, rqd, PBLK_WRITE); |
a4bd217b JG |
63 | |
64 | return ret; | |
65 | } | |
66 | ||
67 | static unsigned long pblk_end_queued_w_bio(struct pblk *pblk, | |
68 | struct nvm_rq *rqd, | |
69 | struct pblk_c_ctx *c_ctx) | |
70 | { | |
71 | list_del(&c_ctx->list); | |
72 | return pblk_end_w_bio(pblk, rqd, c_ctx); | |
73 | } | |
74 | ||
75 | static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd, | |
76 | struct pblk_c_ctx *c_ctx) | |
77 | { | |
78 | struct pblk_c_ctx *c, *r; | |
79 | unsigned long flags; | |
80 | unsigned long pos; | |
81 | ||
880eda54 | 82 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
83 | atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes); |
84 | #endif | |
e99e802f | 85 | pblk_up_rq(pblk, c_ctx->lun_bitmap); |
a4bd217b JG |
86 | |
87 | pos = pblk_rb_sync_init(&pblk->rwb, &flags); | |
88 | if (pos == c_ctx->sentry) { | |
89 | pos = pblk_end_w_bio(pblk, rqd, c_ctx); | |
90 | ||
91 | retry: | |
92 | list_for_each_entry_safe(c, r, &pblk->compl_list, list) { | |
93 | rqd = nvm_rq_from_c_ctx(c); | |
94 | if (c->sentry == pos) { | |
95 | pos = pblk_end_queued_w_bio(pblk, rqd, c); | |
96 | goto retry; | |
97 | } | |
98 | } | |
99 | } else { | |
100 | WARN_ON(nvm_rq_from_c_ctx(c_ctx) != rqd); | |
101 | list_add_tail(&c_ctx->list, &pblk->compl_list); | |
102 | } | |
103 | pblk_rb_sync_end(&pblk->rwb, &flags); | |
104 | } | |
105 | ||
6a3abf5b HH |
106 | /* Map remaining sectors in chunk, starting from ppa */ |
107 | static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa) | |
a4bd217b | 108 | { |
6a3abf5b HH |
109 | struct nvm_tgt_dev *dev = pblk->dev; |
110 | struct nvm_geo *geo = &dev->geo; | |
111 | struct pblk_line *line; | |
112 | struct ppa_addr map_ppa = *ppa; | |
113 | u64 paddr; | |
114 | int done = 0; | |
a4bd217b | 115 | |
cb21665c | 116 | line = pblk_ppa_to_line(pblk, *ppa); |
6a3abf5b | 117 | spin_lock(&line->lock); |
a4bd217b | 118 | |
6a3abf5b HH |
119 | while (!done) { |
120 | paddr = pblk_dev_ppa_to_line_addr(pblk, map_ppa); | |
2942f50f | 121 | |
6a3abf5b HH |
122 | if (!test_and_set_bit(paddr, line->map_bitmap)) |
123 | line->left_msecs--; | |
a4bd217b | 124 | |
6a3abf5b HH |
125 | if (!test_and_set_bit(paddr, line->invalid_bitmap)) |
126 | le32_add_cpu(line->vsc, -1); | |
a4bd217b | 127 | |
6a3abf5b HH |
128 | if (geo->version == NVM_OCSSD_SPEC_12) { |
129 | map_ppa.ppa++; | |
130 | if (map_ppa.g.pg == geo->num_pg) | |
131 | done = 1; | |
132 | } else { | |
133 | map_ppa.m.sec++; | |
134 | if (map_ppa.m.sec == geo->clba) | |
135 | done = 1; | |
a4bd217b | 136 | } |
6a3abf5b | 137 | } |
a4bd217b | 138 | |
48b8d208 | 139 | line->w_err_gc->has_write_err = 1; |
6a3abf5b HH |
140 | spin_unlock(&line->lock); |
141 | } | |
142 | ||
143 | static void pblk_prepare_resubmit(struct pblk *pblk, unsigned int sentry, | |
144 | unsigned int nr_entries) | |
145 | { | |
146 | struct pblk_rb *rb = &pblk->rwb; | |
147 | struct pblk_rb_entry *entry; | |
148 | struct pblk_line *line; | |
149 | struct pblk_w_ctx *w_ctx; | |
150 | struct ppa_addr ppa_l2p; | |
151 | int flags; | |
152 | unsigned int pos, i; | |
153 | ||
154 | spin_lock(&pblk->trans_lock); | |
155 | pos = sentry; | |
156 | for (i = 0; i < nr_entries; i++) { | |
157 | entry = &rb->entries[pos]; | |
158 | w_ctx = &entry->w_ctx; | |
159 | ||
160 | /* Check if the lba has been overwritten */ | |
161 | ppa_l2p = pblk_trans_map_get(pblk, w_ctx->lba); | |
162 | if (!pblk_ppa_comp(ppa_l2p, entry->cacheline)) | |
163 | w_ctx->lba = ADDR_EMPTY; | |
164 | ||
165 | /* Mark up the entry as submittable again */ | |
166 | flags = READ_ONCE(w_ctx->flags); | |
167 | flags |= PBLK_WRITTEN_DATA; | |
168 | /* Release flags on write context. Protect from writes */ | |
169 | smp_store_release(&w_ctx->flags, flags); | |
a4bd217b | 170 | |
2e696f90 | 171 | /* Decrease the reference count to the line as we will |
6a3abf5b | 172 | * re-map these entries |
a4bd217b | 173 | */ |
cb21665c | 174 | line = pblk_ppa_to_line(pblk, w_ctx->ppa); |
6a3abf5b HH |
175 | kref_put(&line->ref, pblk_line_put); |
176 | ||
177 | pos = (pos + 1) & (rb->nr_entries - 1); | |
a4bd217b | 178 | } |
6a3abf5b HH |
179 | spin_unlock(&pblk->trans_lock); |
180 | } | |
a4bd217b | 181 | |
6a3abf5b HH |
182 | static void pblk_queue_resubmit(struct pblk *pblk, struct pblk_c_ctx *c_ctx) |
183 | { | |
184 | struct pblk_c_ctx *r_ctx; | |
185 | ||
186 | r_ctx = kzalloc(sizeof(struct pblk_c_ctx), GFP_KERNEL); | |
187 | if (!r_ctx) | |
188 | return; | |
189 | ||
190 | r_ctx->lun_bitmap = NULL; | |
191 | r_ctx->sentry = c_ctx->sentry; | |
192 | r_ctx->nr_valid = c_ctx->nr_valid; | |
193 | r_ctx->nr_padded = c_ctx->nr_padded; | |
194 | ||
195 | spin_lock(&pblk->resubmit_lock); | |
196 | list_add_tail(&r_ctx->list, &pblk->resubmit_list); | |
197 | spin_unlock(&pblk->resubmit_lock); | |
198 | ||
880eda54 | 199 | #ifdef CONFIG_NVM_PBLK_DEBUG |
6a3abf5b HH |
200 | atomic_long_add(c_ctx->nr_valid, &pblk->recov_writes); |
201 | #endif | |
202 | } | |
203 | ||
204 | static void pblk_submit_rec(struct work_struct *work) | |
205 | { | |
206 | struct pblk_rec_ctx *recovery = | |
207 | container_of(work, struct pblk_rec_ctx, ws_rec); | |
208 | struct pblk *pblk = recovery->pblk; | |
209 | struct nvm_rq *rqd = recovery->rqd; | |
210 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); | |
d68a9344 | 211 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
6a3abf5b HH |
212 | |
213 | pblk_log_write_err(pblk, rqd); | |
214 | ||
6a3abf5b HH |
215 | pblk_map_remaining(pblk, ppa_list); |
216 | pblk_queue_resubmit(pblk, c_ctx); | |
217 | ||
e99e802f | 218 | pblk_up_rq(pblk, c_ctx->lun_bitmap); |
6a3abf5b HH |
219 | if (c_ctx->nr_padded) |
220 | pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid, | |
221 | c_ctx->nr_padded); | |
222 | bio_put(rqd->bio); | |
223 | pblk_free_rqd(pblk, rqd, PBLK_WRITE); | |
224 | mempool_free(recovery, &pblk->rec_pool); | |
225 | ||
226 | atomic_dec(&pblk->inflight_io); | |
227 | } | |
228 | ||
229 | ||
230 | static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd) | |
231 | { | |
232 | struct pblk_rec_ctx *recovery; | |
233 | ||
234 | recovery = mempool_alloc(&pblk->rec_pool, GFP_ATOMIC); | |
235 | if (!recovery) { | |
4e495a46 | 236 | pblk_err(pblk, "could not allocate recovery work\n"); |
6a3abf5b | 237 | return; |
a4bd217b JG |
238 | } |
239 | ||
6a3abf5b HH |
240 | recovery->pblk = pblk; |
241 | recovery->rqd = rqd; | |
242 | ||
a4bd217b | 243 | INIT_WORK(&recovery->ws_rec, pblk_submit_rec); |
ef576494 | 244 | queue_work(pblk->close_wq, &recovery->ws_rec); |
a4bd217b JG |
245 | } |
246 | ||
247 | static void pblk_end_io_write(struct nvm_rq *rqd) | |
248 | { | |
249 | struct pblk *pblk = rqd->private; | |
250 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); | |
251 | ||
252 | if (rqd->error) { | |
6a3abf5b HH |
253 | pblk_end_w_fail(pblk, rqd); |
254 | return; | |
4c44abf4 HH |
255 | } else { |
256 | if (trace_pblk_chunk_state_enabled()) | |
257 | pblk_check_chunk_state_update(pblk, rqd); | |
880eda54 | 258 | #ifdef CONFIG_NVM_PBLK_DEBUG |
4e4cbee9 | 259 | WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n"); |
a4bd217b | 260 | #endif |
4c44abf4 | 261 | } |
a4bd217b JG |
262 | |
263 | pblk_complete_write(pblk, rqd, c_ctx); | |
588726d3 | 264 | atomic_dec(&pblk->inflight_io); |
a4bd217b JG |
265 | } |
266 | ||
dd2a4343 JG |
267 | static void pblk_end_io_write_meta(struct nvm_rq *rqd) |
268 | { | |
269 | struct pblk *pblk = rqd->private; | |
dd2a4343 JG |
270 | struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd); |
271 | struct pblk_line *line = m_ctx->private; | |
272 | struct pblk_emeta *emeta = line->emeta; | |
d68a9344 | 273 | struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
dd2a4343 JG |
274 | int sync; |
275 | ||
43241cfe | 276 | pblk_up_chunk(pblk, ppa_list[0]); |
dd2a4343 JG |
277 | |
278 | if (rqd->error) { | |
279 | pblk_log_write_err(pblk, rqd); | |
4e495a46 | 280 | pblk_err(pblk, "metadata I/O failed. Line %d\n", line->id); |
48b8d208 | 281 | line->w_err_gc->has_write_err = 1; |
4c44abf4 HH |
282 | } else { |
283 | if (trace_pblk_chunk_state_enabled()) | |
284 | pblk_check_chunk_state_update(pblk, rqd); | |
dd2a4343 | 285 | } |
dd2a4343 JG |
286 | |
287 | sync = atomic_add_return(rqd->nr_ppas, &emeta->sync); | |
288 | if (sync == emeta->nr_entries) | |
b84ae4a8 JG |
289 | pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws, |
290 | GFP_ATOMIC, pblk->close_wq); | |
dd2a4343 | 291 | |
e2cddf20 | 292 | pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); |
588726d3 JG |
293 | |
294 | atomic_dec(&pblk->inflight_io); | |
dd2a4343 JG |
295 | } |
296 | ||
a4bd217b | 297 | static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd, |
dd2a4343 JG |
298 | unsigned int nr_secs, |
299 | nvm_end_io_fn(*end_io)) | |
a4bd217b JG |
300 | { |
301 | struct nvm_tgt_dev *dev = pblk->dev; | |
302 | ||
303 | /* Setup write request */ | |
304 | rqd->opcode = NVM_OP_PWRITE; | |
305 | rqd->nr_ppas = nr_secs; | |
d7b68016 | 306 | rqd->is_seq = 1; |
a4bd217b | 307 | rqd->private = pblk; |
dd2a4343 | 308 | rqd->end_io = end_io; |
a4bd217b JG |
309 | |
310 | rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, | |
311 | &rqd->dma_meta_list); | |
312 | if (!rqd->meta_list) | |
313 | return -ENOMEM; | |
314 | ||
a4bd217b JG |
315 | rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; |
316 | rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; | |
317 | ||
318 | return 0; | |
319 | } | |
320 | ||
321 | static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd, | |
1e82123d | 322 | struct ppa_addr *erase_ppa) |
a4bd217b JG |
323 | { |
324 | struct pblk_line_meta *lm = &pblk->lm; | |
d624f371 | 325 | struct pblk_line *e_line = pblk_line_get_erase(pblk); |
1e82123d | 326 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); |
a4bd217b JG |
327 | unsigned int valid = c_ctx->nr_valid; |
328 | unsigned int padded = c_ctx->nr_padded; | |
329 | unsigned int nr_secs = valid + padded; | |
330 | unsigned long *lun_bitmap; | |
1e82123d | 331 | int ret; |
a4bd217b JG |
332 | |
333 | lun_bitmap = kzalloc(lm->lun_bitmap_len, GFP_KERNEL); | |
d624f371 JG |
334 | if (!lun_bitmap) |
335 | return -ENOMEM; | |
a4bd217b JG |
336 | c_ctx->lun_bitmap = lun_bitmap; |
337 | ||
dd2a4343 | 338 | ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write); |
a4bd217b JG |
339 | if (ret) { |
340 | kfree(lun_bitmap); | |
d624f371 | 341 | return ret; |
a4bd217b JG |
342 | } |
343 | ||
588726d3 | 344 | if (likely(!e_line || !atomic_read(&e_line->left_eblks))) |
a4bd217b JG |
345 | pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0); |
346 | else | |
347 | pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, | |
d624f371 | 348 | valid, erase_ppa); |
a4bd217b | 349 | |
d624f371 | 350 | return 0; |
a4bd217b JG |
351 | } |
352 | ||
a4bd217b JG |
353 | static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail, |
354 | unsigned int secs_to_flush) | |
355 | { | |
356 | int secs_to_sync; | |
357 | ||
358 | secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush); | |
359 | ||
880eda54 | 360 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
361 | if ((!secs_to_sync && secs_to_flush) |
362 | || (secs_to_sync < 0) | |
363 | || (secs_to_sync > secs_avail && !secs_to_flush)) { | |
4e495a46 | 364 | pblk_err(pblk, "bad sector calculation (a:%d,s:%d,f:%d)\n", |
a4bd217b JG |
365 | secs_avail, secs_to_sync, secs_to_flush); |
366 | } | |
367 | #endif | |
368 | ||
369 | return secs_to_sync; | |
370 | } | |
371 | ||
dd2a4343 JG |
372 | int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line) |
373 | { | |
374 | struct nvm_tgt_dev *dev = pblk->dev; | |
375 | struct nvm_geo *geo = &dev->geo; | |
376 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; | |
377 | struct pblk_line_meta *lm = &pblk->lm; | |
378 | struct pblk_emeta *emeta = meta_line->emeta; | |
d68a9344 | 379 | struct ppa_addr *ppa_list; |
dd2a4343 | 380 | struct pblk_g_ctx *m_ctx; |
dd2a4343 JG |
381 | struct bio *bio; |
382 | struct nvm_rq *rqd; | |
383 | void *data; | |
384 | u64 paddr; | |
385 | int rq_ppas = pblk->min_write_pgs; | |
386 | int id = meta_line->id; | |
387 | int rq_len; | |
388 | int i, j; | |
389 | int ret; | |
390 | ||
e2cddf20 | 391 | rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT); |
2942f50f | 392 | |
dd2a4343 JG |
393 | m_ctx = nvm_rq_to_pdu(rqd); |
394 | m_ctx->private = meta_line; | |
395 | ||
e46f4e48 | 396 | rq_len = rq_ppas * geo->csecs; |
dd2a4343 JG |
397 | data = ((void *)emeta->buf) + emeta->mem; |
398 | ||
de54e703 JG |
399 | bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len, |
400 | l_mg->emeta_alloc_type, GFP_KERNEL); | |
dd2a4343 | 401 | if (IS_ERR(bio)) { |
4e495a46 | 402 | pblk_err(pblk, "failed to map emeta io"); |
dd2a4343 JG |
403 | ret = PTR_ERR(bio); |
404 | goto fail_free_rqd; | |
405 | } | |
406 | bio->bi_iter.bi_sector = 0; /* internal bio */ | |
407 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); | |
408 | rqd->bio = bio; | |
409 | ||
410 | ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta); | |
411 | if (ret) | |
412 | goto fail_free_bio; | |
413 | ||
d68a9344 | 414 | ppa_list = nvm_rq_to_ppa_list(rqd); |
dd2a4343 JG |
415 | for (i = 0; i < rqd->nr_ppas; ) { |
416 | spin_lock(&meta_line->lock); | |
417 | paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas); | |
418 | spin_unlock(&meta_line->lock); | |
419 | for (j = 0; j < rq_ppas; j++, i++, paddr++) | |
d68a9344 | 420 | ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id); |
dd2a4343 JG |
421 | } |
422 | ||
d8adaa3b | 423 | spin_lock(&l_mg->close_lock); |
dd2a4343 | 424 | emeta->mem += rq_len; |
d8adaa3b | 425 | if (emeta->mem >= lm->emeta_len[0]) |
dd2a4343 | 426 | list_del(&meta_line->list); |
d8adaa3b | 427 | spin_unlock(&l_mg->close_lock); |
dd2a4343 | 428 | |
43241cfe | 429 | pblk_down_chunk(pblk, ppa_list[0]); |
3eaa11e2 | 430 | |
dd2a4343 JG |
431 | ret = pblk_submit_io(pblk, rqd); |
432 | if (ret) { | |
4e495a46 | 433 | pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); |
dd2a4343 JG |
434 | goto fail_rollback; |
435 | } | |
436 | ||
437 | return NVM_IO_OK; | |
438 | ||
439 | fail_rollback: | |
43241cfe | 440 | pblk_up_chunk(pblk, ppa_list[0]); |
dd2a4343 JG |
441 | spin_lock(&l_mg->close_lock); |
442 | pblk_dealloc_page(pblk, meta_line, rq_ppas); | |
443 | list_add(&meta_line->list, &meta_line->list); | |
444 | spin_unlock(&l_mg->close_lock); | |
445 | fail_free_bio: | |
55e836d4 | 446 | bio_put(bio); |
dd2a4343 | 447 | fail_free_rqd: |
e2cddf20 | 448 | pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT); |
dd2a4343 JG |
449 | return ret; |
450 | } | |
451 | ||
1e82123d JG |
452 | static inline bool pblk_valid_meta_ppa(struct pblk *pblk, |
453 | struct pblk_line *meta_line, | |
454 | struct nvm_rq *data_rqd) | |
455 | { | |
456 | struct nvm_tgt_dev *dev = pblk->dev; | |
457 | struct nvm_geo *geo = &dev->geo; | |
458 | struct pblk_c_ctx *data_c_ctx = nvm_rq_to_pdu(data_rqd); | |
459 | struct pblk_line *data_line = pblk_line_get_data(pblk); | |
460 | struct ppa_addr ppa, ppa_opt; | |
461 | u64 paddr; | |
462 | int pos_opt; | |
463 | ||
464 | /* Schedule a metadata I/O that is half the distance from the data I/O | |
465 | * with regards to the number of LUNs forming the pblk instance. This | |
466 | * balances LUN conflicts across every I/O. | |
467 | * | |
468 | * When the LUN configuration changes (e.g., due to GC), this distance | |
469 | * can align, which would result on metadata and data I/Os colliding. In | |
470 | * this case, modify the distance to not be optimal, but move the | |
471 | * optimal in the right direction. | |
472 | */ | |
473 | paddr = pblk_lookup_page(pblk, meta_line); | |
474 | ppa = addr_to_gen_ppa(pblk, paddr, 0); | |
475 | ppa_opt = addr_to_gen_ppa(pblk, paddr + data_line->meta_distance, 0); | |
476 | pos_opt = pblk_ppa_to_pos(geo, ppa_opt); | |
477 | ||
478 | if (test_bit(pos_opt, data_c_ctx->lun_bitmap) || | |
479 | test_bit(pos_opt, data_line->blk_bitmap)) | |
480 | return true; | |
481 | ||
482 | if (unlikely(pblk_ppa_comp(ppa_opt, ppa))) | |
483 | data_line->meta_distance--; | |
484 | ||
485 | return false; | |
486 | } | |
487 | ||
488 | static struct pblk_line *pblk_should_submit_meta_io(struct pblk *pblk, | |
489 | struct nvm_rq *data_rqd) | |
dd2a4343 JG |
490 | { |
491 | struct pblk_line_meta *lm = &pblk->lm; | |
492 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; | |
493 | struct pblk_line *meta_line; | |
494 | ||
495 | spin_lock(&l_mg->close_lock); | |
dd2a4343 JG |
496 | if (list_empty(&l_mg->emeta_list)) { |
497 | spin_unlock(&l_mg->close_lock); | |
1e82123d | 498 | return NULL; |
dd2a4343 JG |
499 | } |
500 | meta_line = list_first_entry(&l_mg->emeta_list, struct pblk_line, list); | |
d8adaa3b JG |
501 | if (meta_line->emeta->mem >= lm->emeta_len[0]) { |
502 | spin_unlock(&l_mg->close_lock); | |
503 | return NULL; | |
504 | } | |
dd2a4343 JG |
505 | spin_unlock(&l_mg->close_lock); |
506 | ||
1e82123d JG |
507 | if (!pblk_valid_meta_ppa(pblk, meta_line, data_rqd)) |
508 | return NULL; | |
dd2a4343 | 509 | |
1e82123d | 510 | return meta_line; |
dd2a4343 JG |
511 | } |
512 | ||
d624f371 JG |
513 | static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd) |
514 | { | |
d624f371 | 515 | struct ppa_addr erase_ppa; |
1e82123d | 516 | struct pblk_line *meta_line; |
d624f371 JG |
517 | int err; |
518 | ||
26f76dce | 519 | pblk_ppa_set_empty(&erase_ppa); |
d624f371 JG |
520 | |
521 | /* Assign lbas to ppas and populate request structure */ | |
1e82123d | 522 | err = pblk_setup_w_rq(pblk, rqd, &erase_ppa); |
d624f371 | 523 | if (err) { |
4e495a46 | 524 | pblk_err(pblk, "could not setup write request: %d\n", err); |
d624f371 JG |
525 | return NVM_IO_ERR; |
526 | } | |
527 | ||
1e82123d | 528 | meta_line = pblk_should_submit_meta_io(pblk, rqd); |
dd2a4343 | 529 | |
1e82123d JG |
530 | /* Submit data write for current data line */ |
531 | err = pblk_submit_io(pblk, rqd); | |
532 | if (err) { | |
4e495a46 | 533 | pblk_err(pblk, "data I/O submission failed: %d\n", err); |
1e82123d JG |
534 | return NVM_IO_ERR; |
535 | } | |
d624f371 | 536 | |
26f76dce | 537 | if (!pblk_ppa_empty(erase_ppa)) { |
1e82123d | 538 | /* Submit erase for next data line */ |
dd2a4343 JG |
539 | if (pblk_blk_erase_async(pblk, erase_ppa)) { |
540 | struct pblk_line *e_line = pblk_line_get_erase(pblk); | |
541 | struct nvm_tgt_dev *dev = pblk->dev; | |
542 | struct nvm_geo *geo = &dev->geo; | |
543 | int bit; | |
544 | ||
545 | atomic_inc(&e_line->left_eblks); | |
546 | bit = pblk_ppa_to_pos(geo, erase_ppa); | |
547 | WARN_ON(!test_and_clear_bit(bit, e_line->erase_bitmap)); | |
548 | } | |
d624f371 JG |
549 | } |
550 | ||
1e82123d JG |
551 | if (meta_line) { |
552 | /* Submit metadata write for previous data line */ | |
553 | err = pblk_submit_meta_io(pblk, meta_line); | |
554 | if (err) { | |
4e495a46 MB |
555 | pblk_err(pblk, "metadata I/O submission failed: %d", |
556 | err); | |
1e82123d JG |
557 | return NVM_IO_ERR; |
558 | } | |
559 | } | |
560 | ||
d624f371 JG |
561 | return NVM_IO_OK; |
562 | } | |
563 | ||
564 | static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd) | |
565 | { | |
566 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); | |
567 | struct bio *bio = rqd->bio; | |
568 | ||
569 | if (c_ctx->nr_padded) | |
cd8ddbf7 JG |
570 | pblk_bio_free_pages(pblk, bio, c_ctx->nr_valid, |
571 | c_ctx->nr_padded); | |
d624f371 JG |
572 | } |
573 | ||
a4bd217b JG |
574 | static int pblk_submit_write(struct pblk *pblk) |
575 | { | |
576 | struct bio *bio; | |
577 | struct nvm_rq *rqd; | |
a4bd217b JG |
578 | unsigned int secs_avail, secs_to_sync, secs_to_com; |
579 | unsigned int secs_to_flush; | |
580 | unsigned long pos; | |
6a3abf5b | 581 | unsigned int resubmit; |
a4bd217b | 582 | |
6a3abf5b HH |
583 | spin_lock(&pblk->resubmit_lock); |
584 | resubmit = !list_empty(&pblk->resubmit_list); | |
585 | spin_unlock(&pblk->resubmit_lock); | |
586 | ||
587 | /* Resubmit failed writes first */ | |
588 | if (resubmit) { | |
589 | struct pblk_c_ctx *r_ctx; | |
590 | ||
591 | spin_lock(&pblk->resubmit_lock); | |
592 | r_ctx = list_first_entry(&pblk->resubmit_list, | |
593 | struct pblk_c_ctx, list); | |
594 | list_del(&r_ctx->list); | |
595 | spin_unlock(&pblk->resubmit_lock); | |
596 | ||
597 | secs_avail = r_ctx->nr_valid; | |
598 | pos = r_ctx->sentry; | |
599 | ||
600 | pblk_prepare_resubmit(pblk, pos, secs_avail); | |
601 | secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail, | |
602 | secs_avail); | |
a4bd217b | 603 | |
6a3abf5b HH |
604 | kfree(r_ctx); |
605 | } else { | |
606 | /* If there are no sectors in the cache, | |
607 | * flushes (bios without data) will be cleared on | |
608 | * the cache threads | |
609 | */ | |
610 | secs_avail = pblk_rb_read_count(&pblk->rwb); | |
611 | if (!secs_avail) | |
612 | return 1; | |
613 | ||
614 | secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb); | |
615 | if (!secs_to_flush && secs_avail < pblk->min_write_pgs) | |
616 | return 1; | |
617 | ||
618 | secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail, | |
619 | secs_to_flush); | |
620 | if (secs_to_sync > pblk->max_write_pgs) { | |
4e495a46 | 621 | pblk_err(pblk, "bad buffer sync calculation\n"); |
6a3abf5b HH |
622 | return 1; |
623 | } | |
624 | ||
625 | secs_to_com = (secs_to_sync > secs_avail) ? | |
626 | secs_avail : secs_to_sync; | |
627 | pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com); | |
628 | } | |
a4bd217b | 629 | |
875d94f3 JG |
630 | bio = bio_alloc(GFP_KERNEL, secs_to_sync); |
631 | ||
632 | bio->bi_iter.bi_sector = 0; /* internal bio */ | |
633 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); | |
634 | ||
e2cddf20 | 635 | rqd = pblk_alloc_rqd(pblk, PBLK_WRITE); |
875d94f3 JG |
636 | rqd->bio = bio; |
637 | ||
638 | if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync, | |
d624f371 | 639 | secs_avail)) { |
4e495a46 | 640 | pblk_err(pblk, "corrupted write bio\n"); |
a4bd217b JG |
641 | goto fail_put_bio; |
642 | } | |
643 | ||
d624f371 | 644 | if (pblk_submit_io_set(pblk, rqd)) |
a4bd217b | 645 | goto fail_free_bio; |
a4bd217b | 646 | |
880eda54 | 647 | #ifdef CONFIG_NVM_PBLK_DEBUG |
a4bd217b JG |
648 | atomic_long_add(secs_to_sync, &pblk->sub_writes); |
649 | #endif | |
650 | ||
651 | return 0; | |
652 | ||
653 | fail_free_bio: | |
d624f371 | 654 | pblk_free_write_rqd(pblk, rqd); |
a4bd217b JG |
655 | fail_put_bio: |
656 | bio_put(bio); | |
e2cddf20 | 657 | pblk_free_rqd(pblk, rqd, PBLK_WRITE); |
a4bd217b JG |
658 | |
659 | return 1; | |
660 | } | |
661 | ||
662 | int pblk_write_ts(void *data) | |
663 | { | |
664 | struct pblk *pblk = data; | |
665 | ||
666 | while (!kthread_should_stop()) { | |
667 | if (!pblk_submit_write(pblk)) | |
668 | continue; | |
669 | set_current_state(TASK_INTERRUPTIBLE); | |
670 | io_schedule(); | |
671 | } | |
672 | ||
673 | return 0; | |
674 | } |