Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
86db1e29 JA |
2 | /* |
3 | * Functions related to mapping data to requests | |
4 | */ | |
5 | #include <linux/kernel.h> | |
68db0cf1 | 6 | #include <linux/sched/task_stack.h> |
86db1e29 JA |
7 | #include <linux/module.h> |
8 | #include <linux/bio.h> | |
9 | #include <linux/blkdev.h> | |
26e49cfc | 10 | #include <linux/uio.h> |
86db1e29 JA |
11 | |
12 | #include "blk.h" | |
13 | ||
130879f1 | 14 | struct bio_map_data { |
f3256075 CH |
15 | bool is_our_pages : 1; |
16 | bool is_null_mapped : 1; | |
130879f1 CH |
17 | struct iov_iter iter; |
18 | struct iovec iov[]; | |
19 | }; | |
20 | ||
21 | static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, | |
22 | gfp_t gfp_mask) | |
23 | { | |
24 | struct bio_map_data *bmd; | |
25 | ||
26 | if (data->nr_segs > UIO_MAXIOV) | |
27 | return NULL; | |
28 | ||
29 | bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); | |
30 | if (!bmd) | |
31 | return NULL; | |
130879f1 | 32 | bmd->iter = *data; |
0a2481cd | 33 | if (iter_is_iovec(data)) { |
de4f5fed JA |
34 | memcpy(bmd->iov, iter_iov(data), sizeof(struct iovec) * data->nr_segs); |
35 | bmd->iter.__iov = bmd->iov; | |
0a2481cd | 36 | } |
130879f1 CH |
37 | return bmd; |
38 | } | |
39 | ||
40 | /** | |
41 | * bio_copy_from_iter - copy all pages from iov_iter to bio | |
42 | * @bio: The &struct bio which describes the I/O as destination | |
43 | * @iter: iov_iter as source | |
44 | * | |
45 | * Copy all pages from iov_iter to bio. | |
46 | * Returns 0 on success, or error on failure. | |
47 | */ | |
48 | static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) | |
49 | { | |
50 | struct bio_vec *bvec; | |
51 | struct bvec_iter_all iter_all; | |
52 | ||
53 | bio_for_each_segment_all(bvec, bio, iter_all) { | |
54 | ssize_t ret; | |
55 | ||
56 | ret = copy_page_from_iter(bvec->bv_page, | |
57 | bvec->bv_offset, | |
58 | bvec->bv_len, | |
59 | iter); | |
60 | ||
61 | if (!iov_iter_count(iter)) | |
62 | break; | |
63 | ||
64 | if (ret < bvec->bv_len) | |
65 | return -EFAULT; | |
66 | } | |
67 | ||
68 | return 0; | |
69 | } | |
70 | ||
71 | /** | |
72 | * bio_copy_to_iter - copy all pages from bio to iov_iter | |
73 | * @bio: The &struct bio which describes the I/O as source | |
74 | * @iter: iov_iter as destination | |
75 | * | |
76 | * Copy all pages from bio to iov_iter. | |
77 | * Returns 0 on success, or error on failure. | |
78 | */ | |
79 | static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) | |
80 | { | |
81 | struct bio_vec *bvec; | |
82 | struct bvec_iter_all iter_all; | |
83 | ||
84 | bio_for_each_segment_all(bvec, bio, iter_all) { | |
85 | ssize_t ret; | |
86 | ||
87 | ret = copy_page_to_iter(bvec->bv_page, | |
88 | bvec->bv_offset, | |
89 | bvec->bv_len, | |
90 | &iter); | |
91 | ||
92 | if (!iov_iter_count(&iter)) | |
93 | break; | |
94 | ||
95 | if (ret < bvec->bv_len) | |
96 | return -EFAULT; | |
97 | } | |
98 | ||
99 | return 0; | |
100 | } | |
101 | ||
102 | /** | |
103 | * bio_uncopy_user - finish previously mapped bio | |
104 | * @bio: bio being terminated | |
105 | * | |
106 | * Free pages allocated from bio_copy_user_iov() and write back data | |
107 | * to user space in case of a read. | |
108 | */ | |
109 | static int bio_uncopy_user(struct bio *bio) | |
110 | { | |
111 | struct bio_map_data *bmd = bio->bi_private; | |
112 | int ret = 0; | |
113 | ||
3310eeba | 114 | if (!bmd->is_null_mapped) { |
130879f1 CH |
115 | /* |
116 | * if we're in a workqueue, the request is orphaned, so | |
117 | * don't copy into a random user address space, just free | |
118 | * and return -EINTR so user space doesn't expect any data. | |
119 | */ | |
120 | if (!current->mm) | |
121 | ret = -EINTR; | |
122 | else if (bio_data_dir(bio) == READ) | |
123 | ret = bio_copy_to_iter(bio, bmd->iter); | |
124 | if (bmd->is_our_pages) | |
125 | bio_free_pages(bio); | |
126 | } | |
127 | kfree(bmd); | |
130879f1 CH |
128 | return ret; |
129 | } | |
130 | ||
7589ad67 CH |
131 | static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, |
132 | struct iov_iter *iter, gfp_t gfp_mask) | |
130879f1 CH |
133 | { |
134 | struct bio_map_data *bmd; | |
135 | struct page *page; | |
393bb12e | 136 | struct bio *bio; |
130879f1 CH |
137 | int i = 0, ret; |
138 | int nr_pages; | |
139 | unsigned int len = iter->count; | |
140 | unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; | |
141 | ||
142 | bmd = bio_alloc_map_data(iter, gfp_mask); | |
143 | if (!bmd) | |
7589ad67 | 144 | return -ENOMEM; |
130879f1 CH |
145 | |
146 | /* | |
147 | * We need to do a deep copy of the iov_iter including the iovecs. | |
148 | * The caller provided iov might point to an on-stack or otherwise | |
149 | * shortlived one. | |
150 | */ | |
f3256075 | 151 | bmd->is_our_pages = !map_data; |
03859717 | 152 | bmd->is_null_mapped = (map_data && map_data->null_mapped); |
130879f1 | 153 | |
5f7136db | 154 | nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); |
130879f1 CH |
155 | |
156 | ret = -ENOMEM; | |
066ff571 | 157 | bio = bio_kmalloc(nr_pages, gfp_mask); |
130879f1 CH |
158 | if (!bio) |
159 | goto out_bmd; | |
066ff571 | 160 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); |
130879f1 CH |
161 | |
162 | if (map_data) { | |
f5d632d1 | 163 | nr_pages = 1U << map_data->page_order; |
130879f1 CH |
164 | i = map_data->offset / PAGE_SIZE; |
165 | } | |
166 | while (len) { | |
167 | unsigned int bytes = PAGE_SIZE; | |
168 | ||
169 | bytes -= offset; | |
170 | ||
171 | if (bytes > len) | |
172 | bytes = len; | |
173 | ||
174 | if (map_data) { | |
175 | if (i == map_data->nr_entries * nr_pages) { | |
176 | ret = -ENOMEM; | |
7589ad67 | 177 | goto cleanup; |
130879f1 CH |
178 | } |
179 | ||
180 | page = map_data->pages[i / nr_pages]; | |
181 | page += (i % nr_pages); | |
182 | ||
183 | i++; | |
184 | } else { | |
ce288e05 | 185 | page = alloc_page(GFP_NOIO | gfp_mask); |
130879f1 CH |
186 | if (!page) { |
187 | ret = -ENOMEM; | |
7589ad67 | 188 | goto cleanup; |
130879f1 CH |
189 | } |
190 | } | |
191 | ||
7589ad67 | 192 | if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { |
130879f1 CH |
193 | if (!map_data) |
194 | __free_page(page); | |
195 | break; | |
196 | } | |
197 | ||
198 | len -= bytes; | |
199 | offset = 0; | |
200 | } | |
201 | ||
130879f1 CH |
202 | if (map_data) |
203 | map_data->offset += bio->bi_iter.bi_size; | |
204 | ||
205 | /* | |
206 | * success | |
207 | */ | |
208 | if ((iov_iter_rw(iter) == WRITE && | |
209 | (!map_data || !map_data->null_mapped)) || | |
210 | (map_data && map_data->from_user)) { | |
211 | ret = bio_copy_from_iter(bio, iter); | |
212 | if (ret) | |
213 | goto cleanup; | |
214 | } else { | |
215 | if (bmd->is_our_pages) | |
216 | zero_fill_bio(bio); | |
217 | iov_iter_advance(iter, bio->bi_iter.bi_size); | |
218 | } | |
219 | ||
220 | bio->bi_private = bmd; | |
7589ad67 | 221 | |
393bb12e | 222 | ret = blk_rq_append_bio(rq, bio); |
7589ad67 CH |
223 | if (ret) |
224 | goto cleanup; | |
7589ad67 | 225 | return 0; |
130879f1 CH |
226 | cleanup: |
227 | if (!map_data) | |
228 | bio_free_pages(bio); | |
066ff571 CH |
229 | bio_uninit(bio); |
230 | kfree(bio); | |
130879f1 CH |
231 | out_bmd: |
232 | kfree(bmd); | |
7589ad67 | 233 | return ret; |
130879f1 CH |
234 | } |
235 | ||
32f1c71b | 236 | static void blk_mq_map_bio_put(struct bio *bio) |
8af870aa JA |
237 | { |
238 | if (bio->bi_opf & REQ_ALLOC_CACHE) { | |
239 | bio_put(bio); | |
240 | } else { | |
241 | bio_uninit(bio); | |
242 | kfree(bio); | |
243 | } | |
244 | } | |
245 | ||
ab89e8e7 KJ |
246 | static struct bio *blk_rq_map_bio_alloc(struct request *rq, |
247 | unsigned int nr_vecs, gfp_t gfp_mask) | |
130879f1 | 248 | { |
393bb12e | 249 | struct bio *bio; |
130879f1 | 250 | |
46930b7c | 251 | if (rq->cmd_flags & REQ_ALLOC_CACHE && (nr_vecs <= BIO_INLINE_VECS)) { |
7e2e355d | 252 | bio = bio_alloc_bioset(NULL, nr_vecs, rq->cmd_flags, gfp_mask, |
8af870aa JA |
253 | &fs_bio_set); |
254 | if (!bio) | |
ab89e8e7 | 255 | return NULL; |
8af870aa JA |
256 | } else { |
257 | bio = bio_kmalloc(nr_vecs, gfp_mask); | |
258 | if (!bio) | |
ab89e8e7 | 259 | return NULL; |
8af870aa JA |
260 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); |
261 | } | |
ab89e8e7 KJ |
262 | return bio; |
263 | } | |
264 | ||
265 | static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, | |
266 | gfp_t gfp_mask) | |
267 | { | |
f62e52d1 | 268 | iov_iter_extraction_t extraction_flags = 0; |
ab89e8e7 KJ |
269 | unsigned int max_sectors = queue_max_hw_sectors(rq->q); |
270 | unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); | |
271 | struct bio *bio; | |
272 | int ret; | |
273 | int j; | |
274 | ||
275 | if (!iov_iter_count(iter)) | |
276 | return -EINVAL; | |
277 | ||
278 | bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); | |
279 | if (bio == NULL) | |
280 | return -ENOMEM; | |
130879f1 | 281 | |
7ee4ccf5 | 282 | if (blk_queue_pci_p2pdma(rq->q)) |
f62e52d1 | 283 | extraction_flags |= ITER_ALLOW_P2PDMA; |
403b6fb8 DH |
284 | if (iov_iter_extract_will_pin(iter)) |
285 | bio_set_flag(bio, BIO_PAGE_PINNED); | |
7ee4ccf5 | 286 | |
130879f1 | 287 | while (iov_iter_count(iter)) { |
403b6fb8 DH |
288 | struct page *stack_pages[UIO_FASTIOV]; |
289 | struct page **pages = stack_pages; | |
130879f1 | 290 | ssize_t bytes; |
91e5adda | 291 | size_t offs; |
130879f1 CH |
292 | int npages; |
293 | ||
403b6fb8 DH |
294 | if (nr_vecs > ARRAY_SIZE(stack_pages)) |
295 | pages = NULL; | |
296 | ||
297 | bytes = iov_iter_extract_pages(iter, &pages, LONG_MAX, | |
298 | nr_vecs, extraction_flags, &offs); | |
130879f1 CH |
299 | if (unlikely(bytes <= 0)) { |
300 | ret = bytes ? bytes : -EFAULT; | |
301 | goto out_unmap; | |
302 | } | |
303 | ||
304 | npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); | |
305 | ||
7ab89db9 | 306 | if (unlikely(offs & queue_dma_alignment(rq->q))) |
130879f1 | 307 | j = 0; |
7ab89db9 | 308 | else { |
130879f1 CH |
309 | for (j = 0; j < npages; j++) { |
310 | struct page *page = pages[j]; | |
311 | unsigned int n = PAGE_SIZE - offs; | |
312 | bool same_page = false; | |
313 | ||
314 | if (n > bytes) | |
315 | n = bytes; | |
316 | ||
7589ad67 | 317 | if (!bio_add_hw_page(rq->q, bio, page, n, offs, |
e4581105 | 318 | max_sectors, &same_page)) { |
130879f1 | 319 | if (same_page) |
403b6fb8 | 320 | bio_release_page(bio, page); |
130879f1 CH |
321 | break; |
322 | } | |
323 | ||
130879f1 CH |
324 | bytes -= n; |
325 | offs = 0; | |
326 | } | |
130879f1 CH |
327 | } |
328 | /* | |
329 | * release the pages we didn't map into the bio, if any | |
330 | */ | |
331 | while (j < npages) | |
403b6fb8 | 332 | bio_release_page(bio, pages[j++]); |
e88811bc JA |
333 | if (pages != stack_pages) |
334 | kvfree(pages); | |
130879f1 | 335 | /* couldn't stuff something into bio? */ |
480cb846 AV |
336 | if (bytes) { |
337 | iov_iter_revert(iter, bytes); | |
130879f1 | 338 | break; |
480cb846 | 339 | } |
130879f1 CH |
340 | } |
341 | ||
393bb12e | 342 | ret = blk_rq_append_bio(rq, bio); |
7589ad67 | 343 | if (ret) |
393bb12e | 344 | goto out_unmap; |
7589ad67 CH |
345 | return 0; |
346 | ||
130879f1 CH |
347 | out_unmap: |
348 | bio_release_pages(bio, false); | |
32f1c71b | 349 | blk_mq_map_bio_put(bio); |
7589ad67 | 350 | return ret; |
130879f1 CH |
351 | } |
352 | ||
130879f1 CH |
353 | static void bio_invalidate_vmalloc_pages(struct bio *bio) |
354 | { | |
f358afc5 | 355 | #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE |
130879f1 CH |
356 | if (bio->bi_private && !op_is_write(bio_op(bio))) { |
357 | unsigned long i, len = 0; | |
358 | ||
359 | for (i = 0; i < bio->bi_vcnt; i++) | |
360 | len += bio->bi_io_vec[i].bv_len; | |
361 | invalidate_kernel_vmap_range(bio->bi_private, len); | |
362 | } | |
363 | #endif | |
364 | } | |
365 | ||
366 | static void bio_map_kern_endio(struct bio *bio) | |
367 | { | |
368 | bio_invalidate_vmalloc_pages(bio); | |
066ff571 CH |
369 | bio_uninit(bio); |
370 | kfree(bio); | |
130879f1 CH |
371 | } |
372 | ||
373 | /** | |
374 | * bio_map_kern - map kernel address into bio | |
375 | * @q: the struct request_queue for the bio | |
376 | * @data: pointer to buffer to map | |
377 | * @len: length in bytes | |
378 | * @gfp_mask: allocation flags for bio allocation | |
379 | * | |
380 | * Map the kernel address into a bio suitable for io to a block | |
381 | * device. Returns an error pointer in case of error. | |
382 | */ | |
383 | static struct bio *bio_map_kern(struct request_queue *q, void *data, | |
384 | unsigned int len, gfp_t gfp_mask) | |
385 | { | |
386 | unsigned long kaddr = (unsigned long)data; | |
387 | unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
388 | unsigned long start = kaddr >> PAGE_SHIFT; | |
389 | const int nr_pages = end - start; | |
390 | bool is_vmalloc = is_vmalloc_addr(data); | |
391 | struct page *page; | |
392 | int offset, i; | |
393 | struct bio *bio; | |
394 | ||
066ff571 | 395 | bio = bio_kmalloc(nr_pages, gfp_mask); |
130879f1 CH |
396 | if (!bio) |
397 | return ERR_PTR(-ENOMEM); | |
066ff571 | 398 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); |
130879f1 CH |
399 | |
400 | if (is_vmalloc) { | |
401 | flush_kernel_vmap_range(data, len); | |
402 | bio->bi_private = data; | |
403 | } | |
404 | ||
405 | offset = offset_in_page(kaddr); | |
406 | for (i = 0; i < nr_pages; i++) { | |
407 | unsigned int bytes = PAGE_SIZE - offset; | |
408 | ||
409 | if (len <= 0) | |
410 | break; | |
411 | ||
412 | if (bytes > len) | |
413 | bytes = len; | |
414 | ||
415 | if (!is_vmalloc) | |
416 | page = virt_to_page(data); | |
417 | else | |
418 | page = vmalloc_to_page(data); | |
419 | if (bio_add_pc_page(q, bio, page, bytes, | |
420 | offset) < bytes) { | |
421 | /* we don't support partial mappings */ | |
066ff571 CH |
422 | bio_uninit(bio); |
423 | kfree(bio); | |
130879f1 CH |
424 | return ERR_PTR(-EINVAL); |
425 | } | |
426 | ||
427 | data += bytes; | |
428 | len -= bytes; | |
429 | offset = 0; | |
430 | } | |
431 | ||
432 | bio->bi_end_io = bio_map_kern_endio; | |
433 | return bio; | |
434 | } | |
435 | ||
436 | static void bio_copy_kern_endio(struct bio *bio) | |
437 | { | |
438 | bio_free_pages(bio); | |
066ff571 CH |
439 | bio_uninit(bio); |
440 | kfree(bio); | |
130879f1 CH |
441 | } |
442 | ||
443 | static void bio_copy_kern_endio_read(struct bio *bio) | |
444 | { | |
445 | char *p = bio->bi_private; | |
446 | struct bio_vec *bvec; | |
447 | struct bvec_iter_all iter_all; | |
448 | ||
449 | bio_for_each_segment_all(bvec, bio, iter_all) { | |
d24920e2 | 450 | memcpy_from_bvec(p, bvec); |
130879f1 CH |
451 | p += bvec->bv_len; |
452 | } | |
453 | ||
454 | bio_copy_kern_endio(bio); | |
455 | } | |
456 | ||
457 | /** | |
458 | * bio_copy_kern - copy kernel address into bio | |
459 | * @q: the struct request_queue for the bio | |
460 | * @data: pointer to buffer to copy | |
461 | * @len: length in bytes | |
462 | * @gfp_mask: allocation flags for bio and page allocation | |
463 | * @reading: data direction is READ | |
464 | * | |
465 | * copy the kernel address into a bio suitable for io to a block | |
466 | * device. Returns an error pointer in case of error. | |
467 | */ | |
468 | static struct bio *bio_copy_kern(struct request_queue *q, void *data, | |
469 | unsigned int len, gfp_t gfp_mask, int reading) | |
470 | { | |
471 | unsigned long kaddr = (unsigned long)data; | |
472 | unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; | |
473 | unsigned long start = kaddr >> PAGE_SHIFT; | |
474 | struct bio *bio; | |
475 | void *p = data; | |
476 | int nr_pages = 0; | |
477 | ||
478 | /* | |
479 | * Overflow, abort | |
480 | */ | |
481 | if (end < start) | |
482 | return ERR_PTR(-EINVAL); | |
483 | ||
484 | nr_pages = end - start; | |
066ff571 | 485 | bio = bio_kmalloc(nr_pages, gfp_mask); |
130879f1 CH |
486 | if (!bio) |
487 | return ERR_PTR(-ENOMEM); | |
066ff571 | 488 | bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); |
130879f1 CH |
489 | |
490 | while (len) { | |
491 | struct page *page; | |
492 | unsigned int bytes = PAGE_SIZE; | |
493 | ||
494 | if (bytes > len) | |
495 | bytes = len; | |
496 | ||
cc8f7fe1 | 497 | page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); |
130879f1 CH |
498 | if (!page) |
499 | goto cleanup; | |
500 | ||
501 | if (!reading) | |
502 | memcpy(page_address(page), p, bytes); | |
503 | ||
504 | if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) | |
505 | break; | |
506 | ||
507 | len -= bytes; | |
508 | p += bytes; | |
509 | } | |
510 | ||
511 | if (reading) { | |
512 | bio->bi_end_io = bio_copy_kern_endio_read; | |
513 | bio->bi_private = data; | |
514 | } else { | |
515 | bio->bi_end_io = bio_copy_kern_endio; | |
516 | } | |
517 | ||
518 | return bio; | |
519 | ||
520 | cleanup: | |
521 | bio_free_pages(bio); | |
066ff571 CH |
522 | bio_uninit(bio); |
523 | kfree(bio); | |
130879f1 CH |
524 | return ERR_PTR(-ENOMEM); |
525 | } | |
526 | ||
98d61d5b | 527 | /* |
0abc2a10 JA |
528 | * Append a bio to a passthrough request. Only works if the bio can be merged |
529 | * into the request based on the driver constraints. | |
98d61d5b | 530 | */ |
393bb12e | 531 | int blk_rq_append_bio(struct request *rq, struct bio *bio) |
86db1e29 | 532 | { |
14ccb66b CH |
533 | struct bvec_iter iter; |
534 | struct bio_vec bv; | |
535 | unsigned int nr_segs = 0; | |
0abc2a10 | 536 | |
393bb12e | 537 | bio_for_each_bvec(bv, bio, iter) |
14ccb66b CH |
538 | nr_segs++; |
539 | ||
98d61d5b | 540 | if (!rq->bio) { |
393bb12e | 541 | blk_rq_bio_prep(rq, bio, nr_segs); |
98d61d5b | 542 | } else { |
393bb12e | 543 | if (!ll_back_merge_fn(rq, bio, nr_segs)) |
98d61d5b | 544 | return -EINVAL; |
393bb12e CH |
545 | rq->biotail->bi_next = bio; |
546 | rq->biotail = bio; | |
547 | rq->__data_len += (bio)->bi_iter.bi_size; | |
548 | bio_crypt_free_ctx(bio); | |
86db1e29 | 549 | } |
98d61d5b | 550 | |
86db1e29 JA |
551 | return 0; |
552 | } | |
98d61d5b | 553 | EXPORT_SYMBOL(blk_rq_append_bio); |
86db1e29 | 554 | |
37987547 KJ |
555 | /* Prepare bio for passthrough IO given ITER_BVEC iter */ |
556 | static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter) | |
557 | { | |
558 | struct request_queue *q = rq->q; | |
559 | size_t nr_iter = iov_iter_count(iter); | |
560 | size_t nr_segs = iter->nr_segs; | |
561 | struct bio_vec *bvecs, *bvprvp = NULL; | |
aa261f20 | 562 | const struct queue_limits *lim = &q->limits; |
37987547 KJ |
563 | unsigned int nsegs = 0, bytes = 0; |
564 | struct bio *bio; | |
565 | size_t i; | |
566 | ||
567 | if (!nr_iter || (nr_iter >> SECTOR_SHIFT) > queue_max_hw_sectors(q)) | |
568 | return -EINVAL; | |
569 | if (nr_segs > queue_max_segments(q)) | |
570 | return -EINVAL; | |
571 | ||
572 | /* no iovecs to alloc, as we already have a BVEC iterator */ | |
573 | bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL); | |
574 | if (bio == NULL) | |
575 | return -ENOMEM; | |
576 | ||
577 | bio_iov_bvec_set(bio, (struct iov_iter *)iter); | |
578 | blk_rq_bio_prep(rq, bio, nr_segs); | |
579 | ||
580 | /* loop to perform a bunch of sanity checks */ | |
581 | bvecs = (struct bio_vec *)iter->bvec; | |
582 | for (i = 0; i < nr_segs; i++) { | |
583 | struct bio_vec *bv = &bvecs[i]; | |
584 | ||
585 | /* | |
586 | * If the queue doesn't support SG gaps and adding this | |
587 | * offset would create a gap, fallback to copy. | |
588 | */ | |
589 | if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv->bv_offset)) { | |
590 | blk_mq_map_bio_put(bio); | |
591 | return -EREMOTEIO; | |
592 | } | |
593 | /* check full condition */ | |
594 | if (nsegs >= nr_segs || bytes > UINT_MAX - bv->bv_len) | |
595 | goto put_bio; | |
596 | if (bytes + bv->bv_len > nr_iter) | |
597 | goto put_bio; | |
598 | if (bv->bv_offset + bv->bv_len > PAGE_SIZE) | |
599 | goto put_bio; | |
600 | ||
601 | nsegs++; | |
602 | bytes += bv->bv_len; | |
603 | bvprvp = bv; | |
604 | } | |
605 | return 0; | |
606 | put_bio: | |
607 | blk_mq_map_bio_put(bio); | |
608 | return -EINVAL; | |
609 | } | |
610 | ||
86db1e29 | 611 | /** |
aebf526b | 612 | * blk_rq_map_user_iov - map user data to a request, for passthrough requests |
86db1e29 JA |
613 | * @q: request queue where request should be inserted |
614 | * @rq: request to map data to | |
152e283f | 615 | * @map_data: pointer to the rq_map_data holding pages (if necessary) |
26e49cfc | 616 | * @iter: iovec iterator |
a3bce90e | 617 | * @gfp_mask: memory allocation flags |
86db1e29 JA |
618 | * |
619 | * Description: | |
710027a4 | 620 | * Data will be mapped directly for zero copy I/O, if possible. Otherwise |
86db1e29 JA |
621 | * a kernel bounce buffer is used. |
622 | * | |
710027a4 | 623 | * A matching blk_rq_unmap_user() must be issued at the end of I/O, while |
86db1e29 | 624 | * still in process context. |
86db1e29 JA |
625 | */ |
626 | int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, | |
26e49cfc KO |
627 | struct rq_map_data *map_data, |
628 | const struct iov_iter *iter, gfp_t gfp_mask) | |
86db1e29 | 629 | { |
37987547 | 630 | bool copy = false, map_bvec = false; |
357f435d | 631 | unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); |
4d6af73d CH |
632 | struct bio *bio = NULL; |
633 | struct iov_iter i; | |
69e0927b | 634 | int ret = -EINVAL; |
86db1e29 | 635 | |
357f435d AV |
636 | if (map_data) |
637 | copy = true; | |
393bb12e CH |
638 | else if (blk_queue_may_bounce(q)) |
639 | copy = true; | |
357f435d AV |
640 | else if (iov_iter_alignment(iter) & align) |
641 | copy = true; | |
37987547 KJ |
642 | else if (iov_iter_is_bvec(iter)) |
643 | map_bvec = true; | |
d46aa786 | 644 | else if (!user_backed_iter(iter)) |
37987547 | 645 | copy = true; |
357f435d AV |
646 | else if (queue_virt_boundary(q)) |
647 | copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); | |
afdc1a78 | 648 | |
37987547 KJ |
649 | if (map_bvec) { |
650 | ret = blk_rq_map_user_bvec(rq, iter); | |
651 | if (!ret) | |
652 | return 0; | |
653 | if (ret != -EREMOTEIO) | |
654 | goto fail; | |
655 | /* fall back to copying the data on limits mismatches */ | |
656 | copy = true; | |
657 | } | |
658 | ||
4d6af73d CH |
659 | i = *iter; |
660 | do { | |
7589ad67 CH |
661 | if (copy) |
662 | ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); | |
663 | else | |
664 | ret = bio_map_user_iov(rq, &i, gfp_mask); | |
4d6af73d CH |
665 | if (ret) |
666 | goto unmap_rq; | |
667 | if (!bio) | |
668 | bio = rq->bio; | |
669 | } while (iov_iter_count(&i)); | |
86db1e29 | 670 | |
86db1e29 | 671 | return 0; |
4d6af73d CH |
672 | |
673 | unmap_rq: | |
3b7995a9 | 674 | blk_rq_unmap_user(bio); |
a0ac402c | 675 | fail: |
4d6af73d | 676 | rq->bio = NULL; |
69e0927b | 677 | return ret; |
86db1e29 | 678 | } |
152e283f | 679 | EXPORT_SYMBOL(blk_rq_map_user_iov); |
86db1e29 | 680 | |
ddad8dd0 CH |
681 | int blk_rq_map_user(struct request_queue *q, struct request *rq, |
682 | struct rq_map_data *map_data, void __user *ubuf, | |
683 | unsigned long len, gfp_t gfp_mask) | |
684 | { | |
26e49cfc | 685 | struct iov_iter i; |
d46aa786 | 686 | int ret = import_ubuf(rq_data_dir(rq), ubuf, len, &i); |
ddad8dd0 | 687 | |
8f7e885a AV |
688 | if (unlikely(ret < 0)) |
689 | return ret; | |
ddad8dd0 | 690 | |
26e49cfc | 691 | return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); |
ddad8dd0 CH |
692 | } |
693 | EXPORT_SYMBOL(blk_rq_map_user); | |
694 | ||
55765402 AG |
695 | int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data, |
696 | void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask, | |
697 | bool vec, int iov_count, bool check_iter_count, int rw) | |
698 | { | |
699 | int ret = 0; | |
700 | ||
701 | if (vec) { | |
702 | struct iovec fast_iov[UIO_FASTIOV]; | |
703 | struct iovec *iov = fast_iov; | |
704 | struct iov_iter iter; | |
705 | ||
706 | ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len, | |
707 | UIO_FASTIOV, &iov, &iter); | |
708 | if (ret < 0) | |
709 | return ret; | |
710 | ||
711 | if (iov_count) { | |
712 | /* SG_IO howto says that the shorter of the two wins */ | |
713 | iov_iter_truncate(&iter, buf_len); | |
714 | if (check_iter_count && !iov_iter_count(&iter)) { | |
715 | kfree(iov); | |
716 | return -EINVAL; | |
717 | } | |
718 | } | |
719 | ||
720 | ret = blk_rq_map_user_iov(req->q, req, map_data, &iter, | |
721 | gfp_mask); | |
722 | kfree(iov); | |
723 | } else if (buf_len) { | |
724 | ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len, | |
725 | gfp_mask); | |
726 | } | |
727 | return ret; | |
728 | } | |
729 | EXPORT_SYMBOL(blk_rq_map_user_io); | |
730 | ||
86db1e29 JA |
731 | /** |
732 | * blk_rq_unmap_user - unmap a request with user data | |
733 | * @bio: start of bio list | |
734 | * | |
735 | * Description: | |
736 | * Unmap a rq previously mapped by blk_rq_map_user(). The caller must | |
737 | * supply the original rq->bio from the blk_rq_map_user() return, since | |
710027a4 | 738 | * the I/O completion may have changed rq->bio. |
86db1e29 JA |
739 | */ |
740 | int blk_rq_unmap_user(struct bio *bio) | |
741 | { | |
393bb12e | 742 | struct bio *next_bio; |
86db1e29 JA |
743 | int ret = 0, ret2; |
744 | ||
745 | while (bio) { | |
3310eeba | 746 | if (bio->bi_private) { |
393bb12e | 747 | ret2 = bio_uncopy_user(bio); |
7b63c052 CH |
748 | if (ret2 && !ret) |
749 | ret = ret2; | |
3310eeba | 750 | } else { |
393bb12e | 751 | bio_release_pages(bio, bio_data_dir(bio) == READ); |
7b63c052 | 752 | } |
86db1e29 | 753 | |
393bb12e | 754 | next_bio = bio; |
86db1e29 | 755 | bio = bio->bi_next; |
32f1c71b | 756 | blk_mq_map_bio_put(next_bio); |
86db1e29 JA |
757 | } |
758 | ||
759 | return ret; | |
760 | } | |
86db1e29 JA |
761 | EXPORT_SYMBOL(blk_rq_unmap_user); |
762 | ||
763 | /** | |
aebf526b | 764 | * blk_rq_map_kern - map kernel data to a request, for passthrough requests |
86db1e29 JA |
765 | * @q: request queue where request should be inserted |
766 | * @rq: request to fill | |
767 | * @kbuf: the kernel buffer | |
768 | * @len: length of user data | |
769 | * @gfp_mask: memory allocation flags | |
68154e90 FT |
770 | * |
771 | * Description: | |
772 | * Data will be mapped directly if possible. Otherwise a bounce | |
e227867f | 773 | * buffer is used. Can be called multiple times to append multiple |
3a5a3927 | 774 | * buffers. |
86db1e29 JA |
775 | */ |
776 | int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, | |
777 | unsigned int len, gfp_t gfp_mask) | |
778 | { | |
68154e90 | 779 | int reading = rq_data_dir(rq) == READ; |
14417799 | 780 | unsigned long addr = (unsigned long) kbuf; |
393bb12e | 781 | struct bio *bio; |
3a5a3927 | 782 | int ret; |
86db1e29 | 783 | |
ae03bf63 | 784 | if (len > (queue_max_hw_sectors(q) << 9)) |
86db1e29 JA |
785 | return -EINVAL; |
786 | if (!len || !kbuf) | |
787 | return -EINVAL; | |
788 | ||
393bb12e CH |
789 | if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || |
790 | blk_queue_may_bounce(q)) | |
68154e90 FT |
791 | bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); |
792 | else | |
793 | bio = bio_map_kern(q, kbuf, len, gfp_mask); | |
794 | ||
86db1e29 JA |
795 | if (IS_ERR(bio)) |
796 | return PTR_ERR(bio); | |
797 | ||
aebf526b CH |
798 | bio->bi_opf &= ~REQ_OP_MASK; |
799 | bio->bi_opf |= req_op(rq); | |
86db1e29 | 800 | |
393bb12e | 801 | ret = blk_rq_append_bio(rq, bio); |
066ff571 CH |
802 | if (unlikely(ret)) { |
803 | bio_uninit(bio); | |
804 | kfree(bio); | |
805 | } | |
393bb12e | 806 | return ret; |
86db1e29 | 807 | } |
86db1e29 | 808 | EXPORT_SYMBOL(blk_rq_map_kern); |