Commit | Line | Data |
---|---|---|
3dcf60bc | 1 | // SPDX-License-Identifier: GPL-2.0 |
86db1e29 JA |
2 | /* |
3 | * Functions related to setting various queue properties from drivers | |
4 | */ | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/module.h> | |
7 | #include <linux/bio.h> | |
8 | #include <linux/blkdev.h> | |
320ae51f | 9 | #include <linux/blk-mq.h> |
cf4aebc2 | 10 | #include <linux/sched/sysctl.h> |
86db1e29 JA |
11 | |
12 | #include "blk.h" | |
bd166ef1 | 13 | #include "blk-mq-sched.h" |
86db1e29 | 14 | |
86db1e29 JA |
15 | /** |
16 | * blk_end_sync_rq - executes a completion event on a request | |
17 | * @rq: request to complete | |
710027a4 | 18 | * @error: end I/O status of the request |
86db1e29 | 19 | */ |
2a842aca | 20 | static void blk_end_sync_rq(struct request *rq, blk_status_t error) |
86db1e29 JA |
21 | { |
22 | struct completion *waiting = rq->end_io_data; | |
23 | ||
fb9b16e1 | 24 | rq->end_io_data = (void *)(uintptr_t)error; |
86db1e29 JA |
25 | |
26 | /* | |
27 | * complete last, if this is a stack request the process (and thus | |
28 | * the rq pointer) could be invalid right after this complete() | |
29 | */ | |
30 | complete(waiting); | |
31 | } | |
86db1e29 JA |
32 | |
33 | /** | |
8eeed0b5 | 34 | * blk_execute_rq_nowait - insert a request to I/O scheduler for execution |
86db1e29 JA |
35 | * @bd_disk: matching gendisk |
36 | * @rq: request to insert | |
37 | * @at_head: insert request at head or tail of queue | |
38 | * @done: I/O completion handler | |
39 | * | |
40 | * Description: | |
710027a4 | 41 | * Insert a fully prepared request at the back of the I/O scheduler queue |
86db1e29 | 42 | * for execution. Don't wait for completion. |
e81ca6fe MR |
43 | * |
44 | * Note: | |
45 | * This function will invoke @done directly if the queue is dead. | |
86db1e29 | 46 | */ |
8eeed0b5 GJ |
47 | void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq, |
48 | int at_head, rq_end_io_fn *done) | |
86db1e29 | 49 | { |
8ba61435 | 50 | WARN_ON(irqs_disabled()); |
57292b58 | 51 | WARN_ON(!blk_rq_is_passthrough(rq)); |
e81ca6fe MR |
52 | |
53 | rq->rq_disk = bd_disk; | |
54 | rq->end_io = done; | |
320ae51f | 55 | |
b5af37ab | 56 | blk_account_io_start(rq); |
48d9b0d4 | 57 | |
43a5e4e2 ML |
58 | /* |
59 | * don't check dying flag for MQ because the request won't | |
68bdf1ac | 60 | * be reused after dying flag is set |
43a5e4e2 | 61 | */ |
a1ce35fa | 62 | blk_mq_sched_insert_request(rq, at_head, true, false); |
86db1e29 JA |
63 | } |
64 | EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); | |
65 | ||
c01b5a81 KB |
66 | static bool blk_rq_is_poll(struct request *rq) |
67 | { | |
3e08773c CH |
68 | if (!rq->mq_hctx) |
69 | return false; | |
70 | if (rq->mq_hctx->type != HCTX_TYPE_POLL) | |
71 | return false; | |
72 | if (WARN_ON_ONCE(!rq->bio)) | |
73 | return false; | |
74 | return true; | |
c01b5a81 KB |
75 | } |
76 | ||
77 | static void blk_rq_poll_completion(struct request *rq, struct completion *wait) | |
78 | { | |
79 | do { | |
5a72e899 | 80 | bio_poll(rq->bio, NULL, 0); |
c01b5a81 KB |
81 | cond_resched(); |
82 | } while (!completion_done(wait)); | |
83 | } | |
84 | ||
86db1e29 JA |
85 | /** |
86 | * blk_execute_rq - insert a request into queue for execution | |
86db1e29 JA |
87 | * @bd_disk: matching gendisk |
88 | * @rq: request to insert | |
89 | * @at_head: insert request at head or tail of queue | |
90 | * | |
91 | * Description: | |
710027a4 | 92 | * Insert a fully prepared request at the back of the I/O scheduler queue |
86db1e29 | 93 | * for execution and wait for completion. |
fb9b16e1 | 94 | * Return: The blk_status_t result provided to blk_mq_end_request(). |
86db1e29 | 95 | */ |
fb9b16e1 | 96 | blk_status_t blk_execute_rq(struct gendisk *bd_disk, struct request *rq, int at_head) |
86db1e29 JA |
97 | { |
98 | DECLARE_COMPLETION_ONSTACK(wait); | |
4b197769 | 99 | unsigned long hang_check; |
86db1e29 | 100 | |
86db1e29 | 101 | rq->end_io_data = &wait; |
8eeed0b5 | 102 | blk_execute_rq_nowait(bd_disk, rq, at_head, blk_end_sync_rq); |
4b197769 ML |
103 | |
104 | /* Prevent hang_check timer from firing at us during very long I/O */ | |
105 | hang_check = sysctl_hung_task_timeout_secs; | |
c01b5a81 KB |
106 | |
107 | if (blk_rq_is_poll(rq)) | |
108 | blk_rq_poll_completion(rq, &wait); | |
109 | else if (hang_check) | |
5577022f | 110 | while (!wait_for_completion_io_timeout(&wait, hang_check * (HZ/2))); |
4b197769 | 111 | else |
5577022f | 112 | wait_for_completion_io(&wait); |
fb9b16e1 KB |
113 | |
114 | return (blk_status_t)(uintptr_t)rq->end_io_data; | |
86db1e29 | 115 | } |
86db1e29 | 116 | EXPORT_SYMBOL(blk_execute_rq); |