Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5ead97c8 JF |
2 | /* |
3 | * Xen hypercall batching. | |
4 | * | |
5 | * Xen allows multiple hypercalls to be issued at once, using the | |
6 | * multicall interface. This allows the cost of trapping into the | |
7 | * hypervisor to be amortized over several calls. | |
8 | * | |
9 | * This file implements a simple interface for multicalls. There's a | |
10 | * per-cpu buffer of outstanding multicalls. When you want to queue a | |
11 | * multicall for issuing, you can allocate a multicall slot for the | |
12 | * call and its arguments, along with storage for space which is | |
13 | * pointed to by the arguments (for passing pointers to structures, | |
14 | * etc). When the multicall is actually issued, all the space for the | |
15 | * commands and allocated memory is freed for reuse. | |
16 | * | |
17 | * Multicalls are flushed whenever any of the buffers get full, or | |
18 | * when explicitly requested. There's no way to get per-multicall | |
19 | * return results back. It will BUG if any of the multicalls fail. | |
20 | * | |
21 | * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 | |
22 | */ | |
23 | #include <linux/percpu.h> | |
f120f13e | 24 | #include <linux/hardirq.h> |
994025ca | 25 | #include <linux/debugfs.h> |
5ead97c8 JF |
26 | |
27 | #include <asm/xen/hypercall.h> | |
28 | ||
29 | #include "multicalls.h" | |
994025ca JF |
30 | #include "debugfs.h" |
31 | ||
32 | #define MC_BATCH 32 | |
5ead97c8 | 33 | |
ffc78767 | 34 | #define MC_DEBUG 0 |
a122d623 | 35 | |
400d3494 | 36 | #define MC_ARGS (MC_BATCH * 16) |
5ead97c8 | 37 | |
994025ca | 38 | |
5ead97c8 | 39 | struct mc_buffer { |
2a6f6d09 | 40 | unsigned mcidx, argidx, cbidx; |
5ead97c8 | 41 | struct multicall_entry entries[MC_BATCH]; |
a122d623 JF |
42 | #if MC_DEBUG |
43 | struct multicall_entry debug[MC_BATCH]; | |
b93d51dc | 44 | void *caller[MC_BATCH]; |
a122d623 | 45 | #endif |
400d3494 | 46 | unsigned char args[MC_ARGS]; |
91e0c5f3 JF |
47 | struct callback { |
48 | void (*fn)(void *); | |
49 | void *data; | |
50 | } callbacks[MC_BATCH]; | |
5ead97c8 JF |
51 | }; |
52 | ||
53 | static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); | |
54 | DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); | |
55 | ||
56 | void xen_mc_flush(void) | |
57 | { | |
89cbc767 | 58 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
eac303bf | 59 | struct multicall_entry *mc; |
5ead97c8 JF |
60 | int ret = 0; |
61 | unsigned long flags; | |
91e0c5f3 | 62 | int i; |
5ead97c8 | 63 | |
f120f13e JF |
64 | BUG_ON(preemptible()); |
65 | ||
5ead97c8 JF |
66 | /* Disable interrupts in case someone comes in and queues |
67 | something in the middle */ | |
68 | local_irq_save(flags); | |
69 | ||
c796f213 JF |
70 | trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx); |
71 | ||
eac303bf JF |
72 | switch (b->mcidx) { |
73 | case 0: | |
74 | /* no-op */ | |
75 | BUG_ON(b->argidx != 0); | |
76 | break; | |
77 | ||
78 | case 1: | |
79 | /* Singleton multicall - bypass multicall machinery | |
80 | and just do the call directly. */ | |
81 | mc = &b->entries[0]; | |
82 | ||
83 | mc->result = privcmd_call(mc->op, | |
84 | mc->args[0], mc->args[1], mc->args[2], | |
85 | mc->args[3], mc->args[4]); | |
86 | ret = mc->result < 0; | |
87 | break; | |
88 | ||
89 | default: | |
a122d623 JF |
90 | #if MC_DEBUG |
91 | memcpy(b->debug, b->entries, | |
92 | b->mcidx * sizeof(struct multicall_entry)); | |
93 | #endif | |
94 | ||
5ead97c8 JF |
95 | if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) |
96 | BUG(); | |
97 | for (i = 0; i < b->mcidx; i++) | |
98 | if (b->entries[i].result < 0) | |
99 | ret++; | |
a122d623 JF |
100 | |
101 | #if MC_DEBUG | |
102 | if (ret) { | |
103 | printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", | |
104 | ret, smp_processor_id()); | |
8ba6c2b0 | 105 | dump_stack(); |
7ebed39f | 106 | for (i = 0; i < b->mcidx; i++) { |
b93d51dc | 107 | printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n", |
a122d623 JF |
108 | i+1, b->mcidx, |
109 | b->debug[i].op, | |
110 | b->debug[i].args[0], | |
b93d51dc IC |
111 | b->entries[i].result, |
112 | b->caller[i]); | |
a122d623 JF |
113 | } |
114 | } | |
115 | #endif | |
eac303bf | 116 | } |
a122d623 | 117 | |
eac303bf JF |
118 | b->mcidx = 0; |
119 | b->argidx = 0; | |
5ead97c8 | 120 | |
7ebed39f | 121 | for (i = 0; i < b->cbidx; i++) { |
91e0c5f3 JF |
122 | struct callback *cb = &b->callbacks[i]; |
123 | ||
124 | (*cb->fn)(cb->data); | |
125 | } | |
126 | b->cbidx = 0; | |
127 | ||
c9960863 JF |
128 | local_irq_restore(flags); |
129 | ||
3d39e9d0 | 130 | WARN_ON(ret); |
5ead97c8 JF |
131 | } |
132 | ||
133 | struct multicall_space __xen_mc_entry(size_t args) | |
134 | { | |
89cbc767 | 135 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
5ead97c8 | 136 | struct multicall_space ret; |
400d3494 | 137 | unsigned argidx = roundup(b->argidx, sizeof(u64)); |
5ead97c8 | 138 | |
c796f213 JF |
139 | trace_xen_mc_entry_alloc(args); |
140 | ||
f120f13e | 141 | BUG_ON(preemptible()); |
f124c6ae | 142 | BUG_ON(b->argidx >= MC_ARGS); |
5ead97c8 | 143 | |
4a7b005d JF |
144 | if (unlikely(b->mcidx == MC_BATCH || |
145 | (argidx + args) >= MC_ARGS)) { | |
c796f213 JF |
146 | trace_xen_mc_flush_reason((b->mcidx == MC_BATCH) ? |
147 | XEN_MC_FL_BATCH : XEN_MC_FL_ARGS); | |
5ead97c8 | 148 | xen_mc_flush(); |
400d3494 JF |
149 | argidx = roundup(b->argidx, sizeof(u64)); |
150 | } | |
5ead97c8 JF |
151 | |
152 | ret.mc = &b->entries[b->mcidx]; | |
ffc78767 | 153 | #if MC_DEBUG |
b93d51dc IC |
154 | b->caller[b->mcidx] = __builtin_return_address(0); |
155 | #endif | |
5ead97c8 | 156 | b->mcidx++; |
400d3494 JF |
157 | ret.args = &b->args[argidx]; |
158 | b->argidx = argidx + args; | |
159 | ||
f124c6ae | 160 | BUG_ON(b->argidx >= MC_ARGS); |
400d3494 JF |
161 | return ret; |
162 | } | |
163 | ||
164 | struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) | |
165 | { | |
89cbc767 | 166 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
400d3494 JF |
167 | struct multicall_space ret = { NULL, NULL }; |
168 | ||
169 | BUG_ON(preemptible()); | |
f124c6ae | 170 | BUG_ON(b->argidx >= MC_ARGS); |
400d3494 | 171 | |
c796f213 JF |
172 | if (unlikely(b->mcidx == 0 || |
173 | b->entries[b->mcidx - 1].op != op)) { | |
174 | trace_xen_mc_extend_args(op, size, XEN_MC_XE_BAD_OP); | |
175 | goto out; | |
176 | } | |
400d3494 | 177 | |
c796f213 JF |
178 | if (unlikely((b->argidx + size) >= MC_ARGS)) { |
179 | trace_xen_mc_extend_args(op, size, XEN_MC_XE_NO_SPACE); | |
180 | goto out; | |
181 | } | |
400d3494 JF |
182 | |
183 | ret.mc = &b->entries[b->mcidx - 1]; | |
5ead97c8 | 184 | ret.args = &b->args[b->argidx]; |
400d3494 | 185 | b->argidx += size; |
5ead97c8 | 186 | |
f124c6ae | 187 | BUG_ON(b->argidx >= MC_ARGS); |
c796f213 JF |
188 | |
189 | trace_xen_mc_extend_args(op, size, XEN_MC_XE_OK); | |
190 | out: | |
5ead97c8 JF |
191 | return ret; |
192 | } | |
91e0c5f3 JF |
193 | |
194 | void xen_mc_callback(void (*fn)(void *), void *data) | |
195 | { | |
89cbc767 | 196 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
91e0c5f3 JF |
197 | struct callback *cb; |
198 | ||
c796f213 JF |
199 | if (b->cbidx == MC_BATCH) { |
200 | trace_xen_mc_flush_reason(XEN_MC_FL_CALLBACK); | |
91e0c5f3 | 201 | xen_mc_flush(); |
c796f213 JF |
202 | } |
203 | ||
204 | trace_xen_mc_callback(fn, data); | |
91e0c5f3 JF |
205 | |
206 | cb = &b->callbacks[b->cbidx++]; | |
207 | cb->fn = fn; | |
208 | cb->data = data; | |
209 | } |