| 1 | /* |
| 2 | * Xen hypercall batching. |
| 3 | * |
| 4 | * Xen allows multiple hypercalls to be issued at once, using the |
| 5 | * multicall interface. This allows the cost of trapping into the |
| 6 | * hypervisor to be amortized over several calls. |
| 7 | * |
| 8 | * This file implements a simple interface for multicalls. There's a |
| 9 | * per-cpu buffer of outstanding multicalls. When you want to queue a |
| 10 | * multicall for issuing, you can allocate a multicall slot for the |
| 11 | * call and its arguments, along with storage for space which is |
| 12 | * pointed to by the arguments (for passing pointers to structures, |
| 13 | * etc). When the multicall is actually issued, all the space for the |
| 14 | * commands and allocated memory is freed for reuse. |
| 15 | * |
| 16 | * Multicalls are flushed whenever any of the buffers get full, or |
| 17 | * when explicitly requested. There's no way to get per-multicall |
| 18 | * return results back. It will BUG if any of the multicalls fail. |
| 19 | * |
| 20 | * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 |
| 21 | */ |
| 22 | #include <linux/percpu.h> |
| 23 | #include <linux/hardirq.h> |
| 24 | #include <linux/debugfs.h> |
| 25 | |
| 26 | #include <asm/xen/hypercall.h> |
| 27 | |
| 28 | #include "multicalls.h" |
| 29 | #include "debugfs.h" |
| 30 | |
| 31 | #define MC_BATCH 32 |
| 32 | |
| 33 | #define MC_DEBUG 0 |
| 34 | |
| 35 | #define MC_ARGS (MC_BATCH * 16) |
| 36 | |
| 37 | |
| 38 | struct mc_buffer { |
| 39 | unsigned mcidx, argidx, cbidx; |
| 40 | struct multicall_entry entries[MC_BATCH]; |
| 41 | #if MC_DEBUG |
| 42 | struct multicall_entry debug[MC_BATCH]; |
| 43 | void *caller[MC_BATCH]; |
| 44 | #endif |
| 45 | unsigned char args[MC_ARGS]; |
| 46 | struct callback { |
| 47 | void (*fn)(void *); |
| 48 | void *data; |
| 49 | } callbacks[MC_BATCH]; |
| 50 | }; |
| 51 | |
| 52 | static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); |
| 53 | DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); |
| 54 | |
| 55 | void xen_mc_flush(void) |
| 56 | { |
| 57 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
| 58 | struct multicall_entry *mc; |
| 59 | int ret = 0; |
| 60 | unsigned long flags; |
| 61 | int i; |
| 62 | |
| 63 | BUG_ON(preemptible()); |
| 64 | |
| 65 | /* Disable interrupts in case someone comes in and queues |
| 66 | something in the middle */ |
| 67 | local_irq_save(flags); |
| 68 | |
| 69 | trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx); |
| 70 | |
| 71 | switch (b->mcidx) { |
| 72 | case 0: |
| 73 | /* no-op */ |
| 74 | BUG_ON(b->argidx != 0); |
| 75 | break; |
| 76 | |
| 77 | case 1: |
| 78 | /* Singleton multicall - bypass multicall machinery |
| 79 | and just do the call directly. */ |
| 80 | mc = &b->entries[0]; |
| 81 | |
| 82 | mc->result = privcmd_call(mc->op, |
| 83 | mc->args[0], mc->args[1], mc->args[2], |
| 84 | mc->args[3], mc->args[4]); |
| 85 | ret = mc->result < 0; |
| 86 | break; |
| 87 | |
| 88 | default: |
| 89 | #if MC_DEBUG |
| 90 | memcpy(b->debug, b->entries, |
| 91 | b->mcidx * sizeof(struct multicall_entry)); |
| 92 | #endif |
| 93 | |
| 94 | if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) |
| 95 | BUG(); |
| 96 | for (i = 0; i < b->mcidx; i++) |
| 97 | if (b->entries[i].result < 0) |
| 98 | ret++; |
| 99 | |
| 100 | #if MC_DEBUG |
| 101 | if (ret) { |
| 102 | printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", |
| 103 | ret, smp_processor_id()); |
| 104 | dump_stack(); |
| 105 | for (i = 0; i < b->mcidx; i++) { |
| 106 | printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n", |
| 107 | i+1, b->mcidx, |
| 108 | b->debug[i].op, |
| 109 | b->debug[i].args[0], |
| 110 | b->entries[i].result, |
| 111 | b->caller[i]); |
| 112 | } |
| 113 | } |
| 114 | #endif |
| 115 | } |
| 116 | |
| 117 | b->mcidx = 0; |
| 118 | b->argidx = 0; |
| 119 | |
| 120 | for (i = 0; i < b->cbidx; i++) { |
| 121 | struct callback *cb = &b->callbacks[i]; |
| 122 | |
| 123 | (*cb->fn)(cb->data); |
| 124 | } |
| 125 | b->cbidx = 0; |
| 126 | |
| 127 | local_irq_restore(flags); |
| 128 | |
| 129 | WARN_ON(ret); |
| 130 | } |
| 131 | |
| 132 | struct multicall_space __xen_mc_entry(size_t args) |
| 133 | { |
| 134 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
| 135 | struct multicall_space ret; |
| 136 | unsigned argidx = roundup(b->argidx, sizeof(u64)); |
| 137 | |
| 138 | trace_xen_mc_entry_alloc(args); |
| 139 | |
| 140 | BUG_ON(preemptible()); |
| 141 | BUG_ON(b->argidx >= MC_ARGS); |
| 142 | |
| 143 | if (unlikely(b->mcidx == MC_BATCH || |
| 144 | (argidx + args) >= MC_ARGS)) { |
| 145 | trace_xen_mc_flush_reason((b->mcidx == MC_BATCH) ? |
| 146 | XEN_MC_FL_BATCH : XEN_MC_FL_ARGS); |
| 147 | xen_mc_flush(); |
| 148 | argidx = roundup(b->argidx, sizeof(u64)); |
| 149 | } |
| 150 | |
| 151 | ret.mc = &b->entries[b->mcidx]; |
| 152 | #if MC_DEBUG |
| 153 | b->caller[b->mcidx] = __builtin_return_address(0); |
| 154 | #endif |
| 155 | b->mcidx++; |
| 156 | ret.args = &b->args[argidx]; |
| 157 | b->argidx = argidx + args; |
| 158 | |
| 159 | BUG_ON(b->argidx >= MC_ARGS); |
| 160 | return ret; |
| 161 | } |
| 162 | |
| 163 | struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) |
| 164 | { |
| 165 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
| 166 | struct multicall_space ret = { NULL, NULL }; |
| 167 | |
| 168 | BUG_ON(preemptible()); |
| 169 | BUG_ON(b->argidx >= MC_ARGS); |
| 170 | |
| 171 | if (unlikely(b->mcidx == 0 || |
| 172 | b->entries[b->mcidx - 1].op != op)) { |
| 173 | trace_xen_mc_extend_args(op, size, XEN_MC_XE_BAD_OP); |
| 174 | goto out; |
| 175 | } |
| 176 | |
| 177 | if (unlikely((b->argidx + size) >= MC_ARGS)) { |
| 178 | trace_xen_mc_extend_args(op, size, XEN_MC_XE_NO_SPACE); |
| 179 | goto out; |
| 180 | } |
| 181 | |
| 182 | ret.mc = &b->entries[b->mcidx - 1]; |
| 183 | ret.args = &b->args[b->argidx]; |
| 184 | b->argidx += size; |
| 185 | |
| 186 | BUG_ON(b->argidx >= MC_ARGS); |
| 187 | |
| 188 | trace_xen_mc_extend_args(op, size, XEN_MC_XE_OK); |
| 189 | out: |
| 190 | return ret; |
| 191 | } |
| 192 | |
| 193 | void xen_mc_callback(void (*fn)(void *), void *data) |
| 194 | { |
| 195 | struct mc_buffer *b = this_cpu_ptr(&mc_buffer); |
| 196 | struct callback *cb; |
| 197 | |
| 198 | if (b->cbidx == MC_BATCH) { |
| 199 | trace_xen_mc_flush_reason(XEN_MC_FL_CALLBACK); |
| 200 | xen_mc_flush(); |
| 201 | } |
| 202 | |
| 203 | trace_xen_mc_callback(fn, data); |
| 204 | |
| 205 | cb = &b->callbacks[b->cbidx++]; |
| 206 | cb->fn = fn; |
| 207 | cb->data = data; |
| 208 | } |