Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e683014c | 2 | #include <linux/export.h> |
6ede31e0 BP |
3 | #include <linux/preempt.h> |
4 | #include <linux/smp.h> | |
07cde313 | 5 | #include <linux/completion.h> |
6ede31e0 BP |
6 | #include <asm/msr.h> |
7 | ||
8 | static void __rdmsr_on_cpu(void *info) | |
9 | { | |
10 | struct msr_info *rv = info; | |
11 | struct msr *reg; | |
12 | int this_cpu = raw_smp_processor_id(); | |
13 | ||
14 | if (rv->msrs) | |
15 | reg = per_cpu_ptr(rv->msrs, this_cpu); | |
16 | else | |
17 | reg = &rv->reg; | |
18 | ||
19 | rdmsr(rv->msr_no, reg->l, reg->h); | |
20 | } | |
21 | ||
22 | static void __wrmsr_on_cpu(void *info) | |
23 | { | |
24 | struct msr_info *rv = info; | |
25 | struct msr *reg; | |
26 | int this_cpu = raw_smp_processor_id(); | |
27 | ||
28 | if (rv->msrs) | |
29 | reg = per_cpu_ptr(rv->msrs, this_cpu); | |
30 | else | |
31 | reg = &rv->reg; | |
32 | ||
33 | wrmsr(rv->msr_no, reg->l, reg->h); | |
34 | } | |
35 | ||
36 | int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) | |
37 | { | |
38 | int err; | |
39 | struct msr_info rv; | |
40 | ||
41 | memset(&rv, 0, sizeof(rv)); | |
42 | ||
43 | rv.msr_no = msr_no; | |
44 | err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1); | |
45 | *l = rv.reg.l; | |
46 | *h = rv.reg.h; | |
47 | ||
48 | return err; | |
49 | } | |
50 | EXPORT_SYMBOL(rdmsr_on_cpu); | |
51 | ||
1a6b991a JP |
52 | int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) |
53 | { | |
54 | int err; | |
55 | struct msr_info rv; | |
56 | ||
57 | memset(&rv, 0, sizeof(rv)); | |
58 | ||
59 | rv.msr_no = msr_no; | |
60 | err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1); | |
61 | *q = rv.reg.q; | |
62 | ||
63 | return err; | |
64 | } | |
65 | EXPORT_SYMBOL(rdmsrl_on_cpu); | |
66 | ||
6ede31e0 BP |
67 | int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) |
68 | { | |
69 | int err; | |
70 | struct msr_info rv; | |
71 | ||
72 | memset(&rv, 0, sizeof(rv)); | |
73 | ||
74 | rv.msr_no = msr_no; | |
75 | rv.reg.l = l; | |
76 | rv.reg.h = h; | |
77 | err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1); | |
78 | ||
79 | return err; | |
80 | } | |
81 | EXPORT_SYMBOL(wrmsr_on_cpu); | |
82 | ||
1a6b991a JP |
83 | int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) |
84 | { | |
85 | int err; | |
86 | struct msr_info rv; | |
87 | ||
88 | memset(&rv, 0, sizeof(rv)); | |
89 | ||
90 | rv.msr_no = msr_no; | |
91 | rv.reg.q = q; | |
92 | ||
93 | err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1); | |
94 | ||
95 | return err; | |
96 | } | |
97 | EXPORT_SYMBOL(wrmsrl_on_cpu); | |
98 | ||
6ede31e0 BP |
99 | static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no, |
100 | struct msr *msrs, | |
101 | void (*msr_func) (void *info)) | |
102 | { | |
103 | struct msr_info rv; | |
104 | int this_cpu; | |
105 | ||
106 | memset(&rv, 0, sizeof(rv)); | |
107 | ||
108 | rv.msrs = msrs; | |
109 | rv.msr_no = msr_no; | |
110 | ||
111 | this_cpu = get_cpu(); | |
112 | ||
113 | if (cpumask_test_cpu(this_cpu, mask)) | |
114 | msr_func(&rv); | |
115 | ||
116 | smp_call_function_many(mask, msr_func, &rv, 1); | |
117 | put_cpu(); | |
118 | } | |
119 | ||
120 | /* rdmsr on a bunch of CPUs | |
121 | * | |
122 | * @mask: which CPUs | |
123 | * @msr_no: which MSR | |
124 | * @msrs: array of MSR values | |
125 | * | |
126 | */ | |
127 | void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs) | |
128 | { | |
129 | __rwmsr_on_cpus(mask, msr_no, msrs, __rdmsr_on_cpu); | |
130 | } | |
131 | EXPORT_SYMBOL(rdmsr_on_cpus); | |
132 | ||
133 | /* | |
134 | * wrmsr on a bunch of CPUs | |
135 | * | |
136 | * @mask: which CPUs | |
137 | * @msr_no: which MSR | |
138 | * @msrs: array of MSR values | |
139 | * | |
140 | */ | |
141 | void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs) | |
142 | { | |
143 | __rwmsr_on_cpus(mask, msr_no, msrs, __wrmsr_on_cpu); | |
144 | } | |
145 | EXPORT_SYMBOL(wrmsr_on_cpus); | |
146 | ||
07cde313 ED |
147 | struct msr_info_completion { |
148 | struct msr_info msr; | |
149 | struct completion done; | |
150 | }; | |
151 | ||
6ede31e0 BP |
152 | /* These "safe" variants are slower and should be used when the target MSR |
153 | may not actually exist. */ | |
154 | static void __rdmsr_safe_on_cpu(void *info) | |
155 | { | |
07cde313 | 156 | struct msr_info_completion *rv = info; |
6ede31e0 | 157 | |
07cde313 ED |
158 | rv->msr.err = rdmsr_safe(rv->msr.msr_no, &rv->msr.reg.l, &rv->msr.reg.h); |
159 | complete(&rv->done); | |
6ede31e0 BP |
160 | } |
161 | ||
162 | static void __wrmsr_safe_on_cpu(void *info) | |
163 | { | |
164 | struct msr_info *rv = info; | |
165 | ||
166 | rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h); | |
167 | } | |
168 | ||
169 | int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) | |
170 | { | |
07cde313 | 171 | struct msr_info_completion rv; |
545b8c8d | 172 | call_single_data_t csd; |
6ede31e0 | 173 | int err; |
6ede31e0 | 174 | |
545b8c8d PZ |
175 | INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv); |
176 | ||
6ede31e0 | 177 | memset(&rv, 0, sizeof(rv)); |
07cde313 ED |
178 | init_completion(&rv.done); |
179 | rv.msr.msr_no = msr_no; | |
6ede31e0 | 180 | |
07cde313 ED |
181 | err = smp_call_function_single_async(cpu, &csd); |
182 | if (!err) { | |
183 | wait_for_completion(&rv.done); | |
184 | err = rv.msr.err; | |
185 | } | |
186 | *l = rv.msr.reg.l; | |
187 | *h = rv.msr.reg.h; | |
6ede31e0 | 188 | |
07cde313 | 189 | return err; |
6ede31e0 BP |
190 | } |
191 | EXPORT_SYMBOL(rdmsr_safe_on_cpu); | |
192 | ||
193 | int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) | |
194 | { | |
195 | int err; | |
196 | struct msr_info rv; | |
197 | ||
198 | memset(&rv, 0, sizeof(rv)); | |
199 | ||
200 | rv.msr_no = msr_no; | |
201 | rv.reg.l = l; | |
202 | rv.reg.h = h; | |
203 | err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1); | |
204 | ||
205 | return err ? err : rv.err; | |
206 | } | |
207 | EXPORT_SYMBOL(wrmsr_safe_on_cpu); | |
208 | ||
1a6b991a JP |
209 | int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) |
210 | { | |
211 | int err; | |
212 | struct msr_info rv; | |
213 | ||
214 | memset(&rv, 0, sizeof(rv)); | |
215 | ||
216 | rv.msr_no = msr_no; | |
217 | rv.reg.q = q; | |
218 | ||
219 | err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1); | |
220 | ||
221 | return err ? err : rv.err; | |
222 | } | |
223 | EXPORT_SYMBOL(wrmsrl_safe_on_cpu); | |
224 | ||
225 | int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) | |
226 | { | |
9b9a5135 | 227 | u32 low, high; |
1a6b991a | 228 | int err; |
1a6b991a | 229 | |
9b9a5135 ED |
230 | err = rdmsr_safe_on_cpu(cpu, msr_no, &low, &high); |
231 | *q = (u64)high << 32 | low; | |
1a6b991a | 232 | |
9b9a5135 | 233 | return err; |
1a6b991a JP |
234 | } |
235 | EXPORT_SYMBOL(rdmsrl_safe_on_cpu); | |
236 | ||
6ede31e0 BP |
237 | /* |
238 | * These variants are significantly slower, but allows control over | |
239 | * the entire 32-bit GPR set. | |
240 | */ | |
241 | static void __rdmsr_safe_regs_on_cpu(void *info) | |
242 | { | |
243 | struct msr_regs_info *rv = info; | |
244 | ||
245 | rv->err = rdmsr_safe_regs(rv->regs); | |
246 | } | |
247 | ||
248 | static void __wrmsr_safe_regs_on_cpu(void *info) | |
249 | { | |
250 | struct msr_regs_info *rv = info; | |
251 | ||
252 | rv->err = wrmsr_safe_regs(rv->regs); | |
253 | } | |
254 | ||
396a66aa | 255 | int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) |
6ede31e0 BP |
256 | { |
257 | int err; | |
258 | struct msr_regs_info rv; | |
259 | ||
260 | rv.regs = regs; | |
261 | rv.err = -EIO; | |
262 | err = smp_call_function_single(cpu, __rdmsr_safe_regs_on_cpu, &rv, 1); | |
263 | ||
264 | return err ? err : rv.err; | |
265 | } | |
266 | EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu); | |
267 | ||
396a66aa | 268 | int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) |
6ede31e0 BP |
269 | { |
270 | int err; | |
271 | struct msr_regs_info rv; | |
272 | ||
273 | rv.regs = regs; | |
274 | rv.err = -EIO; | |
275 | err = smp_call_function_single(cpu, __wrmsr_safe_regs_on_cpu, &rv, 1); | |
276 | ||
277 | return err ? err : rv.err; | |
278 | } | |
279 | EXPORT_SYMBOL(wrmsr_safe_regs_on_cpu); |