Commit | Line | Data |
---|---|---|
adbb3901 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cbcca5d0 SO |
2 | /* |
3 | * s390 specific pci instructions | |
4 | * | |
5 | * Copyright IBM Corp. 2013 | |
6 | */ | |
7 | ||
8 | #include <linux/export.h> | |
9 | #include <linux/errno.h> | |
10 | #include <linux/delay.h> | |
71ba41c9 | 11 | #include <linux/jump_label.h> |
d09a307f | 12 | #include <asm/asm-extable.h> |
48070c73 | 13 | #include <asm/facility.h> |
cbcca5d0 | 14 | #include <asm/pci_insn.h> |
3d8258e4 | 15 | #include <asm/pci_debug.h> |
81deca12 | 16 | #include <asm/pci_io.h> |
f0bacb7f | 17 | #include <asm/processor.h> |
cbcca5d0 SO |
18 | |
19 | #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ | |
20 | ||
cde8833e NS |
21 | struct zpci_err_insn_data { |
22 | u8 insn; | |
23 | u8 cc; | |
24 | u8 status; | |
25 | union { | |
26 | struct { | |
27 | u64 req; | |
28 | u64 offset; | |
29 | }; | |
30 | struct { | |
31 | u64 addr; | |
32 | u64 len; | |
33 | }; | |
34 | }; | |
35 | } __packed; | |
36 | ||
34fb0e70 | 37 | static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status, |
cde8833e NS |
38 | u64 req, u64 offset) |
39 | { | |
40 | struct zpci_err_insn_data data = { | |
41 | .insn = insn, .cc = cc, .status = status, | |
42 | .req = req, .offset = offset}; | |
43 | ||
34fb0e70 | 44 | zpci_err_hex_level(lvl, &data, sizeof(data)); |
cde8833e NS |
45 | } |
46 | ||
34fb0e70 | 47 | static inline void zpci_err_insn_addr(int lvl, u8 insn, u8 cc, u8 status, |
cde8833e | 48 | u64 addr, u64 len) |
3d8258e4 | 49 | { |
cde8833e NS |
50 | struct zpci_err_insn_data data = { |
51 | .insn = insn, .cc = cc, .status = status, | |
52 | .addr = addr, .len = len}; | |
3d8258e4 | 53 | |
34fb0e70 | 54 | zpci_err_hex_level(lvl, &data, sizeof(data)); |
3d8258e4 SO |
55 | } |
56 | ||
cbcca5d0 SO |
57 | /* Modify PCI Function Controls */ |
58 | static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status) | |
59 | { | |
60 | u8 cc; | |
61 | ||
62 | asm volatile ( | |
63 | " .insn rxy,0xe300000000d0,%[req],%[fib]\n" | |
64 | " ipm %[cc]\n" | |
65 | " srl %[cc],28\n" | |
66 | : [cc] "=d" (cc), [req] "+d" (req), [fib] "+Q" (*fib) | |
67 | : : "cc"); | |
68 | *status = req >> 24 & 0xff; | |
69 | return cc; | |
70 | } | |
71 | ||
4dfbd3ef | 72 | u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status) |
cbcca5d0 | 73 | { |
34fb0e70 | 74 | bool retried = false; |
4dfbd3ef | 75 | u8 cc; |
cbcca5d0 SO |
76 | |
77 | do { | |
4dfbd3ef | 78 | cc = __mpcifc(req, fib, status); |
34fb0e70 | 79 | if (cc == 2) { |
cbcca5d0 | 80 | msleep(ZPCI_INSN_BUSY_DELAY); |
34fb0e70 NS |
81 | if (!retried) { |
82 | zpci_err_insn_req(1, 'M', cc, *status, req, 0); | |
83 | retried = true; | |
84 | } | |
85 | } | |
cbcca5d0 SO |
86 | } while (cc == 2); |
87 | ||
88 | if (cc) | |
34fb0e70 NS |
89 | zpci_err_insn_req(0, 'M', cc, *status, req, 0); |
90 | else if (retried) | |
91 | zpci_err_insn_req(1, 'M', cc, *status, req, 0); | |
3d8258e4 | 92 | |
4dfbd3ef | 93 | return cc; |
cbcca5d0 | 94 | } |
3c5a1b6f | 95 | EXPORT_SYMBOL_GPL(zpci_mod_fc); |
cbcca5d0 SO |
96 | |
97 | /* Refresh PCI Translations */ | |
98 | static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) | |
99 | { | |
d66a4c7f | 100 | union register_pair addr_range = {.even = addr, .odd = range}; |
cbcca5d0 SO |
101 | u8 cc; |
102 | ||
103 | asm volatile ( | |
d66a4c7f | 104 | " .insn rre,0xb9d30000,%[fn],%[addr_range]\n" |
cbcca5d0 SO |
105 | " ipm %[cc]\n" |
106 | " srl %[cc],28\n" | |
107 | : [cc] "=d" (cc), [fn] "+d" (fn) | |
d66a4c7f | 108 | : [addr_range] "d" (addr_range.pair) |
cbcca5d0 SO |
109 | : "cc"); |
110 | *status = fn >> 24 & 0xff; | |
111 | return cc; | |
112 | } | |
113 | ||
9389339f | 114 | int zpci_refresh_trans(u64 fn, u64 addr, u64 range) |
cbcca5d0 | 115 | { |
34fb0e70 | 116 | bool retried = false; |
cbcca5d0 SO |
117 | u8 cc, status; |
118 | ||
119 | do { | |
120 | cc = __rpcit(fn, addr, range, &status); | |
34fb0e70 | 121 | if (cc == 2) { |
cbcca5d0 | 122 | udelay(ZPCI_INSN_BUSY_DELAY); |
34fb0e70 NS |
123 | if (!retried) { |
124 | zpci_err_insn_addr(1, 'R', cc, status, addr, range); | |
125 | retried = true; | |
126 | } | |
127 | } | |
cbcca5d0 SO |
128 | } while (cc == 2); |
129 | ||
130 | if (cc) | |
34fb0e70 NS |
131 | zpci_err_insn_addr(0, 'R', cc, status, addr, range); |
132 | else if (retried) | |
133 | zpci_err_insn_addr(1, 'R', cc, status, addr, range); | |
3d8258e4 | 134 | |
a5f10055 SO |
135 | if (cc == 1 && (status == 4 || status == 16)) |
136 | return -ENOMEM; | |
137 | ||
cbcca5d0 SO |
138 | return (cc) ? -EIO : 0; |
139 | } | |
140 | ||
141 | /* Set Interruption Controls */ | |
062f0024 | 142 | int zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib) |
cbcca5d0 | 143 | { |
48070c73 CB |
144 | if (!test_facility(72)) |
145 | return -EIO; | |
e979ce7b SO |
146 | |
147 | asm volatile( | |
148 | ".insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]\n" | |
149 | : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib)); | |
150 | ||
48070c73 | 151 | return 0; |
cbcca5d0 | 152 | } |
062f0024 | 153 | EXPORT_SYMBOL_GPL(zpci_set_irq_ctrl); |
cbcca5d0 SO |
154 | |
155 | /* PCI Load */ | |
7b411ac6 | 156 | static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) |
cbcca5d0 | 157 | { |
d66a4c7f | 158 | union register_pair req_off = {.even = req, .odd = offset}; |
f0bacb7f | 159 | int cc = -ENXIO; |
cbcca5d0 | 160 | u64 __data; |
cbcca5d0 SO |
161 | |
162 | asm volatile ( | |
d66a4c7f | 163 | " .insn rre,0xb9d20000,%[data],%[req_off]\n" |
f0bacb7f | 164 | "0: ipm %[cc]\n" |
cbcca5d0 | 165 | " srl %[cc],28\n" |
f0bacb7f SO |
166 | "1:\n" |
167 | EX_TABLE(0b, 1b) | |
d66a4c7f NS |
168 | : [cc] "+d" (cc), [data] "=d" (__data), |
169 | [req_off] "+&d" (req_off.pair) :: "cc"); | |
170 | *status = req_off.even >> 24 & 0xff; | |
7b411ac6 HC |
171 | *data = __data; |
172 | return cc; | |
173 | } | |
174 | ||
175 | static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) | |
176 | { | |
177 | u64 __data; | |
178 | int cc; | |
179 | ||
180 | cc = ____pcilg(&__data, req, offset, status); | |
b170bad4 SO |
181 | if (!cc) |
182 | *data = __data; | |
183 | ||
cbcca5d0 SO |
184 | return cc; |
185 | } | |
186 | ||
81deca12 | 187 | int __zpci_load(u64 *data, u64 req, u64 offset) |
cbcca5d0 | 188 | { |
34fb0e70 | 189 | bool retried = false; |
f0bacb7f SO |
190 | u8 status; |
191 | int cc; | |
cbcca5d0 SO |
192 | |
193 | do { | |
194 | cc = __pcilg(data, req, offset, &status); | |
34fb0e70 | 195 | if (cc == 2) { |
cbcca5d0 | 196 | udelay(ZPCI_INSN_BUSY_DELAY); |
34fb0e70 NS |
197 | if (!retried) { |
198 | zpci_err_insn_req(1, 'l', cc, status, req, offset); | |
199 | retried = true; | |
200 | } | |
201 | } | |
cbcca5d0 SO |
202 | } while (cc == 2); |
203 | ||
f0bacb7f | 204 | if (cc) |
34fb0e70 NS |
205 | zpci_err_insn_req(0, 'l', cc, status, req, offset); |
206 | else if (retried) | |
207 | zpci_err_insn_req(1, 'l', cc, status, req, offset); | |
3d8258e4 | 208 | |
f0bacb7f | 209 | return (cc > 0) ? -EIO : cc; |
cbcca5d0 | 210 | } |
81deca12 SO |
211 | EXPORT_SYMBOL_GPL(__zpci_load); |
212 | ||
71ba41c9 SO |
213 | static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr, |
214 | unsigned long len) | |
81deca12 SO |
215 | { |
216 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; | |
4fe20497 | 217 | u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len); |
81deca12 SO |
218 | |
219 | return __zpci_load(data, req, ZPCI_OFFSET(addr)); | |
220 | } | |
71ba41c9 SO |
221 | |
222 | static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) | |
223 | { | |
d66a4c7f | 224 | union register_pair ioaddr_len = {.even = ioaddr, .odd = len}; |
71ba41c9 SO |
225 | int cc = -ENXIO; |
226 | u64 __data; | |
227 | ||
228 | asm volatile ( | |
d66a4c7f | 229 | " .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" |
71ba41c9 SO |
230 | "0: ipm %[cc]\n" |
231 | " srl %[cc],28\n" | |
232 | "1:\n" | |
233 | EX_TABLE(0b, 1b) | |
d66a4c7f NS |
234 | : [cc] "+d" (cc), [data] "=d" (__data), |
235 | [ioaddr_len] "+&d" (ioaddr_len.pair) :: "cc"); | |
236 | *status = ioaddr_len.odd >> 24 & 0xff; | |
71ba41c9 SO |
237 | *data = __data; |
238 | return cc; | |
239 | } | |
240 | ||
241 | int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len) | |
242 | { | |
243 | u8 status; | |
244 | int cc; | |
245 | ||
246 | if (!static_branch_unlikely(&have_mio)) | |
247 | return zpci_load_fh(data, addr, len); | |
248 | ||
249 | cc = __pcilg_mio(data, (__force u64) addr, len, &status); | |
250 | if (cc) | |
34fb0e70 | 251 | zpci_err_insn_addr(0, 'L', cc, status, (__force u64) addr, len); |
71ba41c9 SO |
252 | |
253 | return (cc > 0) ? -EIO : cc; | |
254 | } | |
9389339f | 255 | EXPORT_SYMBOL_GPL(zpci_load); |
cbcca5d0 SO |
256 | |
257 | /* PCI Store */ | |
f0bacb7f | 258 | static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) |
cbcca5d0 | 259 | { |
d66a4c7f | 260 | union register_pair req_off = {.even = req, .odd = offset}; |
f0bacb7f | 261 | int cc = -ENXIO; |
cbcca5d0 SO |
262 | |
263 | asm volatile ( | |
d66a4c7f | 264 | " .insn rre,0xb9d00000,%[data],%[req_off]\n" |
f0bacb7f | 265 | "0: ipm %[cc]\n" |
cbcca5d0 | 266 | " srl %[cc],28\n" |
f0bacb7f SO |
267 | "1:\n" |
268 | EX_TABLE(0b, 1b) | |
d66a4c7f NS |
269 | : [cc] "+d" (cc), [req_off] "+&d" (req_off.pair) |
270 | : [data] "d" (data) | |
cbcca5d0 | 271 | : "cc"); |
d66a4c7f | 272 | *status = req_off.even >> 24 & 0xff; |
cbcca5d0 SO |
273 | return cc; |
274 | } | |
275 | ||
81deca12 | 276 | int __zpci_store(u64 data, u64 req, u64 offset) |
cbcca5d0 | 277 | { |
34fb0e70 | 278 | bool retried = false; |
f0bacb7f SO |
279 | u8 status; |
280 | int cc; | |
cbcca5d0 SO |
281 | |
282 | do { | |
283 | cc = __pcistg(data, req, offset, &status); | |
34fb0e70 | 284 | if (cc == 2) { |
cbcca5d0 | 285 | udelay(ZPCI_INSN_BUSY_DELAY); |
34fb0e70 NS |
286 | if (!retried) { |
287 | zpci_err_insn_req(1, 's', cc, status, req, offset); | |
288 | retried = true; | |
289 | } | |
290 | } | |
cbcca5d0 SO |
291 | } while (cc == 2); |
292 | ||
293 | if (cc) | |
34fb0e70 NS |
294 | zpci_err_insn_req(0, 's', cc, status, req, offset); |
295 | else if (retried) | |
296 | zpci_err_insn_req(1, 's', cc, status, req, offset); | |
3d8258e4 | 297 | |
f0bacb7f | 298 | return (cc > 0) ? -EIO : cc; |
cbcca5d0 | 299 | } |
81deca12 SO |
300 | EXPORT_SYMBOL_GPL(__zpci_store); |
301 | ||
71ba41c9 SO |
302 | static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data, |
303 | unsigned long len) | |
81deca12 SO |
304 | { |
305 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; | |
4fe20497 | 306 | u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len); |
81deca12 SO |
307 | |
308 | return __zpci_store(data, req, ZPCI_OFFSET(addr)); | |
309 | } | |
71ba41c9 SO |
310 | |
311 | static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) | |
312 | { | |
d66a4c7f | 313 | union register_pair ioaddr_len = {.even = ioaddr, .odd = len}; |
71ba41c9 SO |
314 | int cc = -ENXIO; |
315 | ||
316 | asm volatile ( | |
d66a4c7f | 317 | " .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" |
71ba41c9 SO |
318 | "0: ipm %[cc]\n" |
319 | " srl %[cc],28\n" | |
320 | "1:\n" | |
321 | EX_TABLE(0b, 1b) | |
d66a4c7f NS |
322 | : [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) |
323 | : [data] "d" (data) | |
324 | : "cc", "memory"); | |
325 | *status = ioaddr_len.odd >> 24 & 0xff; | |
71ba41c9 SO |
326 | return cc; |
327 | } | |
328 | ||
329 | int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len) | |
330 | { | |
331 | u8 status; | |
332 | int cc; | |
333 | ||
334 | if (!static_branch_unlikely(&have_mio)) | |
335 | return zpci_store_fh(addr, data, len); | |
336 | ||
337 | cc = __pcistg_mio(data, (__force u64) addr, len, &status); | |
338 | if (cc) | |
34fb0e70 | 339 | zpci_err_insn_addr(0, 'S', cc, status, (__force u64) addr, len); |
71ba41c9 SO |
340 | |
341 | return (cc > 0) ? -EIO : cc; | |
342 | } | |
9389339f | 343 | EXPORT_SYMBOL_GPL(zpci_store); |
cbcca5d0 SO |
344 | |
345 | /* PCI Store Block */ | |
f0bacb7f | 346 | static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) |
cbcca5d0 | 347 | { |
f0bacb7f | 348 | int cc = -ENXIO; |
cbcca5d0 SO |
349 | |
350 | asm volatile ( | |
351 | " .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" | |
f0bacb7f | 352 | "0: ipm %[cc]\n" |
cbcca5d0 | 353 | " srl %[cc],28\n" |
f0bacb7f SO |
354 | "1:\n" |
355 | EX_TABLE(0b, 1b) | |
356 | : [cc] "+d" (cc), [req] "+d" (req) | |
cbcca5d0 SO |
357 | : [offset] "d" (offset), [data] "Q" (*data) |
358 | : "cc"); | |
359 | *status = req >> 24 & 0xff; | |
360 | return cc; | |
361 | } | |
362 | ||
81deca12 | 363 | int __zpci_store_block(const u64 *data, u64 req, u64 offset) |
cbcca5d0 | 364 | { |
34fb0e70 | 365 | bool retried = false; |
f0bacb7f SO |
366 | u8 status; |
367 | int cc; | |
cbcca5d0 SO |
368 | |
369 | do { | |
370 | cc = __pcistb(data, req, offset, &status); | |
34fb0e70 | 371 | if (cc == 2) { |
cbcca5d0 | 372 | udelay(ZPCI_INSN_BUSY_DELAY); |
34fb0e70 NS |
373 | if (!retried) { |
374 | zpci_err_insn_req(0, 'b', cc, status, req, offset); | |
375 | retried = true; | |
376 | } | |
377 | } | |
cbcca5d0 SO |
378 | } while (cc == 2); |
379 | ||
380 | if (cc) | |
34fb0e70 NS |
381 | zpci_err_insn_req(0, 'b', cc, status, req, offset); |
382 | else if (retried) | |
383 | zpci_err_insn_req(1, 'b', cc, status, req, offset); | |
3d8258e4 | 384 | |
f0bacb7f | 385 | return (cc > 0) ? -EIO : cc; |
cbcca5d0 | 386 | } |
81deca12 SO |
387 | EXPORT_SYMBOL_GPL(__zpci_store_block); |
388 | ||
71ba41c9 SO |
389 | static inline int zpci_write_block_fh(volatile void __iomem *dst, |
390 | const void *src, unsigned long len) | |
81deca12 SO |
391 | { |
392 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)]; | |
393 | u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); | |
394 | u64 offset = ZPCI_OFFSET(dst); | |
395 | ||
396 | return __zpci_store_block(src, req, offset); | |
397 | } | |
71ba41c9 SO |
398 | |
399 | static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) | |
400 | { | |
401 | int cc = -ENXIO; | |
402 | ||
403 | asm volatile ( | |
404 | " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" | |
405 | "0: ipm %[cc]\n" | |
406 | " srl %[cc],28\n" | |
407 | "1:\n" | |
408 | EX_TABLE(0b, 1b) | |
409 | : [cc] "+d" (cc), [len] "+d" (len) | |
410 | : [ioaddr] "d" (ioaddr), [data] "Q" (*data) | |
411 | : "cc"); | |
412 | *status = len >> 24 & 0xff; | |
413 | return cc; | |
414 | } | |
415 | ||
416 | int zpci_write_block(volatile void __iomem *dst, | |
417 | const void *src, unsigned long len) | |
418 | { | |
419 | u8 status; | |
420 | int cc; | |
421 | ||
422 | if (!static_branch_unlikely(&have_mio)) | |
423 | return zpci_write_block_fh(dst, src, len); | |
424 | ||
425 | cc = __pcistb_mio(src, (__force u64) dst, len, &status); | |
426 | if (cc) | |
34fb0e70 | 427 | zpci_err_insn_addr(0, 'B', cc, status, (__force u64) dst, len); |
71ba41c9 SO |
428 | |
429 | return (cc > 0) ? -EIO : cc; | |
430 | } | |
81deca12 | 431 | EXPORT_SYMBOL_GPL(zpci_write_block); |
71ba41c9 SO |
432 | |
433 | static inline void __pciwb_mio(void) | |
434 | { | |
7b293216 | 435 | asm volatile (".insn rre,0xb9d50000,0,0\n"); |
71ba41c9 SO |
436 | } |
437 | ||
438 | void zpci_barrier(void) | |
439 | { | |
440 | if (static_branch_likely(&have_mio)) | |
441 | __pciwb_mio(); | |
442 | } | |
443 | EXPORT_SYMBOL_GPL(zpci_barrier); |