Commit | Line | Data |
---|---|---|
adbb3901 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cbcca5d0 SO |
2 | /* |
3 | * s390 specific pci instructions | |
4 | * | |
5 | * Copyright IBM Corp. 2013 | |
6 | */ | |
7 | ||
8 | #include <linux/export.h> | |
9 | #include <linux/errno.h> | |
10 | #include <linux/delay.h> | |
71ba41c9 | 11 | #include <linux/jump_label.h> |
48070c73 | 12 | #include <asm/facility.h> |
cbcca5d0 | 13 | #include <asm/pci_insn.h> |
3d8258e4 | 14 | #include <asm/pci_debug.h> |
81deca12 | 15 | #include <asm/pci_io.h> |
f0bacb7f | 16 | #include <asm/processor.h> |
cbcca5d0 SO |
17 | |
18 | #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ | |
19 | ||
3d8258e4 SO |
20 | static inline void zpci_err_insn(u8 cc, u8 status, u64 req, u64 offset) |
21 | { | |
22 | struct { | |
3d8258e4 SO |
23 | u64 req; |
24 | u64 offset; | |
7cc8944e SO |
25 | u8 cc; |
26 | u8 status; | |
27 | } __packed data = {req, offset, cc, status}; | |
3d8258e4 SO |
28 | |
29 | zpci_err_hex(&data, sizeof(data)); | |
30 | } | |
31 | ||
cbcca5d0 SO |
32 | /* Modify PCI Function Controls */ |
33 | static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status) | |
34 | { | |
35 | u8 cc; | |
36 | ||
37 | asm volatile ( | |
38 | " .insn rxy,0xe300000000d0,%[req],%[fib]\n" | |
39 | " ipm %[cc]\n" | |
40 | " srl %[cc],28\n" | |
41 | : [cc] "=d" (cc), [req] "+d" (req), [fib] "+Q" (*fib) | |
42 | : : "cc"); | |
43 | *status = req >> 24 & 0xff; | |
44 | return cc; | |
45 | } | |
46 | ||
4dfbd3ef | 47 | u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status) |
cbcca5d0 | 48 | { |
4dfbd3ef | 49 | u8 cc; |
cbcca5d0 SO |
50 | |
51 | do { | |
4dfbd3ef | 52 | cc = __mpcifc(req, fib, status); |
cbcca5d0 SO |
53 | if (cc == 2) |
54 | msleep(ZPCI_INSN_BUSY_DELAY); | |
55 | } while (cc == 2); | |
56 | ||
57 | if (cc) | |
4dfbd3ef | 58 | zpci_err_insn(cc, *status, req, 0); |
3d8258e4 | 59 | |
4dfbd3ef | 60 | return cc; |
cbcca5d0 SO |
61 | } |
62 | ||
63 | /* Refresh PCI Translations */ | |
64 | static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) | |
65 | { | |
66 | register u64 __addr asm("2") = addr; | |
67 | register u64 __range asm("3") = range; | |
68 | u8 cc; | |
69 | ||
70 | asm volatile ( | |
71 | " .insn rre,0xb9d30000,%[fn],%[addr]\n" | |
72 | " ipm %[cc]\n" | |
73 | " srl %[cc],28\n" | |
74 | : [cc] "=d" (cc), [fn] "+d" (fn) | |
75 | : [addr] "d" (__addr), "d" (__range) | |
76 | : "cc"); | |
77 | *status = fn >> 24 & 0xff; | |
78 | return cc; | |
79 | } | |
80 | ||
9389339f | 81 | int zpci_refresh_trans(u64 fn, u64 addr, u64 range) |
cbcca5d0 SO |
82 | { |
83 | u8 cc, status; | |
84 | ||
85 | do { | |
86 | cc = __rpcit(fn, addr, range, &status); | |
87 | if (cc == 2) | |
88 | udelay(ZPCI_INSN_BUSY_DELAY); | |
89 | } while (cc == 2); | |
90 | ||
91 | if (cc) | |
3d8258e4 SO |
92 | zpci_err_insn(cc, status, addr, range); |
93 | ||
a5f10055 SO |
94 | if (cc == 1 && (status == 4 || status == 16)) |
95 | return -ENOMEM; | |
96 | ||
cbcca5d0 SO |
97 | return (cc) ? -EIO : 0; |
98 | } | |
99 | ||
100 | /* Set Interruption Controls */ | |
e979ce7b | 101 | int __zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib) |
cbcca5d0 | 102 | { |
48070c73 CB |
103 | if (!test_facility(72)) |
104 | return -EIO; | |
e979ce7b SO |
105 | |
106 | asm volatile( | |
107 | ".insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]\n" | |
108 | : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib)); | |
109 | ||
48070c73 | 110 | return 0; |
cbcca5d0 SO |
111 | } |
112 | ||
113 | /* PCI Load */ | |
7b411ac6 | 114 | static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) |
cbcca5d0 SO |
115 | { |
116 | register u64 __req asm("2") = req; | |
117 | register u64 __offset asm("3") = offset; | |
f0bacb7f | 118 | int cc = -ENXIO; |
cbcca5d0 | 119 | u64 __data; |
cbcca5d0 SO |
120 | |
121 | asm volatile ( | |
122 | " .insn rre,0xb9d20000,%[data],%[req]\n" | |
f0bacb7f | 123 | "0: ipm %[cc]\n" |
cbcca5d0 | 124 | " srl %[cc],28\n" |
f0bacb7f SO |
125 | "1:\n" |
126 | EX_TABLE(0b, 1b) | |
127 | : [cc] "+d" (cc), [data] "=d" (__data), [req] "+d" (__req) | |
cbcca5d0 SO |
128 | : "d" (__offset) |
129 | : "cc"); | |
130 | *status = __req >> 24 & 0xff; | |
7b411ac6 HC |
131 | *data = __data; |
132 | return cc; | |
133 | } | |
134 | ||
135 | static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) | |
136 | { | |
137 | u64 __data; | |
138 | int cc; | |
139 | ||
140 | cc = ____pcilg(&__data, req, offset, status); | |
b170bad4 SO |
141 | if (!cc) |
142 | *data = __data; | |
143 | ||
cbcca5d0 SO |
144 | return cc; |
145 | } | |
146 | ||
81deca12 | 147 | int __zpci_load(u64 *data, u64 req, u64 offset) |
cbcca5d0 | 148 | { |
f0bacb7f SO |
149 | u8 status; |
150 | int cc; | |
cbcca5d0 SO |
151 | |
152 | do { | |
153 | cc = __pcilg(data, req, offset, &status); | |
154 | if (cc == 2) | |
155 | udelay(ZPCI_INSN_BUSY_DELAY); | |
156 | } while (cc == 2); | |
157 | ||
f0bacb7f | 158 | if (cc) |
3d8258e4 SO |
159 | zpci_err_insn(cc, status, req, offset); |
160 | ||
f0bacb7f | 161 | return (cc > 0) ? -EIO : cc; |
cbcca5d0 | 162 | } |
81deca12 SO |
163 | EXPORT_SYMBOL_GPL(__zpci_load); |
164 | ||
71ba41c9 SO |
165 | static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr, |
166 | unsigned long len) | |
81deca12 SO |
167 | { |
168 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; | |
169 | u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); | |
170 | ||
171 | return __zpci_load(data, req, ZPCI_OFFSET(addr)); | |
172 | } | |
71ba41c9 SO |
173 | |
174 | static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) | |
175 | { | |
176 | register u64 addr asm("2") = ioaddr; | |
177 | register u64 r3 asm("3") = len; | |
178 | int cc = -ENXIO; | |
179 | u64 __data; | |
180 | ||
181 | asm volatile ( | |
182 | " .insn rre,0xb9d60000,%[data],%[ioaddr]\n" | |
183 | "0: ipm %[cc]\n" | |
184 | " srl %[cc],28\n" | |
185 | "1:\n" | |
186 | EX_TABLE(0b, 1b) | |
187 | : [cc] "+d" (cc), [data] "=d" (__data), "+d" (r3) | |
188 | : [ioaddr] "d" (addr) | |
189 | : "cc"); | |
190 | *status = r3 >> 24 & 0xff; | |
191 | *data = __data; | |
192 | return cc; | |
193 | } | |
194 | ||
195 | int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len) | |
196 | { | |
197 | u8 status; | |
198 | int cc; | |
199 | ||
200 | if (!static_branch_unlikely(&have_mio)) | |
201 | return zpci_load_fh(data, addr, len); | |
202 | ||
203 | cc = __pcilg_mio(data, (__force u64) addr, len, &status); | |
204 | if (cc) | |
205 | zpci_err_insn(cc, status, 0, (__force u64) addr); | |
206 | ||
207 | return (cc > 0) ? -EIO : cc; | |
208 | } | |
9389339f | 209 | EXPORT_SYMBOL_GPL(zpci_load); |
cbcca5d0 SO |
210 | |
211 | /* PCI Store */ | |
f0bacb7f | 212 | static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) |
cbcca5d0 SO |
213 | { |
214 | register u64 __req asm("2") = req; | |
215 | register u64 __offset asm("3") = offset; | |
f0bacb7f | 216 | int cc = -ENXIO; |
cbcca5d0 SO |
217 | |
218 | asm volatile ( | |
219 | " .insn rre,0xb9d00000,%[data],%[req]\n" | |
f0bacb7f | 220 | "0: ipm %[cc]\n" |
cbcca5d0 | 221 | " srl %[cc],28\n" |
f0bacb7f SO |
222 | "1:\n" |
223 | EX_TABLE(0b, 1b) | |
224 | : [cc] "+d" (cc), [req] "+d" (__req) | |
cbcca5d0 SO |
225 | : "d" (__offset), [data] "d" (data) |
226 | : "cc"); | |
227 | *status = __req >> 24 & 0xff; | |
228 | return cc; | |
229 | } | |
230 | ||
81deca12 | 231 | int __zpci_store(u64 data, u64 req, u64 offset) |
cbcca5d0 | 232 | { |
f0bacb7f SO |
233 | u8 status; |
234 | int cc; | |
cbcca5d0 SO |
235 | |
236 | do { | |
237 | cc = __pcistg(data, req, offset, &status); | |
238 | if (cc == 2) | |
239 | udelay(ZPCI_INSN_BUSY_DELAY); | |
240 | } while (cc == 2); | |
241 | ||
242 | if (cc) | |
3d8258e4 SO |
243 | zpci_err_insn(cc, status, req, offset); |
244 | ||
f0bacb7f | 245 | return (cc > 0) ? -EIO : cc; |
cbcca5d0 | 246 | } |
81deca12 SO |
247 | EXPORT_SYMBOL_GPL(__zpci_store); |
248 | ||
71ba41c9 SO |
249 | static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data, |
250 | unsigned long len) | |
81deca12 SO |
251 | { |
252 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; | |
253 | u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); | |
254 | ||
255 | return __zpci_store(data, req, ZPCI_OFFSET(addr)); | |
256 | } | |
71ba41c9 SO |
257 | |
258 | static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) | |
259 | { | |
260 | register u64 addr asm("2") = ioaddr; | |
261 | register u64 r3 asm("3") = len; | |
262 | int cc = -ENXIO; | |
263 | ||
264 | asm volatile ( | |
265 | " .insn rre,0xb9d40000,%[data],%[ioaddr]\n" | |
266 | "0: ipm %[cc]\n" | |
267 | " srl %[cc],28\n" | |
268 | "1:\n" | |
269 | EX_TABLE(0b, 1b) | |
270 | : [cc] "+d" (cc), "+d" (r3) | |
271 | : [data] "d" (data), [ioaddr] "d" (addr) | |
272 | : "cc"); | |
273 | *status = r3 >> 24 & 0xff; | |
274 | return cc; | |
275 | } | |
276 | ||
277 | int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len) | |
278 | { | |
279 | u8 status; | |
280 | int cc; | |
281 | ||
282 | if (!static_branch_unlikely(&have_mio)) | |
283 | return zpci_store_fh(addr, data, len); | |
284 | ||
285 | cc = __pcistg_mio(data, (__force u64) addr, len, &status); | |
286 | if (cc) | |
287 | zpci_err_insn(cc, status, 0, (__force u64) addr); | |
288 | ||
289 | return (cc > 0) ? -EIO : cc; | |
290 | } | |
9389339f | 291 | EXPORT_SYMBOL_GPL(zpci_store); |
cbcca5d0 SO |
292 | |
293 | /* PCI Store Block */ | |
f0bacb7f | 294 | static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) |
cbcca5d0 | 295 | { |
f0bacb7f | 296 | int cc = -ENXIO; |
cbcca5d0 SO |
297 | |
298 | asm volatile ( | |
299 | " .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" | |
f0bacb7f | 300 | "0: ipm %[cc]\n" |
cbcca5d0 | 301 | " srl %[cc],28\n" |
f0bacb7f SO |
302 | "1:\n" |
303 | EX_TABLE(0b, 1b) | |
304 | : [cc] "+d" (cc), [req] "+d" (req) | |
cbcca5d0 SO |
305 | : [offset] "d" (offset), [data] "Q" (*data) |
306 | : "cc"); | |
307 | *status = req >> 24 & 0xff; | |
308 | return cc; | |
309 | } | |
310 | ||
81deca12 | 311 | int __zpci_store_block(const u64 *data, u64 req, u64 offset) |
cbcca5d0 | 312 | { |
f0bacb7f SO |
313 | u8 status; |
314 | int cc; | |
cbcca5d0 SO |
315 | |
316 | do { | |
317 | cc = __pcistb(data, req, offset, &status); | |
318 | if (cc == 2) | |
319 | udelay(ZPCI_INSN_BUSY_DELAY); | |
320 | } while (cc == 2); | |
321 | ||
322 | if (cc) | |
3d8258e4 SO |
323 | zpci_err_insn(cc, status, req, offset); |
324 | ||
f0bacb7f | 325 | return (cc > 0) ? -EIO : cc; |
cbcca5d0 | 326 | } |
81deca12 SO |
327 | EXPORT_SYMBOL_GPL(__zpci_store_block); |
328 | ||
71ba41c9 SO |
329 | static inline int zpci_write_block_fh(volatile void __iomem *dst, |
330 | const void *src, unsigned long len) | |
81deca12 SO |
331 | { |
332 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)]; | |
333 | u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); | |
334 | u64 offset = ZPCI_OFFSET(dst); | |
335 | ||
336 | return __zpci_store_block(src, req, offset); | |
337 | } | |
71ba41c9 SO |
338 | |
339 | static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) | |
340 | { | |
341 | int cc = -ENXIO; | |
342 | ||
343 | asm volatile ( | |
344 | " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" | |
345 | "0: ipm %[cc]\n" | |
346 | " srl %[cc],28\n" | |
347 | "1:\n" | |
348 | EX_TABLE(0b, 1b) | |
349 | : [cc] "+d" (cc), [len] "+d" (len) | |
350 | : [ioaddr] "d" (ioaddr), [data] "Q" (*data) | |
351 | : "cc"); | |
352 | *status = len >> 24 & 0xff; | |
353 | return cc; | |
354 | } | |
355 | ||
356 | int zpci_write_block(volatile void __iomem *dst, | |
357 | const void *src, unsigned long len) | |
358 | { | |
359 | u8 status; | |
360 | int cc; | |
361 | ||
362 | if (!static_branch_unlikely(&have_mio)) | |
363 | return zpci_write_block_fh(dst, src, len); | |
364 | ||
365 | cc = __pcistb_mio(src, (__force u64) dst, len, &status); | |
366 | if (cc) | |
367 | zpci_err_insn(cc, status, 0, (__force u64) dst); | |
368 | ||
369 | return (cc > 0) ? -EIO : cc; | |
370 | } | |
81deca12 | 371 | EXPORT_SYMBOL_GPL(zpci_write_block); |
71ba41c9 SO |
372 | |
373 | static inline void __pciwb_mio(void) | |
374 | { | |
375 | unsigned long unused = 0; | |
376 | ||
377 | asm volatile (".insn rre,0xb9d50000,%[op],%[op]\n" | |
378 | : [op] "+d" (unused)); | |
379 | } | |
380 | ||
381 | void zpci_barrier(void) | |
382 | { | |
383 | if (static_branch_likely(&have_mio)) | |
384 | __pciwb_mio(); | |
385 | } | |
386 | EXPORT_SYMBOL_GPL(zpci_barrier); |