Commit | Line | Data |
---|---|---|
adbb3901 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cd248341 JG |
2 | /* |
3 | * Copyright IBM Corp. 2012 | |
4 | * | |
5 | * Author(s): | |
6 | * Jan Glauber <jang@linux.vnet.ibm.com> | |
7 | * | |
8 | * The System z PCI code is a rewrite from a prototype by | |
9 | * the following people (Kudoz!): | |
bedef755 JG |
10 | * Alexander Schmidt |
11 | * Christoph Raisch | |
12 | * Hannes Hering | |
13 | * Hoang-Nam Nguyen | |
14 | * Jan-Bernd Themann | |
15 | * Stefan Roscher | |
16 | * Thomas Klein | |
cd248341 JG |
17 | */ |
18 | ||
896cb7e6 GS |
19 | #define KMSG_COMPONENT "zpci" |
20 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt | |
cd248341 JG |
21 | |
22 | #include <linux/kernel.h> | |
23 | #include <linux/slab.h> | |
24 | #include <linux/err.h> | |
25 | #include <linux/export.h> | |
26 | #include <linux/delay.h> | |
27 | #include <linux/seq_file.h> | |
71ba41c9 | 28 | #include <linux/jump_label.h> |
cd248341 | 29 | #include <linux/pci.h> |
794b8846 | 30 | #include <linux/printk.h> |
cd248341 | 31 | |
9a4da8a5 JG |
32 | #include <asm/isc.h> |
33 | #include <asm/airq.h> | |
cd248341 JG |
34 | #include <asm/facility.h> |
35 | #include <asm/pci_insn.h> | |
a755a45d | 36 | #include <asm/pci_clp.h> |
828b35f6 | 37 | #include <asm/pci_dma.h> |
cd248341 | 38 | |
05bc1be6 | 39 | #include "pci_bus.h" |
abb95b75 | 40 | #include "pci_iov.h" |
05bc1be6 | 41 | |
cd248341 | 42 | /* list of all detected zpci devices */ |
67f43f38 | 43 | static LIST_HEAD(zpci_list); |
57b5918c | 44 | static DEFINE_SPINLOCK(zpci_list_lock); |
cd248341 | 45 | |
969ae01b | 46 | static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE); |
1f44a225 | 47 | static DEFINE_SPINLOCK(zpci_domain_lock); |
9a4da8a5 | 48 | |
c506fff3 | 49 | #define ZPCI_IOMAP_ENTRIES \ |
c9c13ba4 | 50 | min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2), \ |
c506fff3 SO |
51 | ZPCI_IOMAP_MAX_ENTRIES) |
52 | ||
6cf17f9a PM |
53 | unsigned int s390_pci_no_rid; |
54 | ||
cd248341 | 55 | static DEFINE_SPINLOCK(zpci_iomap_lock); |
c506fff3 | 56 | static unsigned long *zpci_iomap_bitmap; |
cd248341 JG |
57 | struct zpci_iomap_entry *zpci_iomap_start; |
58 | EXPORT_SYMBOL_GPL(zpci_iomap_start); | |
59 | ||
71ba41c9 SO |
60 | DEFINE_STATIC_KEY_FALSE(have_mio); |
61 | ||
d0b08853 JG |
62 | static struct kmem_cache *zdev_fmb_cache; |
63 | ||
98b1d33d MR |
64 | /* AEN structures that must be preserved over KVM module re-insertion */ |
65 | union zpci_sic_iib *zpci_aipb; | |
66 | EXPORT_SYMBOL_GPL(zpci_aipb); | |
67 | struct airq_iv *zpci_aif_sbv; | |
68 | EXPORT_SYMBOL_GPL(zpci_aif_sbv); | |
69 | ||
cd248341 JG |
70 | struct zpci_dev *get_zdev_by_fid(u32 fid) |
71 | { | |
72 | struct zpci_dev *tmp, *zdev = NULL; | |
73 | ||
57b5918c | 74 | spin_lock(&zpci_list_lock); |
cd248341 JG |
75 | list_for_each_entry(tmp, &zpci_list, entry) { |
76 | if (tmp->fid == fid) { | |
77 | zdev = tmp; | |
c122383d | 78 | zpci_zdev_get(zdev); |
cd248341 JG |
79 | break; |
80 | } | |
81 | } | |
57b5918c | 82 | spin_unlock(&zpci_list_lock); |
cd248341 JG |
83 | return zdev; |
84 | } | |
85 | ||
01553d9a SO |
86 | void zpci_remove_reserved_devices(void) |
87 | { | |
88 | struct zpci_dev *tmp, *zdev; | |
89 | enum zpci_state state; | |
90 | LIST_HEAD(remove); | |
91 | ||
92 | spin_lock(&zpci_list_lock); | |
93 | list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) { | |
94 | if (zdev->state == ZPCI_FN_STATE_STANDBY && | |
95 | !clp_get_state(zdev->fid, &state) && | |
96 | state == ZPCI_FN_STATE_RESERVED) | |
97 | list_move_tail(&zdev->entry, &remove); | |
98 | } | |
99 | spin_unlock(&zpci_list_lock); | |
100 | ||
101 | list_for_each_entry_safe(zdev, tmp, &remove, entry) | |
a46044a9 | 102 | zpci_device_reserved(zdev); |
cd248341 JG |
103 | } |
104 | ||
105 | int pci_domain_nr(struct pci_bus *bus) | |
106 | { | |
05bc1be6 | 107 | return ((struct zpci_bus *) bus->sysdata)->domain_nr; |
cd248341 JG |
108 | } |
109 | EXPORT_SYMBOL_GPL(pci_domain_nr); | |
110 | ||
111 | int pci_proc_domain(struct pci_bus *bus) | |
112 | { | |
113 | return pci_domain_nr(bus); | |
114 | } | |
115 | EXPORT_SYMBOL_GPL(pci_proc_domain); | |
116 | ||
828b35f6 JG |
117 | /* Modify PCI: Register I/O address translation parameters */ |
118 | int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, | |
119 | u64 base, u64 limit, u64 iota) | |
120 | { | |
72570834 SO |
121 | u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT); |
122 | struct zpci_fib fib = {0}; | |
1f3f7681 | 123 | u8 cc, status; |
828b35f6 JG |
124 | |
125 | WARN_ON_ONCE(iota & 0x3fff); | |
72570834 SO |
126 | fib.pba = base; |
127 | fib.pal = limit; | |
128 | fib.iota = iota | ZPCI_IOTA_RTTO_FLAG; | |
c68468ed | 129 | fib.gd = zdev->gisa; |
1f3f7681 NS |
130 | cc = zpci_mod_fc(req, &fib, &status); |
131 | if (cc) | |
132 | zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); | |
133 | return cc; | |
828b35f6 | 134 | } |
09340b2f | 135 | EXPORT_SYMBOL_GPL(zpci_register_ioat); |
828b35f6 JG |
136 | |
137 | /* Modify PCI: Unregister I/O address translation parameters */ | |
138 | int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) | |
139 | { | |
72570834 SO |
140 | u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT); |
141 | struct zpci_fib fib = {0}; | |
142 | u8 cc, status; | |
828b35f6 | 143 | |
c68468ed MR |
144 | fib.gd = zdev->gisa; |
145 | ||
72570834 | 146 | cc = zpci_mod_fc(req, &fib, &status); |
1f3f7681 NS |
147 | if (cc) |
148 | zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); | |
149 | return cc; | |
828b35f6 JG |
150 | } |
151 | ||
d0b08853 JG |
152 | /* Modify PCI: Set PCI function measurement parameters */ |
153 | int zpci_fmb_enable_device(struct zpci_dev *zdev) | |
154 | { | |
4e5bd780 SO |
155 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); |
156 | struct zpci_fib fib = {0}; | |
157 | u8 cc, status; | |
d0b08853 | 158 | |
0b7589ec | 159 | if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length) |
d0b08853 JG |
160 | return -EINVAL; |
161 | ||
08b42124 | 162 | zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); |
d0b08853 JG |
163 | if (!zdev->fmb) |
164 | return -ENOMEM; | |
d0b08853 JG |
165 | WARN_ON((u64) zdev->fmb & 0xf); |
166 | ||
6001018a SO |
167 | /* reset software counters */ |
168 | atomic64_set(&zdev->allocated_pages, 0); | |
169 | atomic64_set(&zdev->mapped_pages, 0); | |
170 | atomic64_set(&zdev->unmapped_pages, 0); | |
171 | ||
4e5bd780 | 172 | fib.fmb_addr = virt_to_phys(zdev->fmb); |
c68468ed | 173 | fib.gd = zdev->gisa; |
4e5bd780 SO |
174 | cc = zpci_mod_fc(req, &fib, &status); |
175 | if (cc) { | |
176 | kmem_cache_free(zdev_fmb_cache, zdev->fmb); | |
177 | zdev->fmb = NULL; | |
178 | } | |
179 | return cc ? -EIO : 0; | |
d0b08853 JG |
180 | } |
181 | ||
182 | /* Modify PCI: Disable PCI function measurement */ | |
183 | int zpci_fmb_disable_device(struct zpci_dev *zdev) | |
184 | { | |
4e5bd780 SO |
185 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); |
186 | struct zpci_fib fib = {0}; | |
187 | u8 cc, status; | |
d0b08853 JG |
188 | |
189 | if (!zdev->fmb) | |
190 | return -EINVAL; | |
191 | ||
c68468ed MR |
192 | fib.gd = zdev->gisa; |
193 | ||
d0b08853 | 194 | /* Function measurement is disabled if fmb address is zero */ |
4e5bd780 SO |
195 | cc = zpci_mod_fc(req, &fib, &status); |
196 | if (cc == 3) /* Function already gone. */ | |
197 | cc = 0; | |
d0b08853 | 198 | |
4e5bd780 SO |
199 | if (!cc) { |
200 | kmem_cache_free(zdev_fmb_cache, zdev->fmb); | |
201 | zdev->fmb = NULL; | |
202 | } | |
203 | return cc ? -EIO : 0; | |
d0b08853 JG |
204 | } |
205 | ||
cd248341 JG |
206 | static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) |
207 | { | |
208 | u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); | |
209 | u64 data; | |
210 | int rc; | |
211 | ||
81deca12 | 212 | rc = __zpci_load(&data, req, offset); |
b170bad4 | 213 | if (!rc) { |
5064cd35 SO |
214 | data = le64_to_cpu((__force __le64) data); |
215 | data >>= (8 - len) * 8; | |
cd248341 | 216 | *val = (u32) data; |
b170bad4 | 217 | } else |
cd248341 JG |
218 | *val = 0xffffffff; |
219 | return rc; | |
220 | } | |
221 | ||
222 | static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) | |
223 | { | |
224 | u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); | |
225 | u64 data = val; | |
226 | int rc; | |
227 | ||
5064cd35 SO |
228 | data <<= (8 - len) * 8; |
229 | data = (__force u64) cpu_to_le64(data); | |
81deca12 | 230 | rc = __zpci_store(data, req, offset); |
cd248341 JG |
231 | return rc; |
232 | } | |
233 | ||
cd248341 JG |
234 | resource_size_t pcibios_align_resource(void *data, const struct resource *res, |
235 | resource_size_t size, | |
236 | resource_size_t align) | |
237 | { | |
238 | return 0; | |
239 | } | |
240 | ||
87bc359b JG |
241 | /* combine single writes by using store-block insn */ |
242 | void __iowrite64_copy(void __iomem *to, const void *from, size_t count) | |
243 | { | |
244 | zpci_memcpy_toio(to, from, count); | |
245 | } | |
246 | ||
b02002cc | 247 | static void __iomem *__ioremap(phys_addr_t addr, size_t size, pgprot_t prot) |
71ba41c9 | 248 | { |
a999eb96 | 249 | unsigned long offset, vaddr; |
71ba41c9 | 250 | struct vm_struct *area; |
a999eb96 | 251 | phys_addr_t last_addr; |
71ba41c9 | 252 | |
a999eb96 NS |
253 | last_addr = addr + size - 1; |
254 | if (!size || last_addr < addr) | |
71ba41c9 SO |
255 | return NULL; |
256 | ||
257 | if (!static_branch_unlikely(&have_mio)) | |
a999eb96 | 258 | return (void __iomem *) addr; |
71ba41c9 | 259 | |
a999eb96 NS |
260 | offset = addr & ~PAGE_MASK; |
261 | addr &= PAGE_MASK; | |
71ba41c9 SO |
262 | size = PAGE_ALIGN(size + offset); |
263 | area = get_vm_area(size, VM_IOREMAP); | |
264 | if (!area) | |
265 | return NULL; | |
266 | ||
a999eb96 | 267 | vaddr = (unsigned long) area->addr; |
b02002cc | 268 | if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) { |
a999eb96 | 269 | free_vm_area(area); |
71ba41c9 SO |
270 | return NULL; |
271 | } | |
272 | return (void __iomem *) ((unsigned long) area->addr + offset); | |
273 | } | |
b02002cc NS |
274 | |
275 | void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) | |
276 | { | |
277 | return __ioremap(addr, size, __pgprot(prot)); | |
278 | } | |
279 | EXPORT_SYMBOL(ioremap_prot); | |
280 | ||
281 | void __iomem *ioremap(phys_addr_t addr, size_t size) | |
282 | { | |
283 | return __ioremap(addr, size, PAGE_KERNEL); | |
284 | } | |
71ba41c9 SO |
285 | EXPORT_SYMBOL(ioremap); |
286 | ||
b02002cc NS |
287 | void __iomem *ioremap_wc(phys_addr_t addr, size_t size) |
288 | { | |
289 | return __ioremap(addr, size, pgprot_writecombine(PAGE_KERNEL)); | |
290 | } | |
291 | EXPORT_SYMBOL(ioremap_wc); | |
292 | ||
293 | void __iomem *ioremap_wt(phys_addr_t addr, size_t size) | |
294 | { | |
295 | return __ioremap(addr, size, pgprot_writethrough(PAGE_KERNEL)); | |
296 | } | |
297 | EXPORT_SYMBOL(ioremap_wt); | |
298 | ||
71ba41c9 SO |
299 | void iounmap(volatile void __iomem *addr) |
300 | { | |
301 | if (static_branch_likely(&have_mio)) | |
302 | vunmap((__force void *) ((unsigned long) addr & PAGE_MASK)); | |
303 | } | |
304 | EXPORT_SYMBOL(iounmap); | |
305 | ||
cd248341 | 306 | /* Create a virtual mapping cookie for a PCI BAR */ |
71ba41c9 SO |
307 | static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, |
308 | unsigned long offset, unsigned long max) | |
cd248341 | 309 | { |
198a5278 | 310 | struct zpci_dev *zdev = to_zpci(pdev); |
cd248341 JG |
311 | int idx; |
312 | ||
cd248341 JG |
313 | idx = zdev->bars[bar].map_idx; |
314 | spin_lock(&zpci_iomap_lock); | |
8cfc99b5 | 315 | /* Detect overrun */ |
f5e44f82 SO |
316 | WARN_ON(!++zpci_iomap_start[idx].count); |
317 | zpci_iomap_start[idx].fh = zdev->fh; | |
318 | zpci_iomap_start[idx].bar = bar; | |
cd248341 JG |
319 | spin_unlock(&zpci_iomap_lock); |
320 | ||
9e00caae | 321 | return (void __iomem *) ZPCI_ADDR(idx) + offset; |
cd248341 | 322 | } |
71ba41c9 SO |
323 | |
324 | static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, | |
325 | unsigned long offset, | |
326 | unsigned long max) | |
327 | { | |
328 | unsigned long barsize = pci_resource_len(pdev, bar); | |
329 | struct zpci_dev *zdev = to_zpci(pdev); | |
330 | void __iomem *iova; | |
331 | ||
332 | iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); | |
333 | return iova ? iova + offset : iova; | |
334 | } | |
335 | ||
336 | void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, | |
337 | unsigned long offset, unsigned long max) | |
338 | { | |
c9c13ba4 | 339 | if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) |
71ba41c9 SO |
340 | return NULL; |
341 | ||
342 | if (static_branch_likely(&have_mio)) | |
343 | return pci_iomap_range_mio(pdev, bar, offset, max); | |
344 | else | |
345 | return pci_iomap_range_fh(pdev, bar, offset, max); | |
346 | } | |
d9426083 | 347 | EXPORT_SYMBOL(pci_iomap_range); |
8cfc99b5 MT |
348 | |
349 | void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) | |
350 | { | |
351 | return pci_iomap_range(dev, bar, 0, maxlen); | |
352 | } | |
353 | EXPORT_SYMBOL(pci_iomap); | |
cd248341 | 354 | |
71ba41c9 SO |
355 | static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, |
356 | unsigned long offset, unsigned long max) | |
357 | { | |
358 | unsigned long barsize = pci_resource_len(pdev, bar); | |
359 | struct zpci_dev *zdev = to_zpci(pdev); | |
360 | void __iomem *iova; | |
361 | ||
362 | iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); | |
363 | return iova ? iova + offset : iova; | |
364 | } | |
365 | ||
366 | void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, | |
367 | unsigned long offset, unsigned long max) | |
368 | { | |
c9c13ba4 | 369 | if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) |
71ba41c9 SO |
370 | return NULL; |
371 | ||
372 | if (static_branch_likely(&have_mio)) | |
373 | return pci_iomap_wc_range_mio(pdev, bar, offset, max); | |
374 | else | |
375 | return pci_iomap_range_fh(pdev, bar, offset, max); | |
376 | } | |
377 | EXPORT_SYMBOL(pci_iomap_wc_range); | |
378 | ||
379 | void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) | |
380 | { | |
381 | return pci_iomap_wc_range(dev, bar, 0, maxlen); | |
382 | } | |
383 | EXPORT_SYMBOL(pci_iomap_wc); | |
384 | ||
385 | static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) | |
cd248341 | 386 | { |
9e00caae | 387 | unsigned int idx = ZPCI_IDX(addr); |
cd248341 | 388 | |
cd248341 | 389 | spin_lock(&zpci_iomap_lock); |
8cfc99b5 | 390 | /* Detect underrun */ |
f5e44f82 | 391 | WARN_ON(!zpci_iomap_start[idx].count); |
8cfc99b5 MT |
392 | if (!--zpci_iomap_start[idx].count) { |
393 | zpci_iomap_start[idx].fh = 0; | |
394 | zpci_iomap_start[idx].bar = 0; | |
395 | } | |
cd248341 JG |
396 | spin_unlock(&zpci_iomap_lock); |
397 | } | |
71ba41c9 SO |
398 | |
399 | static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) | |
400 | { | |
401 | iounmap(addr); | |
402 | } | |
403 | ||
404 | void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) | |
405 | { | |
406 | if (static_branch_likely(&have_mio)) | |
407 | pci_iounmap_mio(pdev, addr); | |
408 | else | |
409 | pci_iounmap_fh(pdev, addr); | |
410 | } | |
d9426083 | 411 | EXPORT_SYMBOL(pci_iounmap); |
cd248341 JG |
412 | |
413 | static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, | |
414 | int size, u32 *val) | |
415 | { | |
7dcfe50f | 416 | struct zpci_dev *zdev = zdev_from_bus(bus, devfn); |
cd248341 | 417 | |
44510d6f | 418 | return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; |
cd248341 JG |
419 | } |
420 | ||
421 | static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, | |
422 | int size, u32 val) | |
423 | { | |
7dcfe50f | 424 | struct zpci_dev *zdev = zdev_from_bus(bus, devfn); |
cd248341 | 425 | |
44510d6f | 426 | return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; |
cd248341 JG |
427 | } |
428 | ||
429 | static struct pci_ops pci_root_ops = { | |
430 | .read = pci_read, | |
431 | .write = pci_write, | |
432 | }; | |
433 | ||
1803ba2d | 434 | static void zpci_map_resources(struct pci_dev *pdev) |
cd248341 | 435 | { |
71ba41c9 | 436 | struct zpci_dev *zdev = to_zpci(pdev); |
cd248341 JG |
437 | resource_size_t len; |
438 | int i; | |
439 | ||
c9c13ba4 | 440 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
cd248341 JG |
441 | len = pci_resource_len(pdev, i); |
442 | if (!len) | |
443 | continue; | |
71ba41c9 | 444 | |
c7ff0e91 | 445 | if (zpci_use_mio(zdev)) |
71ba41c9 | 446 | pdev->resource[i].start = |
df057c91 | 447 | (resource_size_t __force) zdev->bars[i].mio_wt; |
71ba41c9 | 448 | else |
c7ff0e91 SO |
449 | pdev->resource[i].start = (resource_size_t __force) |
450 | pci_iomap_range_fh(pdev, i, 0, 0); | |
cd248341 | 451 | pdev->resource[i].end = pdev->resource[i].start + len - 1; |
cd248341 | 452 | } |
cfbb4a7a | 453 | |
abb95b75 | 454 | zpci_iov_map_resources(pdev); |
944239c5 SO |
455 | } |
456 | ||
1803ba2d | 457 | static void zpci_unmap_resources(struct pci_dev *pdev) |
944239c5 | 458 | { |
c7ff0e91 | 459 | struct zpci_dev *zdev = to_zpci(pdev); |
944239c5 SO |
460 | resource_size_t len; |
461 | int i; | |
462 | ||
c7ff0e91 | 463 | if (zpci_use_mio(zdev)) |
71ba41c9 SO |
464 | return; |
465 | ||
c9c13ba4 | 466 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
944239c5 SO |
467 | len = pci_resource_len(pdev, i); |
468 | if (!len) | |
469 | continue; | |
c7ff0e91 SO |
470 | pci_iounmap_fh(pdev, (void __iomem __force *) |
471 | pdev->resource[i].start); | |
944239c5 SO |
472 | } |
473 | } | |
cd248341 | 474 | |
cd248341 JG |
475 | static int zpci_alloc_iomap(struct zpci_dev *zdev) |
476 | { | |
bf19c94d | 477 | unsigned long entry; |
cd248341 JG |
478 | |
479 | spin_lock(&zpci_iomap_lock); | |
c506fff3 SO |
480 | entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); |
481 | if (entry == ZPCI_IOMAP_ENTRIES) { | |
cd248341 JG |
482 | spin_unlock(&zpci_iomap_lock); |
483 | return -ENOSPC; | |
484 | } | |
c506fff3 | 485 | set_bit(entry, zpci_iomap_bitmap); |
cd248341 JG |
486 | spin_unlock(&zpci_iomap_lock); |
487 | return entry; | |
488 | } | |
489 | ||
490 | static void zpci_free_iomap(struct zpci_dev *zdev, int entry) | |
491 | { | |
492 | spin_lock(&zpci_iomap_lock); | |
493 | memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); | |
c506fff3 | 494 | clear_bit(entry, zpci_iomap_bitmap); |
cd248341 JG |
495 | spin_unlock(&zpci_iomap_lock); |
496 | } | |
497 | ||
4fe20497 NS |
498 | static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh) |
499 | { | |
500 | int bar, idx; | |
501 | ||
502 | spin_lock(&zpci_iomap_lock); | |
503 | for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { | |
504 | if (!zdev->bars[bar].size) | |
505 | continue; | |
506 | idx = zdev->bars[bar].map_idx; | |
507 | if (!zpci_iomap_start[idx].count) | |
508 | continue; | |
509 | WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh); | |
510 | } | |
511 | spin_unlock(&zpci_iomap_lock); | |
512 | } | |
513 | ||
514 | void zpci_update_fh(struct zpci_dev *zdev, u32 fh) | |
515 | { | |
516 | if (!fh || zdev->fh == fh) | |
517 | return; | |
518 | ||
519 | zdev->fh = fh; | |
520 | if (zpci_use_mio(zdev)) | |
521 | return; | |
522 | if (zdev->has_resources && zdev_enabled(zdev)) | |
523 | zpci_do_update_iomap_fh(zdev, fh); | |
524 | } | |
525 | ||
7a572a3a SO |
526 | static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, |
527 | unsigned long size, unsigned long flags) | |
528 | { | |
529 | struct resource *r; | |
530 | ||
531 | r = kzalloc(sizeof(*r), GFP_KERNEL); | |
532 | if (!r) | |
533 | return NULL; | |
534 | ||
535 | r->start = start; | |
536 | r->end = r->start + size - 1; | |
537 | r->flags = flags; | |
538 | r->name = zdev->res_name; | |
539 | ||
540 | if (request_resource(&iomem_resource, r)) { | |
541 | kfree(r); | |
542 | return NULL; | |
543 | } | |
544 | return r; | |
545 | } | |
546 | ||
05bc1be6 PM |
547 | int zpci_setup_bus_resources(struct zpci_dev *zdev, |
548 | struct list_head *resources) | |
7a572a3a SO |
549 | { |
550 | unsigned long addr, size, flags; | |
551 | struct resource *res; | |
552 | int i, entry; | |
553 | ||
554 | snprintf(zdev->res_name, sizeof(zdev->res_name), | |
05bc1be6 | 555 | "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR); |
7a572a3a | 556 | |
c9c13ba4 | 557 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
7a572a3a SO |
558 | if (!zdev->bars[i].size) |
559 | continue; | |
560 | entry = zpci_alloc_iomap(zdev); | |
561 | if (entry < 0) | |
562 | return entry; | |
563 | zdev->bars[i].map_idx = entry; | |
564 | ||
565 | /* only MMIO is supported */ | |
566 | flags = IORESOURCE_MEM; | |
567 | if (zdev->bars[i].val & 8) | |
568 | flags |= IORESOURCE_PREFETCH; | |
569 | if (zdev->bars[i].val & 4) | |
570 | flags |= IORESOURCE_MEM_64; | |
571 | ||
c7ff0e91 | 572 | if (zpci_use_mio(zdev)) |
df057c91 | 573 | addr = (unsigned long) zdev->bars[i].mio_wt; |
dcd33b23 SO |
574 | else |
575 | addr = ZPCI_ADDR(entry); | |
7a572a3a SO |
576 | size = 1UL << zdev->bars[i].size; |
577 | ||
578 | res = __alloc_res(zdev, addr, size, flags); | |
579 | if (!res) { | |
580 | zpci_free_iomap(zdev, entry); | |
581 | return -ENOMEM; | |
582 | } | |
583 | zdev->bars[i].res = res; | |
584 | pci_add_resource(resources, res); | |
585 | } | |
a50297cf | 586 | zdev->has_resources = 1; |
7a572a3a SO |
587 | |
588 | return 0; | |
589 | } | |
590 | ||
591 | static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) | |
592 | { | |
593 | int i; | |
594 | ||
c9c13ba4 | 595 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
2b1df724 | 596 | if (!zdev->bars[i].size || !zdev->bars[i].res) |
7a572a3a SO |
597 | continue; |
598 | ||
599 | zpci_free_iomap(zdev, zdev->bars[i].map_idx); | |
600 | release_resource(zdev->bars[i].res); | |
601 | kfree(zdev->bars[i].res); | |
602 | } | |
a50297cf | 603 | zdev->has_resources = 0; |
7a572a3a SO |
604 | } |
605 | ||
06dc660e | 606 | int pcibios_device_add(struct pci_dev *pdev) |
af0a8a84 | 607 | { |
2a671f77 | 608 | struct zpci_dev *zdev = to_zpci(pdev); |
cb809182 SO |
609 | struct resource *res; |
610 | int i; | |
611 | ||
2a671f77 NS |
612 | /* The pdev has a reference to the zdev via its bus */ |
613 | zpci_zdev_get(zdev); | |
7dc20ab1 SO |
614 | if (pdev->is_physfn) |
615 | pdev->no_vf_scan = 1; | |
616 | ||
ef4858c6 | 617 | pdev->dev.groups = zpci_attr_groups; |
5657933d | 618 | pdev->dev.dma_ops = &s390_pci_dma_ops; |
1803ba2d | 619 | zpci_map_resources(pdev); |
cb809182 | 620 | |
c9c13ba4 | 621 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
cb809182 SO |
622 | res = &pdev->resource[i]; |
623 | if (res->parent || !res->flags) | |
624 | continue; | |
625 | pci_claim_resource(pdev, i); | |
626 | } | |
627 | ||
628 | return 0; | |
629 | } | |
630 | ||
1803ba2d SO |
631 | void pcibios_release_device(struct pci_dev *pdev) |
632 | { | |
2a671f77 NS |
633 | struct zpci_dev *zdev = to_zpci(pdev); |
634 | ||
1803ba2d | 635 | zpci_unmap_resources(pdev); |
2a671f77 | 636 | zpci_zdev_put(zdev); |
1803ba2d SO |
637 | } |
638 | ||
cb809182 SO |
639 | int pcibios_enable_device(struct pci_dev *pdev, int mask) |
640 | { | |
198a5278 | 641 | struct zpci_dev *zdev = to_zpci(pdev); |
af0a8a84 | 642 | |
9a99649f | 643 | zpci_debug_init_device(zdev, dev_name(&pdev->dev)); |
af0a8a84 | 644 | zpci_fmb_enable_device(zdev); |
af0a8a84 | 645 | |
d7533232 | 646 | return pci_enable_resources(pdev, mask); |
af0a8a84 SO |
647 | } |
648 | ||
cb809182 | 649 | void pcibios_disable_device(struct pci_dev *pdev) |
944239c5 | 650 | { |
198a5278 | 651 | struct zpci_dev *zdev = to_zpci(pdev); |
944239c5 | 652 | |
944239c5 SO |
653 | zpci_fmb_disable_device(zdev); |
654 | zpci_debug_exit_device(zdev); | |
944239c5 SO |
655 | } |
656 | ||
05bc1be6 | 657 | static int __zpci_register_domain(int domain) |
cd248341 | 658 | { |
969ae01b | 659 | spin_lock(&zpci_domain_lock); |
05bc1be6 | 660 | if (test_bit(domain, zpci_domain)) { |
969ae01b | 661 | spin_unlock(&zpci_domain_lock); |
05bc1be6 PM |
662 | pr_err("Domain %04x is already assigned\n", domain); |
663 | return -EEXIST; | |
969ae01b | 664 | } |
05bc1be6 PM |
665 | set_bit(domain, zpci_domain); |
666 | spin_unlock(&zpci_domain_lock); | |
667 | return domain; | |
668 | } | |
969ae01b | 669 | |
05bc1be6 PM |
670 | static int __zpci_alloc_domain(void) |
671 | { | |
672 | int domain; | |
312e8462 | 673 | |
05bc1be6 | 674 | spin_lock(&zpci_domain_lock); |
969ae01b NS |
675 | /* |
676 | * We can always auto allocate domains below ZPCI_NR_DEVICES. | |
677 | * There is either a free domain or we have reached the maximum in | |
678 | * which case we would have bailed earlier. | |
679 | */ | |
05bc1be6 PM |
680 | domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); |
681 | set_bit(domain, zpci_domain); | |
cd248341 | 682 | spin_unlock(&zpci_domain_lock); |
05bc1be6 | 683 | return domain; |
cd248341 JG |
684 | } |
685 | ||
05bc1be6 | 686 | int zpci_alloc_domain(int domain) |
cd248341 | 687 | { |
05bc1be6 PM |
688 | if (zpci_unique_uid) { |
689 | if (domain) | |
690 | return __zpci_register_domain(domain); | |
691 | pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n"); | |
692 | update_uid_checking(false); | |
693 | } | |
694 | return __zpci_alloc_domain(); | |
cd248341 JG |
695 | } |
696 | ||
05bc1be6 | 697 | void zpci_free_domain(int domain) |
7d594322 | 698 | { |
05bc1be6 PM |
699 | spin_lock(&zpci_domain_lock); |
700 | clear_bit(domain, zpci_domain); | |
701 | spin_unlock(&zpci_domain_lock); | |
7d594322 SO |
702 | } |
703 | ||
7d594322 | 704 | |
a755a45d JG |
705 | int zpci_enable_device(struct zpci_dev *zdev) |
706 | { | |
cc049eec | 707 | u32 fh = zdev->fh; |
1f3f7681 | 708 | int rc = 0; |
828b35f6 | 709 | |
1f3f7681 | 710 | if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES)) |
f7addcdd | 711 | rc = -EIO; |
1f3f7681 | 712 | else |
4fe20497 | 713 | zpci_update_fh(zdev, fh); |
a755a45d JG |
714 | return rc; |
715 | } | |
09340b2f | 716 | EXPORT_SYMBOL_GPL(zpci_enable_device); |
a755a45d | 717 | |
cb65a669 SO |
718 | int zpci_disable_device(struct zpci_dev *zdev) |
719 | { | |
cc049eec | 720 | u32 fh = zdev->fh; |
8256adda NS |
721 | int cc, rc = 0; |
722 | ||
cc049eec NS |
723 | cc = clp_disable_fh(zdev, &fh); |
724 | if (!cc) { | |
4fe20497 | 725 | zpci_update_fh(zdev, fh); |
cc049eec | 726 | } else if (cc == CLP_RC_SETPCIFN_ALRDY) { |
8256adda NS |
727 | pr_info("Disabling PCI function %08x had no effect as it was already disabled\n", |
728 | zdev->fid); | |
729 | /* Function is already disabled - update handle */ | |
cc049eec NS |
730 | rc = clp_refresh_fh(zdev->fid, &fh); |
731 | if (!rc) { | |
4fe20497 | 732 | zpci_update_fh(zdev, fh); |
8256adda | 733 | rc = -EINVAL; |
cc049eec NS |
734 | } |
735 | } else { | |
8256adda NS |
736 | rc = -EIO; |
737 | } | |
738 | return rc; | |
cb65a669 | 739 | } |
09340b2f | 740 | EXPORT_SYMBOL_GPL(zpci_disable_device); |
cb65a669 | 741 | |
da995d53 NS |
742 | /** |
743 | * zpci_hot_reset_device - perform a reset of the given zPCI function | |
744 | * @zdev: the slot which should be reset | |
745 | * | |
746 | * Performs a low level reset of the zPCI function. The reset is low level in | |
747 | * the sense that the zPCI function can be reset without detaching it from the | |
748 | * common PCI subsystem. The reset may be performed while under control of | |
749 | * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation | |
750 | * table is reinstated at the end of the reset. | |
751 | * | |
752 | * After the reset the functions internal state is reset to an initial state | |
753 | * equivalent to its state during boot when first probing a driver. | |
754 | * Consequently after reset the PCI function requires re-initialization via the | |
755 | * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors() | |
756 | * and enabling the function via e.g.pci_enablde_device_flags().The caller | |
757 | * must guard against concurrent reset attempts. | |
758 | * | |
759 | * In most cases this function should not be called directly but through | |
760 | * pci_reset_function() or pci_reset_bus() which handle the save/restore and | |
761 | * locking. | |
762 | * | |
763 | * Return: 0 on success and an error value otherwise | |
764 | */ | |
765 | int zpci_hot_reset_device(struct zpci_dev *zdev) | |
766 | { | |
767 | int rc; | |
768 | ||
769 | zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh); | |
770 | if (zdev_enabled(zdev)) { | |
771 | /* Disables device access, DMAs and IRQs (reset state) */ | |
772 | rc = zpci_disable_device(zdev); | |
773 | /* | |
774 | * Due to a z/VM vs LPAR inconsistency in the error state the | |
775 | * FH may indicate an enabled device but disable says the | |
776 | * device is already disabled don't treat it as an error here. | |
777 | */ | |
778 | if (rc == -EINVAL) | |
779 | rc = 0; | |
780 | if (rc) | |
781 | return rc; | |
782 | } | |
783 | ||
784 | rc = zpci_enable_device(zdev); | |
785 | if (rc) | |
786 | return rc; | |
787 | ||
788 | if (zdev->dma_table) | |
789 | rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, | |
568de506 | 790 | virt_to_phys(zdev->dma_table)); |
da995d53 NS |
791 | else |
792 | rc = zpci_dma_init_device(zdev); | |
793 | if (rc) { | |
794 | zpci_disable_device(zdev); | |
795 | return rc; | |
796 | } | |
797 | ||
798 | return 0; | |
799 | } | |
800 | ||
ba764dd7 NS |
801 | /** |
802 | * zpci_create_device() - Create a new zpci_dev and add it to the zbus | |
803 | * @fid: Function ID of the device to be created | |
804 | * @fh: Current Function Handle of the device to be created | |
805 | * @state: Initial state after creation either Standby or Configured | |
806 | * | |
807 | * Creates a new zpci device and adds it to its, possibly newly created, zbus | |
808 | * as well as zpci_list. | |
809 | * | |
14c87ba8 | 810 | * Returns: the zdev on success or an error pointer otherwise |
ba764dd7 | 811 | */ |
14c87ba8 | 812 | struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) |
cd248341 | 813 | { |
ba764dd7 | 814 | struct zpci_dev *zdev; |
cd248341 JG |
815 | int rc; |
816 | ||
52c79e63 | 817 | zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", fid, fh, state); |
ba764dd7 NS |
818 | zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); |
819 | if (!zdev) | |
14c87ba8 | 820 | return ERR_PTR(-ENOMEM); |
05bc1be6 | 821 | |
ba764dd7 NS |
822 | /* FID and Function Handle are the static/dynamic identifiers */ |
823 | zdev->fid = fid; | |
824 | zdev->fh = fh; | |
cd248341 | 825 | |
ba764dd7 NS |
826 | /* Query function properties and update zdev */ |
827 | rc = clp_query_pci_fn(zdev); | |
f42c2235 | 828 | if (rc) |
ba764dd7 NS |
829 | goto error; |
830 | zdev->state = state; | |
f42c2235 | 831 | |
ba764dd7 | 832 | kref_init(&zdev->kref); |
80ed156a | 833 | mutex_init(&zdev->lock); |
09340b2f | 834 | mutex_init(&zdev->kzdev_lock); |
ba764dd7 NS |
835 | |
836 | rc = zpci_init_iommu(zdev); | |
837 | if (rc) | |
838 | goto error; | |
839 | ||
05bc1be6 | 840 | rc = zpci_bus_device_register(zdev, &pci_root_ops); |
cd248341 | 841 | if (rc) |
a50297cf | 842 | goto error_destroy_iommu; |
ba764dd7 NS |
843 | |
844 | spin_lock(&zpci_list_lock); | |
845 | list_add_tail(&zdev->entry, &zpci_list); | |
846 | spin_unlock(&zpci_list_lock); | |
cd248341 | 847 | |
14c87ba8 | 848 | return zdev; |
cd248341 | 849 | |
ba764dd7 | 850 | error_destroy_iommu: |
f42c2235 | 851 | zpci_destroy_iommu(zdev); |
ba764dd7 NS |
852 | error: |
853 | zpci_dbg(0, "add fid:%x, rc:%d\n", fid, rc); | |
854 | kfree(zdev); | |
14c87ba8 | 855 | return ERR_PTR(rc); |
cd248341 JG |
856 | } |
857 | ||
a46044a9 NS |
858 | bool zpci_is_device_configured(struct zpci_dev *zdev) |
859 | { | |
860 | enum zpci_state state = zdev->state; | |
861 | ||
862 | return state != ZPCI_FN_STATE_RESERVED && | |
863 | state != ZPCI_FN_STATE_STANDBY; | |
864 | } | |
865 | ||
2631f6b6 | 866 | /** |
a7f82c36 | 867 | * zpci_scan_configured_device() - Scan a freshly configured zpci_dev |
2631f6b6 NS |
868 | * @zdev: The zpci_dev to be configured |
869 | * @fh: The general function handle supplied by the platform | |
870 | * | |
61311e32 | 871 | * Given a device in the configuration state Configured, enables, scans and |
a7f82c36 NS |
872 | * adds it to the common code PCI subsystem if possible. If the PCI device is |
873 | * parked because we can not yet create a PCI bus because we have not seen | |
874 | * function 0, it is ignored but will be scanned once function 0 appears. | |
875 | * If any failure occurs, the zpci_dev is left disabled. | |
2631f6b6 NS |
876 | * |
877 | * Return: 0 on success, or an error code otherwise | |
878 | */ | |
a7f82c36 | 879 | int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh) |
2631f6b6 | 880 | { |
2631f6b6 NS |
881 | int rc; |
882 | ||
4fe20497 | 883 | zpci_update_fh(zdev, fh); |
2631f6b6 NS |
884 | /* the PCI function will be scanned once function 0 appears */ |
885 | if (!zdev->zbus->bus) | |
886 | return 0; | |
887 | ||
a50297cf NS |
888 | /* For function 0 on a multi-function bus scan whole bus as we might |
889 | * have to pick up existing functions waiting for it to allow creating | |
890 | * the PCI bus | |
891 | */ | |
892 | if (zdev->devfn == 0 && zdev->zbus->multifunction) | |
893 | rc = zpci_bus_scan_bus(zdev->zbus); | |
894 | else | |
895 | rc = zpci_bus_scan_device(zdev); | |
2631f6b6 | 896 | |
2631f6b6 NS |
897 | return rc; |
898 | } | |
899 | ||
900 | /** | |
901 | * zpci_deconfigure_device() - Deconfigure a zpci_dev | |
902 | * @zdev: The zpci_dev to configure | |
903 | * | |
904 | * Deconfigure a zPCI function that is currently configured and possibly known | |
905 | * to the common code PCI subsystem. | |
906 | * If any failure occurs the device is left as is. | |
907 | * | |
908 | * Return: 0 on success, or an error code otherwise | |
909 | */ | |
910 | int zpci_deconfigure_device(struct zpci_dev *zdev) | |
911 | { | |
912 | int rc; | |
913 | ||
914 | if (zdev->zbus->bus) | |
95b3a8b4 | 915 | zpci_bus_remove_device(zdev, false); |
2631f6b6 | 916 | |
1f3f7681 NS |
917 | if (zdev->dma_table) { |
918 | rc = zpci_dma_exit_device(zdev); | |
919 | if (rc) | |
920 | return rc; | |
921 | } | |
2631f6b6 NS |
922 | if (zdev_enabled(zdev)) { |
923 | rc = zpci_disable_device(zdev); | |
924 | if (rc) | |
925 | return rc; | |
926 | } | |
927 | ||
928 | rc = sclp_pci_deconfigure(zdev->fid); | |
929 | zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc); | |
930 | if (rc) | |
931 | return rc; | |
932 | zdev->state = ZPCI_FN_STATE_STANDBY; | |
933 | ||
934 | return 0; | |
935 | } | |
936 | ||
a46044a9 NS |
937 | /** |
938 | * zpci_device_reserved() - Mark device as resverved | |
939 | * @zdev: the zpci_dev that was reserved | |
940 | * | |
941 | * Handle the case that a given zPCI function was reserved by another system. | |
942 | * After a call to this function the zpci_dev can not be found via | |
943 | * get_zdev_by_fid() anymore but may still be accessible via existing | |
944 | * references though it will not be functional anymore. | |
945 | */ | |
946 | void zpci_device_reserved(struct zpci_dev *zdev) | |
947 | { | |
948 | if (zdev->has_hp_slot) | |
949 | zpci_exit_slot(zdev); | |
950 | /* | |
951 | * Remove device from zpci_list as it is going away. This also | |
952 | * makes sure we ignore subsequent zPCI events for this device. | |
953 | */ | |
954 | spin_lock(&zpci_list_lock); | |
955 | list_del(&zdev->entry); | |
956 | spin_unlock(&zpci_list_lock); | |
957 | zdev->state = ZPCI_FN_STATE_RESERVED; | |
958 | zpci_dbg(3, "rsv fid:%x\n", zdev->fid); | |
959 | zpci_zdev_put(zdev); | |
960 | } | |
961 | ||
05bc1be6 | 962 | void zpci_release_device(struct kref *kref) |
623bd44d | 963 | { |
05bc1be6 | 964 | struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref); |
a9045c22 | 965 | int ret; |
05bc1be6 | 966 | |
2f0230b2 | 967 | if (zdev->zbus->bus) |
95b3a8b4 | 968 | zpci_bus_remove_device(zdev, false); |
44510d6f | 969 | |
1f3f7681 NS |
970 | if (zdev->dma_table) |
971 | zpci_dma_exit_device(zdev); | |
f6576a1b | 972 | if (zdev_enabled(zdev)) |
05bc1be6 | 973 | zpci_disable_device(zdev); |
f6576a1b NS |
974 | |
975 | switch (zdev->state) { | |
a9045c22 NS |
976 | case ZPCI_FN_STATE_CONFIGURED: |
977 | ret = sclp_pci_deconfigure(zdev->fid); | |
978 | zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, ret); | |
979 | fallthrough; | |
05bc1be6 | 980 | case ZPCI_FN_STATE_STANDBY: |
44510d6f | 981 | if (zdev->has_hp_slot) |
05bc1be6 | 982 | zpci_exit_slot(zdev); |
a46044a9 NS |
983 | spin_lock(&zpci_list_lock); |
984 | list_del(&zdev->entry); | |
985 | spin_unlock(&zpci_list_lock); | |
986 | zpci_dbg(3, "rsv fid:%x\n", zdev->fid); | |
987 | fallthrough; | |
988 | case ZPCI_FN_STATE_RESERVED: | |
02368b7c NS |
989 | if (zdev->has_resources) |
990 | zpci_cleanup_bus_resources(zdev); | |
44510d6f PM |
991 | zpci_bus_device_unregister(zdev); |
992 | zpci_destroy_iommu(zdev); | |
05bc1be6 PM |
993 | fallthrough; |
994 | default: | |
995 | break; | |
996 | } | |
05bc1be6 PM |
997 | zpci_dbg(3, "rem fid:%x\n", zdev->fid); |
998 | kfree(zdev); | |
623bd44d SO |
999 | } |
1000 | ||
bd3a1725 MS |
1001 | int zpci_report_error(struct pci_dev *pdev, |
1002 | struct zpci_report_error_header *report) | |
1003 | { | |
1004 | struct zpci_dev *zdev = to_zpci(pdev); | |
1005 | ||
1006 | return sclp_pci_report(report, zdev->fh, zdev->fid); | |
1007 | } | |
1008 | EXPORT_SYMBOL(zpci_report_error); | |
1009 | ||
4cdf2f4e NS |
1010 | /** |
1011 | * zpci_clear_error_state() - Clears the zPCI error state of the device | |
1012 | * @zdev: The zdev for which the zPCI error state should be reset | |
1013 | * | |
1014 | * Clear the zPCI error state of the device. If clearing the zPCI error state | |
1015 | * fails the device is left in the error state. In this case it may make sense | |
1016 | * to call zpci_io_perm_failure() on the associated pdev if it exists. | |
1017 | * | |
1018 | * Returns: 0 on success, -EIO otherwise | |
1019 | */ | |
1020 | int zpci_clear_error_state(struct zpci_dev *zdev) | |
1021 | { | |
1022 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR); | |
1023 | struct zpci_fib fib = {0}; | |
1024 | u8 status; | |
1025 | int cc; | |
1026 | ||
1027 | cc = zpci_mod_fc(req, &fib, &status); | |
1028 | if (cc) { | |
1029 | zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); | |
1030 | return -EIO; | |
1031 | } | |
1032 | ||
1033 | return 0; | |
1034 | } | |
1035 | ||
1036 | /** | |
1037 | * zpci_reset_load_store_blocked() - Re-enables L/S from error state | |
1038 | * @zdev: The zdev for which to unblock load/store access | |
1039 | * | |
1040 | * Re-enables load/store access for a PCI function in the error state while | |
1041 | * keeping DMA blocked. In this state drivers can poke MMIO space to determine | |
1042 | * if error recovery is possible while catching any rogue DMA access from the | |
1043 | * device. | |
1044 | * | |
1045 | * Returns: 0 on success, -EIO otherwise | |
1046 | */ | |
1047 | int zpci_reset_load_store_blocked(struct zpci_dev *zdev) | |
1048 | { | |
1049 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK); | |
1050 | struct zpci_fib fib = {0}; | |
1051 | u8 status; | |
1052 | int cc; | |
1053 | ||
1054 | cc = zpci_mod_fc(req, &fib, &status); | |
1055 | if (cc) { | |
1056 | zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); | |
1057 | return -EIO; | |
1058 | } | |
1059 | ||
1060 | return 0; | |
1061 | } | |
1062 | ||
cd248341 JG |
1063 | static int zpci_mem_init(void) |
1064 | { | |
80c544de SO |
1065 | BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || |
1066 | __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); | |
1067 | ||
d0b08853 | 1068 | zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), |
80c544de | 1069 | __alignof__(struct zpci_fmb), 0, NULL); |
d0b08853 | 1070 | if (!zdev_fmb_cache) |
c506fff3 | 1071 | goto error_fmb; |
d0b08853 | 1072 | |
c506fff3 SO |
1073 | zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, |
1074 | sizeof(*zpci_iomap_start), GFP_KERNEL); | |
cd248341 | 1075 | if (!zpci_iomap_start) |
9a4da8a5 | 1076 | goto error_iomap; |
cd248341 | 1077 | |
c506fff3 SO |
1078 | zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), |
1079 | sizeof(*zpci_iomap_bitmap), GFP_KERNEL); | |
1080 | if (!zpci_iomap_bitmap) | |
1081 | goto error_iomap_bitmap; | |
1082 | ||
b02002cc NS |
1083 | if (static_branch_likely(&have_mio)) |
1084 | clp_setup_writeback_mio(); | |
1085 | ||
c506fff3 SO |
1086 | return 0; |
1087 | error_iomap_bitmap: | |
1088 | kfree(zpci_iomap_start); | |
9a4da8a5 | 1089 | error_iomap: |
d0b08853 | 1090 | kmem_cache_destroy(zdev_fmb_cache); |
c506fff3 | 1091 | error_fmb: |
cd248341 JG |
1092 | return -ENOMEM; |
1093 | } | |
1094 | ||
1095 | static void zpci_mem_exit(void) | |
1096 | { | |
c506fff3 | 1097 | kfree(zpci_iomap_bitmap); |
cd248341 | 1098 | kfree(zpci_iomap_start); |
d0b08853 | 1099 | kmem_cache_destroy(zdev_fmb_cache); |
cd248341 JG |
1100 | } |
1101 | ||
6324b4de | 1102 | static unsigned int s390_pci_probe __initdata = 1; |
fbfe07d4 | 1103 | unsigned int s390_pci_force_floating __initdata; |
aa3b7c29 | 1104 | static unsigned int s390_pci_initialized; |
cd248341 JG |
1105 | |
1106 | char * __init pcibios_setup(char *str) | |
1107 | { | |
257608fb SO |
1108 | if (!strcmp(str, "off")) { |
1109 | s390_pci_probe = 0; | |
cd248341 JG |
1110 | return NULL; |
1111 | } | |
56271303 | 1112 | if (!strcmp(str, "nomio")) { |
3322ba0d | 1113 | S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO; |
56271303 SO |
1114 | return NULL; |
1115 | } | |
fbfe07d4 SO |
1116 | if (!strcmp(str, "force_floating")) { |
1117 | s390_pci_force_floating = 1; | |
1118 | return NULL; | |
1119 | } | |
6cf17f9a PM |
1120 | if (!strcmp(str, "norid")) { |
1121 | s390_pci_no_rid = 1; | |
1122 | return NULL; | |
1123 | } | |
cd248341 JG |
1124 | return str; |
1125 | } | |
1126 | ||
aa3b7c29 SO |
1127 | bool zpci_is_enabled(void) |
1128 | { | |
1129 | return s390_pci_initialized; | |
1130 | } | |
1131 | ||
cd248341 JG |
1132 | static int __init pci_base_init(void) |
1133 | { | |
1134 | int rc; | |
1135 | ||
1e5635d1 | 1136 | if (!s390_pci_probe) |
cd248341 JG |
1137 | return 0; |
1138 | ||
da78693e NS |
1139 | if (!test_facility(69) || !test_facility(71)) { |
1140 | pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n"); | |
cd248341 | 1141 | return 0; |
da78693e | 1142 | } |
cd248341 | 1143 | |
3322ba0d | 1144 | if (MACHINE_HAS_PCI_MIO) { |
71ba41c9 | 1145 | static_branch_enable(&have_mio); |
9964f396 SO |
1146 | ctl_set_bit(2, 5); |
1147 | } | |
71ba41c9 | 1148 | |
d0b08853 JG |
1149 | rc = zpci_debug_init(); |
1150 | if (rc) | |
1f44a225 | 1151 | goto out; |
d0b08853 | 1152 | |
cd248341 JG |
1153 | rc = zpci_mem_init(); |
1154 | if (rc) | |
1155 | goto out_mem; | |
1156 | ||
9a4da8a5 JG |
1157 | rc = zpci_irq_init(); |
1158 | if (rc) | |
1159 | goto out_irq; | |
1160 | ||
828b35f6 JG |
1161 | rc = zpci_dma_init(); |
1162 | if (rc) | |
1163 | goto out_dma; | |
1164 | ||
1d578966 | 1165 | rc = clp_scan_pci_devices(); |
a755a45d JG |
1166 | if (rc) |
1167 | goto out_find; | |
14c87ba8 | 1168 | zpci_bus_scan_busses(); |
a755a45d | 1169 | |
aa3b7c29 | 1170 | s390_pci_initialized = 1; |
cd248341 JG |
1171 | return 0; |
1172 | ||
a755a45d | 1173 | out_find: |
828b35f6 JG |
1174 | zpci_dma_exit(); |
1175 | out_dma: | |
9a4da8a5 JG |
1176 | zpci_irq_exit(); |
1177 | out_irq: | |
cd248341 JG |
1178 | zpci_mem_exit(); |
1179 | out_mem: | |
d0b08853 | 1180 | zpci_debug_exit(); |
1f44a225 | 1181 | out: |
cd248341 JG |
1182 | return rc; |
1183 | } | |
67f43f38 | 1184 | subsys_initcall_sync(pci_base_init); |