Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
4eafad7f AI |
2 | /* |
3 | * Access to PCI I/O memory from user space programs. | |
4 | * | |
5 | * Copyright IBM Corp. 2014 | |
6 | * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com> | |
7 | */ | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/syscalls.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/mm.h> | |
12 | #include <linux/errno.h> | |
13 | #include <linux/pci.h> | |
d09a307f | 14 | #include <asm/asm-extable.h> |
f058599e NS |
15 | #include <asm/pci_io.h> |
16 | #include <asm/pci_debug.h> | |
17 | ||
18 | static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) | |
19 | { | |
20 | struct { | |
21 | u64 offset; | |
22 | u8 cc; | |
23 | u8 status; | |
24 | } data = {offset, cc, status}; | |
25 | ||
26 | zpci_err_hex(&data, sizeof(data)); | |
27 | } | |
28 | ||
29 | static inline int __pcistb_mio_inuser( | |
30 | void __iomem *ioaddr, const void __user *src, | |
31 | u64 len, u8 *status) | |
32 | { | |
33 | int cc = -ENXIO; | |
34 | ||
35 | asm volatile ( | |
36 | " sacf 256\n" | |
37 | "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" | |
38 | "1: ipm %[cc]\n" | |
39 | " srl %[cc],28\n" | |
40 | "2: sacf 768\n" | |
41 | EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) | |
42 | : [cc] "+d" (cc), [len] "+d" (len) | |
43 | : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) | |
44 | : "cc", "memory"); | |
45 | *status = len >> 24 & 0xff; | |
46 | return cc; | |
47 | } | |
48 | ||
49 | static inline int __pcistg_mio_inuser( | |
50 | void __iomem *ioaddr, const void __user *src, | |
51 | u64 ulen, u8 *status) | |
52 | { | |
d66a4c7f | 53 | union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; |
f058599e NS |
54 | int cc = -ENXIO; |
55 | u64 val = 0; | |
56 | u64 cnt = ulen; | |
57 | u8 tmp; | |
58 | ||
59 | /* | |
60 | * copy 0 < @len <= 8 bytes from @src into the right most bytes of | |
61 | * a register, then store it to PCI at @ioaddr while in secondary | |
62 | * address space. pcistg then uses the user mappings. | |
63 | */ | |
64 | asm volatile ( | |
65 | " sacf 256\n" | |
66 | "0: llgc %[tmp],0(%[src])\n" | |
67 | " sllg %[val],%[val],8\n" | |
68 | " aghi %[src],1\n" | |
69 | " ogr %[val],%[tmp]\n" | |
70 | " brctg %[cnt],0b\n" | |
d66a4c7f | 71 | "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" |
f058599e NS |
72 | "2: ipm %[cc]\n" |
73 | " srl %[cc],28\n" | |
74 | "3: sacf 768\n" | |
75 | EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) | |
76 | : | |
77 | [src] "+a" (src), [cnt] "+d" (cnt), | |
78 | [val] "+d" (val), [tmp] "=d" (tmp), | |
d66a4c7f | 79 | [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) |
f058599e | 80 | :: "cc", "memory"); |
d66a4c7f | 81 | *status = ioaddr_len.odd >> 24 & 0xff; |
f058599e NS |
82 | |
83 | /* did we read everything from user memory? */ | |
84 | if (!cc && cnt != 0) | |
85 | cc = -EFAULT; | |
86 | ||
87 | return cc; | |
88 | } | |
89 | ||
90 | static inline int __memcpy_toio_inuser(void __iomem *dst, | |
91 | const void __user *src, size_t n) | |
92 | { | |
93 | int size, rc = 0; | |
94 | u8 status = 0; | |
f058599e NS |
95 | |
96 | if (!src) | |
97 | return -EINVAL; | |
98 | ||
f058599e NS |
99 | while (n > 0) { |
100 | size = zpci_get_max_write_size((u64 __force) dst, | |
101 | (u64 __force) src, n, | |
102 | ZPCI_MAX_WRITE_SIZE); | |
103 | if (size > 8) /* main path */ | |
104 | rc = __pcistb_mio_inuser(dst, src, size, &status); | |
105 | else | |
106 | rc = __pcistg_mio_inuser(dst, src, size, &status); | |
107 | if (rc) | |
108 | break; | |
109 | src += size; | |
110 | dst += size; | |
111 | n -= size; | |
112 | } | |
f058599e NS |
113 | if (rc) |
114 | zpci_err_mmio(rc, status, (__force u64) dst); | |
115 | return rc; | |
116 | } | |
4eafad7f | 117 | |
4eafad7f AI |
118 | SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, |
119 | const void __user *, user_buffer, size_t, length) | |
120 | { | |
121 | u8 local_buf[64]; | |
122 | void __iomem *io_addr; | |
123 | void *buf; | |
a67a88b0 DV |
124 | struct vm_area_struct *vma; |
125 | pte_t *ptep; | |
126 | spinlock_t *ptl; | |
4eafad7f AI |
127 | long ret; |
128 | ||
129 | if (!zpci_is_enabled()) | |
130 | return -ENODEV; | |
131 | ||
132 | if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) | |
133 | return -EINVAL; | |
f058599e NS |
134 | |
135 | /* | |
4631f3ca NS |
136 | * We only support write access to MIO capable devices if we are on |
137 | * a MIO enabled system. Otherwise we would have to check for every | |
138 | * address if it is a special ZPCI_ADDR and would have to do | |
a67a88b0 | 139 | * a pfn lookup which we don't need for MIO capable devices. Currently |
4631f3ca NS |
140 | * ISM devices are the only devices without MIO support and there is no |
141 | * known need for accessing these from userspace. | |
f058599e NS |
142 | */ |
143 | if (static_branch_likely(&have_mio)) { | |
144 | ret = __memcpy_toio_inuser((void __iomem *) mmio_addr, | |
145 | user_buffer, | |
146 | length); | |
147 | return ret; | |
148 | } | |
149 | ||
4eafad7f AI |
150 | if (length > 64) { |
151 | buf = kmalloc(length, GFP_KERNEL); | |
152 | if (!buf) | |
153 | return -ENOMEM; | |
154 | } else | |
155 | buf = local_buf; | |
156 | ||
a67a88b0 DV |
157 | ret = -EFAULT; |
158 | if (copy_from_user(buf, user_buffer, length)) | |
159 | goto out_free; | |
160 | ||
161 | mmap_read_lock(current->mm); | |
162 | ret = -EINVAL; | |
a8b92b8c | 163 | vma = vma_lookup(current->mm, mmio_addr); |
a67a88b0 DV |
164 | if (!vma) |
165 | goto out_unlock_mmap; | |
166 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) | |
167 | goto out_unlock_mmap; | |
168 | ret = -EACCES; | |
169 | if (!(vma->vm_flags & VM_WRITE)) | |
170 | goto out_unlock_mmap; | |
171 | ||
9fd6dad1 | 172 | ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); |
4eafad7f | 173 | if (ret) |
a67a88b0 DV |
174 | goto out_unlock_mmap; |
175 | ||
176 | io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | | |
f058599e | 177 | (mmio_addr & ~PAGE_MASK)); |
4eafad7f | 178 | |
4eafad7f | 179 | if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) |
a67a88b0 | 180 | goto out_unlock_pt; |
4eafad7f | 181 | |
f0483044 | 182 | ret = zpci_memcpy_toio(io_addr, buf, length); |
a67a88b0 DV |
183 | out_unlock_pt: |
184 | pte_unmap_unlock(ptep, ptl); | |
185 | out_unlock_mmap: | |
186 | mmap_read_unlock(current->mm); | |
187 | out_free: | |
4eafad7f AI |
188 | if (buf != local_buf) |
189 | kfree(buf); | |
190 | return ret; | |
191 | } | |
192 | ||
f058599e NS |
193 | static inline int __pcilg_mio_inuser( |
194 | void __user *dst, const void __iomem *ioaddr, | |
195 | u64 ulen, u8 *status) | |
196 | { | |
d66a4c7f | 197 | union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; |
f058599e NS |
198 | u64 cnt = ulen; |
199 | int shift = ulen * 8; | |
200 | int cc = -ENXIO; | |
201 | u64 val, tmp; | |
202 | ||
203 | /* | |
204 | * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in | |
205 | * user space) into a register using pcilg then store these bytes at | |
206 | * user address @dst | |
207 | */ | |
208 | asm volatile ( | |
209 | " sacf 256\n" | |
d66a4c7f | 210 | "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" |
f058599e NS |
211 | "1: ipm %[cc]\n" |
212 | " srl %[cc],28\n" | |
213 | " ltr %[cc],%[cc]\n" | |
214 | " jne 4f\n" | |
215 | "2: ahi %[shift],-8\n" | |
216 | " srlg %[tmp],%[val],0(%[shift])\n" | |
217 | "3: stc %[tmp],0(%[dst])\n" | |
218 | " aghi %[dst],1\n" | |
219 | " brctg %[cnt],2b\n" | |
220 | "4: sacf 768\n" | |
221 | EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) | |
222 | : | |
d66a4c7f NS |
223 | [ioaddr_len] "+&d" (ioaddr_len.pair), |
224 | [cc] "+d" (cc), [val] "=d" (val), | |
f058599e NS |
225 | [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), |
226 | [shift] "+d" (shift) | |
d66a4c7f | 227 | :: "cc", "memory"); |
f058599e NS |
228 | |
229 | /* did we write everything to the user space buffer? */ | |
230 | if (!cc && cnt != 0) | |
231 | cc = -EFAULT; | |
232 | ||
d66a4c7f | 233 | *status = ioaddr_len.odd >> 24 & 0xff; |
f058599e NS |
234 | return cc; |
235 | } | |
236 | ||
237 | static inline int __memcpy_fromio_inuser(void __user *dst, | |
238 | const void __iomem *src, | |
239 | unsigned long n) | |
240 | { | |
241 | int size, rc = 0; | |
242 | u8 status; | |
f058599e | 243 | |
f058599e NS |
244 | while (n > 0) { |
245 | size = zpci_get_max_write_size((u64 __force) src, | |
246 | (u64 __force) dst, n, | |
247 | ZPCI_MAX_READ_SIZE); | |
248 | rc = __pcilg_mio_inuser(dst, src, size, &status); | |
249 | if (rc) | |
250 | break; | |
251 | src += size; | |
252 | dst += size; | |
253 | n -= size; | |
254 | } | |
f058599e NS |
255 | if (rc) |
256 | zpci_err_mmio(rc, status, (__force u64) dst); | |
257 | return rc; | |
258 | } | |
259 | ||
4eafad7f AI |
260 | SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, |
261 | void __user *, user_buffer, size_t, length) | |
262 | { | |
263 | u8 local_buf[64]; | |
264 | void __iomem *io_addr; | |
265 | void *buf; | |
a67a88b0 DV |
266 | struct vm_area_struct *vma; |
267 | pte_t *ptep; | |
268 | spinlock_t *ptl; | |
4eafad7f AI |
269 | long ret; |
270 | ||
271 | if (!zpci_is_enabled()) | |
272 | return -ENODEV; | |
273 | ||
274 | if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) | |
275 | return -EINVAL; | |
f058599e NS |
276 | |
277 | /* | |
4631f3ca NS |
278 | * We only support read access to MIO capable devices if we are on |
279 | * a MIO enabled system. Otherwise we would have to check for every | |
280 | * address if it is a special ZPCI_ADDR and would have to do | |
a67a88b0 | 281 | * a pfn lookup which we don't need for MIO capable devices. Currently |
4631f3ca NS |
282 | * ISM devices are the only devices without MIO support and there is no |
283 | * known need for accessing these from userspace. | |
f058599e NS |
284 | */ |
285 | if (static_branch_likely(&have_mio)) { | |
286 | ret = __memcpy_fromio_inuser( | |
287 | user_buffer, (const void __iomem *)mmio_addr, | |
288 | length); | |
289 | return ret; | |
290 | } | |
291 | ||
4eafad7f AI |
292 | if (length > 64) { |
293 | buf = kmalloc(length, GFP_KERNEL); | |
294 | if (!buf) | |
295 | return -ENOMEM; | |
f058599e | 296 | } else { |
4eafad7f | 297 | buf = local_buf; |
f058599e | 298 | } |
4eafad7f | 299 | |
a67a88b0 DV |
300 | mmap_read_lock(current->mm); |
301 | ret = -EINVAL; | |
a8b92b8c | 302 | vma = vma_lookup(current->mm, mmio_addr); |
a67a88b0 DV |
303 | if (!vma) |
304 | goto out_unlock_mmap; | |
305 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) | |
306 | goto out_unlock_mmap; | |
307 | ret = -EACCES; | |
308 | if (!(vma->vm_flags & VM_WRITE)) | |
309 | goto out_unlock_mmap; | |
310 | ||
9fd6dad1 | 311 | ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); |
4eafad7f | 312 | if (ret) |
a67a88b0 DV |
313 | goto out_unlock_mmap; |
314 | ||
315 | io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | | |
316 | (mmio_addr & ~PAGE_MASK)); | |
4eafad7f | 317 | |
f0483044 SO |
318 | if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { |
319 | ret = -EFAULT; | |
a67a88b0 | 320 | goto out_unlock_pt; |
f0483044 SO |
321 | } |
322 | ret = zpci_memcpy_fromio(buf, io_addr, length); | |
a67a88b0 DV |
323 | |
324 | out_unlock_pt: | |
325 | pte_unmap_unlock(ptep, ptl); | |
326 | out_unlock_mmap: | |
327 | mmap_read_unlock(current->mm); | |
328 | ||
329 | if (!ret && copy_to_user(user_buffer, buf, length)) | |
f0483044 | 330 | ret = -EFAULT; |
4eafad7f | 331 | |
4eafad7f AI |
332 | if (buf != local_buf) |
333 | kfree(buf); | |
334 | return ret; | |
335 | } |