Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
88df125f HC |
2 | /* |
3 | * Access kernel memory without faulting -- s390 specific implementation. | |
4 | * | |
3c1a3bce | 5 | * Copyright IBM Corp. 2009, 2015 |
88df125f | 6 | * |
88df125f HC |
7 | */ |
8 | ||
9 | #include <linux/uaccess.h> | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/types.h> | |
12 | #include <linux/errno.h> | |
7f0bf656 | 13 | #include <linux/gfp.h> |
b2a68c23 | 14 | #include <linux/cpu.h> |
a0616cde | 15 | #include <asm/ctl_reg.h> |
63df41d6 | 16 | #include <asm/io.h> |
78c98f90 | 17 | #include <asm/stacktrace.h> |
88df125f | 18 | |
8a5d8473 | 19 | static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) |
88df125f | 20 | { |
3c1a3bce HC |
21 | unsigned long aligned, offset, count; |
22 | char tmp[8]; | |
88df125f | 23 | |
3c1a3bce HC |
24 | aligned = (unsigned long) dst & ~7UL; |
25 | offset = (unsigned long) dst & 7UL; | |
26 | size = min(8UL - offset, size); | |
27 | count = size - 1; | |
88df125f HC |
28 | asm volatile( |
29 | " bras 1,0f\n" | |
3c1a3bce HC |
30 | " mvc 0(1,%4),0(%5)\n" |
31 | "0: mvc 0(8,%3),0(%0)\n" | |
32 | " ex %1,0(1)\n" | |
33 | " lg %1,0(%3)\n" | |
34 | " lra %0,0(%0)\n" | |
35 | " sturg %1,%0\n" | |
36 | : "+&a" (aligned), "+&a" (count), "=m" (tmp) | |
37 | : "a" (&tmp), "a" (&tmp[offset]), "a" (src) | |
38 | : "cc", "memory", "1"); | |
39 | return size; | |
88df125f HC |
40 | } |
41 | ||
8a5d8473 HC |
42 | /* |
43 | * s390_kernel_write - write to kernel memory bypassing DAT | |
44 | * @dst: destination address | |
45 | * @src: source address | |
46 | * @size: number of bytes to copy | |
47 | * | |
48 | * This function writes to kernel memory bypassing DAT and possible page table | |
49 | * write protection. It writes to the destination using the sturg instruction. | |
3c1a3bce HC |
50 | * Therefore we have a read-modify-write sequence: the function reads eight |
51 | * bytes from destination at an eight byte boundary, modifies the bytes | |
8a5d8473 | 52 | * requested and writes the result back in a loop. |
8a5d8473 | 53 | */ |
a646ef39 MS |
54 | static DEFINE_SPINLOCK(s390_kernel_write_lock); |
55 | ||
cb2cceae | 56 | notrace void *s390_kernel_write(void *dst, const void *src, size_t size) |
88df125f | 57 | { |
cb2cceae | 58 | void *tmp = dst; |
a646ef39 | 59 | unsigned long flags; |
3c1a3bce | 60 | long copied; |
88df125f | 61 | |
a646ef39 | 62 | spin_lock_irqsave(&s390_kernel_write_lock, flags); |
d6df52e9 VG |
63 | if (!(flags & PSW_MASK_DAT)) { |
64 | memcpy(dst, src, size); | |
65 | } else { | |
66 | while (size) { | |
67 | copied = s390_kernel_write_odd(tmp, src, size); | |
68 | tmp += copied; | |
69 | src += copied; | |
70 | size -= copied; | |
71 | } | |
88df125f | 72 | } |
a646ef39 | 73 | spin_unlock_irqrestore(&s390_kernel_write_lock, flags); |
cb2cceae JP |
74 | |
75 | return dst; | |
88df125f | 76 | } |
92fe3132 | 77 | |
13f9bae5 | 78 | static int __no_sanitize_address __memcpy_real(void *dest, void *src, size_t count) |
92fe3132 | 79 | { |
dda74578 | 80 | union register_pair _dst, _src; |
92fe3132 MH |
81 | int rc = -EFAULT; |
82 | ||
dda74578 HC |
83 | _dst.even = (unsigned long) dest; |
84 | _dst.odd = (unsigned long) count; | |
85 | _src.even = (unsigned long) src; | |
86 | _src.odd = (unsigned long) count; | |
92fe3132 | 87 | asm volatile ( |
dda74578 | 88 | "0: mvcle %[dst],%[src],0\n" |
92fe3132 | 89 | "1: jo 0b\n" |
dda74578 | 90 | " lhi %[rc],0\n" |
92fe3132 MH |
91 | "2:\n" |
92 | EX_TABLE(1b,2b) | |
dda74578 HC |
93 | : [rc] "+&d" (rc), [dst] "+&d" (_dst.pair), [src] "+&d" (_src.pair) |
94 | : : "cc", "memory"); | |
b785e0d0 MH |
95 | return rc; |
96 | } | |
97 | ||
13f9bae5 VG |
98 | static unsigned long __no_sanitize_address _memcpy_real(unsigned long dest, |
99 | unsigned long src, | |
100 | unsigned long count) | |
b785e0d0 | 101 | { |
52499d93 | 102 | int irqs_disabled, rc; |
b785e0d0 | 103 | unsigned long flags; |
b785e0d0 MH |
104 | |
105 | if (!count) | |
106 | return 0; | |
13f9bae5 | 107 | flags = arch_local_irq_save(); |
52499d93 HC |
108 | irqs_disabled = arch_irqs_disabled_flags(flags); |
109 | if (!irqs_disabled) | |
110 | trace_hardirqs_off(); | |
13f9bae5 | 111 | __arch_local_irq_stnsm(0xf8); // disable DAT |
ce3dc447 | 112 | rc = __memcpy_real((void *) dest, (void *) src, (size_t) count); |
13f9bae5 VG |
113 | if (flags & PSW_MASK_DAT) |
114 | __arch_local_irq_stosm(0x04); // enable DAT | |
52499d93 | 115 | if (!irqs_disabled) |
0986d977 CB |
116 | trace_hardirqs_on(); |
117 | __arch_local_irq_ssm(flags); | |
92fe3132 MH |
118 | return rc; |
119 | } | |
7dd6b334 | 120 | |
ce3dc447 MS |
121 | /* |
122 | * Copy memory in real mode (kernel to kernel) | |
123 | */ | |
124 | int memcpy_real(void *dest, void *src, size_t count) | |
125 | { | |
7c496e66 HC |
126 | unsigned long _dest = (unsigned long)dest; |
127 | unsigned long _src = (unsigned long)src; | |
128 | unsigned long _count = (unsigned long)count; | |
7f28dad3 VG |
129 | int rc; |
130 | ||
131 | if (S390_lowcore.nodat_stack != 0) { | |
132 | preempt_disable(); | |
7c496e66 HC |
133 | rc = call_on_stack(3, S390_lowcore.nodat_stack, |
134 | unsigned long, _memcpy_real, | |
135 | unsigned long, _dest, | |
136 | unsigned long, _src, | |
137 | unsigned long, _count); | |
7f28dad3 VG |
138 | preempt_enable(); |
139 | return rc; | |
140 | } | |
ce3dc447 MS |
141 | /* |
142 | * This is a really early memcpy_real call, the stacks are | |
143 | * not set up yet. Just call _memcpy_real on the early boot | |
144 | * stack | |
145 | */ | |
7c496e66 | 146 | return _memcpy_real(_dest, _src, _count); |
ce3dc447 MS |
147 | } |
148 | ||
7dd6b334 | 149 | /* |
73bf463e | 150 | * Copy memory in absolute mode (kernel to kernel) |
7dd6b334 | 151 | */ |
73bf463e | 152 | void memcpy_absolute(void *dest, void *src, size_t count) |
7dd6b334 | 153 | { |
73bf463e | 154 | unsigned long cr0, flags, prefix; |
7dd6b334 | 155 | |
73bf463e | 156 | flags = arch_local_irq_save(); |
7dd6b334 MH |
157 | __ctl_store(cr0, 0, 0); |
158 | __ctl_clear_bit(0, 28); /* disable lowcore protection */ | |
73bf463e MH |
159 | prefix = store_prefix(); |
160 | if (prefix) { | |
161 | local_mcck_disable(); | |
162 | set_prefix(0); | |
163 | memcpy(dest, src, count); | |
164 | set_prefix(prefix); | |
165 | local_mcck_enable(); | |
166 | } else { | |
167 | memcpy(dest, src, count); | |
168 | } | |
7dd6b334 | 169 | __ctl_load(cr0, 0, 0); |
73bf463e | 170 | arch_local_irq_restore(flags); |
7dd6b334 | 171 | } |
7f0bf656 MH |
172 | |
173 | /* | |
174 | * Copy memory from kernel (real) to user (virtual) | |
175 | */ | |
211deca6 | 176 | int copy_to_user_real(void __user *dest, void *src, unsigned long count) |
7f0bf656 MH |
177 | { |
178 | int offs = 0, size, rc; | |
179 | char *buf; | |
180 | ||
181 | buf = (char *) __get_free_page(GFP_KERNEL); | |
182 | if (!buf) | |
183 | return -ENOMEM; | |
184 | rc = -EFAULT; | |
185 | while (offs < count) { | |
186 | size = min(PAGE_SIZE, count - offs); | |
187 | if (memcpy_real(buf, src + offs, size)) | |
188 | goto out; | |
189 | if (copy_to_user(dest + offs, buf, size)) | |
190 | goto out; | |
191 | offs += size; | |
192 | } | |
193 | rc = 0; | |
194 | out: | |
195 | free_page((unsigned long) buf); | |
196 | return rc; | |
197 | } | |
198 | ||
b2a68c23 MH |
199 | /* |
200 | * Check if physical address is within prefix or zero page | |
201 | */ | |
1f231e29 | 202 | static int is_swapped(phys_addr_t addr) |
b2a68c23 | 203 | { |
1f231e29 | 204 | phys_addr_t lc; |
b2a68c23 MH |
205 | int cpu; |
206 | ||
c667aeac | 207 | if (addr < sizeof(struct lowcore)) |
b2a68c23 MH |
208 | return 1; |
209 | for_each_online_cpu(cpu) { | |
1f231e29 | 210 | lc = virt_to_phys(lowcore_ptr[cpu]); |
c667aeac | 211 | if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) |
b2a68c23 MH |
212 | continue; |
213 | return 1; | |
214 | } | |
215 | return 0; | |
216 | } | |
217 | ||
b2a68c23 MH |
218 | /* |
219 | * Convert a physical pointer for /dev/mem access | |
220 | * | |
221 | * For swapped prefix pages a new buffer is returned that contains a copy of | |
222 | * the absolute memory. The buffer size is maximum one page large. | |
223 | */ | |
4707a341 | 224 | void *xlate_dev_mem_ptr(phys_addr_t addr) |
b2a68c23 | 225 | { |
1f231e29 AG |
226 | void *ptr = phys_to_virt(addr); |
227 | void *bounce = ptr; | |
b2a68c23 MH |
228 | unsigned long size; |
229 | ||
a73de293 | 230 | cpus_read_lock(); |
b2a68c23 MH |
231 | preempt_disable(); |
232 | if (is_swapped(addr)) { | |
233 | size = PAGE_SIZE - (addr & ~PAGE_MASK); | |
234 | bounce = (void *) __get_free_page(GFP_ATOMIC); | |
235 | if (bounce) | |
1f231e29 | 236 | memcpy_absolute(bounce, ptr, size); |
b2a68c23 MH |
237 | } |
238 | preempt_enable(); | |
a73de293 | 239 | cpus_read_unlock(); |
b2a68c23 MH |
240 | return bounce; |
241 | } | |
242 | ||
243 | /* | |
244 | * Free converted buffer for /dev/mem access (if necessary) | |
245 | */ | |
1f231e29 | 246 | void unxlate_dev_mem_ptr(phys_addr_t addr, void *ptr) |
b2a68c23 | 247 | { |
1f231e29 AG |
248 | if (addr != virt_to_phys(ptr)) |
249 | free_page((unsigned long)ptr); | |
b2a68c23 | 250 | } |