Commit | Line | Data |
---|---|---|
88df125f HC |
1 | /* |
2 | * Access kernel memory without faulting -- s390 specific implementation. | |
3 | * | |
3c1a3bce | 4 | * Copyright IBM Corp. 2009, 2015 |
88df125f HC |
5 | * |
6 | * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>, | |
7 | * | |
8 | */ | |
9 | ||
10 | #include <linux/uaccess.h> | |
11 | #include <linux/kernel.h> | |
12 | #include <linux/types.h> | |
13 | #include <linux/errno.h> | |
7f0bf656 | 14 | #include <linux/gfp.h> |
b2a68c23 | 15 | #include <linux/cpu.h> |
a0616cde | 16 | #include <asm/ctl_reg.h> |
63df41d6 | 17 | #include <asm/io.h> |
88df125f | 18 | |
8a5d8473 | 19 | static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) |
88df125f | 20 | { |
3c1a3bce HC |
21 | unsigned long aligned, offset, count; |
22 | char tmp[8]; | |
88df125f | 23 | |
3c1a3bce HC |
24 | aligned = (unsigned long) dst & ~7UL; |
25 | offset = (unsigned long) dst & 7UL; | |
26 | size = min(8UL - offset, size); | |
27 | count = size - 1; | |
88df125f HC |
28 | asm volatile( |
29 | " bras 1,0f\n" | |
3c1a3bce HC |
30 | " mvc 0(1,%4),0(%5)\n" |
31 | "0: mvc 0(8,%3),0(%0)\n" | |
32 | " ex %1,0(1)\n" | |
33 | " lg %1,0(%3)\n" | |
34 | " lra %0,0(%0)\n" | |
35 | " sturg %1,%0\n" | |
36 | : "+&a" (aligned), "+&a" (count), "=m" (tmp) | |
37 | : "a" (&tmp), "a" (&tmp[offset]), "a" (src) | |
38 | : "cc", "memory", "1"); | |
39 | return size; | |
88df125f HC |
40 | } |
41 | ||
8a5d8473 HC |
42 | /* |
43 | * s390_kernel_write - write to kernel memory bypassing DAT | |
44 | * @dst: destination address | |
45 | * @src: source address | |
46 | * @size: number of bytes to copy | |
47 | * | |
48 | * This function writes to kernel memory bypassing DAT and possible page table | |
49 | * write protection. It writes to the destination using the sturg instruction. | |
3c1a3bce HC |
50 | * Therefore we have a read-modify-write sequence: the function reads eight |
51 | * bytes from destination at an eight byte boundary, modifies the bytes | |
8a5d8473 HC |
52 | * requested and writes the result back in a loop. |
53 | * | |
54 | * Note: this means that this function may not be called concurrently on | |
55 | * several cpus with overlapping words, since this may potentially | |
56 | * cause data corruption. | |
57 | */ | |
58 | void notrace s390_kernel_write(void *dst, const void *src, size_t size) | |
88df125f | 59 | { |
3c1a3bce | 60 | long copied; |
88df125f HC |
61 | |
62 | while (size) { | |
8a5d8473 | 63 | copied = s390_kernel_write_odd(dst, src, size); |
88df125f HC |
64 | dst += copied; |
65 | src += copied; | |
66 | size -= copied; | |
67 | } | |
88df125f | 68 | } |
92fe3132 | 69 | |
b785e0d0 | 70 | static int __memcpy_real(void *dest, void *src, size_t count) |
92fe3132 MH |
71 | { |
72 | register unsigned long _dest asm("2") = (unsigned long) dest; | |
73 | register unsigned long _len1 asm("3") = (unsigned long) count; | |
74 | register unsigned long _src asm("4") = (unsigned long) src; | |
75 | register unsigned long _len2 asm("5") = (unsigned long) count; | |
92fe3132 MH |
76 | int rc = -EFAULT; |
77 | ||
92fe3132 MH |
78 | asm volatile ( |
79 | "0: mvcle %1,%2,0x0\n" | |
80 | "1: jo 0b\n" | |
81 | " lhi %0,0x0\n" | |
82 | "2:\n" | |
83 | EX_TABLE(1b,2b) | |
84 | : "+d" (rc), "+d" (_dest), "+d" (_src), "+d" (_len1), | |
85 | "+d" (_len2), "=m" (*((long *) dest)) | |
86 | : "m" (*((long *) src)) | |
87 | : "cc", "memory"); | |
b785e0d0 MH |
88 | return rc; |
89 | } | |
90 | ||
91 | /* | |
92 | * Copy memory in real mode (kernel to kernel) | |
93 | */ | |
94 | int memcpy_real(void *dest, void *src, size_t count) | |
95 | { | |
52499d93 | 96 | int irqs_disabled, rc; |
b785e0d0 | 97 | unsigned long flags; |
b785e0d0 MH |
98 | |
99 | if (!count) | |
100 | return 0; | |
52499d93 HC |
101 | flags = __arch_local_irq_stnsm(0xf8UL); |
102 | irqs_disabled = arch_irqs_disabled_flags(flags); | |
103 | if (!irqs_disabled) | |
104 | trace_hardirqs_off(); | |
b785e0d0 | 105 | rc = __memcpy_real(dest, src, count); |
52499d93 | 106 | if (!irqs_disabled) |
0986d977 CB |
107 | trace_hardirqs_on(); |
108 | __arch_local_irq_ssm(flags); | |
92fe3132 MH |
109 | return rc; |
110 | } | |
7dd6b334 MH |
111 | |
112 | /* | |
73bf463e | 113 | * Copy memory in absolute mode (kernel to kernel) |
7dd6b334 | 114 | */ |
73bf463e | 115 | void memcpy_absolute(void *dest, void *src, size_t count) |
7dd6b334 | 116 | { |
73bf463e | 117 | unsigned long cr0, flags, prefix; |
7dd6b334 | 118 | |
73bf463e | 119 | flags = arch_local_irq_save(); |
7dd6b334 MH |
120 | __ctl_store(cr0, 0, 0); |
121 | __ctl_clear_bit(0, 28); /* disable lowcore protection */ | |
73bf463e MH |
122 | prefix = store_prefix(); |
123 | if (prefix) { | |
124 | local_mcck_disable(); | |
125 | set_prefix(0); | |
126 | memcpy(dest, src, count); | |
127 | set_prefix(prefix); | |
128 | local_mcck_enable(); | |
129 | } else { | |
130 | memcpy(dest, src, count); | |
131 | } | |
7dd6b334 | 132 | __ctl_load(cr0, 0, 0); |
73bf463e | 133 | arch_local_irq_restore(flags); |
7dd6b334 | 134 | } |
7f0bf656 MH |
135 | |
136 | /* | |
137 | * Copy memory from kernel (real) to user (virtual) | |
138 | */ | |
211deca6 | 139 | int copy_to_user_real(void __user *dest, void *src, unsigned long count) |
7f0bf656 MH |
140 | { |
141 | int offs = 0, size, rc; | |
142 | char *buf; | |
143 | ||
144 | buf = (char *) __get_free_page(GFP_KERNEL); | |
145 | if (!buf) | |
146 | return -ENOMEM; | |
147 | rc = -EFAULT; | |
148 | while (offs < count) { | |
149 | size = min(PAGE_SIZE, count - offs); | |
150 | if (memcpy_real(buf, src + offs, size)) | |
151 | goto out; | |
152 | if (copy_to_user(dest + offs, buf, size)) | |
153 | goto out; | |
154 | offs += size; | |
155 | } | |
156 | rc = 0; | |
157 | out: | |
158 | free_page((unsigned long) buf); | |
159 | return rc; | |
160 | } | |
161 | ||
b2a68c23 MH |
162 | /* |
163 | * Check if physical address is within prefix or zero page | |
164 | */ | |
165 | static int is_swapped(unsigned long addr) | |
166 | { | |
167 | unsigned long lc; | |
168 | int cpu; | |
169 | ||
c667aeac | 170 | if (addr < sizeof(struct lowcore)) |
b2a68c23 MH |
171 | return 1; |
172 | for_each_online_cpu(cpu) { | |
173 | lc = (unsigned long) lowcore_ptr[cpu]; | |
c667aeac | 174 | if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) |
b2a68c23 MH |
175 | continue; |
176 | return 1; | |
177 | } | |
178 | return 0; | |
179 | } | |
180 | ||
b2a68c23 MH |
181 | /* |
182 | * Convert a physical pointer for /dev/mem access | |
183 | * | |
184 | * For swapped prefix pages a new buffer is returned that contains a copy of | |
185 | * the absolute memory. The buffer size is maximum one page large. | |
186 | */ | |
4707a341 | 187 | void *xlate_dev_mem_ptr(phys_addr_t addr) |
b2a68c23 MH |
188 | { |
189 | void *bounce = (void *) addr; | |
190 | unsigned long size; | |
191 | ||
192 | get_online_cpus(); | |
193 | preempt_disable(); | |
194 | if (is_swapped(addr)) { | |
195 | size = PAGE_SIZE - (addr & ~PAGE_MASK); | |
196 | bounce = (void *) __get_free_page(GFP_ATOMIC); | |
197 | if (bounce) | |
73bf463e | 198 | memcpy_absolute(bounce, (void *) addr, size); |
b2a68c23 MH |
199 | } |
200 | preempt_enable(); | |
201 | put_online_cpus(); | |
202 | return bounce; | |
203 | } | |
204 | ||
205 | /* | |
206 | * Free converted buffer for /dev/mem access (if necessary) | |
207 | */ | |
4707a341 | 208 | void unxlate_dev_mem_ptr(phys_addr_t addr, void *buf) |
b2a68c23 MH |
209 | { |
210 | if ((void *) addr != buf) | |
211 | free_page((unsigned long) buf); | |
212 | } |