Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
88df125f HC |
2 | /* |
3 | * Access kernel memory without faulting -- s390 specific implementation. | |
4 | * | |
3c1a3bce | 5 | * Copyright IBM Corp. 2009, 2015 |
88df125f | 6 | * |
88df125f HC |
7 | */ |
8 | ||
9 | #include <linux/uaccess.h> | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/types.h> | |
12 | #include <linux/errno.h> | |
7f0bf656 | 13 | #include <linux/gfp.h> |
b2a68c23 | 14 | #include <linux/cpu.h> |
2f0e8aae | 15 | #include <linux/uio.h> |
b378a982 | 16 | #include <linux/io.h> |
d09a307f | 17 | #include <asm/asm-extable.h> |
a0616cde | 18 | #include <asm/ctl_reg.h> |
4df29d2b | 19 | #include <asm/abs_lowcore.h> |
78c98f90 | 20 | #include <asm/stacktrace.h> |
9267bdd8 | 21 | #include <asm/maccess.h> |
88df125f | 22 | |
2f0e8aae | 23 | unsigned long __bootdata_preserved(__memcpy_real_area); |
8e9205d2 | 24 | pte_t *__bootdata_preserved(memcpy_real_ptep); |
2f0e8aae AG |
25 | static DEFINE_MUTEX(memcpy_real_mutex); |
26 | ||
8a5d8473 | 27 | static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) |
88df125f | 28 | { |
3c1a3bce HC |
29 | unsigned long aligned, offset, count; |
30 | char tmp[8]; | |
88df125f | 31 | |
3c1a3bce HC |
32 | aligned = (unsigned long) dst & ~7UL; |
33 | offset = (unsigned long) dst & 7UL; | |
34 | size = min(8UL - offset, size); | |
35 | count = size - 1; | |
88df125f HC |
36 | asm volatile( |
37 | " bras 1,0f\n" | |
3c1a3bce HC |
38 | " mvc 0(1,%4),0(%5)\n" |
39 | "0: mvc 0(8,%3),0(%0)\n" | |
40 | " ex %1,0(1)\n" | |
41 | " lg %1,0(%3)\n" | |
42 | " lra %0,0(%0)\n" | |
43 | " sturg %1,%0\n" | |
44 | : "+&a" (aligned), "+&a" (count), "=m" (tmp) | |
45 | : "a" (&tmp), "a" (&tmp[offset]), "a" (src) | |
46 | : "cc", "memory", "1"); | |
47 | return size; | |
88df125f HC |
48 | } |
49 | ||
8a5d8473 HC |
50 | /* |
51 | * s390_kernel_write - write to kernel memory bypassing DAT | |
52 | * @dst: destination address | |
53 | * @src: source address | |
54 | * @size: number of bytes to copy | |
55 | * | |
56 | * This function writes to kernel memory bypassing DAT and possible page table | |
57 | * write protection. It writes to the destination using the sturg instruction. | |
3c1a3bce HC |
58 | * Therefore we have a read-modify-write sequence: the function reads eight |
59 | * bytes from destination at an eight byte boundary, modifies the bytes | |
8a5d8473 | 60 | * requested and writes the result back in a loop. |
8a5d8473 | 61 | */ |
a646ef39 MS |
62 | static DEFINE_SPINLOCK(s390_kernel_write_lock); |
63 | ||
cb2cceae | 64 | notrace void *s390_kernel_write(void *dst, const void *src, size_t size) |
88df125f | 65 | { |
cb2cceae | 66 | void *tmp = dst; |
a646ef39 | 67 | unsigned long flags; |
3c1a3bce | 68 | long copied; |
88df125f | 69 | |
a646ef39 | 70 | spin_lock_irqsave(&s390_kernel_write_lock, flags); |
12cf6473 AG |
71 | while (size) { |
72 | copied = s390_kernel_write_odd(tmp, src, size); | |
73 | tmp += copied; | |
74 | src += copied; | |
75 | size -= copied; | |
88df125f | 76 | } |
a646ef39 | 77 | spin_unlock_irqrestore(&s390_kernel_write_lock, flags); |
cb2cceae JP |
78 | |
79 | return dst; | |
88df125f | 80 | } |
92fe3132 | 81 | |
2f0e8aae | 82 | size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) |
b785e0d0 | 83 | { |
2f0e8aae AG |
84 | size_t len, copied, res = 0; |
85 | unsigned long phys, offset; | |
86 | void *chunk; | |
87 | pte_t pte; | |
88 | ||
a984f27e | 89 | BUILD_BUG_ON(MEMCPY_REAL_SIZE != PAGE_SIZE); |
2f0e8aae | 90 | while (count) { |
a984f27e AG |
91 | phys = src & MEMCPY_REAL_MASK; |
92 | offset = src & ~MEMCPY_REAL_MASK; | |
2f0e8aae | 93 | chunk = (void *)(__memcpy_real_area + offset); |
a984f27e | 94 | len = min(count, MEMCPY_REAL_SIZE - offset); |
2f0e8aae AG |
95 | pte = mk_pte_phys(phys, PAGE_KERNEL_RO); |
96 | ||
97 | mutex_lock(&memcpy_real_mutex); | |
98 | if (pte_val(pte) != pte_val(*memcpy_real_ptep)) { | |
99 | __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); | |
100 | set_pte(memcpy_real_ptep, pte); | |
101 | } | |
102 | copied = copy_to_iter(chunk, len, iter); | |
103 | mutex_unlock(&memcpy_real_mutex); | |
104 | ||
105 | count -= copied; | |
106 | src += copied; | |
107 | res += copied; | |
108 | if (copied < len) | |
109 | break; | |
110 | } | |
111 | return res; | |
92fe3132 | 112 | } |
7dd6b334 | 113 | |
303fd988 | 114 | int memcpy_real(void *dest, unsigned long src, size_t count) |
ce3dc447 | 115 | { |
2f0e8aae AG |
116 | struct iov_iter iter; |
117 | struct kvec kvec; | |
118 | ||
119 | kvec.iov_base = dest; | |
120 | kvec.iov_len = count; | |
de4eda9d | 121 | iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count); |
2f0e8aae AG |
122 | if (memcpy_real_iter(&iter, src, count) < count) |
123 | return -EFAULT; | |
124 | return 0; | |
ce3dc447 MS |
125 | } |
126 | ||
7dd6b334 | 127 | /* |
4df29d2b | 128 | * Find CPU that owns swapped prefix page |
7dd6b334 | 129 | */ |
4df29d2b | 130 | static int get_swapped_owner(phys_addr_t addr) |
b2a68c23 | 131 | { |
1f231e29 | 132 | phys_addr_t lc; |
b2a68c23 MH |
133 | int cpu; |
134 | ||
b2a68c23 | 135 | for_each_online_cpu(cpu) { |
1f231e29 | 136 | lc = virt_to_phys(lowcore_ptr[cpu]); |
c667aeac | 137 | if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) |
b2a68c23 | 138 | continue; |
4df29d2b | 139 | return cpu; |
b2a68c23 | 140 | } |
4df29d2b | 141 | return -1; |
b2a68c23 MH |
142 | } |
143 | ||
b2a68c23 MH |
144 | /* |
145 | * Convert a physical pointer for /dev/mem access | |
146 | * | |
147 | * For swapped prefix pages a new buffer is returned that contains a copy of | |
148 | * the absolute memory. The buffer size is maximum one page large. | |
149 | */ | |
4707a341 | 150 | void *xlate_dev_mem_ptr(phys_addr_t addr) |
b2a68c23 | 151 | { |
1f231e29 AG |
152 | void *ptr = phys_to_virt(addr); |
153 | void *bounce = ptr; | |
4df29d2b | 154 | struct lowcore *abs_lc; |
b2a68c23 | 155 | unsigned long size; |
4df29d2b | 156 | int this_cpu, cpu; |
b2a68c23 | 157 | |
a73de293 | 158 | cpus_read_lock(); |
4df29d2b AG |
159 | this_cpu = get_cpu(); |
160 | if (addr >= sizeof(struct lowcore)) { | |
161 | cpu = get_swapped_owner(addr); | |
162 | if (cpu < 0) | |
163 | goto out; | |
164 | } | |
165 | bounce = (void *)__get_free_page(GFP_ATOMIC); | |
166 | if (!bounce) | |
167 | goto out; | |
168 | size = PAGE_SIZE - (addr & ~PAGE_MASK); | |
169 | if (addr < sizeof(struct lowcore)) { | |
2154e0b3 | 170 | abs_lc = get_abs_lowcore(); |
4df29d2b AG |
171 | ptr = (void *)abs_lc + addr; |
172 | memcpy(bounce, ptr, size); | |
2154e0b3 | 173 | put_abs_lowcore(abs_lc); |
4df29d2b AG |
174 | } else if (cpu == this_cpu) { |
175 | ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); | |
176 | memcpy(bounce, ptr, size); | |
177 | } else { | |
178 | memcpy(bounce, ptr, size); | |
b2a68c23 | 179 | } |
4df29d2b AG |
180 | out: |
181 | put_cpu(); | |
a73de293 | 182 | cpus_read_unlock(); |
b2a68c23 MH |
183 | return bounce; |
184 | } | |
185 | ||
186 | /* | |
187 | * Free converted buffer for /dev/mem access (if necessary) | |
188 | */ | |
1f231e29 | 189 | void unxlate_dev_mem_ptr(phys_addr_t addr, void *ptr) |
b2a68c23 | 190 | { |
1f231e29 AG |
191 | if (addr != virt_to_phys(ptr)) |
192 | free_page((unsigned long)ptr); | |
b2a68c23 | 193 | } |