Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
c33fa9f5 | 2 | /* |
3f0acb1e | 3 | * Access kernel or user memory without faulting. |
c33fa9f5 | 4 | */ |
b95f1b31 | 5 | #include <linux/export.h> |
c33fa9f5 | 6 | #include <linux/mm.h> |
7c7fcf76 | 7 | #include <linux/uaccess.h> |
c33fa9f5 | 8 | |
fe557319 CH |
9 | bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, |
10 | size_t size) | |
eab0c608 CH |
11 | { |
12 | return true; | |
13 | } | |
14 | ||
b58294ea CH |
15 | #ifdef HAVE_GET_KERNEL_NOFAULT |
16 | ||
fe557319 | 17 | #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ |
b58294ea CH |
18 | while (len >= sizeof(type)) { \ |
19 | __get_kernel_nofault(dst, src, type, err_label); \ | |
20 | dst += sizeof(type); \ | |
21 | src += sizeof(type); \ | |
22 | len -= sizeof(type); \ | |
23 | } | |
24 | ||
fe557319 | 25 | long copy_from_kernel_nofault(void *dst, const void *src, size_t size) |
b58294ea | 26 | { |
2423de2e AB |
27 | unsigned long align = 0; |
28 | ||
29 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) | |
30 | align = (unsigned long)dst | (unsigned long)src; | |
31 | ||
fe557319 | 32 | if (!copy_from_kernel_nofault_allowed(src, size)) |
2a71e81d | 33 | return -ERANGE; |
b58294ea CH |
34 | |
35 | pagefault_disable(); | |
2423de2e AB |
36 | if (!(align & 7)) |
37 | copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); | |
38 | if (!(align & 3)) | |
39 | copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); | |
40 | if (!(align & 1)) | |
41 | copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); | |
fe557319 | 42 | copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); |
b58294ea CH |
43 | pagefault_enable(); |
44 | return 0; | |
45 | Efault: | |
46 | pagefault_enable(); | |
47 | return -EFAULT; | |
48 | } | |
fe557319 | 49 | EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); |
b58294ea | 50 | |
fe557319 | 51 | #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ |
b58294ea CH |
52 | while (len >= sizeof(type)) { \ |
53 | __put_kernel_nofault(dst, src, type, err_label); \ | |
54 | dst += sizeof(type); \ | |
55 | src += sizeof(type); \ | |
56 | len -= sizeof(type); \ | |
57 | } | |
58 | ||
fe557319 | 59 | long copy_to_kernel_nofault(void *dst, const void *src, size_t size) |
b58294ea | 60 | { |
2423de2e AB |
61 | unsigned long align = 0; |
62 | ||
63 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) | |
64 | align = (unsigned long)dst | (unsigned long)src; | |
65 | ||
b58294ea | 66 | pagefault_disable(); |
2423de2e AB |
67 | if (!(align & 7)) |
68 | copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); | |
69 | if (!(align & 3)) | |
70 | copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); | |
71 | if (!(align & 1)) | |
72 | copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); | |
fe557319 | 73 | copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); |
b58294ea CH |
74 | pagefault_enable(); |
75 | return 0; | |
76 | Efault: | |
77 | pagefault_enable(); | |
78 | return -EFAULT; | |
79 | } | |
80 | ||
81 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) | |
82 | { | |
83 | const void *src = unsafe_addr; | |
84 | ||
85 | if (unlikely(count <= 0)) | |
86 | return 0; | |
fe557319 | 87 | if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) |
2a71e81d | 88 | return -ERANGE; |
b58294ea CH |
89 | |
90 | pagefault_disable(); | |
91 | do { | |
92 | __get_kernel_nofault(dst, src, u8, Efault); | |
93 | dst++; | |
94 | src++; | |
95 | } while (dst[-1] && src - unsafe_addr < count); | |
96 | pagefault_enable(); | |
97 | ||
98 | dst[-1] = '\0'; | |
99 | return src - unsafe_addr; | |
100 | Efault: | |
101 | pagefault_enable(); | |
102 | dst[-1] = '\0'; | |
103 | return -EFAULT; | |
104 | } | |
105 | #else /* HAVE_GET_KERNEL_NOFAULT */ | |
c33fa9f5 | 106 | /** |
fe557319 | 107 | * copy_from_kernel_nofault(): safely attempt to read from kernel-space |
4f6de12b CH |
108 | * @dst: pointer to the buffer that shall take the data |
109 | * @src: address to read from | |
110 | * @size: size of the data chunk | |
111 | * | |
112 | * Safely read from kernel address @src to the buffer at @dst. If a kernel | |
2a71e81d CH |
113 | * fault happens, handle that and return -EFAULT. If @src is not a valid kernel |
114 | * address, return -ERANGE. | |
0ab32b6f AM |
115 | * |
116 | * We ensure that the copy_from_user is executed in atomic context so that | |
c1e8d7c6 | 117 | * do_page_fault() doesn't attempt to take mmap_lock. This makes |
fe557319 | 118 | * copy_from_kernel_nofault() suitable for use within regions where the caller |
c1e8d7c6 | 119 | * already holds mmap_lock, or other locks which nest inside mmap_lock. |
c33fa9f5 | 120 | */ |
fe557319 | 121 | long copy_from_kernel_nofault(void *dst, const void *src, size_t size) |
c33fa9f5 IM |
122 | { |
123 | long ret; | |
b4b8ac52 | 124 | mm_segment_t old_fs = get_fs(); |
c33fa9f5 | 125 | |
fe557319 | 126 | if (!copy_from_kernel_nofault_allowed(src, size)) |
2a71e81d | 127 | return -ERANGE; |
eab0c608 | 128 | |
b4b8ac52 | 129 | set_fs(KERNEL_DS); |
cd030905 CH |
130 | pagefault_disable(); |
131 | ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, | |
132 | size); | |
133 | pagefault_enable(); | |
b4b8ac52 | 134 | set_fs(old_fs); |
c33fa9f5 | 135 | |
cd030905 CH |
136 | if (ret) |
137 | return -EFAULT; | |
138 | return 0; | |
c33fa9f5 | 139 | } |
fe557319 | 140 | EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); |
c33fa9f5 IM |
141 | |
142 | /** | |
fe557319 | 143 | * copy_to_kernel_nofault(): safely attempt to write to a location |
c33fa9f5 IM |
144 | * @dst: address to write to |
145 | * @src: pointer to the data that shall be written | |
146 | * @size: size of the data chunk | |
147 | * | |
148 | * Safely write to address @dst from the buffer at @src. If a kernel fault | |
149 | * happens, handle that and return -EFAULT. | |
150 | */ | |
fe557319 | 151 | long copy_to_kernel_nofault(void *dst, const void *src, size_t size) |
c33fa9f5 IM |
152 | { |
153 | long ret; | |
b4b8ac52 | 154 | mm_segment_t old_fs = get_fs(); |
c33fa9f5 | 155 | |
b4b8ac52 | 156 | set_fs(KERNEL_DS); |
cd030905 CH |
157 | pagefault_disable(); |
158 | ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); | |
159 | pagefault_enable(); | |
b4b8ac52 | 160 | set_fs(old_fs); |
c33fa9f5 | 161 | |
cd030905 CH |
162 | if (ret) |
163 | return -EFAULT; | |
164 | return 0; | |
c33fa9f5 | 165 | } |
dbb7ee0e | 166 | |
4f6de12b | 167 | /** |
c4cb1644 | 168 | * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe |
4f6de12b CH |
169 | * address. |
170 | * @dst: Destination address, in kernel space. This buffer must be at | |
171 | * least @count bytes long. | |
172 | * @unsafe_addr: Unsafe address. | |
173 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
174 | * | |
175 | * Copies a NUL-terminated string from unsafe address to kernel buffer. | |
176 | * | |
177 | * On success, returns the length of the string INCLUDING the trailing NUL. | |
178 | * | |
2a71e81d CH |
179 | * If access fails, returns -EFAULT (some data may have been copied and the |
180 | * trailing NUL added). If @unsafe_addr is not a valid kernel address, return | |
181 | * -ERANGE. | |
4f6de12b CH |
182 | * |
183 | * If @count is smaller than the length of the string, copies @count-1 bytes, | |
184 | * sets the last byte of @dst buffer to NUL and returns @count. | |
185 | */ | |
eab0c608 | 186 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) |
dbb7ee0e AS |
187 | { |
188 | mm_segment_t old_fs = get_fs(); | |
189 | const void *src = unsafe_addr; | |
190 | long ret; | |
191 | ||
192 | if (unlikely(count <= 0)) | |
193 | return 0; | |
fe557319 | 194 | if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) |
2a71e81d | 195 | return -ERANGE; |
dbb7ee0e AS |
196 | |
197 | set_fs(KERNEL_DS); | |
198 | pagefault_disable(); | |
199 | ||
200 | do { | |
bd28b145 | 201 | ret = __get_user(*dst++, (const char __user __force *)src++); |
dbb7ee0e AS |
202 | } while (dst[-1] && ret == 0 && src - unsafe_addr < count); |
203 | ||
204 | dst[-1] = '\0'; | |
205 | pagefault_enable(); | |
206 | set_fs(old_fs); | |
207 | ||
9dd861d5 | 208 | return ret ? -EFAULT : src - unsafe_addr; |
dbb7ee0e | 209 | } |
b58294ea | 210 | #endif /* HAVE_GET_KERNEL_NOFAULT */ |
3d708182 | 211 | |
fc3562d7 | 212 | /** |
c0ee37e8 | 213 | * copy_from_user_nofault(): safely attempt to read from a user-space location |
fc3562d7 CH |
214 | * @dst: pointer to the buffer that shall take the data |
215 | * @src: address to read from. This must be a user address. | |
216 | * @size: size of the data chunk | |
217 | * | |
218 | * Safely read from user address @src to the buffer at @dst. If a kernel fault | |
219 | * happens, handle that and return -EFAULT. | |
220 | */ | |
c0ee37e8 | 221 | long copy_from_user_nofault(void *dst, const void __user *src, size_t size) |
fc3562d7 CH |
222 | { |
223 | long ret = -EFAULT; | |
3d13f313 | 224 | mm_segment_t old_fs = force_uaccess_begin(); |
fc3562d7 | 225 | |
fc3562d7 CH |
226 | if (access_ok(src, size)) { |
227 | pagefault_disable(); | |
228 | ret = __copy_from_user_inatomic(dst, src, size); | |
229 | pagefault_enable(); | |
230 | } | |
3d13f313 | 231 | force_uaccess_end(old_fs); |
fc3562d7 CH |
232 | |
233 | if (ret) | |
234 | return -EFAULT; | |
235 | return 0; | |
236 | } | |
c0ee37e8 | 237 | EXPORT_SYMBOL_GPL(copy_from_user_nofault); |
fc3562d7 CH |
238 | |
239 | /** | |
c0ee37e8 | 240 | * copy_to_user_nofault(): safely attempt to write to a user-space location |
fc3562d7 CH |
241 | * @dst: address to write to |
242 | * @src: pointer to the data that shall be written | |
243 | * @size: size of the data chunk | |
244 | * | |
245 | * Safely write to address @dst from the buffer at @src. If a kernel fault | |
246 | * happens, handle that and return -EFAULT. | |
247 | */ | |
c0ee37e8 | 248 | long copy_to_user_nofault(void __user *dst, const void *src, size_t size) |
fc3562d7 CH |
249 | { |
250 | long ret = -EFAULT; | |
3d13f313 | 251 | mm_segment_t old_fs = force_uaccess_begin(); |
fc3562d7 | 252 | |
fc3562d7 CH |
253 | if (access_ok(dst, size)) { |
254 | pagefault_disable(); | |
255 | ret = __copy_to_user_inatomic(dst, src, size); | |
256 | pagefault_enable(); | |
257 | } | |
3d13f313 | 258 | force_uaccess_end(old_fs); |
fc3562d7 CH |
259 | |
260 | if (ret) | |
261 | return -EFAULT; | |
262 | return 0; | |
263 | } | |
c0ee37e8 | 264 | EXPORT_SYMBOL_GPL(copy_to_user_nofault); |
fc3562d7 | 265 | |
3d708182 | 266 | /** |
bd88bb5d | 267 | * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user |
3d708182 MH |
268 | * address. |
269 | * @dst: Destination address, in kernel space. This buffer must be at | |
270 | * least @count bytes long. | |
271 | * @unsafe_addr: Unsafe user address. | |
272 | * @count: Maximum number of bytes to copy, including the trailing NUL. | |
273 | * | |
274 | * Copies a NUL-terminated string from unsafe user address to kernel buffer. | |
275 | * | |
276 | * On success, returns the length of the string INCLUDING the trailing NUL. | |
277 | * | |
278 | * If access fails, returns -EFAULT (some data may have been copied | |
279 | * and the trailing NUL added). | |
280 | * | |
281 | * If @count is smaller than the length of the string, copies @count-1 bytes, | |
282 | * sets the last byte of @dst buffer to NUL and returns @count. | |
283 | */ | |
bd88bb5d | 284 | long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, |
3d708182 MH |
285 | long count) |
286 | { | |
3d13f313 | 287 | mm_segment_t old_fs; |
3d708182 MH |
288 | long ret; |
289 | ||
290 | if (unlikely(count <= 0)) | |
291 | return 0; | |
292 | ||
3d13f313 | 293 | old_fs = force_uaccess_begin(); |
3d708182 MH |
294 | pagefault_disable(); |
295 | ret = strncpy_from_user(dst, unsafe_addr, count); | |
296 | pagefault_enable(); | |
3d13f313 | 297 | force_uaccess_end(old_fs); |
3d708182 MH |
298 | |
299 | if (ret >= count) { | |
300 | ret = count; | |
301 | dst[ret - 1] = '\0'; | |
302 | } else if (ret > 0) { | |
303 | ret++; | |
304 | } | |
305 | ||
306 | return ret; | |
307 | } | |
308 | ||
309 | /** | |
02dddb16 | 310 | * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. |
3d708182 MH |
311 | * @unsafe_addr: The string to measure. |
312 | * @count: Maximum count (including NUL) | |
313 | * | |
314 | * Get the size of a NUL-terminated string in user space without pagefault. | |
315 | * | |
316 | * Returns the size of the string INCLUDING the terminating NUL. | |
317 | * | |
318 | * If the string is too long, returns a number larger than @count. User | |
319 | * has to check the return value against "> count". | |
320 | * On exception (or invalid count), returns 0. | |
321 | * | |
322 | * Unlike strnlen_user, this can be used from IRQ handler etc. because | |
323 | * it disables pagefaults. | |
324 | */ | |
02dddb16 | 325 | long strnlen_user_nofault(const void __user *unsafe_addr, long count) |
3d708182 | 326 | { |
3d13f313 | 327 | mm_segment_t old_fs; |
3d708182 MH |
328 | int ret; |
329 | ||
3d13f313 | 330 | old_fs = force_uaccess_begin(); |
3d708182 MH |
331 | pagefault_disable(); |
332 | ret = strnlen_user(unsafe_addr, count); | |
333 | pagefault_enable(); | |
3d13f313 | 334 | force_uaccess_end(old_fs); |
3d708182 MH |
335 | |
336 | return ret; | |
337 | } |