| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | #ifndef __LINUX_UACCESS_H__ |
| 3 | #define __LINUX_UACCESS_H__ |
| 4 | |
| 5 | #include <linux/fault-inject-usercopy.h> |
| 6 | #include <linux/instrumented.h> |
| 7 | #include <linux/minmax.h> |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/thread_info.h> |
| 10 | |
| 11 | #include <asm/uaccess.h> |
| 12 | |
| 13 | /* |
| 14 | * Architectures should provide two primitives (raw_copy_{to,from}_user()) |
| 15 | * and get rid of their private instances of copy_{to,from}_user() and |
| 16 | * __copy_{to,from}_user{,_inatomic}(). |
| 17 | * |
| 18 | * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and |
| 19 | * return the amount left to copy. They should assume that access_ok() has |
| 20 | * already been checked (and succeeded); they should *not* zero-pad anything. |
| 21 | * No KASAN or object size checks either - those belong here. |
| 22 | * |
| 23 | * Both of these functions should attempt to copy size bytes starting at from |
| 24 | * into the area starting at to. They must not fetch or store anything |
| 25 | * outside of those areas. Return value must be between 0 (everything |
| 26 | * copied successfully) and size (nothing copied). |
| 27 | * |
| 28 | * If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting |
| 29 | * at to must become equal to the bytes fetched from the corresponding area |
| 30 | * starting at from. All data past to + size - N must be left unmodified. |
| 31 | * |
| 32 | * If copying succeeds, the return value must be 0. If some data cannot be |
| 33 | * fetched, it is permitted to copy less than had been fetched; the only |
| 34 | * hard requirement is that not storing anything at all (i.e. returning size) |
| 35 | * should happen only when nothing could be copied. In other words, you don't |
| 36 | * have to squeeze as much as possible - it is allowed, but not necessary. |
| 37 | * |
| 38 | * For raw_copy_from_user() to always points to kernel memory and no faults |
| 39 | * on store should happen. Interpretation of from is affected by set_fs(). |
| 40 | * For raw_copy_to_user() it's the other way round. |
| 41 | * |
| 42 | * Both can be inlined - it's up to architectures whether it wants to bother |
| 43 | * with that. They should not be used directly; they are used to implement |
| 44 | * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) |
| 45 | * that are used instead. Out of those, __... ones are inlined. Plain |
| 46 | * copy_{to,from}_user() might or might not be inlined. If you want them |
| 47 | * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER. |
| 48 | * |
| 49 | * NOTE: only copy_from_user() zero-pads the destination in case of short copy. |
| 50 | * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything |
| 51 | * at all; their callers absolutely must check the return value. |
| 52 | * |
| 53 | * Biarch ones should also provide raw_copy_in_user() - similar to the above, |
| 54 | * but both source and destination are __user pointers (affected by set_fs() |
| 55 | * as usual) and both source and destination can trigger faults. |
| 56 | */ |
| 57 | |
| 58 | static __always_inline __must_check unsigned long |
| 59 | __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) |
| 60 | { |
| 61 | instrument_copy_from_user(to, from, n); |
| 62 | check_object_size(to, n, false); |
| 63 | return raw_copy_from_user(to, from, n); |
| 64 | } |
| 65 | |
| 66 | static __always_inline __must_check unsigned long |
| 67 | __copy_from_user(void *to, const void __user *from, unsigned long n) |
| 68 | { |
| 69 | might_fault(); |
| 70 | if (should_fail_usercopy()) |
| 71 | return n; |
| 72 | instrument_copy_from_user(to, from, n); |
| 73 | check_object_size(to, n, false); |
| 74 | return raw_copy_from_user(to, from, n); |
| 75 | } |
| 76 | |
| 77 | /** |
| 78 | * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. |
| 79 | * @to: Destination address, in user space. |
| 80 | * @from: Source address, in kernel space. |
| 81 | * @n: Number of bytes to copy. |
| 82 | * |
| 83 | * Context: User context only. |
| 84 | * |
| 85 | * Copy data from kernel space to user space. Caller must check |
| 86 | * the specified block with access_ok() before calling this function. |
| 87 | * The caller should also make sure he pins the user space address |
| 88 | * so that we don't result in page fault and sleep. |
| 89 | */ |
| 90 | static __always_inline __must_check unsigned long |
| 91 | __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) |
| 92 | { |
| 93 | if (should_fail_usercopy()) |
| 94 | return n; |
| 95 | instrument_copy_to_user(to, from, n); |
| 96 | check_object_size(from, n, true); |
| 97 | return raw_copy_to_user(to, from, n); |
| 98 | } |
| 99 | |
| 100 | static __always_inline __must_check unsigned long |
| 101 | __copy_to_user(void __user *to, const void *from, unsigned long n) |
| 102 | { |
| 103 | might_fault(); |
| 104 | if (should_fail_usercopy()) |
| 105 | return n; |
| 106 | instrument_copy_to_user(to, from, n); |
| 107 | check_object_size(from, n, true); |
| 108 | return raw_copy_to_user(to, from, n); |
| 109 | } |
| 110 | |
| 111 | #ifdef INLINE_COPY_FROM_USER |
| 112 | static inline __must_check unsigned long |
| 113 | _copy_from_user(void *to, const void __user *from, unsigned long n) |
| 114 | { |
| 115 | unsigned long res = n; |
| 116 | might_fault(); |
| 117 | if (!should_fail_usercopy() && likely(access_ok(from, n))) { |
| 118 | instrument_copy_from_user(to, from, n); |
| 119 | res = raw_copy_from_user(to, from, n); |
| 120 | } |
| 121 | if (unlikely(res)) |
| 122 | memset(to + (n - res), 0, res); |
| 123 | return res; |
| 124 | } |
| 125 | #else |
| 126 | extern __must_check unsigned long |
| 127 | _copy_from_user(void *, const void __user *, unsigned long); |
| 128 | #endif |
| 129 | |
| 130 | #ifdef INLINE_COPY_TO_USER |
| 131 | static inline __must_check unsigned long |
| 132 | _copy_to_user(void __user *to, const void *from, unsigned long n) |
| 133 | { |
| 134 | might_fault(); |
| 135 | if (should_fail_usercopy()) |
| 136 | return n; |
| 137 | if (access_ok(to, n)) { |
| 138 | instrument_copy_to_user(to, from, n); |
| 139 | n = raw_copy_to_user(to, from, n); |
| 140 | } |
| 141 | return n; |
| 142 | } |
| 143 | #else |
| 144 | extern __must_check unsigned long |
| 145 | _copy_to_user(void __user *, const void *, unsigned long); |
| 146 | #endif |
| 147 | |
| 148 | static __always_inline unsigned long __must_check |
| 149 | copy_from_user(void *to, const void __user *from, unsigned long n) |
| 150 | { |
| 151 | if (likely(check_copy_size(to, n, false))) |
| 152 | n = _copy_from_user(to, from, n); |
| 153 | return n; |
| 154 | } |
| 155 | |
| 156 | static __always_inline unsigned long __must_check |
| 157 | copy_to_user(void __user *to, const void *from, unsigned long n) |
| 158 | { |
| 159 | if (likely(check_copy_size(from, n, true))) |
| 160 | n = _copy_to_user(to, from, n); |
| 161 | return n; |
| 162 | } |
| 163 | |
| 164 | #ifndef copy_mc_to_kernel |
| 165 | /* |
| 166 | * Without arch opt-in this generic copy_mc_to_kernel() will not handle |
| 167 | * #MC (or arch equivalent) during source read. |
| 168 | */ |
| 169 | static inline unsigned long __must_check |
| 170 | copy_mc_to_kernel(void *dst, const void *src, size_t cnt) |
| 171 | { |
| 172 | memcpy(dst, src, cnt); |
| 173 | return 0; |
| 174 | } |
| 175 | #endif |
| 176 | |
| 177 | static __always_inline void pagefault_disabled_inc(void) |
| 178 | { |
| 179 | current->pagefault_disabled++; |
| 180 | } |
| 181 | |
| 182 | static __always_inline void pagefault_disabled_dec(void) |
| 183 | { |
| 184 | current->pagefault_disabled--; |
| 185 | } |
| 186 | |
| 187 | /* |
| 188 | * These routines enable/disable the pagefault handler. If disabled, it will |
| 189 | * not take any locks and go straight to the fixup table. |
| 190 | * |
| 191 | * User access methods will not sleep when called from a pagefault_disabled() |
| 192 | * environment. |
| 193 | */ |
| 194 | static inline void pagefault_disable(void) |
| 195 | { |
| 196 | pagefault_disabled_inc(); |
| 197 | /* |
| 198 | * make sure to have issued the store before a pagefault |
| 199 | * can hit. |
| 200 | */ |
| 201 | barrier(); |
| 202 | } |
| 203 | |
| 204 | static inline void pagefault_enable(void) |
| 205 | { |
| 206 | /* |
| 207 | * make sure to issue those last loads/stores before enabling |
| 208 | * the pagefault handler again. |
| 209 | */ |
| 210 | barrier(); |
| 211 | pagefault_disabled_dec(); |
| 212 | } |
| 213 | |
| 214 | /* |
| 215 | * Is the pagefault handler disabled? If so, user access methods will not sleep. |
| 216 | */ |
| 217 | static inline bool pagefault_disabled(void) |
| 218 | { |
| 219 | return current->pagefault_disabled != 0; |
| 220 | } |
| 221 | |
| 222 | /* |
| 223 | * The pagefault handler is in general disabled by pagefault_disable() or |
| 224 | * when in irq context (via in_atomic()). |
| 225 | * |
| 226 | * This function should only be used by the fault handlers. Other users should |
| 227 | * stick to pagefault_disabled(). |
| 228 | * Please NEVER use preempt_disable() to disable the fault handler. With |
| 229 | * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. |
| 230 | * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. |
| 231 | */ |
| 232 | #define faulthandler_disabled() (pagefault_disabled() || in_atomic()) |
| 233 | |
| 234 | #ifndef ARCH_HAS_NOCACHE_UACCESS |
| 235 | |
| 236 | static inline __must_check unsigned long |
| 237 | __copy_from_user_inatomic_nocache(void *to, const void __user *from, |
| 238 | unsigned long n) |
| 239 | { |
| 240 | return __copy_from_user_inatomic(to, from, n); |
| 241 | } |
| 242 | |
| 243 | #endif /* ARCH_HAS_NOCACHE_UACCESS */ |
| 244 | |
| 245 | extern __must_check int check_zeroed_user(const void __user *from, size_t size); |
| 246 | |
| 247 | /** |
| 248 | * copy_struct_from_user: copy a struct from userspace |
| 249 | * @dst: Destination address, in kernel space. This buffer must be @ksize |
| 250 | * bytes long. |
| 251 | * @ksize: Size of @dst struct. |
| 252 | * @src: Source address, in userspace. |
| 253 | * @usize: (Alleged) size of @src struct. |
| 254 | * |
| 255 | * Copies a struct from userspace to kernel space, in a way that guarantees |
| 256 | * backwards-compatibility for struct syscall arguments (as long as future |
| 257 | * struct extensions are made such that all new fields are *appended* to the |
| 258 | * old struct, and zeroed-out new fields have the same meaning as the old |
| 259 | * struct). |
| 260 | * |
| 261 | * @ksize is just sizeof(*dst), and @usize should've been passed by userspace. |
| 262 | * The recommended usage is something like the following: |
| 263 | * |
| 264 | * SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize) |
| 265 | * { |
| 266 | * int err; |
| 267 | * struct foo karg = {}; |
| 268 | * |
| 269 | * if (usize > PAGE_SIZE) |
| 270 | * return -E2BIG; |
| 271 | * if (usize < FOO_SIZE_VER0) |
| 272 | * return -EINVAL; |
| 273 | * |
| 274 | * err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); |
| 275 | * if (err) |
| 276 | * return err; |
| 277 | * |
| 278 | * // ... |
| 279 | * } |
| 280 | * |
| 281 | * There are three cases to consider: |
| 282 | * * If @usize == @ksize, then it's copied verbatim. |
| 283 | * * If @usize < @ksize, then the userspace has passed an old struct to a |
| 284 | * newer kernel. The rest of the trailing bytes in @dst (@ksize - @usize) |
| 285 | * are to be zero-filled. |
| 286 | * * If @usize > @ksize, then the userspace has passed a new struct to an |
| 287 | * older kernel. The trailing bytes unknown to the kernel (@usize - @ksize) |
| 288 | * are checked to ensure they are zeroed, otherwise -E2BIG is returned. |
| 289 | * |
| 290 | * Returns (in all cases, some data may have been copied): |
| 291 | * * -E2BIG: (@usize > @ksize) and there are non-zero trailing bytes in @src. |
| 292 | * * -EFAULT: access to userspace failed. |
| 293 | */ |
| 294 | static __always_inline __must_check int |
| 295 | copy_struct_from_user(void *dst, size_t ksize, const void __user *src, |
| 296 | size_t usize) |
| 297 | { |
| 298 | size_t size = min(ksize, usize); |
| 299 | size_t rest = max(ksize, usize) - size; |
| 300 | |
| 301 | /* Deal with trailing bytes. */ |
| 302 | if (usize < ksize) { |
| 303 | memset(dst + size, 0, rest); |
| 304 | } else if (usize > ksize) { |
| 305 | int ret = check_zeroed_user(src + size, rest); |
| 306 | if (ret <= 0) |
| 307 | return ret ?: -E2BIG; |
| 308 | } |
| 309 | /* Copy the interoperable parts of the struct. */ |
| 310 | if (copy_from_user(dst, src, size)) |
| 311 | return -EFAULT; |
| 312 | return 0; |
| 313 | } |
| 314 | |
| 315 | bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size); |
| 316 | |
| 317 | long copy_from_kernel_nofault(void *dst, const void *src, size_t size); |
| 318 | long notrace copy_to_kernel_nofault(void *dst, const void *src, size_t size); |
| 319 | |
| 320 | long copy_from_user_nofault(void *dst, const void __user *src, size_t size); |
| 321 | long notrace copy_to_user_nofault(void __user *dst, const void *src, |
| 322 | size_t size); |
| 323 | |
| 324 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, |
| 325 | long count); |
| 326 | |
| 327 | long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, |
| 328 | long count); |
| 329 | long strnlen_user_nofault(const void __user *unsafe_addr, long count); |
| 330 | |
| 331 | #ifndef __get_kernel_nofault |
| 332 | #define __get_kernel_nofault(dst, src, type, label) \ |
| 333 | do { \ |
| 334 | type __user *p = (type __force __user *)(src); \ |
| 335 | type data; \ |
| 336 | if (__get_user(data, p)) \ |
| 337 | goto label; \ |
| 338 | *(type *)dst = data; \ |
| 339 | } while (0) |
| 340 | |
| 341 | #define __put_kernel_nofault(dst, src, type, label) \ |
| 342 | do { \ |
| 343 | type __user *p = (type __force __user *)(dst); \ |
| 344 | type data = *(type *)src; \ |
| 345 | if (__put_user(data, p)) \ |
| 346 | goto label; \ |
| 347 | } while (0) |
| 348 | #endif |
| 349 | |
| 350 | /** |
| 351 | * get_kernel_nofault(): safely attempt to read from a location |
| 352 | * @val: read into this variable |
| 353 | * @ptr: address to read from |
| 354 | * |
| 355 | * Returns 0 on success, or -EFAULT. |
| 356 | */ |
| 357 | #define get_kernel_nofault(val, ptr) ({ \ |
| 358 | const typeof(val) *__gk_ptr = (ptr); \ |
| 359 | copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\ |
| 360 | }) |
| 361 | |
| 362 | #ifndef user_access_begin |
| 363 | #define user_access_begin(ptr,len) access_ok(ptr, len) |
| 364 | #define user_access_end() do { } while (0) |
| 365 | #define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0) |
| 366 | #define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e) |
| 367 | #define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e) |
| 368 | #define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e) |
| 369 | #define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e) |
| 370 | static inline unsigned long user_access_save(void) { return 0UL; } |
| 371 | static inline void user_access_restore(unsigned long flags) { } |
| 372 | #endif |
| 373 | #ifndef user_write_access_begin |
| 374 | #define user_write_access_begin user_access_begin |
| 375 | #define user_write_access_end user_access_end |
| 376 | #endif |
| 377 | #ifndef user_read_access_begin |
| 378 | #define user_read_access_begin user_access_begin |
| 379 | #define user_read_access_end user_access_end |
| 380 | #endif |
| 381 | |
| 382 | #ifdef CONFIG_HARDENED_USERCOPY |
| 383 | void __noreturn usercopy_abort(const char *name, const char *detail, |
| 384 | bool to_user, unsigned long offset, |
| 385 | unsigned long len); |
| 386 | #endif |
| 387 | |
| 388 | #endif /* __LINUX_UACCESS_H__ */ |