2 * mpx.c - Memory Protection eXtensions
4 * Copyright (c) 2014, Intel Corporation.
5 * Qiaowei Ren <qiaowei.ren@intel.com>
6 * Dave Hansen <dave.hansen@intel.com>
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/mm_types.h>
11 #include <linux/syscalls.h>
12 #include <linux/sched/sysctl.h>
16 #include <asm/mmu_context.h>
18 #include <asm/processor.h>
19 #include <asm/fpu/internal.h>
21 #define CREATE_TRACE_POINTS
22 #include <asm/trace/mpx.h>
24 static inline unsigned long mpx_bd_size_bytes(struct mm_struct *mm)
27 return MPX_BD_SIZE_BYTES_64;
29 return MPX_BD_SIZE_BYTES_32;
32 static inline unsigned long mpx_bt_size_bytes(struct mm_struct *mm)
35 return MPX_BT_SIZE_BYTES_64;
37 return MPX_BT_SIZE_BYTES_32;
41 * This is really a simplified "vm_mmap". it only handles MPX
42 * bounds tables (the bounds directory is user-allocated).
44 static unsigned long mpx_mmap(unsigned long len)
46 struct mm_struct *mm = current->mm;
47 unsigned long addr, populate;
49 /* Only bounds table can be allocated here */
50 if (len != mpx_bt_size_bytes(mm))
53 down_write(&mm->mmap_sem);
54 addr = do_mmap(NULL, 0, len, PROT_READ | PROT_WRITE,
55 MAP_ANONYMOUS | MAP_PRIVATE, VM_MPX, 0, &populate, NULL);
56 up_write(&mm->mmap_sem);
58 mm_populate(addr, populate);
69 static int get_reg_offset(struct insn *insn, struct pt_regs *regs,
74 static const int regoff[] = {
75 offsetof(struct pt_regs, ax),
76 offsetof(struct pt_regs, cx),
77 offsetof(struct pt_regs, dx),
78 offsetof(struct pt_regs, bx),
79 offsetof(struct pt_regs, sp),
80 offsetof(struct pt_regs, bp),
81 offsetof(struct pt_regs, si),
82 offsetof(struct pt_regs, di),
84 offsetof(struct pt_regs, r8),
85 offsetof(struct pt_regs, r9),
86 offsetof(struct pt_regs, r10),
87 offsetof(struct pt_regs, r11),
88 offsetof(struct pt_regs, r12),
89 offsetof(struct pt_regs, r13),
90 offsetof(struct pt_regs, r14),
91 offsetof(struct pt_regs, r15),
94 int nr_registers = ARRAY_SIZE(regoff);
96 * Don't possibly decode a 32-bit instructions as
97 * reading a 64-bit-only register.
99 if (IS_ENABLED(CONFIG_X86_64) && !insn->x86_64)
104 regno = X86_MODRM_RM(insn->modrm.value);
105 if (X86_REX_B(insn->rex_prefix.value))
110 regno = X86_SIB_INDEX(insn->sib.value);
111 if (X86_REX_X(insn->rex_prefix.value))
116 regno = X86_SIB_BASE(insn->sib.value);
117 if (X86_REX_B(insn->rex_prefix.value))
122 pr_err("invalid register type");
127 if (regno >= nr_registers) {
128 WARN_ONCE(1, "decoded an instruction with an invalid register");
131 return regoff[regno];
135 * return the address being referenced be instruction
136 * for rm=3 returning the content of the rm reg
137 * for rm!=3 calculates the address using SIB and Disp
139 static void __user *mpx_get_addr_ref(struct insn *insn, struct pt_regs *regs)
141 unsigned long addr, base, indx;
142 int addr_offset, base_offset, indx_offset;
145 insn_get_modrm(insn);
147 sib = insn->sib.value;
149 if (X86_MODRM_MOD(insn->modrm.value) == 3) {
150 addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM);
153 addr = regs_get_register(regs, addr_offset);
155 if (insn->sib.nbytes) {
156 base_offset = get_reg_offset(insn, regs, REG_TYPE_BASE);
160 indx_offset = get_reg_offset(insn, regs, REG_TYPE_INDEX);
164 base = regs_get_register(regs, base_offset);
165 indx = regs_get_register(regs, indx_offset);
166 addr = base + indx * (1 << X86_SIB_SCALE(sib));
168 addr_offset = get_reg_offset(insn, regs, REG_TYPE_RM);
171 addr = regs_get_register(regs, addr_offset);
173 addr += insn->displacement.value;
175 return (void __user *)addr;
177 return (void __user *)-1;
180 static int mpx_insn_decode(struct insn *insn,
181 struct pt_regs *regs)
183 unsigned char buf[MAX_INSN_SIZE];
184 int x86_64 = !test_thread_flag(TIF_IA32);
188 not_copied = copy_from_user(buf, (void __user *)regs->ip, sizeof(buf));
189 nr_copied = sizeof(buf) - not_copied;
191 * The decoder _should_ fail nicely if we pass it a short buffer.
192 * But, let's not depend on that implementation detail. If we
193 * did not get anything, just error out now.
197 insn_init(insn, buf, nr_copied, x86_64);
198 insn_get_length(insn);
200 * copy_from_user() tries to get as many bytes as we could see in
201 * the largest possible instruction. If the instruction we are
202 * after is shorter than that _and_ we attempt to copy from
203 * something unreadable, we might get a short read. This is OK
204 * as long as the read did not stop in the middle of the
205 * instruction. Check to see if we got a partial instruction.
207 if (nr_copied < insn->length)
210 insn_get_opcode(insn);
212 * We only _really_ need to decode bndcl/bndcn/bndcu
213 * Error out on anything else.
215 if (insn->opcode.bytes[0] != 0x0f)
217 if ((insn->opcode.bytes[1] != 0x1a) &&
218 (insn->opcode.bytes[1] != 0x1b))
227 * If a bounds overflow occurs then a #BR is generated. This
228 * function decodes MPX instructions to get violation address
229 * and set this address into extended struct siginfo.
231 * Note that this is not a super precise way of doing this.
232 * Userspace could have, by the time we get here, written
233 * anything it wants in to the instructions. We can not
234 * trust anything about it. They might not be valid
235 * instructions or might encode invalid registers, etc...
237 * The caller is expected to kfree() the returned siginfo_t.
239 siginfo_t *mpx_generate_siginfo(struct pt_regs *regs)
241 const struct mpx_bndreg_state *bndregs;
242 const struct mpx_bndreg *bndreg;
243 siginfo_t *info = NULL;
248 err = mpx_insn_decode(&insn, regs);
253 * We know at this point that we are only dealing with
256 insn_get_modrm(&insn);
257 bndregno = X86_MODRM_REG(insn.modrm.value);
262 /* get bndregs field from current task's xsave area */
263 bndregs = get_xsave_field_ptr(XFEATURE_MASK_BNDREGS);
268 /* now go select the individual register in the set of 4 */
269 bndreg = &bndregs->bndreg[bndregno];
271 info = kzalloc(sizeof(*info), GFP_KERNEL);
277 * The registers are always 64-bit, but the upper 32
278 * bits are ignored in 32-bit mode. Also, note that the
279 * upper bounds are architecturally represented in 1's
282 * The 'unsigned long' cast is because the compiler
283 * complains when casting from integers to different-size
286 info->si_lower = (void __user *)(unsigned long)bndreg->lower_bound;
287 info->si_upper = (void __user *)(unsigned long)~bndreg->upper_bound;
288 info->si_addr_lsb = 0;
289 info->si_signo = SIGSEGV;
291 info->si_code = SEGV_BNDERR;
292 info->si_addr = mpx_get_addr_ref(&insn, regs);
294 * We were not able to extract an address from the instruction,
295 * probably because there was something invalid in it.
297 if (info->si_addr == (void __user *)-1) {
301 trace_mpx_bounds_register_exception(info->si_addr, bndreg);
304 /* info might be NULL, but kfree() handles that */
309 static __user void *mpx_get_bounds_dir(void)
311 const struct mpx_bndcsr *bndcsr;
313 if (!cpu_feature_enabled(X86_FEATURE_MPX))
314 return MPX_INVALID_BOUNDS_DIR;
317 * The bounds directory pointer is stored in a register
318 * only accessible if we first do an xsave.
320 bndcsr = get_xsave_field_ptr(XFEATURE_MASK_BNDCSR);
322 return MPX_INVALID_BOUNDS_DIR;
325 * Make sure the register looks valid by checking the
328 if (!(bndcsr->bndcfgu & MPX_BNDCFG_ENABLE_FLAG))
329 return MPX_INVALID_BOUNDS_DIR;
332 * Lastly, mask off the low bits used for configuration
333 * flags, and return the address of the bounds table.
335 return (void __user *)(unsigned long)
336 (bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK);
339 int mpx_enable_management(void)
341 void __user *bd_base = MPX_INVALID_BOUNDS_DIR;
342 struct mm_struct *mm = current->mm;
346 * runtime in the userspace will be responsible for allocation of
347 * the bounds directory. Then, it will save the base of the bounds
348 * directory into XSAVE/XRSTOR Save Area and enable MPX through
349 * XRSTOR instruction.
351 * The copy_xregs_to_kernel() beneath get_xsave_field_ptr() is
352 * expected to be relatively expensive. Storing the bounds
353 * directory here means that we do not have to do xsave in the
354 * unmap path; we can just use mm->context.bd_addr instead.
356 bd_base = mpx_get_bounds_dir();
357 down_write(&mm->mmap_sem);
359 /* MPX doesn't support addresses above 47 bits yet. */
360 if (find_vma(mm, DEFAULT_MAP_WINDOW)) {
361 pr_warn_once("%s (%d): MPX cannot handle addresses "
362 "above 47-bits. Disabling.",
363 current->comm, current->pid);
367 mm->context.bd_addr = bd_base;
368 if (mm->context.bd_addr == MPX_INVALID_BOUNDS_DIR)
371 up_write(&mm->mmap_sem);
375 int mpx_disable_management(void)
377 struct mm_struct *mm = current->mm;
379 if (!cpu_feature_enabled(X86_FEATURE_MPX))
382 down_write(&mm->mmap_sem);
383 mm->context.bd_addr = MPX_INVALID_BOUNDS_DIR;
384 up_write(&mm->mmap_sem);
388 static int mpx_cmpxchg_bd_entry(struct mm_struct *mm,
389 unsigned long *curval,
390 unsigned long __user *addr,
391 unsigned long old_val, unsigned long new_val)
395 * user_atomic_cmpxchg_inatomic() actually uses sizeof()
396 * the pointer that we pass to it to figure out how much
397 * data to cmpxchg. We have to be careful here not to
398 * pass a pointer to a 64-bit data type when we only want
401 if (is_64bit_mm(mm)) {
402 ret = user_atomic_cmpxchg_inatomic(curval,
403 addr, old_val, new_val);
405 u32 uninitialized_var(curval_32);
406 u32 old_val_32 = old_val;
407 u32 new_val_32 = new_val;
408 u32 __user *addr_32 = (u32 __user *)addr;
410 ret = user_atomic_cmpxchg_inatomic(&curval_32,
411 addr_32, old_val_32, new_val_32);
418 * With 32-bit mode, a bounds directory is 4MB, and the size of each
419 * bounds table is 16KB. With 64-bit mode, a bounds directory is 2GB,
420 * and the size of each bounds table is 4MB.
422 static int allocate_bt(struct mm_struct *mm, long __user *bd_entry)
424 unsigned long expected_old_val = 0;
425 unsigned long actual_old_val = 0;
426 unsigned long bt_addr;
427 unsigned long bd_new_entry;
431 * Carve the virtual space out of userspace for the new
434 bt_addr = mpx_mmap(mpx_bt_size_bytes(mm));
435 if (IS_ERR((void *)bt_addr))
436 return PTR_ERR((void *)bt_addr);
438 * Set the valid flag (kinda like _PAGE_PRESENT in a pte)
440 bd_new_entry = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
443 * Go poke the address of the new bounds table in to the
444 * bounds directory entry out in userspace memory. Note:
445 * we may race with another CPU instantiating the same table.
446 * In that case the cmpxchg will see an unexpected
449 * This can fault, but that's OK because we do not hold
450 * mmap_sem at this point, unlike some of the other part
451 * of the MPX code that have to pagefault_disable().
453 ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val, bd_entry,
454 expected_old_val, bd_new_entry);
459 * The user_atomic_cmpxchg_inatomic() will only return nonzero
460 * for faults, *not* if the cmpxchg itself fails. Now we must
461 * verify that the cmpxchg itself completed successfully.
464 * We expected an empty 'expected_old_val', but instead found
465 * an apparently valid entry. Assume we raced with another
466 * thread to instantiate this table and desclare succecss.
468 if (actual_old_val & MPX_BD_ENTRY_VALID_FLAG) {
473 * We found a non-empty bd_entry but it did not have the
474 * VALID_FLAG set. Return an error which will result in
475 * a SEGV since this probably means that somebody scribbled
476 * some invalid data in to a bounds table.
478 if (expected_old_val != actual_old_val) {
482 trace_mpx_new_bounds_table(bt_addr);
485 vm_munmap(bt_addr, mpx_bt_size_bytes(mm));
490 * When a BNDSTX instruction attempts to save bounds to a bounds
491 * table, it will first attempt to look up the table in the
492 * first-level bounds directory. If it does not find a table in
493 * the directory, a #BR is generated and we get here in order to
494 * allocate a new table.
496 * With 32-bit mode, the size of BD is 4MB, and the size of each
497 * bound table is 16KB. With 64-bit mode, the size of BD is 2GB,
498 * and the size of each bound table is 4MB.
500 static int do_mpx_bt_fault(void)
502 unsigned long bd_entry, bd_base;
503 const struct mpx_bndcsr *bndcsr;
504 struct mm_struct *mm = current->mm;
506 bndcsr = get_xsave_field_ptr(XFEATURE_MASK_BNDCSR);
510 * Mask off the preserve and enable bits
512 bd_base = bndcsr->bndcfgu & MPX_BNDCFG_ADDR_MASK;
514 * The hardware provides the address of the missing or invalid
515 * entry via BNDSTATUS, so we don't have to go look it up.
517 bd_entry = bndcsr->bndstatus & MPX_BNDSTA_ADDR_MASK;
519 * Make sure the directory entry is within where we think
522 if ((bd_entry < bd_base) ||
523 (bd_entry >= bd_base + mpx_bd_size_bytes(mm)))
526 return allocate_bt(mm, (long __user *)bd_entry);
529 int mpx_handle_bd_fault(void)
532 * Userspace never asked us to manage the bounds tables,
535 if (!kernel_managing_mpx_tables(current->mm))
538 return do_mpx_bt_fault();
542 * A thin wrapper around get_user_pages(). Returns 0 if the
543 * fault was resolved or -errno if not.
545 static int mpx_resolve_fault(long __user *addr, int write)
550 gup_ret = get_user_pages((unsigned long)addr, nr_pages,
551 write ? FOLL_WRITE : 0, NULL, NULL);
553 * get_user_pages() returns number of pages gotten.
554 * 0 means we failed to fault in and get anything,
555 * probably because 'addr' is bad.
559 /* Other error, return it */
562 /* must have gup'd a page and gup_ret>0, success */
566 static unsigned long mpx_bd_entry_to_bt_addr(struct mm_struct *mm,
567 unsigned long bd_entry)
569 unsigned long bt_addr = bd_entry;
572 * Bit 0 in a bt_entry is always the valid bit.
574 bt_addr &= ~MPX_BD_ENTRY_VALID_FLAG;
576 * Tables are naturally aligned at 8-byte boundaries
577 * on 64-bit and 4-byte boundaries on 32-bit. The
578 * documentation makes it appear that the low bits
579 * are ignored by the hardware, so we do the same.
585 bt_addr &= ~(align_to_bytes-1);
590 * We only want to do a 4-byte get_user() on 32-bit. Otherwise,
591 * we might run off the end of the bounds table if we are on
592 * a 64-bit kernel and try to get 8 bytes.
594 static int get_user_bd_entry(struct mm_struct *mm, unsigned long *bd_entry_ret,
595 long __user *bd_entry_ptr)
601 return get_user(*bd_entry_ret, bd_entry_ptr);
604 * Note that get_user() uses the type of the *pointer* to
605 * establish the size of the get, not the destination.
607 ret = get_user(bd_entry_32, (u32 __user *)bd_entry_ptr);
608 *bd_entry_ret = bd_entry_32;
613 * Get the base of bounds tables pointed by specific bounds
616 static int get_bt_addr(struct mm_struct *mm,
617 long __user *bd_entry_ptr,
618 unsigned long *bt_addr_result)
622 unsigned long bd_entry;
623 unsigned long bt_addr;
625 if (!access_ok(VERIFY_READ, (bd_entry_ptr), sizeof(*bd_entry_ptr)))
632 ret = get_user_bd_entry(mm, &bd_entry, bd_entry_ptr);
637 ret = mpx_resolve_fault(bd_entry_ptr, need_write);
639 * If we could not resolve the fault, consider it
640 * userspace's fault and error out.
646 valid_bit = bd_entry & MPX_BD_ENTRY_VALID_FLAG;
647 bt_addr = mpx_bd_entry_to_bt_addr(mm, bd_entry);
650 * When the kernel is managing bounds tables, a bounds directory
651 * entry will either have a valid address (plus the valid bit)
652 * *OR* be completely empty. If we see a !valid entry *and* some
653 * data in the address field, we know something is wrong. This
654 * -EINVAL return will cause a SIGSEGV.
656 if (!valid_bit && bt_addr)
659 * Do we have an completely zeroed bt entry? That is OK. It
660 * just means there was no bounds table for this memory. Make
661 * sure to distinguish this from -EINVAL, which will cause
667 *bt_addr_result = bt_addr;
671 static inline int bt_entry_size_bytes(struct mm_struct *mm)
674 return MPX_BT_ENTRY_BYTES_64;
676 return MPX_BT_ENTRY_BYTES_32;
680 * Take a virtual address and turns it in to the offset in bytes
681 * inside of the bounds table where the bounds table entry
682 * controlling 'addr' can be found.
684 static unsigned long mpx_get_bt_entry_offset_bytes(struct mm_struct *mm,
687 unsigned long bt_table_nr_entries;
688 unsigned long offset = addr;
690 if (is_64bit_mm(mm)) {
691 /* Bottom 3 bits are ignored on 64-bit */
693 bt_table_nr_entries = MPX_BT_NR_ENTRIES_64;
695 /* Bottom 2 bits are ignored on 32-bit */
697 bt_table_nr_entries = MPX_BT_NR_ENTRIES_32;
700 * We know the size of the table in to which we are
701 * indexing, and we have eliminated all the low bits
702 * which are ignored for indexing.
704 * Mask out all the high bits which we do not need
705 * to index in to the table. Note that the tables
706 * are always powers of two so this gives us a proper
709 offset &= (bt_table_nr_entries-1);
711 * We now have an entry offset in terms of *entries* in
712 * the table. We need to scale it back up to bytes.
714 offset *= bt_entry_size_bytes(mm);
719 * How much virtual address space does a single bounds
720 * directory entry cover?
722 * Note, we need a long long because 4GB doesn't fit in
723 * to a long on 32-bit.
725 static inline unsigned long bd_entry_virt_space(struct mm_struct *mm)
727 unsigned long long virt_space;
728 unsigned long long GB = (1ULL << 30);
731 * This covers 32-bit emulation as well as 32-bit kernels
732 * running on 64-bit hardware.
734 if (!is_64bit_mm(mm))
735 return (4ULL * GB) / MPX_BD_NR_ENTRIES_32;
738 * 'x86_virt_bits' returns what the hardware is capable
739 * of, and returns the full >32-bit address space when
740 * running 32-bit kernels on 64-bit hardware.
742 virt_space = (1ULL << boot_cpu_data.x86_virt_bits);
743 return virt_space / MPX_BD_NR_ENTRIES_64;
747 * Free the backing physical pages of bounds table 'bt_addr'.
748 * Assume start...end is within that bounds table.
750 static noinline int zap_bt_entries_mapping(struct mm_struct *mm,
751 unsigned long bt_addr,
752 unsigned long start_mapping, unsigned long end_mapping)
754 struct vm_area_struct *vma;
755 unsigned long addr, len;
760 * if we 'end' on a boundary, the offset will be 0 which
761 * is not what we want. Back it up a byte to get the
762 * last bt entry. Then once we have the entry itself,
763 * move 'end' back up by the table entry size.
765 start = bt_addr + mpx_get_bt_entry_offset_bytes(mm, start_mapping);
766 end = bt_addr + mpx_get_bt_entry_offset_bytes(mm, end_mapping - 1);
768 * Move end back up by one entry. Among other things
769 * this ensures that it remains page-aligned and does
770 * not screw up zap_page_range()
772 end += bt_entry_size_bytes(mm);
775 * Find the first overlapping vma. If vma->vm_start > start, there
776 * will be a hole in the bounds table. This -EINVAL return will
779 vma = find_vma(mm, start);
780 if (!vma || vma->vm_start > start)
784 * A NUMA policy on a VM_MPX VMA could cause this bounds table to
785 * be split. So we need to look across the entire 'start -> end'
786 * range of this bounds table, find all of the VM_MPX VMAs, and
790 while (vma && vma->vm_start < end) {
792 * We followed a bounds directory entry down
793 * here. If we find a non-MPX VMA, that's bad,
794 * so stop immediately and return an error. This
795 * probably results in a SIGSEGV.
797 if (!(vma->vm_flags & VM_MPX))
800 len = min(vma->vm_end, end) - addr;
801 zap_page_range(vma, addr, len);
802 trace_mpx_unmap_zap(addr, addr+len);
805 addr = vma->vm_start;
810 static unsigned long mpx_get_bd_entry_offset(struct mm_struct *mm,
814 * There are several ways to derive the bd offsets. We
815 * use the following approach here:
816 * 1. We know the size of the virtual address space
817 * 2. We know the number of entries in a bounds table
818 * 3. We know that each entry covers a fixed amount of
819 * virtual address space.
820 * So, we can just divide the virtual address by the
821 * virtual space used by one entry to determine which
822 * entry "controls" the given virtual address.
824 if (is_64bit_mm(mm)) {
825 int bd_entry_size = 8; /* 64-bit pointer */
827 * Take the 64-bit addressing hole in to account.
829 addr &= ((1UL << boot_cpu_data.x86_virt_bits) - 1);
830 return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
832 int bd_entry_size = 4; /* 32-bit pointer */
834 * 32-bit has no hole so this case needs no mask
836 return (addr / bd_entry_virt_space(mm)) * bd_entry_size;
839 * The two return calls above are exact copies. If we
840 * pull out a single copy and put it in here, gcc won't
841 * realize that we're doing a power-of-2 divide and use
842 * shifts. It uses a real divide. If we put them up
843 * there, it manages to figure it out (gcc 4.8.3).
847 static int unmap_entire_bt(struct mm_struct *mm,
848 long __user *bd_entry, unsigned long bt_addr)
850 unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
851 unsigned long uninitialized_var(actual_old_val);
856 unsigned long cleared_bd_entry = 0;
859 ret = mpx_cmpxchg_bd_entry(mm, &actual_old_val,
860 bd_entry, expected_old_val, cleared_bd_entry);
865 ret = mpx_resolve_fault(bd_entry, need_write);
867 * If we could not resolve the fault, consider it
868 * userspace's fault and error out.
874 * The cmpxchg was performed, check the results.
876 if (actual_old_val != expected_old_val) {
878 * Someone else raced with us to unmap the table.
879 * That is OK, since we were both trying to do
880 * the same thing. Declare success.
885 * Something messed with the bounds directory
886 * entry. We hold mmap_sem for read or write
887 * here, so it could not be a _new_ bounds table
888 * that someone just allocated. Something is
889 * wrong, so pass up the error and SIGSEGV.
894 * Note, we are likely being called under do_munmap() already. To
895 * avoid recursion, do_munmap() will check whether it comes
896 * from one bounds table through VM_MPX flag.
898 return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm), NULL);
901 static int try_unmap_single_bt(struct mm_struct *mm,
902 unsigned long start, unsigned long end)
904 struct vm_area_struct *next;
905 struct vm_area_struct *prev;
907 * "bta" == Bounds Table Area: the area controlled by the
908 * bounds table that we are unmapping.
910 unsigned long bta_start_vaddr = start & ~(bd_entry_virt_space(mm)-1);
911 unsigned long bta_end_vaddr = bta_start_vaddr + bd_entry_virt_space(mm);
912 unsigned long uninitialized_var(bt_addr);
913 void __user *bde_vaddr;
916 * We already unlinked the VMAs from the mm's rbtree so 'start'
917 * is guaranteed to be in a hole. This gets us the first VMA
918 * before the hole in to 'prev' and the next VMA after the hole
921 next = find_vma_prev(mm, start, &prev);
923 * Do not count other MPX bounds table VMAs as neighbors.
924 * Although theoretically possible, we do not allow bounds
925 * tables for bounds tables so our heads do not explode.
926 * If we count them as neighbors here, we may end up with
927 * lots of tables even though we have no actual table
930 while (next && (next->vm_flags & VM_MPX))
931 next = next->vm_next;
932 while (prev && (prev->vm_flags & VM_MPX))
933 prev = prev->vm_prev;
935 * We know 'start' and 'end' lie within an area controlled
936 * by a single bounds table. See if there are any other
937 * VMAs controlled by that bounds table. If there are not
938 * then we can "expand" the are we are unmapping to possibly
939 * cover the entire table.
941 next = find_vma_prev(mm, start, &prev);
942 if ((!prev || prev->vm_end <= bta_start_vaddr) &&
943 (!next || next->vm_start >= bta_end_vaddr)) {
945 * No neighbor VMAs controlled by same bounds
946 * table. Try to unmap the whole thing
948 start = bta_start_vaddr;
952 bde_vaddr = mm->context.bd_addr + mpx_get_bd_entry_offset(mm, start);
953 ret = get_bt_addr(mm, bde_vaddr, &bt_addr);
955 * No bounds table there, so nothing to unmap.
957 if (ret == -ENOENT) {
964 * We are unmapping an entire table. Either because the
965 * unmap that started this whole process was large enough
966 * to cover an entire table, or that the unmap was small
967 * but was the area covered by a bounds table.
969 if ((start == bta_start_vaddr) &&
970 (end == bta_end_vaddr))
971 return unmap_entire_bt(mm, bde_vaddr, bt_addr);
972 return zap_bt_entries_mapping(mm, bt_addr, start, end);
975 static int mpx_unmap_tables(struct mm_struct *mm,
976 unsigned long start, unsigned long end)
978 unsigned long one_unmap_start;
979 trace_mpx_unmap_search(start, end);
981 one_unmap_start = start;
982 while (one_unmap_start < end) {
984 unsigned long next_unmap_start = ALIGN(one_unmap_start+1,
985 bd_entry_virt_space(mm));
986 unsigned long one_unmap_end = end;
988 * if the end is beyond the current bounds table,
989 * move it back so we only deal with a single one
992 if (one_unmap_end > next_unmap_start)
993 one_unmap_end = next_unmap_start;
994 ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end);
998 one_unmap_start = next_unmap_start;
1004 * Free unused bounds tables covered in a virtual address region being
1005 * munmap()ed. Assume end > start.
1007 * This function will be called by do_munmap(), and the VMAs covering
1008 * the virtual address region start...end have already been split if
1009 * necessary, and the 'vma' is the first vma in this range (start -> end).
1011 void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
1012 unsigned long start, unsigned long end)
1017 * Refuse to do anything unless userspace has asked
1018 * the kernel to help manage the bounds tables,
1020 if (!kernel_managing_mpx_tables(current->mm))
1023 * This will look across the entire 'start -> end' range,
1024 * and find all of the non-VM_MPX VMAs.
1026 * To avoid recursion, if a VM_MPX vma is found in the range
1027 * (start->end), we will not continue follow-up work. This
1028 * recursion represents having bounds tables for bounds tables,
1029 * which should not occur normally. Being strict about it here
1030 * helps ensure that we do not have an exploitable stack overflow.
1033 if (vma->vm_flags & VM_MPX)
1036 } while (vma && vma->vm_start < end);
1038 ret = mpx_unmap_tables(mm, start, end);
1040 force_sig(SIGSEGV, current);
1043 /* MPX cannot handle addresses above 47 bits yet. */
1044 unsigned long mpx_unmapped_area_check(unsigned long addr, unsigned long len,
1045 unsigned long flags)
1047 if (!kernel_managing_mpx_tables(current->mm))
1049 if (addr + len <= DEFAULT_MAP_WINDOW)
1051 if (flags & MAP_FIXED)
1055 * Requested len is larger than the whole area we're allowed to map in.
1056 * Resetting hinting address wouldn't do much good -- fail early.
1058 if (len > DEFAULT_MAP_WINDOW)
1061 /* Look for unmap area within DEFAULT_MAP_WINDOW */