1 // SPDX-License-Identifier: GPL-2.0
3 * Tests Memory Protection Keys (see Documentation/core-api/protection-keys.rst)
5 * There are examples in here of:
6 * * how to set protection keys on memory
7 * * how to set/clear bits in pkey registers (the rights register)
8 * * how to handle SEGV_PKUERR signals and extract pkey-relevant
9 * information from the siginfo
12 * make sure KSM and KSM COW breaking works
13 * prefault pages in at malloc, or not
14 * protect MPX bounds tables with protection keys?
15 * make sure VMA splitting/merging is working correctly
16 * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys
17 * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel
18 * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks
21 * gcc -mxsave -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
22 * gcc -mxsave -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
25 #define __SANE_USERSPACE_TYPES__
27 #include <linux/elf.h>
28 #include <linux/futex.h>
31 #include <sys/syscall.h>
41 #include <sys/types.h>
46 #include <sys/ptrace.h>
49 #include "pkey-helpers.h"
56 char dprint_in_signal_buffer[DPRINT_IN_SIGNAL_BUF_SIZE];
58 void cat_into_file(char *str, char *file)
60 int fd = open(file, O_RDWR);
63 dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file);
65 * these need to be raw because they are called under
69 fprintf(stderr, "error opening '%s'\n", str);
74 ret = write(fd, str, strlen(str));
75 if (ret != strlen(str)) {
76 perror("write to file failed");
77 fprintf(stderr, "filename: '%s' str: '%s'\n", file, str);
83 #if CONTROL_TRACING > 0
84 static int warned_tracing;
85 int tracing_root_ok(void)
89 fprintf(stderr, "WARNING: not run as root, "
90 "can not do tracing control\n");
100 #if CONTROL_TRACING > 0
101 #define TRACEDIR "/sys/kernel/debug/tracing"
104 if (!tracing_root_ok())
107 sprintf(pidstr, "%d", getpid());
108 cat_into_file("0", TRACEDIR "/tracing_on");
109 cat_into_file("\n", TRACEDIR "/trace");
111 cat_into_file("function_graph", TRACEDIR "/current_tracer");
112 cat_into_file("1", TRACEDIR "/options/funcgraph-proc");
114 cat_into_file("nop", TRACEDIR "/current_tracer");
116 cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid");
117 cat_into_file("1", TRACEDIR "/tracing_on");
118 dprintf1("enabled tracing\n");
122 void tracing_off(void)
124 #if CONTROL_TRACING > 0
125 if (!tracing_root_ok())
127 cat_into_file("0", "/sys/kernel/debug/tracing/tracing_on");
131 void abort_hooks(void)
133 fprintf(stderr, "running %s()...\n", __func__);
135 #ifdef SLEEP_ON_ABORT
136 sleep(SLEEP_ON_ABORT);
141 * This attempts to have roughly a page of instructions followed by a few
142 * instructions that do a write, and another page of instructions. That
143 * way, we are pretty sure that the write is in the second page of
144 * instructions and has at least a page of padding behind it.
146 * *That* lets us be sure to madvise() away the write instruction, which
147 * will then fault, which makes sure that the fault code handles
148 * execute-only memory properly.
151 /* This way, both 4K and 64K alignment are maintained */
152 __attribute__((__aligned__(65536)))
154 __attribute__((__aligned__(PAGE_SIZE)))
156 void lots_o_noops_around_write(int *write_to_me)
158 dprintf3("running %s()\n", __func__);
160 /* Assume this happens in the second page of instructions: */
161 *write_to_me = __LINE__;
162 /* pad out by another page: */
164 dprintf3("%s() done\n", __func__);
167 void dump_mem(void *dumpme, int len_bytes)
169 char *c = (void *)dumpme;
172 for (i = 0; i < len_bytes; i += sizeof(u64)) {
173 u64 *ptr = (u64 *)(c + i);
174 dprintf1("dump[%03d][@%p]: %016llx\n", i, ptr, *ptr);
178 static u32 hw_pkey_get(int pkey, unsigned long flags)
180 u64 pkey_reg = __read_pkey_reg();
182 dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n",
183 __func__, pkey, flags, 0, 0);
184 dprintf2("%s() raw pkey_reg: %016llx\n", __func__, pkey_reg);
186 return (u32) get_pkey_bits(pkey_reg, pkey);
189 static int hw_pkey_set(int pkey, unsigned long rights, unsigned long flags)
191 u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE);
192 u64 old_pkey_reg = __read_pkey_reg();
195 /* make sure that 'rights' only contains the bits we expect: */
196 assert(!(rights & ~mask));
198 /* modify bits accordingly in old pkey_reg and assign it */
199 new_pkey_reg = set_pkey_bits(old_pkey_reg, pkey, rights);
201 __write_pkey_reg(new_pkey_reg);
203 dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x"
204 " pkey_reg now: %016llx old_pkey_reg: %016llx\n",
205 __func__, pkey, rights, flags, 0, __read_pkey_reg(),
210 void pkey_disable_set(int pkey, int flags)
212 unsigned long syscall_flags = 0;
215 u64 orig_pkey_reg = read_pkey_reg();
217 dprintf1("START->%s(%d, 0x%x)\n", __func__,
219 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
221 pkey_rights = hw_pkey_get(pkey, syscall_flags);
223 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
224 pkey, pkey, pkey_rights);
226 pkey_assert(pkey_rights >= 0);
228 pkey_rights |= flags;
230 ret = hw_pkey_set(pkey, pkey_rights, syscall_flags);
232 /* pkey_reg and flags have the same format */
233 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, pkey, pkey_rights);
234 dprintf1("%s(%d) shadow: 0x%016llx\n",
235 __func__, pkey, shadow_pkey_reg);
237 pkey_assert(ret >= 0);
239 pkey_rights = hw_pkey_get(pkey, syscall_flags);
240 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
241 pkey, pkey, pkey_rights);
243 dprintf1("%s(%d) pkey_reg: 0x%016llx\n",
244 __func__, pkey, read_pkey_reg());
246 pkey_assert(read_pkey_reg() >= orig_pkey_reg);
247 dprintf1("END<---%s(%d, 0x%x)\n", __func__,
251 void pkey_disable_clear(int pkey, int flags)
253 unsigned long syscall_flags = 0;
255 int pkey_rights = hw_pkey_get(pkey, syscall_flags);
256 u64 orig_pkey_reg = read_pkey_reg();
258 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
260 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
261 pkey, pkey, pkey_rights);
262 pkey_assert(pkey_rights >= 0);
264 pkey_rights &= ~flags;
266 ret = hw_pkey_set(pkey, pkey_rights, 0);
267 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, pkey, pkey_rights);
268 pkey_assert(ret >= 0);
270 pkey_rights = hw_pkey_get(pkey, syscall_flags);
271 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
272 pkey, pkey, pkey_rights);
274 dprintf1("%s(%d) pkey_reg: 0x%016llx\n", __func__,
275 pkey, read_pkey_reg());
277 assert(read_pkey_reg() <= orig_pkey_reg);
280 void pkey_write_allow(int pkey)
282 pkey_disable_clear(pkey, PKEY_DISABLE_WRITE);
284 void pkey_write_deny(int pkey)
286 pkey_disable_set(pkey, PKEY_DISABLE_WRITE);
288 void pkey_access_allow(int pkey)
290 pkey_disable_clear(pkey, PKEY_DISABLE_ACCESS);
292 void pkey_access_deny(int pkey)
294 pkey_disable_set(pkey, PKEY_DISABLE_ACCESS);
297 /* Failed address bound checks: */
299 # define SEGV_BNDERR 3
303 # define SEGV_PKUERR 4
306 static char *si_code_str(int si_code)
308 if (si_code == SEGV_MAPERR)
309 return "SEGV_MAPERR";
310 if (si_code == SEGV_ACCERR)
311 return "SEGV_ACCERR";
312 if (si_code == SEGV_BNDERR)
313 return "SEGV_BNDERR";
314 if (si_code == SEGV_PKUERR)
315 return "SEGV_PKUERR";
320 int last_si_pkey = -1;
321 void signal_handler(int signum, siginfo_t *si, void *vucontext)
323 ucontext_t *uctxt = vucontext;
327 #if defined(__i386__) || defined(__x86_64__) /* arch */
334 dprint_in_signal = 1;
335 dprintf1(">>>>===============SIGSEGV============================\n");
336 dprintf1("%s()::%d, pkey_reg: 0x%016llx shadow: %016llx\n",
338 __read_pkey_reg(), shadow_pkey_reg);
340 trapno = uctxt->uc_mcontext.gregs[REG_TRAPNO];
341 ip = uctxt->uc_mcontext.gregs[REG_IP_IDX];
342 fpregs = (char *) uctxt->uc_mcontext.fpregs;
344 dprintf2("%s() trapno: %d ip: 0x%016lx info->si_code: %s/%d\n",
345 __func__, trapno, ip, si_code_str(si->si_code),
348 #if defined(__i386__) || defined(__x86_64__) /* arch */
351 * 32-bit has some extra padding so that userspace can tell whether
352 * the XSTATE header is present in addition to the "legacy" FPU
353 * state. We just assume that it is here.
357 pkey_reg_offset = pkey_reg_xstate_offset();
358 pkey_reg_ptr = (void *)(&fpregs[pkey_reg_offset]);
361 * If we got a PKEY fault, we *HAVE* to have at least one bit set in
364 dprintf1("pkey_reg_xstate_offset: %d\n", pkey_reg_xstate_offset());
366 dump_mem(pkey_reg_ptr - 128, 256);
367 pkey_assert(*pkey_reg_ptr);
370 dprintf1("siginfo: %p\n", si);
371 dprintf1(" fpregs: %p\n", fpregs);
373 if ((si->si_code == SEGV_MAPERR) ||
374 (si->si_code == SEGV_ACCERR) ||
375 (si->si_code == SEGV_BNDERR)) {
376 printf("non-PK si_code, exiting...\n");
380 si_pkey_ptr = siginfo_get_pkey_ptr(si);
381 dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr);
382 dump_mem((u8 *)si_pkey_ptr - 8, 24);
383 siginfo_pkey = *si_pkey_ptr;
384 pkey_assert(siginfo_pkey < NR_PKEYS);
385 last_si_pkey = siginfo_pkey;
388 * need __read_pkey_reg() version so we do not do shadow_pkey_reg
391 dprintf1("signal pkey_reg from pkey_reg: %016llx\n",
393 dprintf1("pkey from siginfo: %016llx\n", siginfo_pkey);
394 #if defined(__i386__) || defined(__x86_64__) /* arch */
395 dprintf1("signal pkey_reg from xsave: %08x\n", *pkey_reg_ptr);
396 *(u64 *)pkey_reg_ptr = 0x00000000;
397 dprintf1("WARNING: set PKEY_REG=0 to allow faulting instruction to continue\n");
398 #elif defined(__powerpc64__) /* arch */
399 /* restore access and let the faulting instruction continue */
400 pkey_access_allow(siginfo_pkey);
403 dprintf1("<<<<==================================================\n");
404 dprint_in_signal = 0;
407 int wait_all_children(void)
410 return waitpid(-1, &status, 0);
415 dprint_in_signal = 1;
416 dprintf2("[%d] SIGCHLD: %d\n", getpid(), x);
417 dprint_in_signal = 0;
420 void setup_sigsegv_handler(void)
423 struct sigaction newact;
424 struct sigaction oldact;
426 /* #PF is mapped to sigsegv */
427 int signum = SIGSEGV;
429 newact.sa_handler = 0;
430 newact.sa_sigaction = signal_handler;
432 /*sigset_t - signals to block while in the handler */
433 /* get the old signal mask. */
434 rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask);
435 pkey_assert(rs == 0);
437 /* call sa_sigaction, not sa_handler*/
438 newact.sa_flags = SA_SIGINFO;
440 newact.sa_restorer = 0; /* void(*)(), obsolete */
441 r = sigaction(signum, &newact, &oldact);
442 r = sigaction(SIGALRM, &newact, &oldact);
446 void setup_handlers(void)
448 signal(SIGCHLD, &sig_chld);
449 setup_sigsegv_handler();
452 pid_t fork_lazy_child(void)
457 pkey_assert(forkret >= 0);
458 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
463 dprintf1("child sleeping...\n");
470 int sys_mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
475 dprintf2("%s(0x%p, %zx, prot=%lx, pkey=%lx)\n", __func__,
476 ptr, size, orig_prot, pkey);
479 sret = syscall(SYS_mprotect_key, ptr, size, orig_prot, pkey);
481 dprintf2("SYS_mprotect_key sret: %d\n", sret);
482 dprintf2("SYS_mprotect_key prot: 0x%lx\n", orig_prot);
483 dprintf2("SYS_mprotect_key failed, errno: %d\n", errno);
484 if (DEBUG_LEVEL >= 2)
485 perror("SYS_mprotect_pkey");
490 int sys_pkey_alloc(unsigned long flags, unsigned long init_val)
492 int ret = syscall(SYS_pkey_alloc, flags, init_val);
493 dprintf1("%s(flags=%lx, init_val=%lx) syscall ret: %d errno: %d\n",
494 __func__, flags, init_val, ret, errno);
501 unsigned long init_val = 0x0;
503 dprintf1("%s()::%d, pkey_reg: 0x%016llx shadow: %016llx\n",
504 __func__, __LINE__, __read_pkey_reg(), shadow_pkey_reg);
505 ret = sys_pkey_alloc(0, init_val);
507 * pkey_alloc() sets PKEY register, so we need to reflect it in
510 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
511 " shadow: 0x%016llx\n",
512 __func__, __LINE__, ret, __read_pkey_reg(),
515 /* clear both the bits: */
516 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, ret,
518 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
519 " shadow: 0x%016llx\n",
521 __LINE__, ret, __read_pkey_reg(),
524 * move the new state in from init_val
525 * (remember, we cheated and init_val == pkey_reg format)
527 shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, ret,
530 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
531 " shadow: 0x%016llx\n",
532 __func__, __LINE__, ret, __read_pkey_reg(),
534 dprintf1("%s()::%d errno: %d\n", __func__, __LINE__, errno);
535 /* for shadow checking: */
537 dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
538 " shadow: 0x%016llx\n",
539 __func__, __LINE__, ret, __read_pkey_reg(),
544 int sys_pkey_free(unsigned long pkey)
546 int ret = syscall(SYS_pkey_free, pkey);
547 dprintf1("%s(pkey=%ld) syscall ret: %d\n", __func__, pkey, ret);
552 * I had a bug where pkey bits could be set by mprotect() but
553 * not cleared. This ensures we get lots of random bit sets
554 * and clears on the vma and pte pkey bits.
556 int alloc_random_pkey(void)
558 int max_nr_pkey_allocs;
561 int alloced_pkeys[NR_PKEYS];
564 memset(alloced_pkeys, 0, sizeof(alloced_pkeys));
566 /* allocate every possible key and make a note of which ones we got */
567 max_nr_pkey_allocs = NR_PKEYS;
568 for (i = 0; i < max_nr_pkey_allocs; i++) {
569 int new_pkey = alloc_pkey();
572 alloced_pkeys[nr_alloced++] = new_pkey;
575 pkey_assert(nr_alloced > 0);
576 /* select a random one out of the allocated ones */
577 random_index = rand() % nr_alloced;
578 ret = alloced_pkeys[random_index];
579 /* now zero it out so we don't free it next */
580 alloced_pkeys[random_index] = 0;
582 /* go through the allocated ones that we did not want and free them */
583 for (i = 0; i < nr_alloced; i++) {
585 if (!alloced_pkeys[i])
587 free_ret = sys_pkey_free(alloced_pkeys[i]);
588 pkey_assert(!free_ret);
590 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
591 " shadow: 0x%016llx\n", __func__,
592 __LINE__, ret, __read_pkey_reg(), shadow_pkey_reg);
596 int mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
599 int nr_iterations = random() % 100;
603 int rpkey = alloc_random_pkey();
604 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
605 dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
606 ptr, size, orig_prot, pkey, ret);
607 if (nr_iterations-- < 0)
610 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
611 " shadow: 0x%016llx\n",
612 __func__, __LINE__, ret, __read_pkey_reg(),
614 sys_pkey_free(rpkey);
615 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
616 " shadow: 0x%016llx\n",
617 __func__, __LINE__, ret, __read_pkey_reg(),
620 pkey_assert(pkey < NR_PKEYS);
622 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
623 dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
624 ptr, size, orig_prot, pkey, ret);
626 dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
627 " shadow: 0x%016llx\n", __func__,
628 __LINE__, ret, __read_pkey_reg(), shadow_pkey_reg);
632 struct pkey_malloc_record {
637 struct pkey_malloc_record *pkey_malloc_records;
638 struct pkey_malloc_record *pkey_last_malloc_record;
639 long nr_pkey_malloc_records;
640 void record_pkey_malloc(void *ptr, long size, int prot)
643 struct pkey_malloc_record *rec = NULL;
645 for (i = 0; i < nr_pkey_malloc_records; i++) {
646 rec = &pkey_malloc_records[i];
647 /* find a free record */
652 /* every record is full */
653 size_t old_nr_records = nr_pkey_malloc_records;
654 size_t new_nr_records = (nr_pkey_malloc_records * 2 + 1);
655 size_t new_size = new_nr_records * sizeof(struct pkey_malloc_record);
656 dprintf2("new_nr_records: %zd\n", new_nr_records);
657 dprintf2("new_size: %zd\n", new_size);
658 pkey_malloc_records = realloc(pkey_malloc_records, new_size);
659 pkey_assert(pkey_malloc_records != NULL);
660 rec = &pkey_malloc_records[nr_pkey_malloc_records];
662 * realloc() does not initialize memory, so zero it from
663 * the first new record all the way to the end.
665 for (i = 0; i < new_nr_records - old_nr_records; i++)
666 memset(rec + i, 0, sizeof(*rec));
668 dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n",
669 (int)(rec - pkey_malloc_records), rec, ptr, size);
673 pkey_last_malloc_record = rec;
674 nr_pkey_malloc_records++;
677 void free_pkey_malloc(void *ptr)
681 dprintf3("%s(%p)\n", __func__, ptr);
682 for (i = 0; i < nr_pkey_malloc_records; i++) {
683 struct pkey_malloc_record *rec = &pkey_malloc_records[i];
684 dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n",
685 ptr, i, rec, rec->ptr, rec->size);
686 if ((ptr < rec->ptr) ||
687 (ptr >= rec->ptr + rec->size))
690 dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n",
691 ptr, i, rec, rec->ptr, rec->size);
692 nr_pkey_malloc_records--;
693 ret = munmap(rec->ptr, rec->size);
694 dprintf3("munmap ret: %d\n", ret);
696 dprintf3("clearing rec->ptr, rec: %p\n", rec);
698 dprintf3("done clearing rec->ptr, rec: %p\n", rec);
705 void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey)
711 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
713 pkey_assert(pkey < NR_PKEYS);
714 ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
715 pkey_assert(ptr != (void *)-1);
716 ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey);
718 record_pkey_malloc(ptr, size, prot);
721 dprintf1("%s() for pkey %d @ %p\n", __func__, pkey, ptr);
725 void *malloc_pkey_anon_huge(long size, int prot, u16 pkey)
730 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
733 * Guarantee we can fit at least one huge page in the resulting
734 * allocation by allocating space for 2:
736 size = ALIGN_UP(size, HPAGE_SIZE * 2);
737 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
738 pkey_assert(ptr != (void *)-1);
739 record_pkey_malloc(ptr, size, prot);
740 mprotect_pkey(ptr, size, prot, pkey);
742 dprintf1("unaligned ptr: %p\n", ptr);
743 ptr = ALIGN_PTR_UP(ptr, HPAGE_SIZE);
744 dprintf1(" aligned ptr: %p\n", ptr);
745 ret = madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE);
746 dprintf1("MADV_HUGEPAGE ret: %d\n", ret);
747 ret = madvise(ptr, HPAGE_SIZE, MADV_WILLNEED);
748 dprintf1("MADV_WILLNEED ret: %d\n", ret);
749 memset(ptr, 0, HPAGE_SIZE);
751 dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey, ptr);
755 int hugetlb_setup_ok;
756 #define SYSFS_FMT_NR_HUGE_PAGES "/sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages"
757 #define GET_NR_HUGE_PAGES 10
758 void setup_hugetlbfs(void)
766 if (geteuid() != 0) {
767 fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n");
771 cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages");
774 * Now go make sure that we got the pages and that they
775 * are PMD-level pages. Someone might have made PUD-level
778 hpagesz_kb = HPAGE_SIZE / 1024;
779 hpagesz_mb = hpagesz_kb / 1024;
780 sprintf(buf, SYSFS_FMT_NR_HUGE_PAGES, hpagesz_kb);
781 fd = open(buf, O_RDONLY);
783 fprintf(stderr, "opening sysfs %ldM hugetlb config: %s\n",
784 hpagesz_mb, strerror(errno));
788 /* -1 to guarantee leaving the trailing \0 */
789 err = read(fd, buf, sizeof(buf)-1);
792 fprintf(stderr, "reading sysfs %ldM hugetlb config: %s\n",
793 hpagesz_mb, strerror(errno));
797 if (atoi(buf) != GET_NR_HUGE_PAGES) {
798 fprintf(stderr, "could not confirm %ldM pages, got: '%s' expected %d\n",
799 hpagesz_mb, buf, GET_NR_HUGE_PAGES);
803 hugetlb_setup_ok = 1;
806 void *malloc_pkey_hugetlb(long size, int prot, u16 pkey)
809 int flags = MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB;
811 if (!hugetlb_setup_ok)
812 return PTR_ERR_ENOTSUP;
814 dprintf1("doing %s(%ld, %x, %x)\n", __func__, size, prot, pkey);
815 size = ALIGN_UP(size, HPAGE_SIZE * 2);
816 pkey_assert(pkey < NR_PKEYS);
817 ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0);
818 pkey_assert(ptr != (void *)-1);
819 mprotect_pkey(ptr, size, prot, pkey);
821 record_pkey_malloc(ptr, size, prot);
823 dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey, ptr);
827 void *malloc_pkey_mmap_dax(long size, int prot, u16 pkey)
832 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
834 pkey_assert(pkey < NR_PKEYS);
835 fd = open("/dax/foo", O_RDWR);
836 pkey_assert(fd >= 0);
838 ptr = mmap(0, size, prot, MAP_SHARED, fd, 0);
839 pkey_assert(ptr != (void *)-1);
841 mprotect_pkey(ptr, size, prot, pkey);
843 record_pkey_malloc(ptr, size, prot);
845 dprintf1("mmap()'d for pkey %d @ %p\n", pkey, ptr);
850 void *(*pkey_malloc[])(long size, int prot, u16 pkey) = {
852 malloc_pkey_with_mprotect,
853 malloc_pkey_with_mprotect_subpage,
854 malloc_pkey_anon_huge,
856 /* can not do direct with the pkey_mprotect() API:
857 malloc_pkey_mmap_direct,
858 malloc_pkey_mmap_dax,
862 void *malloc_pkey(long size, int prot, u16 pkey)
865 static int malloc_type;
866 int nr_malloc_types = ARRAY_SIZE(pkey_malloc);
868 pkey_assert(pkey < NR_PKEYS);
871 pkey_assert(malloc_type < nr_malloc_types);
873 ret = pkey_malloc[malloc_type](size, prot, pkey);
874 pkey_assert(ret != (void *)-1);
877 if (malloc_type >= nr_malloc_types)
878 malloc_type = (random()%nr_malloc_types);
880 /* try again if the malloc_type we tried is unsupported */
881 if (ret == PTR_ERR_ENOTSUP)
887 dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__,
888 size, prot, pkey, ret);
892 int last_pkey_faults;
893 #define UNKNOWN_PKEY -2
894 void expected_pkey_fault(int pkey)
896 dprintf2("%s(): last_pkey_faults: %d pkey_faults: %d\n",
897 __func__, last_pkey_faults, pkey_faults);
898 dprintf2("%s(%d): last_si_pkey: %d\n", __func__, pkey, last_si_pkey);
899 pkey_assert(last_pkey_faults + 1 == pkey_faults);
902 * For exec-only memory, we do not know the pkey in
903 * advance, so skip this check.
905 if (pkey != UNKNOWN_PKEY)
906 pkey_assert(last_si_pkey == pkey);
908 #if defined(__i386__) || defined(__x86_64__) /* arch */
910 * The signal handler shold have cleared out PKEY register to let the
911 * test program continue. We now have to restore it.
913 if (__read_pkey_reg() != 0)
915 if (__read_pkey_reg() != shadow_pkey_reg)
919 __write_pkey_reg(shadow_pkey_reg);
920 dprintf1("%s() set pkey_reg=%016llx to restore state after signal "
921 "nuked it\n", __func__, shadow_pkey_reg);
922 last_pkey_faults = pkey_faults;
926 #define do_not_expect_pkey_fault(msg) do { \
927 if (last_pkey_faults != pkey_faults) \
928 dprintf0("unexpected PKey fault: %s\n", msg); \
929 pkey_assert(last_pkey_faults == pkey_faults); \
932 int test_fds[10] = { -1 };
934 void __save_test_fd(int fd)
936 pkey_assert(fd >= 0);
937 pkey_assert(nr_test_fds < ARRAY_SIZE(test_fds));
938 test_fds[nr_test_fds] = fd;
942 int get_test_read_fd(void)
944 int test_fd = open("/etc/passwd", O_RDONLY);
945 __save_test_fd(test_fd);
949 void close_test_fds(void)
953 for (i = 0; i < nr_test_fds; i++) {
962 #define barrier() __asm__ __volatile__("": : :"memory")
963 __attribute__((noinline)) int read_ptr(int *ptr)
966 * Keep GCC from optimizing this away somehow
972 void test_pkey_alloc_free_attach_pkey0(int *ptr, u16 pkey)
975 int max_nr_pkey_allocs;
976 int alloced_pkeys[NR_PKEYS];
980 pkey_assert(pkey_last_malloc_record);
981 size = pkey_last_malloc_record->size;
983 * This is a bit of a hack. But mprotect() requires
984 * huge-page-aligned sizes when operating on hugetlbfs.
985 * So, make sure that we use something that's a multiple
986 * of a huge page when we can.
988 if (size >= HPAGE_SIZE)
991 /* allocate every possible key and make sure key-0 never got allocated */
992 max_nr_pkey_allocs = NR_PKEYS;
993 for (i = 0; i < max_nr_pkey_allocs; i++) {
994 int new_pkey = alloc_pkey();
995 pkey_assert(new_pkey != 0);
999 alloced_pkeys[nr_alloced++] = new_pkey;
1001 /* free all the allocated keys */
1002 for (i = 0; i < nr_alloced; i++) {
1005 if (!alloced_pkeys[i])
1007 free_ret = sys_pkey_free(alloced_pkeys[i]);
1008 pkey_assert(!free_ret);
1011 /* attach key-0 in various modes */
1012 err = sys_mprotect_pkey(ptr, size, PROT_READ, 0);
1014 err = sys_mprotect_pkey(ptr, size, PROT_WRITE, 0);
1016 err = sys_mprotect_pkey(ptr, size, PROT_EXEC, 0);
1018 err = sys_mprotect_pkey(ptr, size, PROT_READ|PROT_WRITE, 0);
1020 err = sys_mprotect_pkey(ptr, size, PROT_READ|PROT_WRITE|PROT_EXEC, 0);
1024 void test_read_of_write_disabled_region(int *ptr, u16 pkey)
1028 dprintf1("disabling write access to PKEY[1], doing read\n");
1029 pkey_write_deny(pkey);
1030 ptr_contents = read_ptr(ptr);
1031 dprintf1("*ptr: %d\n", ptr_contents);
1034 void test_read_of_access_disabled_region(int *ptr, u16 pkey)
1038 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey, ptr);
1040 pkey_access_deny(pkey);
1041 ptr_contents = read_ptr(ptr);
1042 dprintf1("*ptr: %d\n", ptr_contents);
1043 expected_pkey_fault(pkey);
1046 void test_read_of_access_disabled_region_with_page_already_mapped(int *ptr,
1051 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n",
1053 ptr_contents = read_ptr(ptr);
1054 dprintf1("reading ptr before disabling the read : %d\n",
1057 pkey_access_deny(pkey);
1058 ptr_contents = read_ptr(ptr);
1059 dprintf1("*ptr: %d\n", ptr_contents);
1060 expected_pkey_fault(pkey);
1063 void test_write_of_write_disabled_region_with_page_already_mapped(int *ptr,
1067 dprintf1("disabling write access; after accessing the page, "
1068 "to PKEY[%02d], doing write\n", pkey);
1069 pkey_write_deny(pkey);
1071 expected_pkey_fault(pkey);
1074 void test_write_of_write_disabled_region(int *ptr, u16 pkey)
1076 dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey);
1077 pkey_write_deny(pkey);
1079 expected_pkey_fault(pkey);
1081 void test_write_of_access_disabled_region(int *ptr, u16 pkey)
1083 dprintf1("disabling access to PKEY[%02d], doing write\n", pkey);
1084 pkey_access_deny(pkey);
1086 expected_pkey_fault(pkey);
1089 void test_write_of_access_disabled_region_with_page_already_mapped(int *ptr,
1093 dprintf1("disabling access; after accessing the page, "
1094 " to PKEY[%02d], doing write\n", pkey);
1095 pkey_access_deny(pkey);
1097 expected_pkey_fault(pkey);
1100 void test_kernel_write_of_access_disabled_region(int *ptr, u16 pkey)
1103 int test_fd = get_test_read_fd();
1105 dprintf1("disabling access to PKEY[%02d], "
1106 "having kernel read() to buffer\n", pkey);
1107 pkey_access_deny(pkey);
1108 ret = read(test_fd, ptr, 1);
1109 dprintf1("read ret: %d\n", ret);
1112 void test_kernel_write_of_write_disabled_region(int *ptr, u16 pkey)
1115 int test_fd = get_test_read_fd();
1117 pkey_write_deny(pkey);
1118 ret = read(test_fd, ptr, 100);
1119 dprintf1("read ret: %d\n", ret);
1120 if (ret < 0 && (DEBUG_LEVEL > 0))
1121 perror("verbose read result (OK for this to be bad)");
1125 void test_kernel_gup_of_access_disabled_region(int *ptr, u16 pkey)
1127 int pipe_ret, vmsplice_ret;
1131 pipe_ret = pipe(pipe_fds);
1133 pkey_assert(pipe_ret == 0);
1134 dprintf1("disabling access to PKEY[%02d], "
1135 "having kernel vmsplice from buffer\n", pkey);
1136 pkey_access_deny(pkey);
1138 iov.iov_len = PAGE_SIZE;
1139 vmsplice_ret = vmsplice(pipe_fds[1], &iov, 1, SPLICE_F_GIFT);
1140 dprintf1("vmsplice() ret: %d\n", vmsplice_ret);
1141 pkey_assert(vmsplice_ret == -1);
1147 void test_kernel_gup_write_to_write_disabled_region(int *ptr, u16 pkey)
1149 int ignored = 0xdada;
1151 int some_int = __LINE__;
1153 dprintf1("disabling write to PKEY[%02d], "
1154 "doing futex gunk in buffer\n", pkey);
1156 pkey_write_deny(pkey);
1157 futex_ret = syscall(SYS_futex, ptr, FUTEX_WAIT, some_int-1, NULL,
1159 if (DEBUG_LEVEL > 0)
1161 dprintf1("futex() ret: %d\n", futex_ret);
1164 /* Assumes that all pkeys other than 'pkey' are unallocated */
1165 void test_pkey_syscalls_on_non_allocated_pkey(int *ptr, u16 pkey)
1170 /* Note: 0 is the default pkey, so don't mess with it */
1171 for (i = 1; i < NR_PKEYS; i++) {
1175 dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i);
1176 err = sys_pkey_free(i);
1179 err = sys_pkey_free(i);
1182 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, i);
1187 /* Assumes that all pkeys other than 'pkey' are unallocated */
1188 void test_pkey_syscalls_bad_args(int *ptr, u16 pkey)
1191 int bad_pkey = NR_PKEYS+99;
1193 /* pass a known-invalid pkey in: */
1194 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, bad_pkey);
1198 void become_child(void)
1203 pkey_assert(forkret >= 0);
1204 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
1213 /* Assumes that all pkeys other than 'pkey' are unallocated */
1214 void test_pkey_alloc_exhaust(int *ptr, u16 pkey)
1217 int allocated_pkeys[NR_PKEYS] = {0};
1218 int nr_allocated_pkeys = 0;
1221 for (i = 0; i < NR_PKEYS*3; i++) {
1223 dprintf1("%s() alloc loop: %d\n", __func__, i);
1224 new_pkey = alloc_pkey();
1225 dprintf4("%s()::%d, err: %d pkey_reg: 0x%016llx"
1226 " shadow: 0x%016llx\n",
1227 __func__, __LINE__, err, __read_pkey_reg(),
1229 read_pkey_reg(); /* for shadow checking */
1230 dprintf2("%s() errno: %d ENOSPC: %d\n", __func__, errno, ENOSPC);
1231 if ((new_pkey == -1) && (errno == ENOSPC)) {
1232 dprintf2("%s() failed to allocate pkey after %d tries\n",
1233 __func__, nr_allocated_pkeys);
1236 * Ensure the number of successes never
1237 * exceeds the number of keys supported
1240 pkey_assert(nr_allocated_pkeys < NR_PKEYS);
1241 allocated_pkeys[nr_allocated_pkeys++] = new_pkey;
1245 * Make sure that allocation state is properly
1246 * preserved across fork().
1248 if (i == NR_PKEYS*2)
1252 dprintf3("%s()::%d\n", __func__, __LINE__);
1256 * There are 16 pkeys supported in hardware. Three are
1257 * allocated by the time we get here:
1258 * 1. The default key (0)
1259 * 2. One possibly consumed by an execute-only mapping.
1260 * 3. One allocated by the test code and passed in via
1261 * 'pkey' to this function.
1262 * Ensure that we can allocate at least another 13 (16-3).
1265 * There are either 5, 28, 29 or 32 pkeys supported in
1266 * hardware depending on the page size (4K or 64K) and
1267 * platform (powernv or powervm). Four are allocated by
1268 * the time we get here. These include pkey-0, pkey-1,
1269 * exec-only pkey and the one allocated by the test code.
1270 * Ensure that we can allocate the remaining.
1272 pkey_assert(i >= (NR_PKEYS - get_arch_reserved_keys() - 1));
1274 for (i = 0; i < nr_allocated_pkeys; i++) {
1275 err = sys_pkey_free(allocated_pkeys[i]);
1277 read_pkey_reg(); /* for shadow checking */
1281 void arch_force_pkey_reg_init(void)
1283 #if defined(__i386__) || defined(__x86_64__) /* arch */
1287 * All keys should be allocated and set to allow reads and
1288 * writes, so the register should be all 0. If not, just
1291 if (read_pkey_reg())
1295 * Just allocate an absurd about of memory rather than
1296 * doing the XSAVE size enumeration dance.
1298 buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1300 /* These __builtins require compiling with -mxsave */
1302 /* XSAVE to build a valid buffer: */
1303 __builtin_ia32_xsave(buf, XSTATE_PKEY);
1304 /* Clear XSTATE_BV[PKRU]: */
1305 buf[XSTATE_BV_OFFSET/sizeof(u64)] &= ~XSTATE_PKEY;
1306 /* XRSTOR will likely get PKRU back to the init state: */
1307 __builtin_ia32_xrstor(buf, XSTATE_PKEY);
1315 * This is mostly useless on ppc for now. But it will not
1316 * hurt anything and should give some better coverage as
1317 * a long-running test that continually checks the pkey
1320 void test_pkey_init_state(int *ptr, u16 pkey)
1323 int allocated_pkeys[NR_PKEYS] = {0};
1324 int nr_allocated_pkeys = 0;
1327 for (i = 0; i < NR_PKEYS; i++) {
1328 int new_pkey = alloc_pkey();
1332 allocated_pkeys[nr_allocated_pkeys++] = new_pkey;
1335 dprintf3("%s()::%d\n", __func__, __LINE__);
1337 arch_force_pkey_reg_init();
1340 * Loop for a bit, hoping to get exercise the kernel
1341 * context switch code.
1343 for (i = 0; i < 1000000; i++)
1346 for (i = 0; i < nr_allocated_pkeys; i++) {
1347 err = sys_pkey_free(allocated_pkeys[i]);
1349 read_pkey_reg(); /* for shadow checking */
1354 * pkey 0 is special. It is allocated by default, so you do not
1355 * have to call pkey_alloc() to use it first. Make sure that it
1358 void test_mprotect_with_pkey_0(int *ptr, u16 pkey)
1363 assert(pkey_last_malloc_record);
1364 size = pkey_last_malloc_record->size;
1366 * This is a bit of a hack. But mprotect() requires
1367 * huge-page-aligned sizes when operating on hugetlbfs.
1368 * So, make sure that we use something that's a multiple
1369 * of a huge page when we can.
1371 if (size >= HPAGE_SIZE)
1373 prot = pkey_last_malloc_record->prot;
1376 mprotect_pkey(ptr, size, prot, 0);
1378 /* Make sure that we can set it back to the original pkey. */
1379 mprotect_pkey(ptr, size, prot, pkey);
1382 void test_ptrace_of_child(int *ptr, u16 pkey)
1384 __attribute__((__unused__)) int peek_result;
1390 * This is the "control" for our little expermient. Make sure
1391 * we can always access it when ptracing.
1393 int *plain_ptr_unaligned = malloc(HPAGE_SIZE);
1394 int *plain_ptr = ALIGN_PTR_UP(plain_ptr_unaligned, PAGE_SIZE);
1397 * Fork a child which is an exact copy of this process, of course.
1398 * That means we can do all of our tests via ptrace() and then plain
1399 * memory access and ensure they work differently.
1401 child_pid = fork_lazy_child();
1402 dprintf1("[%d] child pid: %d\n", getpid(), child_pid);
1404 ret = ptrace(PTRACE_ATTACH, child_pid, ignored, ignored);
1407 dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret, __LINE__);
1408 pkey_assert(ret != -1);
1409 ret = waitpid(child_pid, &status, WUNTRACED);
1410 if ((ret != child_pid) || !(WIFSTOPPED(status))) {
1411 fprintf(stderr, "weird waitpid result %ld stat %x\n",
1415 dprintf2("waitpid ret: %ld\n", ret);
1416 dprintf2("waitpid status: %d\n", status);
1418 pkey_access_deny(pkey);
1419 pkey_write_deny(pkey);
1421 /* Write access, untested for now:
1422 ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data);
1423 pkey_assert(ret != -1);
1424 dprintf1("poke at %p: %ld\n", peek_at, ret);
1428 * Try to access the pkey-protected "ptr" via ptrace:
1430 ret = ptrace(PTRACE_PEEKDATA, child_pid, ptr, ignored);
1431 /* expect it to work, without an error: */
1432 pkey_assert(ret != -1);
1433 /* Now access from the current task, and expect an exception: */
1434 peek_result = read_ptr(ptr);
1435 expected_pkey_fault(pkey);
1438 * Try to access the NON-pkey-protected "plain_ptr" via ptrace:
1440 ret = ptrace(PTRACE_PEEKDATA, child_pid, plain_ptr, ignored);
1441 /* expect it to work, without an error: */
1442 pkey_assert(ret != -1);
1443 /* Now access from the current task, and expect NO exception: */
1444 peek_result = read_ptr(plain_ptr);
1445 do_not_expect_pkey_fault("read plain pointer after ptrace");
1447 ret = ptrace(PTRACE_DETACH, child_pid, ignored, 0);
1448 pkey_assert(ret != -1);
1450 ret = kill(child_pid, SIGKILL);
1451 pkey_assert(ret != -1);
1455 free(plain_ptr_unaligned);
1458 void *get_pointer_to_instructions(void)
1462 p1 = ALIGN_PTR_UP(&lots_o_noops_around_write, PAGE_SIZE);
1463 dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write);
1464 /* lots_o_noops_around_write should be page-aligned already */
1465 assert(p1 == &lots_o_noops_around_write);
1467 /* Point 'p1' at the *second* page of the function: */
1471 * Try to ensure we fault this in on next touch to ensure
1472 * we get an instruction fault as opposed to a data one
1474 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1479 void test_executing_on_unreadable_memory(int *ptr, u16 pkey)
1486 p1 = get_pointer_to_instructions();
1487 lots_o_noops_around_write(&scratch);
1488 ptr_contents = read_ptr(p1);
1489 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1491 ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC, (u64)pkey);
1493 pkey_access_deny(pkey);
1495 dprintf2("pkey_reg: %016llx\n", read_pkey_reg());
1498 * Make sure this is an *instruction* fault
1500 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1501 lots_o_noops_around_write(&scratch);
1502 do_not_expect_pkey_fault("executing on PROT_EXEC memory");
1503 expect_fault_on_read_execonly_key(p1, pkey);
1506 void test_implicit_mprotect_exec_only_memory(int *ptr, u16 pkey)
1513 dprintf1("%s() start\n", __func__);
1515 p1 = get_pointer_to_instructions();
1516 lots_o_noops_around_write(&scratch);
1517 ptr_contents = read_ptr(p1);
1518 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1520 /* Use a *normal* mprotect(), not mprotect_pkey(): */
1521 ret = mprotect(p1, PAGE_SIZE, PROT_EXEC);
1525 * Reset the shadow, assuming that the above mprotect()
1526 * correctly changed PKRU, but to an unknown value since
1527 * the actual allocated pkey is unknown.
1529 shadow_pkey_reg = __read_pkey_reg();
1531 dprintf2("pkey_reg: %016llx\n", read_pkey_reg());
1533 /* Make sure this is an *instruction* fault */
1534 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1535 lots_o_noops_around_write(&scratch);
1536 do_not_expect_pkey_fault("executing on PROT_EXEC memory");
1537 expect_fault_on_read_execonly_key(p1, UNKNOWN_PKEY);
1540 * Put the memory back to non-PROT_EXEC. Should clear the
1541 * exec-only pkey off the VMA and allow it to be readable
1542 * again. Go to PROT_NONE first to check for a kernel bug
1543 * that did not clear the pkey when doing PROT_NONE.
1545 ret = mprotect(p1, PAGE_SIZE, PROT_NONE);
1548 ret = mprotect(p1, PAGE_SIZE, PROT_READ|PROT_EXEC);
1550 ptr_contents = read_ptr(p1);
1551 do_not_expect_pkey_fault("plain read on recently PROT_EXEC area");
1554 #if defined(__i386__) || defined(__x86_64__)
1555 void test_ptrace_modifies_pkru(int *ptr, u16 pkey)
1560 int pkey_offset = pkey_reg_xstate_offset();
1561 size_t xsave_size = cpu_max_xsave_size();
1567 new_pkru = ~read_pkey_reg();
1568 /* Don't make PROT_EXEC mappings inaccessible */
1572 pkey_assert(child >= 0);
1573 dprintf3("[%d] fork() ret: %d\n", getpid(), child);
1575 ptrace(PTRACE_TRACEME, 0, 0, 0);
1576 /* Stop and allow the tracer to modify PKRU directly */
1580 * need __read_pkey_reg() version so we do not do shadow_pkey_reg
1583 if (__read_pkey_reg() != new_pkru)
1586 /* Stop and allow the tracer to clear XSTATE_BV for PKRU */
1589 if (__read_pkey_reg() != 0)
1592 /* Stop and allow the tracer to examine PKRU */
1598 pkey_assert(child == waitpid(child, &status, 0));
1599 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1600 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1602 xsave = (void *)malloc(xsave_size);
1603 pkey_assert(xsave > 0);
1605 /* Modify the PKRU register directly */
1606 iov.iov_base = xsave;
1607 iov.iov_len = xsave_size;
1608 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1609 pkey_assert(ret == 0);
1611 pkey_register = (u32 *)(xsave + pkey_offset);
1612 pkey_assert(*pkey_register == read_pkey_reg());
1614 *pkey_register = new_pkru;
1616 ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1617 pkey_assert(ret == 0);
1619 /* Test that the modification is visible in ptrace before any execution */
1620 memset(xsave, 0xCC, xsave_size);
1621 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1622 pkey_assert(ret == 0);
1623 pkey_assert(*pkey_register == new_pkru);
1625 /* Execute the tracee */
1626 ret = ptrace(PTRACE_CONT, child, 0, 0);
1627 pkey_assert(ret == 0);
1629 /* Test that the tracee saw the PKRU value change */
1630 pkey_assert(child == waitpid(child, &status, 0));
1631 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1632 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1634 /* Test that the modification is visible in ptrace after execution */
1635 memset(xsave, 0xCC, xsave_size);
1636 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1637 pkey_assert(ret == 0);
1638 pkey_assert(*pkey_register == new_pkru);
1640 /* Clear the PKRU bit from XSTATE_BV */
1641 xstate_bv = (u64 *)(xsave + 512);
1642 *xstate_bv &= ~(1 << 9);
1644 ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1645 pkey_assert(ret == 0);
1647 /* Test that the modification is visible in ptrace before any execution */
1648 memset(xsave, 0xCC, xsave_size);
1649 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1650 pkey_assert(ret == 0);
1651 pkey_assert(*pkey_register == 0);
1653 ret = ptrace(PTRACE_CONT, child, 0, 0);
1654 pkey_assert(ret == 0);
1656 /* Test that the tracee saw the PKRU value go to 0 */
1657 pkey_assert(child == waitpid(child, &status, 0));
1658 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1659 pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1661 /* Test that the modification is visible in ptrace after execution */
1662 memset(xsave, 0xCC, xsave_size);
1663 ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1664 pkey_assert(ret == 0);
1665 pkey_assert(*pkey_register == 0);
1667 ret = ptrace(PTRACE_CONT, child, 0, 0);
1668 pkey_assert(ret == 0);
1669 pkey_assert(child == waitpid(child, &status, 0));
1670 dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1671 pkey_assert(WIFEXITED(status));
1672 pkey_assert(WEXITSTATUS(status) == 0);
1677 void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey)
1679 int size = PAGE_SIZE;
1682 if (cpu_has_pkeys()) {
1683 dprintf1("SKIP: %s: no CPU support\n", __func__);
1687 sret = syscall(SYS_mprotect_key, ptr, size, PROT_READ, pkey);
1688 pkey_assert(sret < 0);
1691 void (*pkey_tests[])(int *ptr, u16 pkey) = {
1692 test_read_of_write_disabled_region,
1693 test_read_of_access_disabled_region,
1694 test_read_of_access_disabled_region_with_page_already_mapped,
1695 test_write_of_write_disabled_region,
1696 test_write_of_write_disabled_region_with_page_already_mapped,
1697 test_write_of_access_disabled_region,
1698 test_write_of_access_disabled_region_with_page_already_mapped,
1699 test_kernel_write_of_access_disabled_region,
1700 test_kernel_write_of_write_disabled_region,
1701 test_kernel_gup_of_access_disabled_region,
1702 test_kernel_gup_write_to_write_disabled_region,
1703 test_executing_on_unreadable_memory,
1704 test_implicit_mprotect_exec_only_memory,
1705 test_mprotect_with_pkey_0,
1706 test_ptrace_of_child,
1707 test_pkey_init_state,
1708 test_pkey_syscalls_on_non_allocated_pkey,
1709 test_pkey_syscalls_bad_args,
1710 test_pkey_alloc_exhaust,
1711 test_pkey_alloc_free_attach_pkey0,
1712 #if defined(__i386__) || defined(__x86_64__)
1713 test_ptrace_modifies_pkru,
1717 void run_tests_once(void)
1720 int prot = PROT_READ|PROT_WRITE;
1722 for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) {
1724 int orig_pkey_faults = pkey_faults;
1726 dprintf1("======================\n");
1727 dprintf1("test %d preparing...\n", test_nr);
1730 pkey = alloc_random_pkey();
1731 dprintf1("test %d starting with pkey: %d\n", test_nr, pkey);
1732 ptr = malloc_pkey(PAGE_SIZE, prot, pkey);
1733 dprintf1("test %d starting...\n", test_nr);
1734 pkey_tests[test_nr](ptr, pkey);
1735 dprintf1("freeing test memory: %p\n", ptr);
1736 free_pkey_malloc(ptr);
1737 sys_pkey_free(pkey);
1739 dprintf1("pkey_faults: %d\n", pkey_faults);
1740 dprintf1("orig_pkey_faults: %d\n", orig_pkey_faults);
1745 printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr);
1746 dprintf1("======================\n\n");
1751 void pkey_setup_shadow(void)
1753 shadow_pkey_reg = __read_pkey_reg();
1758 int nr_iterations = 22;
1759 int pkeys_supported = is_pkeys_supported();
1761 srand((unsigned int)time(NULL));
1765 printf("has pkeys: %d\n", pkeys_supported);
1767 if (!pkeys_supported) {
1768 int size = PAGE_SIZE;
1771 printf("running PKEY tests for unsupported CPU/OS\n");
1773 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1774 assert(ptr != (void *)-1);
1775 test_mprotect_pkey_on_unsupported_cpu(ptr, 1);
1779 pkey_setup_shadow();
1780 printf("startup pkey_reg: %016llx\n", read_pkey_reg());
1783 while (nr_iterations-- > 0)
1786 printf("done (all tests OK)\n");