tools/testing/selftests/vm/split_huge_page_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * A test of splitting PMD THPs and PTE-mapped THPs from a specified virtual
   4  * address range in a process via <debugfs>/split_huge_pages interface.
   5  */
   6
   7 #define _GNU_SOURCE
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <inttypes.h>
  13 #include <string.h>
  14 #include <fcntl.h>
  15 #include <sys/mman.h>
  16 #include <sys/mount.h>
  17 #include <malloc.h>
  18 #include <stdbool.h>
  19 #include "vm_util.h"
  20
  21 uint64_t pagesize;
  22 unsigned int pageshift;
  23 uint64_t pmd_pagesize;
  24
  25 #define SPLIT_DEBUGFS "/sys/kernel/debug/split_huge_pages"
  26 #define INPUT_MAX 80
  27
  28 #define PID_FMT "%d,0x%lx,0x%lx"
  29 #define PATH_FMT "%s,0x%lx,0x%lx"
  30
  31 #define PFN_MASK     ((1UL<<55)-1)
  32 #define KPF_THP      (1UL<<22)
  33
  34 int is_backed_by_thp(char *vaddr, int pagemap_file, int kpageflags_file)
  35 {
  36         uint64_t paddr;
  37         uint64_t page_flags;
  38
  39         if (pagemap_file) {
  40                 pread(pagemap_file, &paddr, sizeof(paddr),
  41                         ((long)vaddr >> pageshift) * sizeof(paddr));
  42
  43                 if (kpageflags_file) {
  44                         pread(kpageflags_file, &page_flags, sizeof(page_flags),
  45                                 (paddr & PFN_MASK) * sizeof(page_flags));
  46
  47                         return !!(page_flags & KPF_THP);
  48                 }
  49         }
  50         return 0;
  51 }
  52
  53 static int write_file(const char *path, const char *buf, size_t buflen)
  54 {
  55         int fd;
  56         ssize_t numwritten;
  57
  58         fd = open(path, O_WRONLY);
  59         if (fd == -1)
  60                 return 0;
  61
  62         numwritten = write(fd, buf, buflen - 1);
  63         close(fd);
  64         if (numwritten < 1)
  65                 return 0;
  66
  67         return (unsigned int) numwritten;
  68 }
  69
  70 static void write_debugfs(const char *fmt, ...)
  71 {
  72         char input[INPUT_MAX];
  73         int ret;
  74         va_list argp;
  75
  76         va_start(argp, fmt);
  77         ret = vsnprintf(input, INPUT_MAX, fmt, argp);
  78         va_end(argp);
  79
  80         if (ret >= INPUT_MAX) {
  81                 printf("%s: Debugfs input is too long\n", __func__);
  82                 exit(EXIT_FAILURE);
  83         }
  84
  85         if (!write_file(SPLIT_DEBUGFS, input, ret + 1)) {
  86                 perror(SPLIT_DEBUGFS);
  87                 exit(EXIT_FAILURE);
  88         }
  89 }
  90
  91 void split_pmd_thp(void)
  92 {
  93         char *one_page;
  94         size_t len = 4 * pmd_pagesize;
  95         size_t i;
  96
  97         one_page = memalign(pmd_pagesize, len);
  98
  99         if (!one_page) {
 100                 printf("Fail to allocate memory\n");
 101                 exit(EXIT_FAILURE);
 102         }
 103
 104         madvise(one_page, len, MADV_HUGEPAGE);
 105
 106         for (i = 0; i < len; i++)
 107                 one_page[i] = (char)i;
 108
 109         if (!check_huge_anon(one_page, 1, pmd_pagesize)) {
 110                 printf("No THP is allocated\n");
 111                 exit(EXIT_FAILURE);
 112         }
 113
 114         /* split all THPs */
 115         write_debugfs(PID_FMT, getpid(), (uint64_t)one_page,
 116                 (uint64_t)one_page + len);
 117
 118         for (i = 0; i < len; i++)
 119                 if (one_page[i] != (char)i) {
 120                         printf("%ld byte corrupted\n", i);
 121                         exit(EXIT_FAILURE);
 122                 }
 123
 124
 125         if (check_huge_anon(one_page, 0, pmd_pagesize)) {
 126                 printf("Still AnonHugePages not split\n");
 127                 exit(EXIT_FAILURE);
 128         }
 129
 130         printf("Split huge pages successful\n");
 131         free(one_page);
 132 }
 133
 134 void split_pte_mapped_thp(void)
 135 {
 136         char *one_page, *pte_mapped, *pte_mapped2;
 137         size_t len = 4 * pmd_pagesize;
 138         uint64_t thp_size;
 139         size_t i;
 140         const char *pagemap_template = "/proc/%d/pagemap";
 141         const char *kpageflags_proc = "/proc/kpageflags";
 142         char pagemap_proc[255];
 143         int pagemap_fd;
 144         int kpageflags_fd;
 145
 146         if (snprintf(pagemap_proc, 255, pagemap_template, getpid()) < 0) {
 147                 perror("get pagemap proc error");
 148                 exit(EXIT_FAILURE);
 149         }
 150         pagemap_fd = open(pagemap_proc, O_RDONLY);
 151
 152         if (pagemap_fd == -1) {
 153                 perror("read pagemap:");
 154                 exit(EXIT_FAILURE);
 155         }
 156
 157         kpageflags_fd = open(kpageflags_proc, O_RDONLY);
 158
 159         if (kpageflags_fd == -1) {
 160                 perror("read kpageflags:");
 161                 exit(EXIT_FAILURE);
 162         }
 163
 164         one_page = mmap((void *)(1UL << 30), len, PROT_READ | PROT_WRITE,
 165                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 166
 167         madvise(one_page, len, MADV_HUGEPAGE);
 168
 169         for (i = 0; i < len; i++)
 170                 one_page[i] = (char)i;
 171
 172         if (!check_huge_anon(one_page, 1, pmd_pagesize)) {
 173                 printf("No THP is allocated\n");
 174                 exit(EXIT_FAILURE);
 175         }
 176
 177         /* remap the first pagesize of first THP */
 178         pte_mapped = mremap(one_page, pagesize, pagesize, MREMAP_MAYMOVE);
 179
 180         /* remap the Nth pagesize of Nth THP */
 181         for (i = 1; i < 4; i++) {
 182                 pte_mapped2 = mremap(one_page + pmd_pagesize * i + pagesize * i,
 183                                      pagesize, pagesize,
 184                                      MREMAP_MAYMOVE|MREMAP_FIXED,
 185                                      pte_mapped + pagesize * i);
 186                 if (pte_mapped2 == (char *)-1) {
 187                         perror("mremap failed");
 188                         exit(EXIT_FAILURE);
 189                 }
 190         }
 191
 192         /* smap does not show THPs after mremap, use kpageflags instead */
 193         thp_size = 0;
 194         for (i = 0; i < pagesize * 4; i++)
 195                 if (i % pagesize == 0 &&
 196                     is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd))
 197                         thp_size++;
 198
 199         if (thp_size != 4) {
 200                 printf("Some THPs are missing during mremap\n");
 201                 exit(EXIT_FAILURE);
 202         }
 203
 204         /* split all remapped THPs */
 205         write_debugfs(PID_FMT, getpid(), (uint64_t)pte_mapped,
 206                       (uint64_t)pte_mapped + pagesize * 4);
 207
 208         /* smap does not show THPs after mremap, use kpageflags instead */
 209         thp_size = 0;
 210         for (i = 0; i < pagesize * 4; i++) {
 211                 if (pte_mapped[i] != (char)i) {
 212                         printf("%ld byte corrupted\n", i);
 213                         exit(EXIT_FAILURE);
 214                 }
 215                 if (i % pagesize == 0 &&
 216                     is_backed_by_thp(&pte_mapped[i], pagemap_fd, kpageflags_fd))
 217                         thp_size++;
 218         }
 219
 220         if (thp_size) {
 221                 printf("Still %ld THPs not split\n", thp_size);
 222                 exit(EXIT_FAILURE);
 223         }
 224
 225         printf("Split PTE-mapped huge pages successful\n");
 226         munmap(one_page, len);
 227         close(pagemap_fd);
 228         close(kpageflags_fd);
 229 }
 230
 231 void split_file_backed_thp(void)
 232 {
 233         int status;
 234         int fd;
 235         ssize_t num_written;
 236         char tmpfs_template[] = "/tmp/thp_split_XXXXXX";
 237         const char *tmpfs_loc = mkdtemp(tmpfs_template);
 238         char testfile[INPUT_MAX];
 239         uint64_t pgoff_start = 0, pgoff_end = 1024;
 240
 241         printf("Please enable pr_debug in split_huge_pages_in_file() if you need more info.\n");
 242
 243         status = mount("tmpfs", tmpfs_loc, "tmpfs", 0, "huge=always,size=4m");
 244
 245         if (status) {
 246                 printf("Unable to create a tmpfs for testing\n");
 247                 exit(EXIT_FAILURE);
 248         }
 249
 250         status = snprintf(testfile, INPUT_MAX, "%s/thp_file", tmpfs_loc);
 251         if (status >= INPUT_MAX) {
 252                 printf("Fail to create file-backed THP split testing file\n");
 253                 goto cleanup;
 254         }
 255
 256         fd = open(testfile, O_CREAT|O_WRONLY);
 257         if (fd == -1) {
 258                 perror("Cannot open testing file\n");
 259                 goto cleanup;
 260         }
 261
 262         /* write something to the file, so a file-backed THP can be allocated */
 263         num_written = write(fd, tmpfs_loc, strlen(tmpfs_loc) + 1);
 264         close(fd);
 265
 266         if (num_written < 1) {
 267                 printf("Fail to write data to testing file\n");
 268                 goto cleanup;
 269         }
 270
 271         /* split the file-backed THP */
 272         write_debugfs(PATH_FMT, testfile, pgoff_start, pgoff_end);
 273
 274         status = unlink(testfile);
 275         if (status)
 276                 perror("Cannot remove testing file\n");
 277
 278 cleanup:
 279         status = umount(tmpfs_loc);
 280         if (status) {
 281                 printf("Unable to umount %s\n", tmpfs_loc);
 282                 exit(EXIT_FAILURE);
 283         }
 284         status = rmdir(tmpfs_loc);
 285         if (status) {
 286                 perror("cannot remove tmp dir");
 287                 exit(EXIT_FAILURE);
 288         }
 289
 290         printf("file-backed THP split test done, please check dmesg for more information\n");
 291 }
 292
 293 int main(int argc, char **argv)
 294 {
 295         if (geteuid() != 0) {
 296                 printf("Please run the benchmark as root\n");
 297                 exit(EXIT_FAILURE);
 298         }
 299
 300         pagesize = getpagesize();
 301         pageshift = ffs(pagesize) - 1;
 302         pmd_pagesize = read_pmd_pagesize();
 303
 304         split_pmd_thp();
 305         split_pte_mapped_thp();
 306         split_file_backed_thp();
 307
 308         return 0;
 309 }