tools/testing/selftests/mm/gup_test.c

   1 #include <fcntl.h>
   2 #include <errno.h>
   3 #include <stdio.h>
   4 #include <stdlib.h>
   5 #include <unistd.h>
   6 #include <dirent.h>
   7 #include <sys/ioctl.h>
   8 #include <sys/mman.h>
   9 #include <sys/stat.h>
  10 #include <sys/types.h>
  11 #include <pthread.h>
  12 #include <assert.h>
  13 #include <mm/gup_test.h>
  14 #include "../kselftest.h"
  15 #include "vm_util.h"
  16
  17 #define MB (1UL << 20)
  18
  19 /* Just the flags we need, copied from mm.h: */
  20 #define FOLL_WRITE      0x01    /* check pte is writable */
  21 #define FOLL_TOUCH      0x02    /* mark page accessed */
  22
  23 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
  24
  25 static unsigned long cmd = GUP_FAST_BENCHMARK;
  26 static int gup_fd, repeats = 1;
  27 static unsigned long size = 128 * MB;
  28 /* Serialize prints */
  29 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
  30
  31 static char *cmd_to_str(unsigned long cmd)
  32 {
  33         switch (cmd) {
  34         case GUP_FAST_BENCHMARK:
  35                 return "GUP_FAST_BENCHMARK";
  36         case PIN_FAST_BENCHMARK:
  37                 return "PIN_FAST_BENCHMARK";
  38         case PIN_LONGTERM_BENCHMARK:
  39                 return "PIN_LONGTERM_BENCHMARK";
  40         case GUP_BASIC_TEST:
  41                 return "GUP_BASIC_TEST";
  42         case PIN_BASIC_TEST:
  43                 return "PIN_BASIC_TEST";
  44         case DUMP_USER_PAGES_TEST:
  45                 return "DUMP_USER_PAGES_TEST";
  46         }
  47         return "Unknown command";
  48 }
  49
  50 void *gup_thread(void *data)
  51 {
  52         struct gup_test gup = *(struct gup_test *)data;
  53         int i, status;
  54
  55         /* Only report timing information on the *_BENCHMARK commands: */
  56         if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
  57              (cmd == PIN_LONGTERM_BENCHMARK)) {
  58                 for (i = 0; i < repeats; i++) {
  59                         gup.size = size;
  60                         status = ioctl(gup_fd, cmd, &gup);
  61                         if (status)
  62                                 break;
  63
  64                         pthread_mutex_lock(&print_mutex);
  65                         ksft_print_msg("%s: Time: get:%lld put:%lld us",
  66                                        cmd_to_str(cmd), gup.get_delta_usec,
  67                                        gup.put_delta_usec);
  68                         if (gup.size != size)
  69                                 ksft_print_msg(", truncated (size: %lld)", gup.size);
  70                         ksft_print_msg("\n");
  71                         pthread_mutex_unlock(&print_mutex);
  72                 }
  73         } else {
  74                 gup.size = size;
  75                 status = ioctl(gup_fd, cmd, &gup);
  76                 if (status)
  77                         goto return_;
  78
  79                 pthread_mutex_lock(&print_mutex);
  80                 ksft_print_msg("%s: done\n", cmd_to_str(cmd));
  81                 if (gup.size != size)
  82                         ksft_print_msg("Truncated (size: %lld)\n", gup.size);
  83                 pthread_mutex_unlock(&print_mutex);
  84         }
  85
  86 return_:
  87         ksft_test_result(!status, "ioctl status %d\n", status);
  88         return NULL;
  89 }
  90
  91 int main(int argc, char **argv)
  92 {
  93         struct gup_test gup = { 0 };
  94         int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
  95         int flags = MAP_PRIVATE, touch = 0;
  96         char *file = "/dev/zero";
  97         pthread_t *tid;
  98         char *p;
  99
 100         while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
 101                 switch (opt) {
 102                 case 'a':
 103                         cmd = PIN_FAST_BENCHMARK;
 104                         break;
 105                 case 'b':
 106                         cmd = PIN_BASIC_TEST;
 107                         break;
 108                 case 'L':
 109                         cmd = PIN_LONGTERM_BENCHMARK;
 110                         break;
 111                 case 'c':
 112                         cmd = DUMP_USER_PAGES_TEST;
 113                         /*
 114                          * Dump page 0 (index 1). May be overridden later, by
 115                          * user's non-option arguments.
 116                          *
 117                          * .which_pages is zero-based, so that zero can mean "do
 118                          * nothing".
 119                          */
 120                         gup.which_pages[0] = 1;
 121                         break;
 122                 case 'p':
 123                         /* works only with DUMP_USER_PAGES_TEST */
 124                         gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
 125                         break;
 126                 case 'F':
 127                         /* strtol, so you can pass flags in hex form */
 128                         gup.gup_flags = strtol(optarg, 0, 0);
 129                         break;
 130                 case 'j':
 131                         nthreads = atoi(optarg);
 132                         break;
 133                 case 'm':
 134                         size = atoi(optarg) * MB;
 135                         break;
 136                 case 'r':
 137                         repeats = atoi(optarg);
 138                         break;
 139                 case 'n':
 140                         nr_pages = atoi(optarg);
 141                         break;
 142                 case 't':
 143                         thp = 1;
 144                         break;
 145                 case 'T':
 146                         thp = 0;
 147                         break;
 148                 case 'U':
 149                         cmd = GUP_BASIC_TEST;
 150                         break;
 151                 case 'u':
 152                         cmd = GUP_FAST_BENCHMARK;
 153                         break;
 154                 case 'w':
 155                         write = 1;
 156                         break;
 157                 case 'W':
 158                         write = 0;
 159                         break;
 160                 case 'f':
 161                         file = optarg;
 162                         break;
 163                 case 'S':
 164                         flags &= ~MAP_PRIVATE;
 165                         flags |= MAP_SHARED;
 166                         break;
 167                 case 'H':
 168                         flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
 169                         break;
 170                 case 'z':
 171                         /* fault pages in gup, do not fault in userland */
 172                         touch = 1;
 173                         break;
 174                 default:
 175                         ksft_exit_fail_msg("Wrong argument\n");
 176                 }
 177         }
 178
 179         if (optind < argc) {
 180                 int extra_arg_count = 0;
 181                 /*
 182                  * For example:
 183                  *
 184                  *   ./gup_test -c 0 1 0x1001
 185                  *
 186                  * ...to dump pages 0, 1, and 4097
 187                  */
 188
 189                 while ((optind < argc) &&
 190                        (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
 191                         /*
 192                          * Do the 1-based indexing here, so that the user can
 193                          * use normal 0-based indexing on the command line.
 194                          */
 195                         long page_index = strtol(argv[optind], 0, 0) + 1;
 196
 197                         gup.which_pages[extra_arg_count] = page_index;
 198                         extra_arg_count++;
 199                         optind++;
 200                 }
 201         }
 202
 203         ksft_print_header();
 204         ksft_set_plan(nthreads);
 205
 206         filed = open(file, O_RDWR|O_CREAT);
 207         if (filed < 0)
 208                 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno));
 209
 210         gup.nr_pages_per_call = nr_pages;
 211         if (write)
 212                 gup.gup_flags |= FOLL_WRITE;
 213
 214         gup_fd = open(GUP_TEST_FILE, O_RDWR);
 215         if (gup_fd == -1) {
 216                 switch (errno) {
 217                 case EACCES:
 218                         if (getuid())
 219                                 ksft_print_msg("Please run this test as root\n");
 220                         break;
 221                 case ENOENT:
 222                         if (opendir("/sys/kernel/debug") == NULL)
 223                                 ksft_print_msg("mount debugfs at /sys/kernel/debug\n");
 224                         ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n");
 225                         break;
 226                 default:
 227                         ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno));
 228                         break;
 229                 }
 230                 ksft_test_result_skip("Please run this test as root\n");
 231                 return ksft_exit_pass();
 232         }
 233
 234         p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
 235         if (p == MAP_FAILED)
 236                 ksft_exit_fail_msg("mmap: %s\n", strerror(errno));
 237         gup.addr = (unsigned long)p;
 238
 239         if (thp == 1)
 240                 madvise(p, size, MADV_HUGEPAGE);
 241         else if (thp == 0)
 242                 madvise(p, size, MADV_NOHUGEPAGE);
 243
 244         /*
 245          * FOLL_TOUCH, in gup_test, is used as an either/or case: either
 246          * fault pages in from the kernel via FOLL_TOUCH, or fault them
 247          * in here, from user space. This allows comparison of performance
 248          * between those two cases.
 249          */
 250         if (touch) {
 251                 gup.gup_flags |= FOLL_TOUCH;
 252         } else {
 253                 for (; (unsigned long)p < gup.addr + size; p += psize())
 254                         p[0] = 0;
 255         }
 256
 257         tid = malloc(sizeof(pthread_t) * nthreads);
 258         assert(tid);
 259         for (i = 0; i < nthreads; i++) {
 260                 ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
 261                 assert(ret == 0);
 262         }
 263         for (i = 0; i < nthreads; i++) {
 264                 ret = pthread_join(tid[i], NULL);
 265                 assert(ret == 0);
 266         }
 267
 268         free(tid);
 269
 270         return ksft_exit_pass();
 271 }