tools/testing/selftests/vm/hmm-tests.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * HMM stands for Heterogeneous Memory Management, it is a helper layer inside
   4  * the linux kernel to help device drivers mirror a process address space in
   5  * the device. This allows the device to use the same address space which
   6  * makes communication and data exchange a lot easier.
   7  *
   8  * This framework's sole purpose is to exercise various code paths inside
   9  * the kernel to make sure that HMM performs as expected and to flush out any
  10  * bugs.
  11  */
  12
  13 #include "../kselftest_harness.h"
  14
  15 #include <errno.h>
  16 #include <fcntl.h>
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <stdint.h>
  20 #include <unistd.h>
  21 #include <strings.h>
  22 #include <time.h>
  23 #include <pthread.h>
  24 #include <sys/types.h>
  25 #include <sys/stat.h>
  26 #include <sys/mman.h>
  27 #include <sys/ioctl.h>
  28
  29
  30 /*
  31  * This is a private UAPI to the kernel test module so it isn't exported
  32  * in the usual include/uapi/... directory.
  33  */
  34 #include <lib/test_hmm_uapi.h>
  35 #include <mm/gup_test.h>
  36
  37 struct hmm_buffer {
  38         void            *ptr;
  39         void            *mirror;
  40         unsigned long   size;
  41         int             fd;
  42         uint64_t        cpages;
  43         uint64_t        faults;
  44 };
  45
  46 enum {
  47         HMM_PRIVATE_DEVICE_ONE,
  48         HMM_PRIVATE_DEVICE_TWO,
  49         HMM_COHERENCE_DEVICE_ONE,
  50         HMM_COHERENCE_DEVICE_TWO,
  51 };
  52
  53 #define TWOMEG          (1 << 21)
  54 #define HMM_BUFFER_SIZE (1024 << 12)
  55 #define HMM_PATH_MAX    64
  56 #define NTIMES          10
  57
  58 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
  59 /* Just the flags we need, copied from mm.h: */
  60 #define FOLL_WRITE      0x01    /* check pte is writable */
  61 #define FOLL_LONGTERM   0x10000 /* mapping lifetime is indefinite */
  62
  63 FIXTURE(hmm)
  64 {
  65         int             fd;
  66         unsigned int    page_size;
  67         unsigned int    page_shift;
  68 };
  69
  70 FIXTURE_VARIANT(hmm)
  71 {
  72         int     device_number;
  73 };
  74
  75 FIXTURE_VARIANT_ADD(hmm, hmm_device_private)
  76 {
  77         .device_number = HMM_PRIVATE_DEVICE_ONE,
  78 };
  79
  80 FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)
  81 {
  82         .device_number = HMM_COHERENCE_DEVICE_ONE,
  83 };
  84
  85 FIXTURE(hmm2)
  86 {
  87         int             fd0;
  88         int             fd1;
  89         unsigned int    page_size;
  90         unsigned int    page_shift;
  91 };
  92
  93 FIXTURE_VARIANT(hmm2)
  94 {
  95         int     device_number0;
  96         int     device_number1;
  97 };
  98
  99 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)
 100 {
 101         .device_number0 = HMM_PRIVATE_DEVICE_ONE,
 102         .device_number1 = HMM_PRIVATE_DEVICE_TWO,
 103 };
 104
 105 FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)
 106 {
 107         .device_number0 = HMM_COHERENCE_DEVICE_ONE,
 108         .device_number1 = HMM_COHERENCE_DEVICE_TWO,
 109 };
 110
 111 static int hmm_open(int unit)
 112 {
 113         char pathname[HMM_PATH_MAX];
 114         int fd;
 115
 116         snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit);
 117         fd = open(pathname, O_RDWR, 0);
 118         if (fd < 0)
 119                 fprintf(stderr, "could not open hmm dmirror driver (%s)\n",
 120                         pathname);
 121         return fd;
 122 }
 123
 124 static bool hmm_is_coherent_type(int dev_num)
 125 {
 126         return (dev_num >= HMM_COHERENCE_DEVICE_ONE);
 127 }
 128
 129 FIXTURE_SETUP(hmm)
 130 {
 131         self->page_size = sysconf(_SC_PAGE_SIZE);
 132         self->page_shift = ffs(self->page_size) - 1;
 133
 134         self->fd = hmm_open(variant->device_number);
 135         if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))
 136                 SKIP(exit(0), "DEVICE_COHERENT not available");
 137         ASSERT_GE(self->fd, 0);
 138 }
 139
 140 FIXTURE_SETUP(hmm2)
 141 {
 142         self->page_size = sysconf(_SC_PAGE_SIZE);
 143         self->page_shift = ffs(self->page_size) - 1;
 144
 145         self->fd0 = hmm_open(variant->device_number0);
 146         if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))
 147                 SKIP(exit(0), "DEVICE_COHERENT not available");
 148         ASSERT_GE(self->fd0, 0);
 149         self->fd1 = hmm_open(variant->device_number1);
 150         ASSERT_GE(self->fd1, 0);
 151 }
 152
 153 FIXTURE_TEARDOWN(hmm)
 154 {
 155         int ret = close(self->fd);
 156
 157         ASSERT_EQ(ret, 0);
 158         self->fd = -1;
 159 }
 160
 161 FIXTURE_TEARDOWN(hmm2)
 162 {
 163         int ret = close(self->fd0);
 164
 165         ASSERT_EQ(ret, 0);
 166         self->fd0 = -1;
 167
 168         ret = close(self->fd1);
 169         ASSERT_EQ(ret, 0);
 170         self->fd1 = -1;
 171 }
 172
 173 static int hmm_dmirror_cmd(int fd,
 174                            unsigned long request,
 175                            struct hmm_buffer *buffer,
 176                            unsigned long npages)
 177 {
 178         struct hmm_dmirror_cmd cmd;
 179         int ret;
 180
 181         /* Simulate a device reading system memory. */
 182         cmd.addr = (__u64)buffer->ptr;
 183         cmd.ptr = (__u64)buffer->mirror;
 184         cmd.npages = npages;
 185
 186         for (;;) {
 187                 ret = ioctl(fd, request, &cmd);
 188                 if (ret == 0)
 189                         break;
 190                 if (errno == EINTR)
 191                         continue;
 192                 return -errno;
 193         }
 194         buffer->cpages = cmd.cpages;
 195         buffer->faults = cmd.faults;
 196
 197         return 0;
 198 }
 199
 200 static void hmm_buffer_free(struct hmm_buffer *buffer)
 201 {
 202         if (buffer == NULL)
 203                 return;
 204
 205         if (buffer->ptr)
 206                 munmap(buffer->ptr, buffer->size);
 207         free(buffer->mirror);
 208         free(buffer);
 209 }
 210
 211 /*
 212  * Create a temporary file that will be deleted on close.
 213  */
 214 static int hmm_create_file(unsigned long size)
 215 {
 216         char path[HMM_PATH_MAX];
 217         int fd;
 218
 219         strcpy(path, "/tmp");
 220         fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600);
 221         if (fd >= 0) {
 222                 int r;
 223
 224                 do {
 225                         r = ftruncate(fd, size);
 226                 } while (r == -1 && errno == EINTR);
 227                 if (!r)
 228                         return fd;
 229                 close(fd);
 230         }
 231         return -1;
 232 }
 233
 234 /*
 235  * Return a random unsigned number.
 236  */
 237 static unsigned int hmm_random(void)
 238 {
 239         static int fd = -1;
 240         unsigned int r;
 241
 242         if (fd < 0) {
 243                 fd = open("/dev/urandom", O_RDONLY);
 244                 if (fd < 0) {
 245                         fprintf(stderr, "%s:%d failed to open /dev/urandom\n",
 246                                         __FILE__, __LINE__);
 247                         return ~0U;
 248                 }
 249         }
 250         read(fd, &r, sizeof(r));
 251         return r;
 252 }
 253
 254 static void hmm_nanosleep(unsigned int n)
 255 {
 256         struct timespec t;
 257
 258         t.tv_sec = 0;
 259         t.tv_nsec = n;
 260         nanosleep(&t, NULL);
 261 }
 262
 263 static int hmm_migrate_sys_to_dev(int fd,
 264                                    struct hmm_buffer *buffer,
 265                                    unsigned long npages)
 266 {
 267         return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);
 268 }
 269
 270 static int hmm_migrate_dev_to_sys(int fd,
 271                                    struct hmm_buffer *buffer,
 272                                    unsigned long npages)
 273 {
 274         return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);
 275 }
 276
 277 /*
 278  * Simple NULL test of device open/close.
 279  */
 280 TEST_F(hmm, open_close)
 281 {
 282 }
 283
 284 /*
 285  * Read private anonymous memory.
 286  */
 287 TEST_F(hmm, anon_read)
 288 {
 289         struct hmm_buffer *buffer;
 290         unsigned long npages;
 291         unsigned long size;
 292         unsigned long i;
 293         int *ptr;
 294         int ret;
 295         int val;
 296
 297         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 298         ASSERT_NE(npages, 0);
 299         size = npages << self->page_shift;
 300
 301         buffer = malloc(sizeof(*buffer));
 302         ASSERT_NE(buffer, NULL);
 303
 304         buffer->fd = -1;
 305         buffer->size = size;
 306         buffer->mirror = malloc(size);
 307         ASSERT_NE(buffer->mirror, NULL);
 308
 309         buffer->ptr = mmap(NULL, size,
 310                            PROT_READ | PROT_WRITE,
 311                            MAP_PRIVATE | MAP_ANONYMOUS,
 312                            buffer->fd, 0);
 313         ASSERT_NE(buffer->ptr, MAP_FAILED);
 314
 315         /*
 316          * Initialize buffer in system memory but leave the first two pages
 317          * zero (pte_none and pfn_zero).
 318          */
 319         i = 2 * self->page_size / sizeof(*ptr);
 320         for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 321                 ptr[i] = i;
 322
 323         /* Set buffer permission to read-only. */
 324         ret = mprotect(buffer->ptr, size, PROT_READ);
 325         ASSERT_EQ(ret, 0);
 326
 327         /* Populate the CPU page table with a special zero page. */
 328         val = *(int *)(buffer->ptr + self->page_size);
 329         ASSERT_EQ(val, 0);
 330
 331         /* Simulate a device reading system memory. */
 332         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 333         ASSERT_EQ(ret, 0);
 334         ASSERT_EQ(buffer->cpages, npages);
 335         ASSERT_EQ(buffer->faults, 1);
 336
 337         /* Check what the device read. */
 338         ptr = buffer->mirror;
 339         for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i)
 340                 ASSERT_EQ(ptr[i], 0);
 341         for (; i < size / sizeof(*ptr); ++i)
 342                 ASSERT_EQ(ptr[i], i);
 343
 344         hmm_buffer_free(buffer);
 345 }
 346
 347 /*
 348  * Read private anonymous memory which has been protected with
 349  * mprotect() PROT_NONE.
 350  */
 351 TEST_F(hmm, anon_read_prot)
 352 {
 353         struct hmm_buffer *buffer;
 354         unsigned long npages;
 355         unsigned long size;
 356         unsigned long i;
 357         int *ptr;
 358         int ret;
 359
 360         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 361         ASSERT_NE(npages, 0);
 362         size = npages << self->page_shift;
 363
 364         buffer = malloc(sizeof(*buffer));
 365         ASSERT_NE(buffer, NULL);
 366
 367         buffer->fd = -1;
 368         buffer->size = size;
 369         buffer->mirror = malloc(size);
 370         ASSERT_NE(buffer->mirror, NULL);
 371
 372         buffer->ptr = mmap(NULL, size,
 373                            PROT_READ | PROT_WRITE,
 374                            MAP_PRIVATE | MAP_ANONYMOUS,
 375                            buffer->fd, 0);
 376         ASSERT_NE(buffer->ptr, MAP_FAILED);
 377
 378         /* Initialize buffer in system memory. */
 379         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 380                 ptr[i] = i;
 381
 382         /* Initialize mirror buffer so we can verify it isn't written. */
 383         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 384                 ptr[i] = -i;
 385
 386         /* Protect buffer from reading. */
 387         ret = mprotect(buffer->ptr, size, PROT_NONE);
 388         ASSERT_EQ(ret, 0);
 389
 390         /* Simulate a device reading system memory. */
 391         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 392         ASSERT_EQ(ret, -EFAULT);
 393
 394         /* Allow CPU to read the buffer so we can check it. */
 395         ret = mprotect(buffer->ptr, size, PROT_READ);
 396         ASSERT_EQ(ret, 0);
 397         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 398                 ASSERT_EQ(ptr[i], i);
 399
 400         /* Check what the device read. */
 401         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 402                 ASSERT_EQ(ptr[i], -i);
 403
 404         hmm_buffer_free(buffer);
 405 }
 406
 407 /*
 408  * Write private anonymous memory.
 409  */
 410 TEST_F(hmm, anon_write)
 411 {
 412         struct hmm_buffer *buffer;
 413         unsigned long npages;
 414         unsigned long size;
 415         unsigned long i;
 416         int *ptr;
 417         int ret;
 418
 419         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 420         ASSERT_NE(npages, 0);
 421         size = npages << self->page_shift;
 422
 423         buffer = malloc(sizeof(*buffer));
 424         ASSERT_NE(buffer, NULL);
 425
 426         buffer->fd = -1;
 427         buffer->size = size;
 428         buffer->mirror = malloc(size);
 429         ASSERT_NE(buffer->mirror, NULL);
 430
 431         buffer->ptr = mmap(NULL, size,
 432                            PROT_READ | PROT_WRITE,
 433                            MAP_PRIVATE | MAP_ANONYMOUS,
 434                            buffer->fd, 0);
 435         ASSERT_NE(buffer->ptr, MAP_FAILED);
 436
 437         /* Initialize data that the device will write to buffer->ptr. */
 438         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 439                 ptr[i] = i;
 440
 441         /* Simulate a device writing system memory. */
 442         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 443         ASSERT_EQ(ret, 0);
 444         ASSERT_EQ(buffer->cpages, npages);
 445         ASSERT_EQ(buffer->faults, 1);
 446
 447         /* Check what the device wrote. */
 448         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 449                 ASSERT_EQ(ptr[i], i);
 450
 451         hmm_buffer_free(buffer);
 452 }
 453
 454 /*
 455  * Write private anonymous memory which has been protected with
 456  * mprotect() PROT_READ.
 457  */
 458 TEST_F(hmm, anon_write_prot)
 459 {
 460         struct hmm_buffer *buffer;
 461         unsigned long npages;
 462         unsigned long size;
 463         unsigned long i;
 464         int *ptr;
 465         int ret;
 466
 467         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 468         ASSERT_NE(npages, 0);
 469         size = npages << self->page_shift;
 470
 471         buffer = malloc(sizeof(*buffer));
 472         ASSERT_NE(buffer, NULL);
 473
 474         buffer->fd = -1;
 475         buffer->size = size;
 476         buffer->mirror = malloc(size);
 477         ASSERT_NE(buffer->mirror, NULL);
 478
 479         buffer->ptr = mmap(NULL, size,
 480                            PROT_READ,
 481                            MAP_PRIVATE | MAP_ANONYMOUS,
 482                            buffer->fd, 0);
 483         ASSERT_NE(buffer->ptr, MAP_FAILED);
 484
 485         /* Simulate a device reading a zero page of memory. */
 486         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);
 487         ASSERT_EQ(ret, 0);
 488         ASSERT_EQ(buffer->cpages, 1);
 489         ASSERT_EQ(buffer->faults, 1);
 490
 491         /* Initialize data that the device will write to buffer->ptr. */
 492         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 493                 ptr[i] = i;
 494
 495         /* Simulate a device writing system memory. */
 496         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 497         ASSERT_EQ(ret, -EPERM);
 498
 499         /* Check what the device wrote. */
 500         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 501                 ASSERT_EQ(ptr[i], 0);
 502
 503         /* Now allow writing and see that the zero page is replaced. */
 504         ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);
 505         ASSERT_EQ(ret, 0);
 506
 507         /* Simulate a device writing system memory. */
 508         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 509         ASSERT_EQ(ret, 0);
 510         ASSERT_EQ(buffer->cpages, npages);
 511         ASSERT_EQ(buffer->faults, 1);
 512
 513         /* Check what the device wrote. */
 514         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 515                 ASSERT_EQ(ptr[i], i);
 516
 517         hmm_buffer_free(buffer);
 518 }
 519
 520 /*
 521  * Check that a device writing an anonymous private mapping
 522  * will copy-on-write if a child process inherits the mapping.
 523  */
 524 TEST_F(hmm, anon_write_child)
 525 {
 526         struct hmm_buffer *buffer;
 527         unsigned long npages;
 528         unsigned long size;
 529         unsigned long i;
 530         int *ptr;
 531         pid_t pid;
 532         int child_fd;
 533         int ret;
 534
 535         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 536         ASSERT_NE(npages, 0);
 537         size = npages << self->page_shift;
 538
 539         buffer = malloc(sizeof(*buffer));
 540         ASSERT_NE(buffer, NULL);
 541
 542         buffer->fd = -1;
 543         buffer->size = size;
 544         buffer->mirror = malloc(size);
 545         ASSERT_NE(buffer->mirror, NULL);
 546
 547         buffer->ptr = mmap(NULL, size,
 548                            PROT_READ | PROT_WRITE,
 549                            MAP_PRIVATE | MAP_ANONYMOUS,
 550                            buffer->fd, 0);
 551         ASSERT_NE(buffer->ptr, MAP_FAILED);
 552
 553         /* Initialize buffer->ptr so we can tell if it is written. */
 554         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 555                 ptr[i] = i;
 556
 557         /* Initialize data that the device will write to buffer->ptr. */
 558         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 559                 ptr[i] = -i;
 560
 561         pid = fork();
 562         if (pid == -1)
 563                 ASSERT_EQ(pid, 0);
 564         if (pid != 0) {
 565                 waitpid(pid, &ret, 0);
 566                 ASSERT_EQ(WIFEXITED(ret), 1);
 567
 568                 /* Check that the parent's buffer did not change. */
 569                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 570                         ASSERT_EQ(ptr[i], i);
 571                 return;
 572         }
 573
 574         /* Check that we see the parent's values. */
 575         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 576                 ASSERT_EQ(ptr[i], i);
 577         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 578                 ASSERT_EQ(ptr[i], -i);
 579
 580         /* The child process needs its own mirror to its own mm. */
 581         child_fd = hmm_open(0);
 582         ASSERT_GE(child_fd, 0);
 583
 584         /* Simulate a device writing system memory. */
 585         ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
 586         ASSERT_EQ(ret, 0);
 587         ASSERT_EQ(buffer->cpages, npages);
 588         ASSERT_EQ(buffer->faults, 1);
 589
 590         /* Check what the device wrote. */
 591         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 592                 ASSERT_EQ(ptr[i], -i);
 593
 594         close(child_fd);
 595         exit(0);
 596 }
 597
 598 /*
 599  * Check that a device writing an anonymous shared mapping
 600  * will not copy-on-write if a child process inherits the mapping.
 601  */
 602 TEST_F(hmm, anon_write_child_shared)
 603 {
 604         struct hmm_buffer *buffer;
 605         unsigned long npages;
 606         unsigned long size;
 607         unsigned long i;
 608         int *ptr;
 609         pid_t pid;
 610         int child_fd;
 611         int ret;
 612
 613         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 614         ASSERT_NE(npages, 0);
 615         size = npages << self->page_shift;
 616
 617         buffer = malloc(sizeof(*buffer));
 618         ASSERT_NE(buffer, NULL);
 619
 620         buffer->fd = -1;
 621         buffer->size = size;
 622         buffer->mirror = malloc(size);
 623         ASSERT_NE(buffer->mirror, NULL);
 624
 625         buffer->ptr = mmap(NULL, size,
 626                            PROT_READ | PROT_WRITE,
 627                            MAP_SHARED | MAP_ANONYMOUS,
 628                            buffer->fd, 0);
 629         ASSERT_NE(buffer->ptr, MAP_FAILED);
 630
 631         /* Initialize buffer->ptr so we can tell if it is written. */
 632         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 633                 ptr[i] = i;
 634
 635         /* Initialize data that the device will write to buffer->ptr. */
 636         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 637                 ptr[i] = -i;
 638
 639         pid = fork();
 640         if (pid == -1)
 641                 ASSERT_EQ(pid, 0);
 642         if (pid != 0) {
 643                 waitpid(pid, &ret, 0);
 644                 ASSERT_EQ(WIFEXITED(ret), 1);
 645
 646                 /* Check that the parent's buffer did change. */
 647                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 648                         ASSERT_EQ(ptr[i], -i);
 649                 return;
 650         }
 651
 652         /* Check that we see the parent's values. */
 653         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 654                 ASSERT_EQ(ptr[i], i);
 655         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 656                 ASSERT_EQ(ptr[i], -i);
 657
 658         /* The child process needs its own mirror to its own mm. */
 659         child_fd = hmm_open(0);
 660         ASSERT_GE(child_fd, 0);
 661
 662         /* Simulate a device writing system memory. */
 663         ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
 664         ASSERT_EQ(ret, 0);
 665         ASSERT_EQ(buffer->cpages, npages);
 666         ASSERT_EQ(buffer->faults, 1);
 667
 668         /* Check what the device wrote. */
 669         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 670                 ASSERT_EQ(ptr[i], -i);
 671
 672         close(child_fd);
 673         exit(0);
 674 }
 675
 676 /*
 677  * Write private anonymous huge page.
 678  */
 679 TEST_F(hmm, anon_write_huge)
 680 {
 681         struct hmm_buffer *buffer;
 682         unsigned long npages;
 683         unsigned long size;
 684         unsigned long i;
 685         void *old_ptr;
 686         void *map;
 687         int *ptr;
 688         int ret;
 689
 690         size = 2 * TWOMEG;
 691
 692         buffer = malloc(sizeof(*buffer));
 693         ASSERT_NE(buffer, NULL);
 694
 695         buffer->fd = -1;
 696         buffer->size = size;
 697         buffer->mirror = malloc(size);
 698         ASSERT_NE(buffer->mirror, NULL);
 699
 700         buffer->ptr = mmap(NULL, size,
 701                            PROT_READ | PROT_WRITE,
 702                            MAP_PRIVATE | MAP_ANONYMOUS,
 703                            buffer->fd, 0);
 704         ASSERT_NE(buffer->ptr, MAP_FAILED);
 705
 706         size = TWOMEG;
 707         npages = size >> self->page_shift;
 708         map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 709         ret = madvise(map, size, MADV_HUGEPAGE);
 710         ASSERT_EQ(ret, 0);
 711         old_ptr = buffer->ptr;
 712         buffer->ptr = map;
 713
 714         /* Initialize data that the device will write to buffer->ptr. */
 715         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 716                 ptr[i] = i;
 717
 718         /* Simulate a device writing system memory. */
 719         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 720         ASSERT_EQ(ret, 0);
 721         ASSERT_EQ(buffer->cpages, npages);
 722         ASSERT_EQ(buffer->faults, 1);
 723
 724         /* Check what the device wrote. */
 725         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 726                 ASSERT_EQ(ptr[i], i);
 727
 728         buffer->ptr = old_ptr;
 729         hmm_buffer_free(buffer);
 730 }
 731
 732 /*
 733  * Read numeric data from raw and tagged kernel status files.  Used to read
 734  * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).
 735  */
 736 static long file_read_ulong(char *file, const char *tag)
 737 {
 738         int fd;
 739         char buf[2048];
 740         int len;
 741         char *p, *q;
 742         long val;
 743
 744         fd = open(file, O_RDONLY);
 745         if (fd < 0) {
 746                 /* Error opening the file */
 747                 return -1;
 748         }
 749
 750         len = read(fd, buf, sizeof(buf));
 751         close(fd);
 752         if (len < 0) {
 753                 /* Error in reading the file */
 754                 return -1;
 755         }
 756         if (len == sizeof(buf)) {
 757                 /* Error file is too large */
 758                 return -1;
 759         }
 760         buf[len] = '\0';
 761
 762         /* Search for a tag if provided */
 763         if (tag) {
 764                 p = strstr(buf, tag);
 765                 if (!p)
 766                         return -1; /* looks like the line we want isn't there */
 767                 p += strlen(tag);
 768         } else
 769                 p = buf;
 770
 771         val = strtol(p, &q, 0);
 772         if (*q != ' ') {
 773                 /* Error parsing the file */
 774                 return -1;
 775         }
 776
 777         return val;
 778 }
 779
 780 /*
 781  * Write huge TLBFS page.
 782  */
 783 TEST_F(hmm, anon_write_hugetlbfs)
 784 {
 785         struct hmm_buffer *buffer;
 786         unsigned long npages;
 787         unsigned long size;
 788         unsigned long default_hsize;
 789         unsigned long i;
 790         int *ptr;
 791         int ret;
 792
 793         default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");
 794         if (default_hsize < 0 || default_hsize*1024 < default_hsize)
 795                 SKIP(return, "Huge page size could not be determined");
 796         default_hsize = default_hsize*1024; /* KB to B */
 797
 798         size = ALIGN(TWOMEG, default_hsize);
 799         npages = size >> self->page_shift;
 800
 801         buffer = malloc(sizeof(*buffer));
 802         ASSERT_NE(buffer, NULL);
 803
 804         buffer->ptr = mmap(NULL, size,
 805                                    PROT_READ | PROT_WRITE,
 806                                    MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
 807                                    -1, 0);
 808         if (buffer->ptr == MAP_FAILED) {
 809                 free(buffer);
 810                 SKIP(return, "Huge page could not be allocated");
 811         }
 812
 813         buffer->fd = -1;
 814         buffer->size = size;
 815         buffer->mirror = malloc(size);
 816         ASSERT_NE(buffer->mirror, NULL);
 817
 818         /* Initialize data that the device will write to buffer->ptr. */
 819         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 820                 ptr[i] = i;
 821
 822         /* Simulate a device writing system memory. */
 823         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 824         ASSERT_EQ(ret, 0);
 825         ASSERT_EQ(buffer->cpages, npages);
 826         ASSERT_EQ(buffer->faults, 1);
 827
 828         /* Check what the device wrote. */
 829         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 830                 ASSERT_EQ(ptr[i], i);
 831
 832         munmap(buffer->ptr, buffer->size);
 833         buffer->ptr = NULL;
 834         hmm_buffer_free(buffer);
 835 }
 836
 837 /*
 838  * Read mmap'ed file memory.
 839  */
 840 TEST_F(hmm, file_read)
 841 {
 842         struct hmm_buffer *buffer;
 843         unsigned long npages;
 844         unsigned long size;
 845         unsigned long i;
 846         int *ptr;
 847         int ret;
 848         int fd;
 849         ssize_t len;
 850
 851         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 852         ASSERT_NE(npages, 0);
 853         size = npages << self->page_shift;
 854
 855         fd = hmm_create_file(size);
 856         ASSERT_GE(fd, 0);
 857
 858         buffer = malloc(sizeof(*buffer));
 859         ASSERT_NE(buffer, NULL);
 860
 861         buffer->fd = fd;
 862         buffer->size = size;
 863         buffer->mirror = malloc(size);
 864         ASSERT_NE(buffer->mirror, NULL);
 865
 866         /* Write initial contents of the file. */
 867         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 868                 ptr[i] = i;
 869         len = pwrite(fd, buffer->mirror, size, 0);
 870         ASSERT_EQ(len, size);
 871         memset(buffer->mirror, 0, size);
 872
 873         buffer->ptr = mmap(NULL, size,
 874                            PROT_READ,
 875                            MAP_SHARED,
 876                            buffer->fd, 0);
 877         ASSERT_NE(buffer->ptr, MAP_FAILED);
 878
 879         /* Simulate a device reading system memory. */
 880         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 881         ASSERT_EQ(ret, 0);
 882         ASSERT_EQ(buffer->cpages, npages);
 883         ASSERT_EQ(buffer->faults, 1);
 884
 885         /* Check what the device read. */
 886         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 887                 ASSERT_EQ(ptr[i], i);
 888
 889         hmm_buffer_free(buffer);
 890 }
 891
 892 /*
 893  * Write mmap'ed file memory.
 894  */
 895 TEST_F(hmm, file_write)
 896 {
 897         struct hmm_buffer *buffer;
 898         unsigned long npages;
 899         unsigned long size;
 900         unsigned long i;
 901         int *ptr;
 902         int ret;
 903         int fd;
 904         ssize_t len;
 905
 906         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 907         ASSERT_NE(npages, 0);
 908         size = npages << self->page_shift;
 909
 910         fd = hmm_create_file(size);
 911         ASSERT_GE(fd, 0);
 912
 913         buffer = malloc(sizeof(*buffer));
 914         ASSERT_NE(buffer, NULL);
 915
 916         buffer->fd = fd;
 917         buffer->size = size;
 918         buffer->mirror = malloc(size);
 919         ASSERT_NE(buffer->mirror, NULL);
 920
 921         buffer->ptr = mmap(NULL, size,
 922                            PROT_READ | PROT_WRITE,
 923                            MAP_SHARED,
 924                            buffer->fd, 0);
 925         ASSERT_NE(buffer->ptr, MAP_FAILED);
 926
 927         /* Initialize data that the device will write to buffer->ptr. */
 928         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 929                 ptr[i] = i;
 930
 931         /* Simulate a device writing system memory. */
 932         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 933         ASSERT_EQ(ret, 0);
 934         ASSERT_EQ(buffer->cpages, npages);
 935         ASSERT_EQ(buffer->faults, 1);
 936
 937         /* Check what the device wrote. */
 938         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 939                 ASSERT_EQ(ptr[i], i);
 940
 941         /* Check that the device also wrote the file. */
 942         len = pread(fd, buffer->mirror, size, 0);
 943         ASSERT_EQ(len, size);
 944         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 945                 ASSERT_EQ(ptr[i], i);
 946
 947         hmm_buffer_free(buffer);
 948 }
 949
 950 /*
 951  * Migrate anonymous memory to device private memory.
 952  */
 953 TEST_F(hmm, migrate)
 954 {
 955         struct hmm_buffer *buffer;
 956         unsigned long npages;
 957         unsigned long size;
 958         unsigned long i;
 959         int *ptr;
 960         int ret;
 961
 962         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 963         ASSERT_NE(npages, 0);
 964         size = npages << self->page_shift;
 965
 966         buffer = malloc(sizeof(*buffer));
 967         ASSERT_NE(buffer, NULL);
 968
 969         buffer->fd = -1;
 970         buffer->size = size;
 971         buffer->mirror = malloc(size);
 972         ASSERT_NE(buffer->mirror, NULL);
 973
 974         buffer->ptr = mmap(NULL, size,
 975                            PROT_READ | PROT_WRITE,
 976                            MAP_PRIVATE | MAP_ANONYMOUS,
 977                            buffer->fd, 0);
 978         ASSERT_NE(buffer->ptr, MAP_FAILED);
 979
 980         /* Initialize buffer in system memory. */
 981         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 982                 ptr[i] = i;
 983
 984         /* Migrate memory to device. */
 985         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
 986         ASSERT_EQ(ret, 0);
 987         ASSERT_EQ(buffer->cpages, npages);
 988
 989         /* Check what the device read. */
 990         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 991                 ASSERT_EQ(ptr[i], i);
 992
 993         hmm_buffer_free(buffer);
 994 }
 995
 996 /*
 997  * Migrate anonymous memory to device private memory and fault some of it back
 998  * to system memory, then try migrating the resulting mix of system and device
 999  * private memory to the device.
1000  */
1001 TEST_F(hmm, migrate_fault)
1002 {
1003         struct hmm_buffer *buffer;
1004         unsigned long npages;
1005         unsigned long size;
1006         unsigned long i;
1007         int *ptr;
1008         int ret;
1009
1010         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1011         ASSERT_NE(npages, 0);
1012         size = npages << self->page_shift;
1013
1014         buffer = malloc(sizeof(*buffer));
1015         ASSERT_NE(buffer, NULL);
1016
1017         buffer->fd = -1;
1018         buffer->size = size;
1019         buffer->mirror = malloc(size);
1020         ASSERT_NE(buffer->mirror, NULL);
1021
1022         buffer->ptr = mmap(NULL, size,
1023                            PROT_READ | PROT_WRITE,
1024                            MAP_PRIVATE | MAP_ANONYMOUS,
1025                            buffer->fd, 0);
1026         ASSERT_NE(buffer->ptr, MAP_FAILED);
1027
1028         /* Initialize buffer in system memory. */
1029         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1030                 ptr[i] = i;
1031
1032         /* Migrate memory to device. */
1033         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1034         ASSERT_EQ(ret, 0);
1035         ASSERT_EQ(buffer->cpages, npages);
1036
1037         /* Check what the device read. */
1038         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1039                 ASSERT_EQ(ptr[i], i);
1040
1041         /* Fault half the pages back to system memory and check them. */
1042         for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1043                 ASSERT_EQ(ptr[i], i);
1044
1045         /* Migrate memory to the device again. */
1046         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1047         ASSERT_EQ(ret, 0);
1048         ASSERT_EQ(buffer->cpages, npages);
1049
1050         /* Check what the device read. */
1051         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1052                 ASSERT_EQ(ptr[i], i);
1053
1054         hmm_buffer_free(buffer);
1055 }
1056
1057 TEST_F(hmm, migrate_release)
1058 {
1059         struct hmm_buffer *buffer;
1060         unsigned long npages;
1061         unsigned long size;
1062         unsigned long i;
1063         int *ptr;
1064         int ret;
1065
1066         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1067         ASSERT_NE(npages, 0);
1068         size = npages << self->page_shift;
1069
1070         buffer = malloc(sizeof(*buffer));
1071         ASSERT_NE(buffer, NULL);
1072
1073         buffer->fd = -1;
1074         buffer->size = size;
1075         buffer->mirror = malloc(size);
1076         ASSERT_NE(buffer->mirror, NULL);
1077
1078         buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1079                            MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
1080         ASSERT_NE(buffer->ptr, MAP_FAILED);
1081
1082         /* Initialize buffer in system memory. */
1083         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1084                 ptr[i] = i;
1085
1086         /* Migrate memory to device. */
1087         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1088         ASSERT_EQ(ret, 0);
1089         ASSERT_EQ(buffer->cpages, npages);
1090
1091         /* Check what the device read. */
1092         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1093                 ASSERT_EQ(ptr[i], i);
1094
1095         /* Release device memory. */
1096         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages);
1097         ASSERT_EQ(ret, 0);
1098
1099         /* Fault pages back to system memory and check them. */
1100         for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1101                 ASSERT_EQ(ptr[i], i);
1102
1103         hmm_buffer_free(buffer);
1104 }
1105
1106 /*
1107  * Migrate anonymous shared memory to device private memory.
1108  */
1109 TEST_F(hmm, migrate_shared)
1110 {
1111         struct hmm_buffer *buffer;
1112         unsigned long npages;
1113         unsigned long size;
1114         int ret;
1115
1116         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1117         ASSERT_NE(npages, 0);
1118         size = npages << self->page_shift;
1119
1120         buffer = malloc(sizeof(*buffer));
1121         ASSERT_NE(buffer, NULL);
1122
1123         buffer->fd = -1;
1124         buffer->size = size;
1125         buffer->mirror = malloc(size);
1126         ASSERT_NE(buffer->mirror, NULL);
1127
1128         buffer->ptr = mmap(NULL, size,
1129                            PROT_READ | PROT_WRITE,
1130                            MAP_SHARED | MAP_ANONYMOUS,
1131                            buffer->fd, 0);
1132         ASSERT_NE(buffer->ptr, MAP_FAILED);
1133
1134         /* Migrate memory to device. */
1135         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1136         ASSERT_EQ(ret, -ENOENT);
1137
1138         hmm_buffer_free(buffer);
1139 }
1140
1141 /*
1142  * Try to migrate various memory types to device private memory.
1143  */
1144 TEST_F(hmm2, migrate_mixed)
1145 {
1146         struct hmm_buffer *buffer;
1147         unsigned long npages;
1148         unsigned long size;
1149         int *ptr;
1150         unsigned char *p;
1151         int ret;
1152         int val;
1153
1154         npages = 6;
1155         size = npages << self->page_shift;
1156
1157         buffer = malloc(sizeof(*buffer));
1158         ASSERT_NE(buffer, NULL);
1159
1160         buffer->fd = -1;
1161         buffer->size = size;
1162         buffer->mirror = malloc(size);
1163         ASSERT_NE(buffer->mirror, NULL);
1164
1165         /* Reserve a range of addresses. */
1166         buffer->ptr = mmap(NULL, size,
1167                            PROT_NONE,
1168                            MAP_PRIVATE | MAP_ANONYMOUS,
1169                            buffer->fd, 0);
1170         ASSERT_NE(buffer->ptr, MAP_FAILED);
1171         p = buffer->ptr;
1172
1173         /* Migrating a protected area should be an error. */
1174         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1175         ASSERT_EQ(ret, -EINVAL);
1176
1177         /* Punch a hole after the first page address. */
1178         ret = munmap(buffer->ptr + self->page_size, self->page_size);
1179         ASSERT_EQ(ret, 0);
1180
1181         /* We expect an error if the vma doesn't cover the range. */
1182         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3);
1183         ASSERT_EQ(ret, -EINVAL);
1184
1185         /* Page 2 will be a read-only zero page. */
1186         ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1187                                 PROT_READ);
1188         ASSERT_EQ(ret, 0);
1189         ptr = (int *)(buffer->ptr + 2 * self->page_size);
1190         val = *ptr + 3;
1191         ASSERT_EQ(val, 3);
1192
1193         /* Page 3 will be read-only. */
1194         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1195                                 PROT_READ | PROT_WRITE);
1196         ASSERT_EQ(ret, 0);
1197         ptr = (int *)(buffer->ptr + 3 * self->page_size);
1198         *ptr = val;
1199         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1200                                 PROT_READ);
1201         ASSERT_EQ(ret, 0);
1202
1203         /* Page 4-5 will be read-write. */
1204         ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size,
1205                                 PROT_READ | PROT_WRITE);
1206         ASSERT_EQ(ret, 0);
1207         ptr = (int *)(buffer->ptr + 4 * self->page_size);
1208         *ptr = val;
1209         ptr = (int *)(buffer->ptr + 5 * self->page_size);
1210         *ptr = val;
1211
1212         /* Now try to migrate pages 2-5 to device 1. */
1213         buffer->ptr = p + 2 * self->page_size;
1214         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4);
1215         ASSERT_EQ(ret, 0);
1216         ASSERT_EQ(buffer->cpages, 4);
1217
1218         /* Page 5 won't be migrated to device 0 because it's on device 1. */
1219         buffer->ptr = p + 5 * self->page_size;
1220         ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1221         ASSERT_EQ(ret, -ENOENT);
1222         buffer->ptr = p;
1223
1224         buffer->ptr = p;
1225         hmm_buffer_free(buffer);
1226 }
1227
1228 /*
1229  * Migrate anonymous memory to device memory and back to system memory
1230  * multiple times. In case of private zone configuration, this is done
1231  * through fault pages accessed by CPU. In case of coherent zone configuration,
1232  * the pages from the device should be explicitly migrated back to system memory.
1233  * The reason is Coherent device zone has coherent access by CPU, therefore
1234  * it will not generate any page fault.
1235  */
1236 TEST_F(hmm, migrate_multiple)
1237 {
1238         struct hmm_buffer *buffer;
1239         unsigned long npages;
1240         unsigned long size;
1241         unsigned long i;
1242         unsigned long c;
1243         int *ptr;
1244         int ret;
1245
1246         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1247         ASSERT_NE(npages, 0);
1248         size = npages << self->page_shift;
1249
1250         for (c = 0; c < NTIMES; c++) {
1251                 buffer = malloc(sizeof(*buffer));
1252                 ASSERT_NE(buffer, NULL);
1253
1254                 buffer->fd = -1;
1255                 buffer->size = size;
1256                 buffer->mirror = malloc(size);
1257                 ASSERT_NE(buffer->mirror, NULL);
1258
1259                 buffer->ptr = mmap(NULL, size,
1260                                    PROT_READ | PROT_WRITE,
1261                                    MAP_PRIVATE | MAP_ANONYMOUS,
1262                                    buffer->fd, 0);
1263                 ASSERT_NE(buffer->ptr, MAP_FAILED);
1264
1265                 /* Initialize buffer in system memory. */
1266                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1267                         ptr[i] = i;
1268
1269                 /* Migrate memory to device. */
1270                 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1271                 ASSERT_EQ(ret, 0);
1272                 ASSERT_EQ(buffer->cpages, npages);
1273
1274                 /* Check what the device read. */
1275                 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1276                         ASSERT_EQ(ptr[i], i);
1277
1278                 /* Migrate back to system memory and check them. */
1279                 if (hmm_is_coherent_type(variant->device_number)) {
1280                         ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
1281                         ASSERT_EQ(ret, 0);
1282                         ASSERT_EQ(buffer->cpages, npages);
1283                 }
1284
1285                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1286                         ASSERT_EQ(ptr[i], i);
1287
1288                 hmm_buffer_free(buffer);
1289         }
1290 }
1291
1292 /*
1293  * Read anonymous memory multiple times.
1294  */
1295 TEST_F(hmm, anon_read_multiple)
1296 {
1297         struct hmm_buffer *buffer;
1298         unsigned long npages;
1299         unsigned long size;
1300         unsigned long i;
1301         unsigned long c;
1302         int *ptr;
1303         int ret;
1304
1305         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1306         ASSERT_NE(npages, 0);
1307         size = npages << self->page_shift;
1308
1309         for (c = 0; c < NTIMES; c++) {
1310                 buffer = malloc(sizeof(*buffer));
1311                 ASSERT_NE(buffer, NULL);
1312
1313                 buffer->fd = -1;
1314                 buffer->size = size;
1315                 buffer->mirror = malloc(size);
1316                 ASSERT_NE(buffer->mirror, NULL);
1317
1318                 buffer->ptr = mmap(NULL, size,
1319                                    PROT_READ | PROT_WRITE,
1320                                    MAP_PRIVATE | MAP_ANONYMOUS,
1321                                    buffer->fd, 0);
1322                 ASSERT_NE(buffer->ptr, MAP_FAILED);
1323
1324                 /* Initialize buffer in system memory. */
1325                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1326                         ptr[i] = i + c;
1327
1328                 /* Simulate a device reading system memory. */
1329                 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1330                                       npages);
1331                 ASSERT_EQ(ret, 0);
1332                 ASSERT_EQ(buffer->cpages, npages);
1333                 ASSERT_EQ(buffer->faults, 1);
1334
1335                 /* Check what the device read. */
1336                 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1337                         ASSERT_EQ(ptr[i], i + c);
1338
1339                 hmm_buffer_free(buffer);
1340         }
1341 }
1342
1343 void *unmap_buffer(void *p)
1344 {
1345         struct hmm_buffer *buffer = p;
1346
1347         /* Delay for a bit and then unmap buffer while it is being read. */
1348         hmm_nanosleep(hmm_random() % 32000);
1349         munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);
1350         buffer->ptr = NULL;
1351
1352         return NULL;
1353 }
1354
1355 /*
1356  * Try reading anonymous memory while it is being unmapped.
1357  */
1358 TEST_F(hmm, anon_teardown)
1359 {
1360         unsigned long npages;
1361         unsigned long size;
1362         unsigned long c;
1363         void *ret;
1364
1365         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1366         ASSERT_NE(npages, 0);
1367         size = npages << self->page_shift;
1368
1369         for (c = 0; c < NTIMES; ++c) {
1370                 pthread_t thread;
1371                 struct hmm_buffer *buffer;
1372                 unsigned long i;
1373                 int *ptr;
1374                 int rc;
1375
1376                 buffer = malloc(sizeof(*buffer));
1377                 ASSERT_NE(buffer, NULL);
1378
1379                 buffer->fd = -1;
1380                 buffer->size = size;
1381                 buffer->mirror = malloc(size);
1382                 ASSERT_NE(buffer->mirror, NULL);
1383
1384                 buffer->ptr = mmap(NULL, size,
1385                                    PROT_READ | PROT_WRITE,
1386                                    MAP_PRIVATE | MAP_ANONYMOUS,
1387                                    buffer->fd, 0);
1388                 ASSERT_NE(buffer->ptr, MAP_FAILED);
1389
1390                 /* Initialize buffer in system memory. */
1391                 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1392                         ptr[i] = i + c;
1393
1394                 rc = pthread_create(&thread, NULL, unmap_buffer, buffer);
1395                 ASSERT_EQ(rc, 0);
1396
1397                 /* Simulate a device reading system memory. */
1398                 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1399                                      npages);
1400                 if (rc == 0) {
1401                         ASSERT_EQ(buffer->cpages, npages);
1402                         ASSERT_EQ(buffer->faults, 1);
1403
1404                         /* Check what the device read. */
1405                         for (i = 0, ptr = buffer->mirror;
1406                              i < size / sizeof(*ptr);
1407                              ++i)
1408                                 ASSERT_EQ(ptr[i], i + c);
1409                 }
1410
1411                 pthread_join(thread, &ret);
1412                 hmm_buffer_free(buffer);
1413         }
1414 }
1415
1416 /*
1417  * Test memory snapshot without faulting in pages accessed by the device.
1418  */
1419 TEST_F(hmm, mixedmap)
1420 {
1421         struct hmm_buffer *buffer;
1422         unsigned long npages;
1423         unsigned long size;
1424         unsigned char *m;
1425         int ret;
1426
1427         npages = 1;
1428         size = npages << self->page_shift;
1429
1430         buffer = malloc(sizeof(*buffer));
1431         ASSERT_NE(buffer, NULL);
1432
1433         buffer->fd = -1;
1434         buffer->size = size;
1435         buffer->mirror = malloc(npages);
1436         ASSERT_NE(buffer->mirror, NULL);
1437
1438
1439         /* Reserve a range of addresses. */
1440         buffer->ptr = mmap(NULL, size,
1441                            PROT_READ | PROT_WRITE,
1442                            MAP_PRIVATE,
1443                            self->fd, 0);
1444         ASSERT_NE(buffer->ptr, MAP_FAILED);
1445
1446         /* Simulate a device snapshotting CPU pagetables. */
1447         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1448         ASSERT_EQ(ret, 0);
1449         ASSERT_EQ(buffer->cpages, npages);
1450
1451         /* Check what the device saw. */
1452         m = buffer->mirror;
1453         ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);
1454
1455         hmm_buffer_free(buffer);
1456 }
1457
1458 /*
1459  * Test memory snapshot without faulting in pages accessed by the device.
1460  */
1461 TEST_F(hmm2, snapshot)
1462 {
1463         struct hmm_buffer *buffer;
1464         unsigned long npages;
1465         unsigned long size;
1466         int *ptr;
1467         unsigned char *p;
1468         unsigned char *m;
1469         int ret;
1470         int val;
1471
1472         npages = 7;
1473         size = npages << self->page_shift;
1474
1475         buffer = malloc(sizeof(*buffer));
1476         ASSERT_NE(buffer, NULL);
1477
1478         buffer->fd = -1;
1479         buffer->size = size;
1480         buffer->mirror = malloc(npages);
1481         ASSERT_NE(buffer->mirror, NULL);
1482
1483         /* Reserve a range of addresses. */
1484         buffer->ptr = mmap(NULL, size,
1485                            PROT_NONE,
1486                            MAP_PRIVATE | MAP_ANONYMOUS,
1487                            buffer->fd, 0);
1488         ASSERT_NE(buffer->ptr, MAP_FAILED);
1489         p = buffer->ptr;
1490
1491         /* Punch a hole after the first page address. */
1492         ret = munmap(buffer->ptr + self->page_size, self->page_size);
1493         ASSERT_EQ(ret, 0);
1494
1495         /* Page 2 will be read-only zero page. */
1496         ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1497                                 PROT_READ);
1498         ASSERT_EQ(ret, 0);
1499         ptr = (int *)(buffer->ptr + 2 * self->page_size);
1500         val = *ptr + 3;
1501         ASSERT_EQ(val, 3);
1502
1503         /* Page 3 will be read-only. */
1504         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1505                                 PROT_READ | PROT_WRITE);
1506         ASSERT_EQ(ret, 0);
1507         ptr = (int *)(buffer->ptr + 3 * self->page_size);
1508         *ptr = val;
1509         ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1510                                 PROT_READ);
1511         ASSERT_EQ(ret, 0);
1512
1513         /* Page 4-6 will be read-write. */
1514         ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size,
1515                                 PROT_READ | PROT_WRITE);
1516         ASSERT_EQ(ret, 0);
1517         ptr = (int *)(buffer->ptr + 4 * self->page_size);
1518         *ptr = val;
1519
1520         /* Page 5 will be migrated to device 0. */
1521         buffer->ptr = p + 5 * self->page_size;
1522         ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1523         ASSERT_EQ(ret, 0);
1524         ASSERT_EQ(buffer->cpages, 1);
1525
1526         /* Page 6 will be migrated to device 1. */
1527         buffer->ptr = p + 6 * self->page_size;
1528         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1);
1529         ASSERT_EQ(ret, 0);
1530         ASSERT_EQ(buffer->cpages, 1);
1531
1532         /* Simulate a device snapshotting CPU pagetables. */
1533         buffer->ptr = p;
1534         ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1535         ASSERT_EQ(ret, 0);
1536         ASSERT_EQ(buffer->cpages, npages);
1537
1538         /* Check what the device saw. */
1539         m = buffer->mirror;
1540         ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR);
1541         ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR);
1542         ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);
1543         ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);
1544         ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);
1545         if (!hmm_is_coherent_type(variant->device_number0)) {
1546                 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
1547                                 HMM_DMIRROR_PROT_WRITE);
1548                 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
1549         } else {
1550                 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |
1551                                 HMM_DMIRROR_PROT_WRITE);
1552                 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |
1553                                 HMM_DMIRROR_PROT_WRITE);
1554         }
1555
1556         hmm_buffer_free(buffer);
1557 }
1558
1559 /*
1560  * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
1561  * should be mapped by a large page table entry.
1562  */
1563 TEST_F(hmm, compound)
1564 {
1565         struct hmm_buffer *buffer;
1566         unsigned long npages;
1567         unsigned long size;
1568         unsigned long default_hsize;
1569         int *ptr;
1570         unsigned char *m;
1571         int ret;
1572         unsigned long i;
1573
1574         /* Skip test if we can't allocate a hugetlbfs page. */
1575
1576         default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");
1577         if (default_hsize < 0 || default_hsize*1024 < default_hsize)
1578                 SKIP(return, "Huge page size could not be determined");
1579         default_hsize = default_hsize*1024; /* KB to B */
1580
1581         size = ALIGN(TWOMEG, default_hsize);
1582         npages = size >> self->page_shift;
1583
1584         buffer = malloc(sizeof(*buffer));
1585         ASSERT_NE(buffer, NULL);
1586
1587         buffer->ptr = mmap(NULL, size,
1588                                    PROT_READ | PROT_WRITE,
1589                                    MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
1590                                    -1, 0);
1591         if (buffer->ptr == MAP_FAILED) {
1592                 free(buffer);
1593                 return;
1594         }
1595
1596         buffer->size = size;
1597         buffer->mirror = malloc(npages);
1598         ASSERT_NE(buffer->mirror, NULL);
1599
1600         /* Initialize the pages the device will snapshot in buffer->ptr. */
1601         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1602                 ptr[i] = i;
1603
1604         /* Simulate a device snapshotting CPU pagetables. */
1605         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1606         ASSERT_EQ(ret, 0);
1607         ASSERT_EQ(buffer->cpages, npages);
1608
1609         /* Check what the device saw. */
1610         m = buffer->mirror;
1611         for (i = 0; i < npages; ++i)
1612                 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |
1613                                 HMM_DMIRROR_PROT_PMD);
1614
1615         /* Make the region read-only. */
1616         ret = mprotect(buffer->ptr, size, PROT_READ);
1617         ASSERT_EQ(ret, 0);
1618
1619         /* Simulate a device snapshotting CPU pagetables. */
1620         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1621         ASSERT_EQ(ret, 0);
1622         ASSERT_EQ(buffer->cpages, npages);
1623
1624         /* Check what the device saw. */
1625         m = buffer->mirror;
1626         for (i = 0; i < npages; ++i)
1627                 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
1628                                 HMM_DMIRROR_PROT_PMD);
1629
1630         munmap(buffer->ptr, buffer->size);
1631         buffer->ptr = NULL;
1632         hmm_buffer_free(buffer);
1633 }
1634
1635 /*
1636  * Test two devices reading the same memory (double mapped).
1637  */
1638 TEST_F(hmm2, double_map)
1639 {
1640         struct hmm_buffer *buffer;
1641         unsigned long npages;
1642         unsigned long size;
1643         unsigned long i;
1644         int *ptr;
1645         int ret;
1646
1647         npages = 6;
1648         size = npages << self->page_shift;
1649
1650         buffer = malloc(sizeof(*buffer));
1651         ASSERT_NE(buffer, NULL);
1652
1653         buffer->fd = -1;
1654         buffer->size = size;
1655         buffer->mirror = malloc(npages);
1656         ASSERT_NE(buffer->mirror, NULL);
1657
1658         /* Reserve a range of addresses. */
1659         buffer->ptr = mmap(NULL, size,
1660                            PROT_READ | PROT_WRITE,
1661                            MAP_PRIVATE | MAP_ANONYMOUS,
1662                            buffer->fd, 0);
1663         ASSERT_NE(buffer->ptr, MAP_FAILED);
1664
1665         /* Initialize buffer in system memory. */
1666         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1667                 ptr[i] = i;
1668
1669         /* Make region read-only. */
1670         ret = mprotect(buffer->ptr, size, PROT_READ);
1671         ASSERT_EQ(ret, 0);
1672
1673         /* Simulate device 0 reading system memory. */
1674         ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1675         ASSERT_EQ(ret, 0);
1676         ASSERT_EQ(buffer->cpages, npages);
1677         ASSERT_EQ(buffer->faults, 1);
1678
1679         /* Check what the device read. */
1680         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1681                 ASSERT_EQ(ptr[i], i);
1682
1683         /* Simulate device 1 reading system memory. */
1684         ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages);
1685         ASSERT_EQ(ret, 0);
1686         ASSERT_EQ(buffer->cpages, npages);
1687         ASSERT_EQ(buffer->faults, 1);
1688
1689         /* Check what the device read. */
1690         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1691                 ASSERT_EQ(ptr[i], i);
1692
1693         /* Migrate pages to device 1 and try to read from device 0. */
1694         ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1695         ASSERT_EQ(ret, 0);
1696         ASSERT_EQ(buffer->cpages, npages);
1697
1698         ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1699         ASSERT_EQ(ret, 0);
1700         ASSERT_EQ(buffer->cpages, npages);
1701         ASSERT_EQ(buffer->faults, 1);
1702
1703         /* Check what device 0 read. */
1704         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1705                 ASSERT_EQ(ptr[i], i);
1706
1707         hmm_buffer_free(buffer);
1708 }
1709
1710 /*
1711  * Basic check of exclusive faulting.
1712  */
1713 TEST_F(hmm, exclusive)
1714 {
1715         struct hmm_buffer *buffer;
1716         unsigned long npages;
1717         unsigned long size;
1718         unsigned long i;
1719         int *ptr;
1720         int ret;
1721
1722         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1723         ASSERT_NE(npages, 0);
1724         size = npages << self->page_shift;
1725
1726         buffer = malloc(sizeof(*buffer));
1727         ASSERT_NE(buffer, NULL);
1728
1729         buffer->fd = -1;
1730         buffer->size = size;
1731         buffer->mirror = malloc(size);
1732         ASSERT_NE(buffer->mirror, NULL);
1733
1734         buffer->ptr = mmap(NULL, size,
1735                            PROT_READ | PROT_WRITE,
1736                            MAP_PRIVATE | MAP_ANONYMOUS,
1737                            buffer->fd, 0);
1738         ASSERT_NE(buffer->ptr, MAP_FAILED);
1739
1740         /* Initialize buffer in system memory. */
1741         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1742                 ptr[i] = i;
1743
1744         /* Map memory exclusively for device access. */
1745         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1746         ASSERT_EQ(ret, 0);
1747         ASSERT_EQ(buffer->cpages, npages);
1748
1749         /* Check what the device read. */
1750         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1751                 ASSERT_EQ(ptr[i], i);
1752
1753         /* Fault pages back to system memory and check them. */
1754         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1755                 ASSERT_EQ(ptr[i]++, i);
1756
1757         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1758                 ASSERT_EQ(ptr[i], i+1);
1759
1760         /* Check atomic access revoked */
1761         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages);
1762         ASSERT_EQ(ret, 0);
1763
1764         hmm_buffer_free(buffer);
1765 }
1766
1767 TEST_F(hmm, exclusive_mprotect)
1768 {
1769         struct hmm_buffer *buffer;
1770         unsigned long npages;
1771         unsigned long size;
1772         unsigned long i;
1773         int *ptr;
1774         int ret;
1775
1776         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1777         ASSERT_NE(npages, 0);
1778         size = npages << self->page_shift;
1779
1780         buffer = malloc(sizeof(*buffer));
1781         ASSERT_NE(buffer, NULL);
1782
1783         buffer->fd = -1;
1784         buffer->size = size;
1785         buffer->mirror = malloc(size);
1786         ASSERT_NE(buffer->mirror, NULL);
1787
1788         buffer->ptr = mmap(NULL, size,
1789                            PROT_READ | PROT_WRITE,
1790                            MAP_PRIVATE | MAP_ANONYMOUS,
1791                            buffer->fd, 0);
1792         ASSERT_NE(buffer->ptr, MAP_FAILED);
1793
1794         /* Initialize buffer in system memory. */
1795         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1796                 ptr[i] = i;
1797
1798         /* Map memory exclusively for device access. */
1799         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1800         ASSERT_EQ(ret, 0);
1801         ASSERT_EQ(buffer->cpages, npages);
1802
1803         /* Check what the device read. */
1804         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1805                 ASSERT_EQ(ptr[i], i);
1806
1807         ret = mprotect(buffer->ptr, size, PROT_READ);
1808         ASSERT_EQ(ret, 0);
1809
1810         /* Simulate a device writing system memory. */
1811         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
1812         ASSERT_EQ(ret, -EPERM);
1813
1814         hmm_buffer_free(buffer);
1815 }
1816
1817 /*
1818  * Check copy-on-write works.
1819  */
1820 TEST_F(hmm, exclusive_cow)
1821 {
1822         struct hmm_buffer *buffer;
1823         unsigned long npages;
1824         unsigned long size;
1825         unsigned long i;
1826         int *ptr;
1827         int ret;
1828
1829         npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1830         ASSERT_NE(npages, 0);
1831         size = npages << self->page_shift;
1832
1833         buffer = malloc(sizeof(*buffer));
1834         ASSERT_NE(buffer, NULL);
1835
1836         buffer->fd = -1;
1837         buffer->size = size;
1838         buffer->mirror = malloc(size);
1839         ASSERT_NE(buffer->mirror, NULL);
1840
1841         buffer->ptr = mmap(NULL, size,
1842                            PROT_READ | PROT_WRITE,
1843                            MAP_PRIVATE | MAP_ANONYMOUS,
1844                            buffer->fd, 0);
1845         ASSERT_NE(buffer->ptr, MAP_FAILED);
1846
1847         /* Initialize buffer in system memory. */
1848         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1849                 ptr[i] = i;
1850
1851         /* Map memory exclusively for device access. */
1852         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1853         ASSERT_EQ(ret, 0);
1854         ASSERT_EQ(buffer->cpages, npages);
1855
1856         fork();
1857
1858         /* Fault pages back to system memory and check them. */
1859         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1860                 ASSERT_EQ(ptr[i]++, i);
1861
1862         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1863                 ASSERT_EQ(ptr[i], i+1);
1864
1865         hmm_buffer_free(buffer);
1866 }
1867
1868 static int gup_test_exec(int gup_fd, unsigned long addr, int cmd,
1869                          int npages, int size, int flags)
1870 {
1871         struct gup_test gup = {
1872                 .nr_pages_per_call      = npages,
1873                 .addr                   = addr,
1874                 .gup_flags              = FOLL_WRITE | flags,
1875                 .size                   = size,
1876         };
1877
1878         if (ioctl(gup_fd, cmd, &gup)) {
1879                 perror("ioctl on error\n");
1880                 return errno;
1881         }
1882
1883         return 0;
1884 }
1885
1886 /*
1887  * Test get user device pages through gup_test. Setting PIN_LONGTERM flag.
1888  * This should trigger a migration back to system memory for both, private
1889  * and coherent type pages.
1890  * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added
1891  * to your configuration before you run it.
1892  */
1893 TEST_F(hmm, hmm_gup_test)
1894 {
1895         struct hmm_buffer *buffer;
1896         int gup_fd;
1897         unsigned long npages;
1898         unsigned long size;
1899         unsigned long i;
1900         int *ptr;
1901         int ret;
1902         unsigned char *m;
1903
1904         gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
1905         if (gup_fd == -1)
1906                 SKIP(return, "Skipping test, could not find gup_test driver");
1907
1908         npages = 4;
1909         size = npages << self->page_shift;
1910
1911         buffer = malloc(sizeof(*buffer));
1912         ASSERT_NE(buffer, NULL);
1913
1914         buffer->fd = -1;
1915         buffer->size = size;
1916         buffer->mirror = malloc(size);
1917         ASSERT_NE(buffer->mirror, NULL);
1918
1919         buffer->ptr = mmap(NULL, size,
1920                            PROT_READ | PROT_WRITE,
1921                            MAP_PRIVATE | MAP_ANONYMOUS,
1922                            buffer->fd, 0);
1923         ASSERT_NE(buffer->ptr, MAP_FAILED);
1924
1925         /* Initialize buffer in system memory. */
1926         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1927                 ptr[i] = i;
1928
1929         /* Migrate memory to device. */
1930         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1931         ASSERT_EQ(ret, 0);
1932         ASSERT_EQ(buffer->cpages, npages);
1933         /* Check what the device read. */
1934         for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1935                 ASSERT_EQ(ptr[i], i);
1936
1937         ASSERT_EQ(gup_test_exec(gup_fd,
1938                                 (unsigned long)buffer->ptr,
1939                                 GUP_BASIC_TEST, 1, self->page_size, 0), 0);
1940         ASSERT_EQ(gup_test_exec(gup_fd,
1941                                 (unsigned long)buffer->ptr + 1 * self->page_size,
1942                                 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0);
1943         ASSERT_EQ(gup_test_exec(gup_fd,
1944                                 (unsigned long)buffer->ptr + 2 * self->page_size,
1945                                 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0);
1946         ASSERT_EQ(gup_test_exec(gup_fd,
1947                                 (unsigned long)buffer->ptr + 3 * self->page_size,
1948                                 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0);
1949
1950         /* Take snapshot to CPU pagetables */
1951         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1952         ASSERT_EQ(ret, 0);
1953         ASSERT_EQ(buffer->cpages, npages);
1954         m = buffer->mirror;
1955         if (hmm_is_coherent_type(variant->device_number)) {
1956                 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);
1957                 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);
1958         } else {
1959                 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);
1960                 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);
1961         }
1962         ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);
1963         ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]);
1964         /*
1965          * Check again the content on the pages. Make sure there's no
1966          * corrupted data.
1967          */
1968         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1969                 ASSERT_EQ(ptr[i], i);
1970
1971         close(gup_fd);
1972         hmm_buffer_free(buffer);
1973 }
1974
1975 /*
1976  * Test copy-on-write in device pages.
1977  * In case of writing to COW private page(s), a page fault will migrate pages
1978  * back to system memory first. Then, these pages will be duplicated. In case
1979  * of COW device coherent type, pages are duplicated directly from device
1980  * memory.
1981  */
1982 TEST_F(hmm, hmm_cow_in_device)
1983 {
1984         struct hmm_buffer *buffer;
1985         unsigned long npages;
1986         unsigned long size;
1987         unsigned long i;
1988         int *ptr;
1989         int ret;
1990         unsigned char *m;
1991         pid_t pid;
1992         int status;
1993
1994         npages = 4;
1995         size = npages << self->page_shift;
1996
1997         buffer = malloc(sizeof(*buffer));
1998         ASSERT_NE(buffer, NULL);
1999
2000         buffer->fd = -1;
2001         buffer->size = size;
2002         buffer->mirror = malloc(size);
2003         ASSERT_NE(buffer->mirror, NULL);
2004
2005         buffer->ptr = mmap(NULL, size,
2006                            PROT_READ | PROT_WRITE,
2007                            MAP_PRIVATE | MAP_ANONYMOUS,
2008                            buffer->fd, 0);
2009         ASSERT_NE(buffer->ptr, MAP_FAILED);
2010
2011         /* Initialize buffer in system memory. */
2012         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2013                 ptr[i] = i;
2014
2015         /* Migrate memory to device. */
2016
2017         ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2018         ASSERT_EQ(ret, 0);
2019         ASSERT_EQ(buffer->cpages, npages);
2020
2021         pid = fork();
2022         if (pid == -1)
2023                 ASSERT_EQ(pid, 0);
2024         if (!pid) {
2025                 /* Child process waitd for SIGTERM from the parent. */
2026                 while (1) {
2027                 }
2028                 perror("Should not reach this\n");
2029                 exit(0);
2030         }
2031         /* Parent process writes to COW pages(s) and gets a
2032          * new copy in system. In case of device private pages,
2033          * this write causes a migration to system mem first.
2034          */
2035         for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2036                 ptr[i] = i;
2037
2038         /* Terminate child and wait */
2039         EXPECT_EQ(0, kill(pid, SIGTERM));
2040         EXPECT_EQ(pid, waitpid(pid, &status, 0));
2041         EXPECT_NE(0, WIFSIGNALED(status));
2042         EXPECT_EQ(SIGTERM, WTERMSIG(status));
2043
2044         /* Take snapshot to CPU pagetables */
2045         ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
2046         ASSERT_EQ(ret, 0);
2047         ASSERT_EQ(buffer->cpages, npages);
2048         m = buffer->mirror;
2049         for (i = 0; i < npages; i++)
2050                 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]);
2051
2052         hmm_buffer_free(buffer);
2053 }
2054 TEST_HARNESS_MAIN