drivers/gpu/drm/nouveau/nouveau_dmem.c

   1 /*
   2  * Copyright 2018 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "nouveau_dmem.h"
  23 #include "nouveau_drv.h"
  24 #include "nouveau_chan.h"
  25 #include "nouveau_dma.h"
  26 #include "nouveau_mem.h"
  27 #include "nouveau_bo.h"
  28
  29 #include <nvif/class.h>
  30 #include <nvif/object.h>
  31 #include <nvif/if000c.h>
  32 #include <nvif/if500b.h>
  33 #include <nvif/if900b.h>
  34
  35 #include <linux/sched/mm.h>
  36 #include <linux/hmm.h>
  37
  38 /*
  39  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  40  * it in vram while in use. We likely want to overhaul memory management for
  41  * nouveau to be more page like (not necessarily with system page size but a
  42  * bigger page size) at lowest level and have some shim layer on top that would
  43  * provide the same functionality as TTM.
  44  */
  45 #define DMEM_CHUNK_SIZE (2UL << 20)
  46 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  47
  48 enum nouveau_aper {
  49         NOUVEAU_APER_VIRT,
  50         NOUVEAU_APER_VRAM,
  51         NOUVEAU_APER_HOST,
  52 };
  53
  54 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  55                                       enum nouveau_aper, u64 dst_addr,
  56                                       enum nouveau_aper, u64 src_addr);
  57
  58 struct nouveau_dmem_chunk {
  59         struct list_head list;
  60         struct nouveau_bo *bo;
  61         struct nouveau_drm *drm;
  62         unsigned long pfn_first;
  63         unsigned long callocated;
  64         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
  65         spinlock_t lock;
  66 };
  67
  68 struct nouveau_dmem_migrate {
  69         nouveau_migrate_copy_t copy_func;
  70         struct nouveau_channel *chan;
  71 };
  72
  73 struct nouveau_dmem {
  74         struct nouveau_drm *drm;
  75         struct dev_pagemap pagemap;
  76         struct nouveau_dmem_migrate migrate;
  77         struct list_head chunk_free;
  78         struct list_head chunk_full;
  79         struct list_head chunk_empty;
  80         struct mutex mutex;
  81 };
  82
  83 static inline struct nouveau_dmem *page_to_dmem(struct page *page)
  84 {
  85         return container_of(page->pgmap, struct nouveau_dmem, pagemap);
  86 }
  87
  88 static unsigned long nouveau_dmem_page_addr(struct page *page)
  89 {
  90         struct nouveau_dmem_chunk *chunk = page->zone_device_data;
  91         unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
  92
  93         return (idx << PAGE_SHIFT) + chunk->bo->offset;
  94 }
  95
  96 static void nouveau_dmem_page_free(struct page *page)
  97 {
  98         struct nouveau_dmem_chunk *chunk = page->zone_device_data;
  99         unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
 100
 101         /*
 102          * FIXME:
 103          *
 104          * This is really a bad example, we need to overhaul nouveau memory
 105          * management to be more page focus and allow lighter locking scheme
 106          * to be use in the process.
 107          */
 108         spin_lock(&chunk->lock);
 109         clear_bit(idx, chunk->bitmap);
 110         WARN_ON(!chunk->callocated);
 111         chunk->callocated--;
 112         /*
 113          * FIXME when chunk->callocated reach 0 we should add the chunk to
 114          * a reclaim list so that it can be freed in case of memory pressure.
 115          */
 116         spin_unlock(&chunk->lock);
 117 }
 118
 119 static void nouveau_dmem_fence_done(struct nouveau_fence **fence)
 120 {
 121         if (fence) {
 122                 nouveau_fence_wait(*fence, true, false);
 123                 nouveau_fence_unref(fence);
 124         } else {
 125                 /*
 126                  * FIXME wait for channel to be IDLE before calling finalizing
 127                  * the hmem object.
 128                  */
 129         }
 130 }
 131
 132 static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm,
 133                 struct vm_fault *vmf, struct migrate_vma *args,
 134                 dma_addr_t *dma_addr)
 135 {
 136         struct device *dev = drm->dev->dev;
 137         struct page *dpage, *spage;
 138
 139         spage = migrate_pfn_to_page(args->src[0]);
 140         if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE))
 141                 return 0;
 142
 143         dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address);
 144         if (!dpage)
 145                 return VM_FAULT_SIGBUS;
 146         lock_page(dpage);
 147
 148         *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
 149         if (dma_mapping_error(dev, *dma_addr))
 150                 goto error_free_page;
 151
 152         if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,
 153                         NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))
 154                 goto error_dma_unmap;
 155
 156         args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
 157         return 0;
 158
 159 error_dma_unmap:
 160         dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 161 error_free_page:
 162         __free_page(dpage);
 163         return VM_FAULT_SIGBUS;
 164 }
 165
 166 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
 167 {
 168         struct nouveau_dmem *dmem = page_to_dmem(vmf->page);
 169         struct nouveau_drm *drm = dmem->drm;
 170         struct nouveau_fence *fence;
 171         unsigned long src = 0, dst = 0;
 172         dma_addr_t dma_addr = 0;
 173         vm_fault_t ret;
 174         struct migrate_vma args = {
 175                 .vma            = vmf->vma,
 176                 .start          = vmf->address,
 177                 .end            = vmf->address + PAGE_SIZE,
 178                 .src            = &src,
 179                 .dst            = &dst,
 180                 .src_owner      = drm->dev,
 181         };
 182
 183         /*
 184          * FIXME what we really want is to find some heuristic to migrate more
 185          * than just one page on CPU fault. When such fault happens it is very
 186          * likely that more surrounding page will CPU fault too.
 187          */
 188         if (migrate_vma_setup(&args) < 0)
 189                 return VM_FAULT_SIGBUS;
 190         if (!args.cpages)
 191                 return 0;
 192
 193         ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr);
 194         if (ret || dst == 0)
 195                 goto done;
 196
 197         nouveau_fence_new(dmem->migrate.chan, false, &fence);
 198         migrate_vma_pages(&args);
 199         nouveau_dmem_fence_done(&fence);
 200         dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 201 done:
 202         migrate_vma_finalize(&args);
 203         return ret;
 204 }
 205
 206 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
 207         .page_free              = nouveau_dmem_page_free,
 208         .migrate_to_ram         = nouveau_dmem_migrate_to_ram,
 209 };
 210
 211 static int
 212 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
 213 {
 214         struct nouveau_dmem_chunk *chunk;
 215         int ret;
 216
 217         if (drm->dmem == NULL)
 218                 return -EINVAL;
 219
 220         mutex_lock(&drm->dmem->mutex);
 221         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 222                                          struct nouveau_dmem_chunk,
 223                                          list);
 224         if (chunk == NULL) {
 225                 mutex_unlock(&drm->dmem->mutex);
 226                 return -ENOMEM;
 227         }
 228
 229         list_del(&chunk->list);
 230         mutex_unlock(&drm->dmem->mutex);
 231
 232         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
 233                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
 234                              &chunk->bo);
 235         if (ret)
 236                 goto out;
 237
 238         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 239         if (ret) {
 240                 nouveau_bo_ref(NULL, &chunk->bo);
 241                 goto out;
 242         }
 243
 244         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
 245         spin_lock_init(&chunk->lock);
 246
 247 out:
 248         mutex_lock(&drm->dmem->mutex);
 249         if (chunk->bo)
 250                 list_add(&chunk->list, &drm->dmem->chunk_empty);
 251         else
 252                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 253         mutex_unlock(&drm->dmem->mutex);
 254
 255         return ret;
 256 }
 257
 258 static struct nouveau_dmem_chunk *
 259 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
 260 {
 261         struct nouveau_dmem_chunk *chunk;
 262
 263         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
 264                                          struct nouveau_dmem_chunk,
 265                                          list);
 266         if (chunk)
 267                 return chunk;
 268
 269         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 270                                          struct nouveau_dmem_chunk,
 271                                          list);
 272         if (chunk->bo)
 273                 return chunk;
 274
 275         return NULL;
 276 }
 277
 278 static int
 279 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
 280                          unsigned long npages,
 281                          unsigned long *pages)
 282 {
 283         struct nouveau_dmem_chunk *chunk;
 284         unsigned long c;
 285         int ret;
 286
 287         memset(pages, 0xff, npages * sizeof(*pages));
 288
 289         mutex_lock(&drm->dmem->mutex);
 290         for (c = 0; c < npages;) {
 291                 unsigned long i;
 292
 293                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
 294                 if (chunk == NULL) {
 295                         mutex_unlock(&drm->dmem->mutex);
 296                         ret = nouveau_dmem_chunk_alloc(drm);
 297                         if (ret) {
 298                                 if (c)
 299                                         return 0;
 300                                 return ret;
 301                         }
 302                         mutex_lock(&drm->dmem->mutex);
 303                         continue;
 304                 }
 305
 306                 spin_lock(&chunk->lock);
 307                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
 308                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
 309                         pages[c] = chunk->pfn_first + i;
 310                         set_bit(i, chunk->bitmap);
 311                         chunk->callocated++;
 312                         c++;
 313
 314                         i = find_next_zero_bit(chunk->bitmap,
 315                                         DMEM_CHUNK_NPAGES, i);
 316                 }
 317                 spin_unlock(&chunk->lock);
 318         }
 319         mutex_unlock(&drm->dmem->mutex);
 320
 321         return 0;
 322 }
 323
 324 static struct page *
 325 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
 326 {
 327         unsigned long pfns[1];
 328         struct page *page;
 329         int ret;
 330
 331         /* FIXME stop all the miss-match API ... */
 332         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
 333         if (ret)
 334                 return NULL;
 335
 336         page = pfn_to_page(pfns[0]);
 337         get_page(page);
 338         lock_page(page);
 339         return page;
 340 }
 341
 342 static void
 343 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
 344 {
 345         unlock_page(page);
 346         put_page(page);
 347 }
 348
 349 void
 350 nouveau_dmem_resume(struct nouveau_drm *drm)
 351 {
 352         struct nouveau_dmem_chunk *chunk;
 353         int ret;
 354
 355         if (drm->dmem == NULL)
 356                 return;
 357
 358         mutex_lock(&drm->dmem->mutex);
 359         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 360                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 361                 /* FIXME handle pin failure */
 362                 WARN_ON(ret);
 363         }
 364         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 365                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 366                 /* FIXME handle pin failure */
 367                 WARN_ON(ret);
 368         }
 369         mutex_unlock(&drm->dmem->mutex);
 370 }
 371
 372 void
 373 nouveau_dmem_suspend(struct nouveau_drm *drm)
 374 {
 375         struct nouveau_dmem_chunk *chunk;
 376
 377         if (drm->dmem == NULL)
 378                 return;
 379
 380         mutex_lock(&drm->dmem->mutex);
 381         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 382                 nouveau_bo_unpin(chunk->bo);
 383         }
 384         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 385                 nouveau_bo_unpin(chunk->bo);
 386         }
 387         mutex_unlock(&drm->dmem->mutex);
 388 }
 389
 390 void
 391 nouveau_dmem_fini(struct nouveau_drm *drm)
 392 {
 393         struct nouveau_dmem_chunk *chunk, *tmp;
 394
 395         if (drm->dmem == NULL)
 396                 return;
 397
 398         mutex_lock(&drm->dmem->mutex);
 399
 400         WARN_ON(!list_empty(&drm->dmem->chunk_free));
 401         WARN_ON(!list_empty(&drm->dmem->chunk_full));
 402
 403         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
 404                 if (chunk->bo) {
 405                         nouveau_bo_unpin(chunk->bo);
 406                         nouveau_bo_ref(NULL, &chunk->bo);
 407                 }
 408                 list_del(&chunk->list);
 409                 kfree(chunk);
 410         }
 411
 412         mutex_unlock(&drm->dmem->mutex);
 413 }
 414
 415 static int
 416 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
 417                     enum nouveau_aper dst_aper, u64 dst_addr,
 418                     enum nouveau_aper src_aper, u64 src_addr)
 419 {
 420         struct nouveau_channel *chan = drm->dmem->migrate.chan;
 421         u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
 422                          (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
 423                          (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
 424                          (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
 425                          (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
 426         int ret;
 427
 428         ret = RING_SPACE(chan, 13);
 429         if (ret)
 430                 return ret;
 431
 432         if (src_aper != NOUVEAU_APER_VIRT) {
 433                 switch (src_aper) {
 434                 case NOUVEAU_APER_VRAM:
 435                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
 436                         break;
 437                 case NOUVEAU_APER_HOST:
 438                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
 439                         break;
 440                 default:
 441                         return -EINVAL;
 442                 }
 443                 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
 444         }
 445
 446         if (dst_aper != NOUVEAU_APER_VIRT) {
 447                 switch (dst_aper) {
 448                 case NOUVEAU_APER_VRAM:
 449                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
 450                         break;
 451                 case NOUVEAU_APER_HOST:
 452                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
 453                         break;
 454                 default:
 455                         return -EINVAL;
 456                 }
 457                 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
 458         }
 459
 460         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 461         OUT_RING  (chan, upper_32_bits(src_addr));
 462         OUT_RING  (chan, lower_32_bits(src_addr));
 463         OUT_RING  (chan, upper_32_bits(dst_addr));
 464         OUT_RING  (chan, lower_32_bits(dst_addr));
 465         OUT_RING  (chan, PAGE_SIZE);
 466         OUT_RING  (chan, PAGE_SIZE);
 467         OUT_RING  (chan, PAGE_SIZE);
 468         OUT_RING  (chan, npages);
 469         BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 470         OUT_RING  (chan, launch_dma);
 471         return 0;
 472 }
 473
 474 static int
 475 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
 476 {
 477         switch (drm->ttm.copy.oclass) {
 478         case PASCAL_DMA_COPY_A:
 479         case PASCAL_DMA_COPY_B:
 480         case  VOLTA_DMA_COPY_A:
 481         case TURING_DMA_COPY_A:
 482                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
 483                 drm->dmem->migrate.chan = drm->ttm.chan;
 484                 return 0;
 485         default:
 486                 break;
 487         }
 488         return -ENODEV;
 489 }
 490
 491 void
 492 nouveau_dmem_init(struct nouveau_drm *drm)
 493 {
 494         struct device *device = drm->dev->dev;
 495         struct resource *res;
 496         unsigned long i, size, pfn_first;
 497         int ret;
 498
 499         /* This only make sense on PASCAL or newer */
 500         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
 501                 return;
 502
 503         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
 504                 return;
 505
 506         drm->dmem->drm = drm;
 507         mutex_init(&drm->dmem->mutex);
 508         INIT_LIST_HEAD(&drm->dmem->chunk_free);
 509         INIT_LIST_HEAD(&drm->dmem->chunk_full);
 510         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
 511
 512         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
 513
 514         /* Initialize migration dma helpers before registering memory */
 515         ret = nouveau_dmem_migrate_init(drm);
 516         if (ret)
 517                 goto out_free;
 518
 519         /*
 520          * FIXME we need some kind of policy to decide how much VRAM we
 521          * want to register with HMM. For now just register everything
 522          * and latter if we want to do thing like over commit then we
 523          * could revisit this.
 524          */
 525         res = devm_request_free_mem_region(device, &iomem_resource, size);
 526         if (IS_ERR(res))
 527                 goto out_free;
 528         drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 529         drm->dmem->pagemap.res = *res;
 530         drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops;
 531         drm->dmem->pagemap.owner = drm->dev;
 532         if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap)))
 533                 goto out_free;
 534
 535         pfn_first = res->start >> PAGE_SHIFT;
 536         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
 537                 struct nouveau_dmem_chunk *chunk;
 538                 struct page *page;
 539                 unsigned long j;
 540
 541                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 542                 if (chunk == NULL) {
 543                         nouveau_dmem_fini(drm);
 544                         return;
 545                 }
 546
 547                 chunk->drm = drm;
 548                 chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES);
 549                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 550
 551                 page = pfn_to_page(chunk->pfn_first);
 552                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page)
 553                         page->zone_device_data = chunk;
 554         }
 555
 556         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
 557         return;
 558 out_free:
 559         kfree(drm->dmem);
 560         drm->dmem = NULL;
 561 }
 562
 563 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,
 564                 unsigned long src, dma_addr_t *dma_addr)
 565 {
 566         struct device *dev = drm->dev->dev;
 567         struct page *dpage, *spage;
 568
 569         spage = migrate_pfn_to_page(src);
 570         if (!spage || !(src & MIGRATE_PFN_MIGRATE))
 571                 goto out;
 572
 573         dpage = nouveau_dmem_page_alloc_locked(drm);
 574         if (!dpage)
 575                 return 0;
 576
 577         *dma_addr = dma_map_page(dev, spage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
 578         if (dma_mapping_error(dev, *dma_addr))
 579                 goto out_free_page;
 580
 581         if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_VRAM,
 582                         nouveau_dmem_page_addr(dpage), NOUVEAU_APER_HOST,
 583                         *dma_addr))
 584                 goto out_dma_unmap;
 585
 586         return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
 587
 588 out_dma_unmap:
 589         dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
 590 out_free_page:
 591         nouveau_dmem_page_free_locked(drm, dpage);
 592 out:
 593         return 0;
 594 }
 595
 596 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm,
 597                 struct migrate_vma *args, dma_addr_t *dma_addrs)
 598 {
 599         struct nouveau_fence *fence;
 600         unsigned long addr = args->start, nr_dma = 0, i;
 601
 602         for (i = 0; addr < args->end; i++) {
 603                 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i],
 604                                 dma_addrs + nr_dma);
 605                 if (args->dst[i])
 606                         nr_dma++;
 607                 addr += PAGE_SIZE;
 608         }
 609
 610         nouveau_fence_new(drm->dmem->migrate.chan, false, &fence);
 611         migrate_vma_pages(args);
 612         nouveau_dmem_fence_done(&fence);
 613
 614         while (nr_dma--) {
 615                 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE,
 616                                 DMA_BIDIRECTIONAL);
 617         }
 618         /*
 619          * FIXME optimization: update GPU page table to point to newly migrated
 620          * memory.
 621          */
 622         migrate_vma_finalize(args);
 623 }
 624
 625 int
 626 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
 627                          struct vm_area_struct *vma,
 628                          unsigned long start,
 629                          unsigned long end)
 630 {
 631         unsigned long npages = (end - start) >> PAGE_SHIFT;
 632         unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages);
 633         dma_addr_t *dma_addrs;
 634         struct migrate_vma args = {
 635                 .vma            = vma,
 636                 .start          = start,
 637         };
 638         unsigned long c, i;
 639         int ret = -ENOMEM;
 640
 641         args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL);
 642         if (!args.src)
 643                 goto out;
 644         args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL);
 645         if (!args.dst)
 646                 goto out_free_src;
 647
 648         dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL);
 649         if (!dma_addrs)
 650                 goto out_free_dst;
 651
 652         for (i = 0; i < npages; i += c) {
 653                 c = min(SG_MAX_SINGLE_ALLOC, npages);
 654                 args.end = start + (c << PAGE_SHIFT);
 655                 ret = migrate_vma_setup(&args);
 656                 if (ret)
 657                         goto out_free_dma;
 658
 659                 if (args.cpages)
 660                         nouveau_dmem_migrate_chunk(drm, &args, dma_addrs);
 661                 args.start = args.end;
 662         }
 663
 664         ret = 0;
 665 out_free_dma:
 666         kfree(dma_addrs);
 667 out_free_dst:
 668         kfree(args.dst);
 669 out_free_src:
 670         kfree(args.src);
 671 out:
 672         return ret;
 673 }
 674
 675 void
 676 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
 677                          struct hmm_range *range)
 678 {
 679         unsigned long i, npages;
 680
 681         npages = (range->end - range->start) >> PAGE_SHIFT;
 682         for (i = 0; i < npages; ++i) {
 683                 struct page *page;
 684                 uint64_t addr;
 685
 686                 page = hmm_device_entry_to_page(range, range->pfns[i]);
 687                 if (page == NULL)
 688                         continue;
 689
 690                 if (!is_device_private_page(page))
 691                         continue;
 692
 693                 addr = nouveau_dmem_page_addr(page);
 694                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
 695                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
 696                 range->pfns[i] |= NVIF_VMM_PFNMAP_V0_VRAM;
 697         }
 698 }