drivers/gpu/drm/xe/tests/xe_migrate.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2020-2022 Intel Corporation
   4  */
   5
   6 #include <kunit/test.h>
   7 #include <kunit/visibility.h>
   8
   9 #include "tests/xe_migrate_test.h"
  10 #include "tests/xe_pci_test.h"
  11
  12 #include "xe_pci.h"
  13 #include "xe_pm.h"
  14
  15 static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
  16                                 const char *str, struct kunit *test)
  17 {
  18         long ret;
  19
  20         if (IS_ERR(fence)) {
  21                 KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
  22                            PTR_ERR(fence));
  23                 return true;
  24         }
  25         if (!fence)
  26                 return true;
  27
  28         ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
  29         if (ret <= 0) {
  30                 KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
  31                 return true;
  32         }
  33
  34         return false;
  35 }
  36
  37 static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe,
  38                           struct xe_bb *bb, u32 second_idx, const char *str,
  39                           struct kunit *test)
  40 {
  41         u64 batch_base = xe_migrate_batch_base(m, xe->info.has_usm);
  42         struct xe_sched_job *job = xe_bb_create_migration_job(m->q, bb,
  43                                                               batch_base,
  44                                                               second_idx);
  45         struct dma_fence *fence;
  46
  47         if (IS_ERR(job)) {
  48                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
  49                            PTR_ERR(job));
  50                 return PTR_ERR(job);
  51         }
  52
  53         xe_sched_job_arm(job);
  54         fence = dma_fence_get(&job->drm.s_fence->finished);
  55         xe_sched_job_push(job);
  56
  57         if (sanity_fence_failed(xe, fence, str, test))
  58                 return -ETIMEDOUT;
  59
  60         dma_fence_put(fence);
  61         kunit_info(test, "%s: Job completed\n", str);
  62         return 0;
  63 }
  64
  65 static void
  66 sanity_populate_cb(struct xe_migrate_pt_update *pt_update,
  67                    struct xe_tile *tile, struct iosys_map *map, void *dst,
  68                    u32 qword_ofs, u32 num_qwords,
  69                    const struct xe_vm_pgtable_update *update)
  70 {
  71         struct migrate_test_params *p =
  72                 to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));
  73         int i;
  74         u64 *ptr = dst;
  75         u64 value;
  76
  77         for (i = 0; i < num_qwords; i++) {
  78                 value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL;
  79                 if (map)
  80                         xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
  81                                   sizeof(u64), u64, value);
  82                 else
  83                         ptr[i] = value;
  84         }
  85
  86         kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU");
  87         if (p->force_gpu && map)
  88                 KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n");
  89 }
  90
  91 static const struct xe_migrate_pt_update_ops sanity_ops = {
  92         .populate = sanity_populate_cb,
  93 };
  94
  95 #define check(_retval, _expected, str, _test)                           \
  96         do { if ((_retval) != (_expected)) {                            \
  97                         KUNIT_FAIL(_test, "Sanity check failed: " str   \
  98                                    " expected %llx, got %llx\n",        \
  99                                    (u64)(_expected), (u64)(_retval));   \
 100                 } } while (0)
 101
 102 static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
 103                       struct kunit *test, u32 region)
 104 {
 105         struct xe_device *xe = tile_to_xe(m->tile);
 106         u64 retval, expected = 0;
 107         bool big = bo->size >= SZ_2M;
 108         struct dma_fence *fence;
 109         const char *str = big ? "Copying big bo" : "Copying small bo";
 110         int err;
 111
 112         struct xe_bo *remote = xe_bo_create_locked(xe, m->tile, NULL,
 113                                                    bo->size,
 114                                                    ttm_bo_type_kernel,
 115                                                    region |
 116                                                    XE_BO_FLAG_NEEDS_CPU_ACCESS);
 117         if (IS_ERR(remote)) {
 118                 KUNIT_FAIL(test, "Failed to allocate remote bo for %s: %pe\n",
 119                            str, remote);
 120                 return;
 121         }
 122
 123         err = xe_bo_validate(remote, NULL, false);
 124         if (err) {
 125                 KUNIT_FAIL(test, "Failed to validate system bo for %s: %i\n",
 126                            str, err);
 127                 goto out_unlock;
 128         }
 129
 130         err = xe_bo_vmap(remote);
 131         if (err) {
 132                 KUNIT_FAIL(test, "Failed to vmap system bo for %s: %i\n",
 133                            str, err);
 134                 goto out_unlock;
 135         }
 136
 137         xe_map_memset(xe, &remote->vmap, 0, 0xd0, remote->size);
 138         fence = xe_migrate_clear(m, remote, remote->ttm.resource);
 139         if (!sanity_fence_failed(xe, fence, big ? "Clearing remote big bo" :
 140                                  "Clearing remote small bo", test)) {
 141                 retval = xe_map_rd(xe, &remote->vmap, 0, u64);
 142                 check(retval, expected, "remote first offset should be cleared",
 143                       test);
 144                 retval = xe_map_rd(xe, &remote->vmap, remote->size - 8, u64);
 145                 check(retval, expected, "remote last offset should be cleared",
 146                       test);
 147         }
 148         dma_fence_put(fence);
 149
 150         /* Try to copy 0xc0 from remote to vram with 2MB or 64KiB/4KiB pages */
 151         xe_map_memset(xe, &remote->vmap, 0, 0xc0, remote->size);
 152         xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size);
 153
 154         expected = 0xc0c0c0c0c0c0c0c0;
 155         fence = xe_migrate_copy(m, remote, bo, remote->ttm.resource,
 156                                 bo->ttm.resource, false);
 157         if (!sanity_fence_failed(xe, fence, big ? "Copying big bo remote -> vram" :
 158                                  "Copying small bo remote -> vram", test)) {
 159                 retval = xe_map_rd(xe, &bo->vmap, 0, u64);
 160                 check(retval, expected,
 161                       "remote -> vram bo first offset should be copied", test);
 162                 retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64);
 163                 check(retval, expected,
 164                       "remote -> vram bo offset should be copied", test);
 165         }
 166         dma_fence_put(fence);
 167
 168         /* And other way around.. slightly hacky.. */
 169         xe_map_memset(xe, &remote->vmap, 0, 0xd0, remote->size);
 170         xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size);
 171
 172         fence = xe_migrate_copy(m, bo, remote, bo->ttm.resource,
 173                                 remote->ttm.resource, false);
 174         if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> remote" :
 175                                  "Copying small bo vram -> remote", test)) {
 176                 retval = xe_map_rd(xe, &remote->vmap, 0, u64);
 177                 check(retval, expected,
 178                       "vram -> remote bo first offset should be copied", test);
 179                 retval = xe_map_rd(xe, &remote->vmap, bo->size - 8, u64);
 180                 check(retval, expected,
 181                       "vram -> remote bo last offset should be copied", test);
 182         }
 183         dma_fence_put(fence);
 184
 185         xe_bo_vunmap(remote);
 186 out_unlock:
 187         xe_bo_unlock(remote);
 188         xe_bo_put(remote);
 189 }
 190
 191 static void test_copy_sysmem(struct xe_migrate *m, struct xe_bo *bo,
 192                              struct kunit *test)
 193 {
 194         test_copy(m, bo, test, XE_BO_FLAG_SYSTEM);
 195 }
 196
 197 static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo,
 198                            struct kunit *test)
 199 {
 200         u32 region;
 201
 202         if (bo->ttm.resource->mem_type == XE_PL_SYSTEM)
 203                 return;
 204
 205         if (bo->ttm.resource->mem_type == XE_PL_VRAM0)
 206                 region = XE_BO_FLAG_VRAM1;
 207         else
 208                 region = XE_BO_FLAG_VRAM0;
 209         test_copy(m, bo, test, region);
 210 }
 211
 212 static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt,
 213                            struct kunit *test, bool force_gpu)
 214 {
 215         struct xe_device *xe = tile_to_xe(m->tile);
 216         struct dma_fence *fence;
 217         u64 retval, expected;
 218         ktime_t then, now;
 219         int i;
 220
 221         struct xe_vm_pgtable_update update = {
 222                 .ofs = 1,
 223                 .qwords = 0x10,
 224                 .pt_bo = pt,
 225         };
 226         struct xe_migrate_pt_update pt_update = {
 227                 .ops = &sanity_ops,
 228         };
 229         struct migrate_test_params p = {
 230                 .base.id = XE_TEST_LIVE_MIGRATE,
 231                 .force_gpu = force_gpu,
 232         };
 233
 234         test->priv = &p;
 235         /* Test xe_migrate_update_pgtables() updates the pagetable as expected */
 236         expected = 0xf0f0f0f0f0f0f0f0ULL;
 237         xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size);
 238
 239         then = ktime_get();
 240         fence = xe_migrate_update_pgtables(m, m->q->vm, NULL, m->q, &update, 1,
 241                                            NULL, 0, &pt_update);
 242         now = ktime_get();
 243         if (sanity_fence_failed(xe, fence, "Migration pagetable update", test))
 244                 return;
 245
 246         kunit_info(test, "Updating without syncing took %llu us,\n",
 247                    (unsigned long long)ktime_to_us(ktime_sub(now, then)));
 248
 249         dma_fence_put(fence);
 250         retval = xe_map_rd(xe, &pt->vmap, 0, u64);
 251         check(retval, expected, "PTE[0] must stay untouched", test);
 252
 253         for (i = 0; i < update.qwords; i++) {
 254                 retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64);
 255                 check(retval, i * 0x1111111111111111ULL, "PTE update", test);
 256         }
 257
 258         retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords),
 259                            u64);
 260         check(retval, expected, "PTE[0x11] must stay untouched", test);
 261 }
 262
 263 static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
 264 {
 265         struct xe_tile *tile = m->tile;
 266         struct xe_device *xe = tile_to_xe(tile);
 267         struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny;
 268         struct xe_res_cursor src_it;
 269         struct dma_fence *fence;
 270         u64 retval, expected;
 271         struct xe_bb *bb;
 272         int err;
 273         u8 id = tile->id;
 274
 275         err = xe_bo_vmap(bo);
 276         if (err) {
 277                 KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
 278                            PTR_ERR(bo));
 279                 return;
 280         }
 281
 282         big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M,
 283                                    ttm_bo_type_kernel,
 284                                    XE_BO_FLAG_VRAM_IF_DGFX(tile) |
 285                                    XE_BO_FLAG_PINNED);
 286         if (IS_ERR(big)) {
 287                 KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
 288                 goto vunmap;
 289         }
 290
 291         pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE,
 292                                   ttm_bo_type_kernel,
 293                                   XE_BO_FLAG_VRAM_IF_DGFX(tile) |
 294                                   XE_BO_FLAG_PINNED);
 295         if (IS_ERR(pt)) {
 296                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 297                            PTR_ERR(pt));
 298                 goto free_big;
 299         }
 300
 301         tiny = xe_bo_create_pin_map(xe, tile, m->q->vm,
 302                                     2 * SZ_4K,
 303                                     ttm_bo_type_kernel,
 304                                     XE_BO_FLAG_VRAM_IF_DGFX(tile) |
 305                                     XE_BO_FLAG_PINNED);
 306         if (IS_ERR(tiny)) {
 307                 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
 308                            PTR_ERR(pt));
 309                 goto free_pt;
 310         }
 311
 312         bb = xe_bb_new(tile->primary_gt, 32, xe->info.has_usm);
 313         if (IS_ERR(bb)) {
 314                 KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
 315                            PTR_ERR(bb));
 316                 goto free_tiny;
 317         }
 318
 319         kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
 320                    (unsigned long)xe_bo_main_addr(m->q->vm->pt_root[id]->bo, XE_PAGE_SIZE),
 321                    (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE));
 322
 323         /* First part of the test, are we updating our pagetable bo with a new entry? */
 324         xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
 325                   0xdeaddeadbeefbeef);
 326         expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0);
 327         if (m->q->vm->flags & XE_VM_FLAG_64K)
 328                 expected |= XE_PTE_PS64;
 329         if (xe_bo_is_vram(pt))
 330                 xe_res_first(pt->ttm.resource, 0, pt->size, &src_it);
 331         else
 332                 xe_res_first_sg(xe_bo_sg(pt), 0, pt->size, &src_it);
 333
 334         emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), false,
 335                  &src_it, XE_PAGE_SIZE, pt->ttm.resource);
 336
 337         run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test);
 338
 339         retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
 340                            u64);
 341         check(retval, expected, "PTE entry write", test);
 342
 343         /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
 344         bb->len = 0;
 345         bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
 346         xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
 347         expected = 0;
 348
 349         emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
 350                    IS_DGFX(xe));
 351         run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable",
 352                        test);
 353
 354         retval = xe_map_rd(xe, &pt->vmap, 0, u32);
 355         check(retval, expected, "Write to PT after adding PTE", test);
 356
 357         /* Sanity checks passed, try the full ones! */
 358
 359         /* Clear a small bo */
 360         kunit_info(test, "Clearing small buffer object\n");
 361         xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size);
 362         expected = 0;
 363         fence = xe_migrate_clear(m, tiny, tiny->ttm.resource);
 364         if (sanity_fence_failed(xe, fence, "Clearing small bo", test))
 365                 goto out;
 366
 367         dma_fence_put(fence);
 368         retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
 369         check(retval, expected, "Command clear small first value", test);
 370         retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32);
 371         check(retval, expected, "Command clear small last value", test);
 372
 373         kunit_info(test, "Copying small buffer object to system\n");
 374         test_copy_sysmem(m, tiny, test);
 375         if (xe->info.tile_count > 1) {
 376                 kunit_info(test, "Copying small buffer object to other vram\n");
 377                 test_copy_vram(m, tiny, test);
 378         }
 379
 380         /* Clear a big bo */
 381         kunit_info(test, "Clearing big buffer object\n");
 382         xe_map_memset(xe, &big->vmap, 0, 0x11, big->size);
 383         expected = 0;
 384         fence = xe_migrate_clear(m, big, big->ttm.resource);
 385         if (sanity_fence_failed(xe, fence, "Clearing big bo", test))
 386                 goto out;
 387
 388         dma_fence_put(fence);
 389         retval = xe_map_rd(xe, &big->vmap, 0, u32);
 390         check(retval, expected, "Command clear big first value", test);
 391         retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32);
 392         check(retval, expected, "Command clear big last value", test);
 393
 394         kunit_info(test, "Copying big buffer object to system\n");
 395         test_copy_sysmem(m, big, test);
 396         if (xe->info.tile_count > 1) {
 397                 kunit_info(test, "Copying big buffer object to other vram\n");
 398                 test_copy_vram(m, big, test);
 399         }
 400
 401         kunit_info(test, "Testing page table update using CPU if GPU idle.\n");
 402         test_pt_update(m, pt, test, false);
 403         kunit_info(test, "Testing page table update using GPU\n");
 404         test_pt_update(m, pt, test, true);
 405
 406 out:
 407         xe_bb_free(bb, NULL);
 408 free_tiny:
 409         xe_bo_unpin(tiny);
 410         xe_bo_put(tiny);
 411 free_pt:
 412         xe_bo_unpin(pt);
 413         xe_bo_put(pt);
 414 free_big:
 415         xe_bo_unpin(big);
 416         xe_bo_put(big);
 417 vunmap:
 418         xe_bo_vunmap(m->pt_bo);
 419 }
 420
 421 static int migrate_test_run_device(struct xe_device *xe)
 422 {
 423         struct kunit *test = xe_cur_kunit();
 424         struct xe_tile *tile;
 425         int id;
 426
 427         xe_pm_runtime_get(xe);
 428
 429         for_each_tile(tile, xe, id) {
 430                 struct xe_migrate *m = tile->migrate;
 431
 432                 kunit_info(test, "Testing tile id %d.\n", id);
 433                 xe_vm_lock(m->q->vm, true);
 434                 xe_migrate_sanity_test(m, test);
 435                 xe_vm_unlock(m->q->vm);
 436         }
 437
 438         xe_pm_runtime_put(xe);
 439
 440         return 0;
 441 }
 442
 443 void xe_migrate_sanity_kunit(struct kunit *test)
 444 {
 445         xe_call_for_each_device(migrate_test_run_device);
 446 }
 447 EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit);