[linux-2.6-block.git] / drivers / gpu / drm / xe / tests / xe_migrate.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2020-2022 Intel Corporation
 */

#include <kunit/test.h>
#include <kunit/visibility.h>

#include "tests/xe_migrate_test.h"
#include "tests/xe_pci_test.h"

#include "xe_pci.h"

static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence,
				const char *str, struct kunit *test)
{
	long ret;

	if (IS_ERR(fence)) {
		KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
			   PTR_ERR(fence));
		return true;
	}
	if (!fence)
		return true;

	ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
	if (ret <= 0) {
		KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
		return true;
	}

	return false;
}

static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe,
			  struct xe_bb *bb, u32 second_idx, const char *str,
			  struct kunit *test)
{
	u64 batch_base = xe_migrate_batch_base(m, xe->info.supports_usm);
	struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb,
							      batch_base,
							      second_idx);
	struct dma_fence *fence;

	if (IS_ERR(job)) {
		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
			   PTR_ERR(job));
		return PTR_ERR(job);
	}

	xe_sched_job_arm(job);
	fence = dma_fence_get(&job->drm.s_fence->finished);
	xe_sched_job_push(job);

	if (sanity_fence_failed(xe, fence, str, test))
		return -ETIMEDOUT;

	dma_fence_put(fence);
	kunit_info(test, "%s: Job completed\n", str);
	return 0;
}

static void
sanity_populate_cb(struct xe_migrate_pt_update *pt_update,
		   struct xe_tile *tile, struct iosys_map *map, void *dst,
		   u32 qword_ofs, u32 num_qwords,
		   const struct xe_vm_pgtable_update *update)
{
	struct migrate_test_params *p =
		to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));
	int i;
	u64 *ptr = dst;
	u64 value;

	for (i = 0; i < num_qwords; i++) {
		value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL;
		if (map)
			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
				  sizeof(u64), u64, value);
		else
			ptr[i] = value;
	}

	kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU");
	if (p->force_gpu && map)
		KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n");
}

static const struct xe_migrate_pt_update_ops sanity_ops = {
	.populate = sanity_populate_cb,
};

#define check(_retval, _expected, str, _test)				\
	do { if ((_retval) != (_expected)) {				\
			KUNIT_FAIL(_test, "Sanity check failed: " str	\
				   " expected %llx, got %llx\n",	\
				   (u64)(_expected), (u64)(_retval));	\
		} } while (0)

static void test_copy(struct xe_migrate *m, struct xe_bo *bo,
		      struct kunit *test)
{
	struct xe_device *xe = tile_to_xe(m->tile);
	u64 retval, expected = 0;
	bool big = bo->size >= SZ_2M;
	struct dma_fence *fence;
	const char *str = big ? "Copying big bo" : "Copying small bo";
	int err;

	struct xe_bo *sysmem = xe_bo_create_locked(xe, m->tile, NULL,
						   bo->size,
						   ttm_bo_type_kernel,
						   XE_BO_CREATE_SYSTEM_BIT |
						   XE_BO_NEEDS_CPU_ACCESS);
	if (IS_ERR(sysmem)) {
		KUNIT_FAIL(test, "Failed to allocate sysmem bo for %s: %li\n",
			   str, PTR_ERR(sysmem));
		return;
	}

	err = xe_bo_validate(sysmem, NULL, false);
	if (err) {
		KUNIT_FAIL(test, "Failed to validate system bo for %s: %li\n",
			   str, err);
		goto out_unlock;
	}

	err = xe_bo_vmap(sysmem);
	if (err) {
		KUNIT_FAIL(test, "Failed to vmap system bo for %s: %li\n",
			   str, err);
		goto out_unlock;
	}

	xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
	fence = xe_migrate_clear(m, sysmem, sysmem->ttm.resource);
	if (!sanity_fence_failed(xe, fence, big ? "Clearing sysmem big bo" :
				 "Clearing sysmem small bo", test)) {
		retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
		check(retval, expected, "sysmem first offset should be cleared",
		      test);
		retval = xe_map_rd(xe, &sysmem->vmap, sysmem->size - 8, u64);
		check(retval, expected, "sysmem last offset should be cleared",
		      test);
	}
	dma_fence_put(fence);

	/* Try to copy 0xc0 from sysmem to vram with 2MB or 64KiB/4KiB pages */
	xe_map_memset(xe, &sysmem->vmap, 0, 0xc0, sysmem->size);
	xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size);

	expected = 0xc0c0c0c0c0c0c0c0;
	fence = xe_migrate_copy(m, sysmem, bo, sysmem->ttm.resource,
				bo->ttm.resource);
	if (!sanity_fence_failed(xe, fence, big ? "Copying big bo sysmem -> vram" :
				 "Copying small bo sysmem -> vram", test)) {
		retval = xe_map_rd(xe, &bo->vmap, 0, u64);
		check(retval, expected,
		      "sysmem -> vram bo first offset should be copied", test);
		retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64);
		check(retval, expected,
		      "sysmem -> vram bo offset should be copied", test);
	}
	dma_fence_put(fence);

	/* And other way around.. slightly hacky.. */
	xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
	xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size);

	fence = xe_migrate_copy(m, bo, sysmem, bo->ttm.resource,
				sysmem->ttm.resource);
	if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> sysmem" :
				 "Copying small bo vram -> sysmem", test)) {
		retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
		check(retval, expected,
		      "vram -> sysmem bo first offset should be copied", test);
		retval = xe_map_rd(xe, &sysmem->vmap, bo->size - 8, u64);
		check(retval, expected,
		      "vram -> sysmem bo last offset should be copied", test);
	}
	dma_fence_put(fence);

	xe_bo_vunmap(sysmem);
out_unlock:
	xe_bo_unlock_no_vm(sysmem);
	xe_bo_put(sysmem);
}

static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt,
			   struct kunit *test, bool force_gpu)
{
	struct xe_device *xe = tile_to_xe(m->tile);
	struct dma_fence *fence;
	u64 retval, expected;
	ktime_t then, now;
	int i;

	struct xe_vm_pgtable_update update = {
		.ofs = 1,
		.qwords = 0x10,
		.pt_bo = pt,
	};
	struct xe_migrate_pt_update pt_update = {
		.ops = &sanity_ops,
	};
	struct migrate_test_params p = {
		.base.id = XE_TEST_LIVE_MIGRATE,
		.force_gpu = force_gpu,
	};

	test->priv = &p;
	/* Test xe_migrate_update_pgtables() updates the pagetable as expected */
	expected = 0xf0f0f0f0f0f0f0f0ULL;
	xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size);

	then = ktime_get();
	fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1,
					   NULL, 0, &pt_update);
	now = ktime_get();
	if (sanity_fence_failed(xe, fence, "Migration pagetable update", test))
		return;

	kunit_info(test, "Updating without syncing took %llu us,\n",
		   (unsigned long long)ktime_to_us(ktime_sub(now, then)));

	dma_fence_put(fence);
	retval = xe_map_rd(xe, &pt->vmap, 0, u64);
	check(retval, expected, "PTE[0] must stay untouched", test);

	for (i = 0; i < update.qwords; i++) {
		retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64);
		check(retval, i * 0x1111111111111111ULL, "PTE update", test);
	}

	retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords),
			   u64);
	check(retval, expected, "PTE[0x11] must stay untouched", test);
}

static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
{
	struct xe_tile *tile = m->tile;
	struct xe_device *xe = tile_to_xe(tile);
	struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny;
	struct xe_res_cursor src_it;
	struct dma_fence *fence;
	u64 retval, expected;
	struct xe_bb *bb;
	int err;
	u8 id = tile->id;

	err = xe_bo_vmap(bo);
	if (err) {
		KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
			   PTR_ERR(bo));
		return;
	}

	big = xe_bo_create_pin_map(xe, tile, m->eng->vm, SZ_4M,
				   ttm_bo_type_kernel,
				   XE_BO_CREATE_VRAM_IF_DGFX(tile) |
				   XE_BO_CREATE_PINNED_BIT);
	if (IS_ERR(big)) {
		KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
		goto vunmap;
	}

	pt = xe_bo_create_pin_map(xe, tile, m->eng->vm, XE_PAGE_SIZE,
				  ttm_bo_type_kernel,
				  XE_BO_CREATE_VRAM_IF_DGFX(tile) |
				  XE_BO_CREATE_PINNED_BIT);
	if (IS_ERR(pt)) {
		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
			   PTR_ERR(pt));
		goto free_big;
	}

	tiny = xe_bo_create_pin_map(xe, tile, m->eng->vm,
				    2 * SZ_4K,
				    ttm_bo_type_kernel,
				    XE_BO_CREATE_VRAM_IF_DGFX(tile) |
				    XE_BO_CREATE_PINNED_BIT);
	if (IS_ERR(tiny)) {
		KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
			   PTR_ERR(pt));
		goto free_pt;
	}

	bb = xe_bb_new(tile->primary_gt, 32, xe->info.supports_usm);
	if (IS_ERR(bb)) {
		KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
			   PTR_ERR(bb));
		goto free_tiny;
	}

	kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
		   (unsigned long)xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, XE_PAGE_SIZE),
		   (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE));

	/* First part of the test, are we updating our pagetable bo with a new entry? */
	xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
		  0xdeaddeadbeefbeef);
	expected = xe_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0);
	if (m->eng->vm->flags & XE_VM_FLAG_64K)
		expected |= XE_PTE_PS64;
	if (xe_bo_is_vram(pt))
		xe_res_first(pt->ttm.resource, 0, pt->size, &src_it);
	else
		xe_res_first_sg(xe_bo_get_sg(pt), 0, pt->size, &src_it);

	emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt),
		 &src_it, XE_PAGE_SIZE, pt);

	run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test);

	retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
			   u64);
	check(retval, expected, "PTE entry write", test);

	/* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
	bb->len = 0;
	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
	xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
	expected = 0;

	emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
		   IS_DGFX(xe));
	run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable",
		       test);

	retval = xe_map_rd(xe, &pt->vmap, 0, u32);
	check(retval, expected, "Write to PT after adding PTE", test);

	/* Sanity checks passed, try the full ones! */

	/* Clear a small bo */
	kunit_info(test, "Clearing small buffer object\n");
	xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size);
	expected = 0;
	fence = xe_migrate_clear(m, tiny, tiny->ttm.resource);
	if (sanity_fence_failed(xe, fence, "Clearing small bo", test))
		goto out;

	dma_fence_put(fence);
	retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
	check(retval, expected, "Command clear small first value", test);
	retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32);
	check(retval, expected, "Command clear small last value", test);

	kunit_info(test, "Copying small buffer object to system\n");
	test_copy(m, tiny, test);

	/* Clear a big bo */
	kunit_info(test, "Clearing big buffer object\n");
	xe_map_memset(xe, &big->vmap, 0, 0x11, big->size);
	expected = 0;
	fence = xe_migrate_clear(m, big, big->ttm.resource);
	if (sanity_fence_failed(xe, fence, "Clearing big bo", test))
		goto out;

	dma_fence_put(fence);
	retval = xe_map_rd(xe, &big->vmap, 0, u32);
	check(retval, expected, "Command clear big first value", test);
	retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32);
	check(retval, expected, "Command clear big last value", test);

	kunit_info(test, "Copying big buffer object to system\n");
	test_copy(m, big, test);

	kunit_info(test, "Testing page table update using CPU if GPU idle.\n");
	test_pt_update(m, pt, test, false);
	kunit_info(test, "Testing page table update using GPU\n");
	test_pt_update(m, pt, test, true);

out:
	xe_bb_free(bb, NULL);
free_tiny:
	xe_bo_unpin(tiny);
	xe_bo_put(tiny);
free_pt:
	xe_bo_unpin(pt);
	xe_bo_put(pt);
free_big:
	xe_bo_unpin(big);
	xe_bo_put(big);
vunmap:
	xe_bo_vunmap(m->pt_bo);
}

static int migrate_test_run_device(struct xe_device *xe)
{
	struct kunit *test = xe_cur_kunit();
	struct xe_tile *tile;
	int id;

	for_each_tile(tile, xe, id) {
		struct xe_migrate *m = tile->migrate;
		struct ww_acquire_ctx ww;

		kunit_info(test, "Testing tile id %d.\n", id);
		xe_vm_lock(m->eng->vm, &ww, 0, true);
		xe_device_mem_access_get(xe);
		xe_migrate_sanity_test(m, test);
		xe_device_mem_access_put(xe);
		xe_vm_unlock(m->eng->vm, &ww);
	}

	return 0;
}

void xe_migrate_sanity_kunit(struct kunit *test)
{
	xe_call_for_each_device(migrate_test_run_device);
}
EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit);
Commit	Line	Data
dd08ebf6 MB	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright © 2020-2022 Intel Corporation
	4	*/
	5
	6	#include <kunit/test.h>
60d5c6ab	7	#include <kunit/visibility.h>
dd08ebf6	8
353dfaaa	9	#include "tests/xe_migrate_test.h"
af049be5	10	#include "tests/xe_pci_test.h"
353dfaaa	11
dd08ebf6 MB	12	#include "xe_pci.h"
	13
	14	static bool sanity_fence_failed(struct xe_device xe, struct dma_fence fence,
	15	const char str, struct kunit test)
	16	{
	17	long ret;
	18
	19	if (IS_ERR(fence)) {
	20	KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str,
	21	PTR_ERR(fence));
	22	return true;
	23	}
	24	if (!fence)
	25	return true;
	26
	27	ret = dma_fence_wait_timeout(fence, false, 5 * HZ);
	28	if (ret <= 0) {
	29	KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret);
	30	return true;
	31	}
	32
	33	return false;
	34	}
	35
	36	static int run_sanity_job(struct xe_migrate m, struct xe_device xe,
	37	struct xe_bb bb, u32 second_idx, const char str,
	38	struct kunit *test)
	39	{
f7339fe7	40	u64 batch_base = xe_migrate_batch_base(m, xe->info.supports_usm);
dd08ebf6	41	struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb,
f7339fe7	42	batch_base,
dd08ebf6 MB	43	second_idx);
	44	struct dma_fence *fence;
	45
	46	if (IS_ERR(job)) {
	47	KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
	48	PTR_ERR(job));
	49	return PTR_ERR(job);
	50	}
	51
	52	xe_sched_job_arm(job);
	53	fence = dma_fence_get(&job->drm.s_fence->finished);
	54	xe_sched_job_push(job);
	55
	56	if (sanity_fence_failed(xe, fence, str, test))
	57	return -ETIMEDOUT;
	58
	59	dma_fence_put(fence);
	60	kunit_info(test, "%s: Job completed\n", str);
	61	return 0;
	62	}
	63
	64	static void
	65	sanity_populate_cb(struct xe_migrate_pt_update *pt_update,
876611c2	66	struct xe_tile tile, struct iosys_map map, void *dst,
dd08ebf6 MB	67	u32 qword_ofs, u32 num_qwords,
	68	const struct xe_vm_pgtable_update *update)
	69	{
7cba3396 TH	70	struct migrate_test_params *p =
7cba3396 TH	71	to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));
dd08ebf6 MB	72	int i;
dd08ebf6 MB	73	u64 *ptr = dst;
17a28ea2 TH	74	u64 value;
	75
	76	for (i = 0; i < num_qwords; i++) {
	77	value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL;
	78	if (map)
876611c2	79	xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
17a28ea2 TH	80	sizeof(u64), u64, value);
	81	else
	82	ptr[i] = value;
	83	}
7cba3396 TH	84
	85	kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU");
	86	if (p->force_gpu && map)
	87	KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n");
dd08ebf6 MB	88	}
	89
	90	static const struct xe_migrate_pt_update_ops sanity_ops = {
	91	.populate = sanity_populate_cb,
	92	};
	93
	94	#define check(_retval, _expected, str, _test) \
	95	do { if ((_retval) != (_expected)) { \
	96	KUNIT_FAIL(_test, "Sanity check failed: " str \
	97	" expected %llx, got %llx\n", \
	98	(u64)(_expected), (u64)(_retval)); \
	99	} } while (0)
	100
	101	static void test_copy(struct xe_migrate m, struct xe_bo bo,
	102	struct kunit *test)
	103	{
08dea767	104	struct xe_device *xe = tile_to_xe(m->tile);
11a2407e	105	u64 retval, expected = 0;
dd08ebf6 MB	106	bool big = bo->size >= SZ_2M;
	107	struct dma_fence *fence;
	108	const char *str = big ? "Copying big bo" : "Copying small bo";
	109	int err;
	110
08dea767	111	struct xe_bo *sysmem = xe_bo_create_locked(xe, m->tile, NULL,
dd08ebf6 MB	112	bo->size,
dd08ebf6 MB	113	ttm_bo_type_kernel,
6a024f1b MA	114	XE_BO_CREATE_SYSTEM_BIT \|
6a024f1b MA	115	XE_BO_NEEDS_CPU_ACCESS);
dd08ebf6 MB	116	if (IS_ERR(sysmem)) {
	117	KUNIT_FAIL(test, "Failed to allocate sysmem bo for %s: %li\n",
	118	str, PTR_ERR(sysmem));
	119	return;
	120	}
	121
	122	err = xe_bo_validate(sysmem, NULL, false);
	123	if (err) {
	124	KUNIT_FAIL(test, "Failed to validate system bo for %s: %li\n",
	125	str, err);
	126	goto out_unlock;
	127	}
	128
	129	err = xe_bo_vmap(sysmem);
	130	if (err) {
	131	KUNIT_FAIL(test, "Failed to vmap system bo for %s: %li\n",
	132	str, err);
	133	goto out_unlock;
	134	}
	135
	136	xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
11a2407e	137	fence = xe_migrate_clear(m, sysmem, sysmem->ttm.resource);
dd08ebf6 MB	138	if (!sanity_fence_failed(xe, fence, big ? "Clearing sysmem big bo" :
	139	"Clearing sysmem small bo", test)) {
	140	retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
	141	check(retval, expected, "sysmem first offset should be cleared",
	142	test);
	143	retval = xe_map_rd(xe, &sysmem->vmap, sysmem->size - 8, u64);
	144	check(retval, expected, "sysmem last offset should be cleared",
	145	test);
	146	}
	147	dma_fence_put(fence);
	148
2a8477f7	149	/* Try to copy 0xc0 from sysmem to vram with 2MB or 64KiB/4KiB pages */
dd08ebf6 MB	150	xe_map_memset(xe, &sysmem->vmap, 0, 0xc0, sysmem->size);
	151	xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size);
	152
370997d1	153	expected = 0xc0c0c0c0c0c0c0c0;
3690a01b	154	fence = xe_migrate_copy(m, sysmem, bo, sysmem->ttm.resource,
dd08ebf6 MB	155	bo->ttm.resource);
	156	if (!sanity_fence_failed(xe, fence, big ? "Copying big bo sysmem -> vram" :
	157	"Copying small bo sysmem -> vram", test)) {
	158	retval = xe_map_rd(xe, &bo->vmap, 0, u64);
	159	check(retval, expected,
	160	"sysmem -> vram bo first offset should be copied", test);
	161	retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64);
	162	check(retval, expected,
	163	"sysmem -> vram bo offset should be copied", test);
	164	}
	165	dma_fence_put(fence);
	166
	167	/* And other way around.. slightly hacky.. */
	168	xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size);
	169	xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size);
	170
3690a01b	171	fence = xe_migrate_copy(m, bo, sysmem, bo->ttm.resource,
dd08ebf6 MB	172	sysmem->ttm.resource);
	173	if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> sysmem" :
	174	"Copying small bo vram -> sysmem", test)) {
	175	retval = xe_map_rd(xe, &sysmem->vmap, 0, u64);
	176	check(retval, expected,
	177	"vram -> sysmem bo first offset should be copied", test);
	178	retval = xe_map_rd(xe, &sysmem->vmap, bo->size - 8, u64);
	179	check(retval, expected,
	180	"vram -> sysmem bo last offset should be copied", test);
	181	}
	182	dma_fence_put(fence);
	183
	184	xe_bo_vunmap(sysmem);
	185	out_unlock:
	186	xe_bo_unlock_no_vm(sysmem);
	187	xe_bo_put(sysmem);
	188	}
	189
	190	static void test_pt_update(struct xe_migrate m, struct xe_bo pt,
7cba3396	191	struct kunit *test, bool force_gpu)
dd08ebf6	192	{
08dea767	193	struct xe_device *xe = tile_to_xe(m->tile);
dd08ebf6 MB	194	struct dma_fence *fence;
dd08ebf6 MB	195	u64 retval, expected;
7cba3396	196	ktime_t then, now;
dd08ebf6 MB	197	int i;
	198
	199	struct xe_vm_pgtable_update update = {
	200	.ofs = 1,
	201	.qwords = 0x10,
	202	.pt_bo = pt,
	203	};
	204	struct xe_migrate_pt_update pt_update = {
	205	.ops = &sanity_ops,
	206	};
7cba3396 TH	207	struct migrate_test_params p = {
	208	.base.id = XE_TEST_LIVE_MIGRATE,
	209	.force_gpu = force_gpu,
	210	};
dd08ebf6	211
7cba3396	212	test->priv = &p;
dd08ebf6 MB	213	/* Test xe_migrate_update_pgtables() updates the pagetable as expected */
	214	expected = 0xf0f0f0f0f0f0f0f0ULL;
	215	xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size);
	216
7cba3396	217	then = ktime_get();
dd08ebf6 MB	218	fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1,
dd08ebf6 MB	219	NULL, 0, &pt_update);
7cba3396	220	now = ktime_get();
dd08ebf6 MB	221	if (sanity_fence_failed(xe, fence, "Migration pagetable update", test))
	222	return;
	223
7cba3396 TH	224	kunit_info(test, "Updating without syncing took %llu us,\n",
	225	(unsigned long long)ktime_to_us(ktime_sub(now, then)));
	226
dd08ebf6 MB	227	dma_fence_put(fence);
	228	retval = xe_map_rd(xe, &pt->vmap, 0, u64);
	229	check(retval, expected, "PTE[0] must stay untouched", test);
	230
	231	for (i = 0; i < update.qwords; i++) {
	232	retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64);
	233	check(retval, i * 0x1111111111111111ULL, "PTE update", test);
	234	}
	235
	236	retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords),
	237	u64);
	238	check(retval, expected, "PTE[0x11] must stay untouched", test);
	239	}
	240
	241	static void xe_migrate_sanity_test(struct xe_migrate m, struct kunit test)
	242	{
08dea767 MR	243	struct xe_tile *tile = m->tile;
08dea767 MR	244	struct xe_device *xe = tile_to_xe(tile);
dd08ebf6 MB	245	struct xe_bo pt, bo = m->pt_bo, big, tiny;
	246	struct xe_res_cursor src_it;
	247	struct dma_fence *fence;
	248	u64 retval, expected;
	249	struct xe_bb *bb;
	250	int err;
08dea767	251	u8 id = tile->id;
dd08ebf6 MB	252
	253	err = xe_bo_vmap(bo);
	254	if (err) {
	255	KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n",
	256	PTR_ERR(bo));
	257	return;
	258	}
	259
876611c2	260	big = xe_bo_create_pin_map(xe, tile, m->eng->vm, SZ_4M,
dd08ebf6	261	ttm_bo_type_kernel,
876611c2	262	XE_BO_CREATE_VRAM_IF_DGFX(tile) \|
dd08ebf6 MB	263	XE_BO_CREATE_PINNED_BIT);
	264	if (IS_ERR(big)) {
	265	KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big));
	266	goto vunmap;
	267	}
	268
876611c2	269	pt = xe_bo_create_pin_map(xe, tile, m->eng->vm, XE_PAGE_SIZE,
dd08ebf6	270	ttm_bo_type_kernel,
876611c2	271	XE_BO_CREATE_VRAM_IF_DGFX(tile) \|
dd08ebf6 MB	272	XE_BO_CREATE_PINNED_BIT);
	273	if (IS_ERR(pt)) {
	274	KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
	275	PTR_ERR(pt));
	276	goto free_big;
	277	}
	278
876611c2	279	tiny = xe_bo_create_pin_map(xe, tile, m->eng->vm,
dd08ebf6 MB	280	2 * SZ_4K,
dd08ebf6 MB	281	ttm_bo_type_kernel,
876611c2	282	XE_BO_CREATE_VRAM_IF_DGFX(tile) \|
dd08ebf6 MB	283	XE_BO_CREATE_PINNED_BIT);
	284	if (IS_ERR(tiny)) {
	285	KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n",
	286	PTR_ERR(pt));
	287	goto free_pt;
	288	}
	289
f6929e80	290	bb = xe_bb_new(tile->primary_gt, 32, xe->info.supports_usm);
dd08ebf6 MB	291	if (IS_ERR(bb)) {
	292	KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n",
	293	PTR_ERR(bb));
	294	goto free_tiny;
	295	}
	296
857912c3	297	kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n",
58e19acf LDM	298	(unsigned long)xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, XE_PAGE_SIZE),
58e19acf LDM	299	(unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE));
dd08ebf6 MB	300
dd08ebf6 MB	301	/* First part of the test, are we updating our pagetable bo with a new entry? */
58e19acf LDM	302	xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
58e19acf LDM	303	0xdeaddeadbeefbeef);
b06d47be	304	expected = xe_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0);
0d39b6da	305	if (m->eng->vm->flags & XE_VM_FLAG_64K)
58e19acf	306	expected \|= XE_PTE_PS64;
9922bb40 TH	307	if (xe_bo_is_vram(pt))
	308	xe_res_first(pt->ttm.resource, 0, pt->size, &src_it);
	309	else
	310	xe_res_first_sg(xe_bo_get_sg(pt), 0, pt->size, &src_it);
	311
dd08ebf6	312	emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt),
58e19acf	313	&src_it, XE_PAGE_SIZE, pt);
9922bb40	314
dd08ebf6 MB	315	run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test);
dd08ebf6 MB	316
58e19acf	317	retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1),
dd08ebf6 MB	318	u64);
	319	check(retval, expected, "PTE entry write", test);
	320
	321	/* Now try to write data to our newly mapped 'pagetable', see if it succeeds */
	322	bb->len = 0;
	323	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
	324	xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead);
11a2407e	325	expected = 0;
dd08ebf6	326
f6929e80	327	emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4,
11a2407e	328	IS_DGFX(xe));
dd08ebf6 MB	329	run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable",
	330	test);
	331
	332	retval = xe_map_rd(xe, &pt->vmap, 0, u32);
	333	check(retval, expected, "Write to PT after adding PTE", test);
	334
	335	/* Sanity checks passed, try the full ones! */
	336
	337	/* Clear a small bo */
	338	kunit_info(test, "Clearing small buffer object\n");
	339	xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size);
11a2407e BV	340	expected = 0;
11a2407e BV	341	fence = xe_migrate_clear(m, tiny, tiny->ttm.resource);
dd08ebf6 MB	342	if (sanity_fence_failed(xe, fence, "Clearing small bo", test))
	343	goto out;
	344
	345	dma_fence_put(fence);
	346	retval = xe_map_rd(xe, &tiny->vmap, 0, u32);
	347	check(retval, expected, "Command clear small first value", test);
	348	retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32);
	349	check(retval, expected, "Command clear small last value", test);
	350
3690a01b TH	351	kunit_info(test, "Copying small buffer object to system\n");
3690a01b TH	352	test_copy(m, tiny, test);
dd08ebf6	353
11a2407e	354	/* Clear a big bo */
dd08ebf6 MB	355	kunit_info(test, "Clearing big buffer object\n");
dd08ebf6 MB	356	xe_map_memset(xe, &big->vmap, 0, 0x11, big->size);
11a2407e BV	357	expected = 0;
11a2407e BV	358	fence = xe_migrate_clear(m, big, big->ttm.resource);
dd08ebf6 MB	359	if (sanity_fence_failed(xe, fence, "Clearing big bo", test))
	360	goto out;
	361
	362	dma_fence_put(fence);
	363	retval = xe_map_rd(xe, &big->vmap, 0, u32);
	364	check(retval, expected, "Command clear big first value", test);
	365	retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32);
	366	check(retval, expected, "Command clear big last value", test);
	367
3690a01b TH	368	kunit_info(test, "Copying big buffer object to system\n");
3690a01b TH	369	test_copy(m, big, test);
dd08ebf6	370
7cba3396 TH	371	kunit_info(test, "Testing page table update using CPU if GPU idle.\n");
	372	test_pt_update(m, pt, test, false);
	373	kunit_info(test, "Testing page table update using GPU\n");
	374	test_pt_update(m, pt, test, true);
dd08ebf6 MB	375
	376	out:
	377	xe_bb_free(bb, NULL);
	378	free_tiny:
	379	xe_bo_unpin(tiny);
	380	xe_bo_put(tiny);
	381	free_pt:
	382	xe_bo_unpin(pt);
	383	xe_bo_put(pt);
	384	free_big:
	385	xe_bo_unpin(big);
	386	xe_bo_put(big);
	387	vunmap:
	388	xe_bo_vunmap(m->pt_bo);
	389	}
	390
	391	static int migrate_test_run_device(struct xe_device *xe)
	392	{
	393	struct kunit *test = xe_cur_kunit();
08dea767	394	struct xe_tile *tile;
dd08ebf6 MB	395	int id;
dd08ebf6 MB	396
08dea767 MR	397	for_each_tile(tile, xe, id) {
08dea767 MR	398	struct xe_migrate *m = tile->migrate;
dd08ebf6 MB	399	struct ww_acquire_ctx ww;
dd08ebf6 MB	400
08dea767	401	kunit_info(test, "Testing tile id %d.\n", id);
dd08ebf6	402	xe_vm_lock(m->eng->vm, &ww, 0, true);
907a319c	403	xe_device_mem_access_get(xe);
dd08ebf6	404	xe_migrate_sanity_test(m, test);
907a319c	405	xe_device_mem_access_put(xe);
dd08ebf6 MB	406	xe_vm_unlock(m->eng->vm, &ww);
	407	}
	408
	409	return 0;
	410	}
	411
	412	void xe_migrate_sanity_kunit(struct kunit *test)
	413	{
	414	xe_call_for_each_device(migrate_test_run_device);
	415	}
60d5c6ab	416	EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit);