Commit | Line | Data |
---|---|---|
dd08ebf6 MB |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2020 Intel Corporation | |
4 | */ | |
5 | #include "xe_migrate.h" | |
6 | ||
7 | #include "xe_bb.h" | |
8 | #include "xe_bo.h" | |
9 | #include "xe_engine.h" | |
10 | #include "xe_ggtt.h" | |
11 | #include "xe_gt.h" | |
12 | #include "xe_hw_engine.h" | |
13 | #include "xe_lrc.h" | |
14 | #include "xe_map.h" | |
15 | #include "xe_mocs.h" | |
16 | #include "xe_pt.h" | |
17 | #include "xe_res_cursor.h" | |
18 | #include "xe_sched_job.h" | |
19 | #include "xe_sync.h" | |
20 | #include "xe_trace.h" | |
21 | #include "xe_vm.h" | |
22 | ||
23 | #include <linux/sizes.h> | |
24 | #include <drm/drm_managed.h> | |
25 | #include <drm/ttm/ttm_tt.h> | |
26 | #include <drm/xe_drm.h> | |
27 | ||
28 | #include "gt/intel_gpu_commands.h" | |
29 | ||
e9d285ff TH |
30 | /** |
31 | * struct xe_migrate - migrate context. | |
32 | */ | |
dd08ebf6 | 33 | struct xe_migrate { |
e9d285ff | 34 | /** @eng: Default engine used for migration */ |
dd08ebf6 | 35 | struct xe_engine *eng; |
e9d285ff | 36 | /** @gt: Backpointer to the gt this struct xe_migrate belongs to. */ |
dd08ebf6 | 37 | struct xe_gt *gt; |
e9d285ff | 38 | /** @job_mutex: Timeline mutex for @eng. */ |
dd08ebf6 | 39 | struct mutex job_mutex; |
e9d285ff | 40 | /** @pt_bo: Page-table buffer object. */ |
dd08ebf6 | 41 | struct xe_bo *pt_bo; |
e9d285ff TH |
42 | /** |
43 | * @cleared_bo: Zeroed out bo used as a source for CCS metadata clears | |
44 | */ | |
dd08ebf6 | 45 | struct xe_bo *cleared_bo; |
e9d285ff | 46 | /** @batch_base_ofs: VM offset of the migration batch buffer */ |
dd08ebf6 | 47 | u64 batch_base_ofs; |
e9d285ff | 48 | /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ |
dd08ebf6 | 49 | u64 usm_batch_base_ofs; |
e9d285ff | 50 | /** @cleared_vram_ofs: VM offset of @cleared_bo. */ |
dd08ebf6 | 51 | u64 cleared_vram_ofs; |
e9d285ff TH |
52 | /** |
53 | * @fence: dma-fence representing the last migration job batch. | |
54 | * Protected by @job_mutex. | |
55 | */ | |
dd08ebf6 | 56 | struct dma_fence *fence; |
e9d285ff TH |
57 | /** |
58 | * @vm_update_sa: For integrated, used to suballocate page-tables | |
59 | * out of the pt_bo. | |
60 | */ | |
dd08ebf6 MB |
61 | struct drm_suballoc_manager vm_update_sa; |
62 | }; | |
63 | ||
64 | #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ | |
65 | #define NUM_KERNEL_PDE 17 | |
66 | #define NUM_PT_SLOTS 32 | |
67 | #define NUM_PT_PER_BLIT (MAX_PREEMPTDISABLE_TRANSFER / SZ_2M) | |
68 | ||
e9d285ff TH |
69 | /** |
70 | * xe_gt_migrate_engine() - Get this gt's migrate engine. | |
71 | * @gt: The gt. | |
72 | * | |
73 | * Returns the default migrate engine of this gt. | |
74 | * TODO: Perhaps this function is slightly misplaced, and even unneeded? | |
75 | * | |
76 | * Return: The default migrate engine | |
77 | */ | |
dd08ebf6 MB |
78 | struct xe_engine *xe_gt_migrate_engine(struct xe_gt *gt) |
79 | { | |
80 | return gt->migrate->eng; | |
81 | } | |
82 | ||
83 | static void xe_migrate_fini(struct drm_device *dev, void *arg) | |
84 | { | |
85 | struct xe_migrate *m = arg; | |
86 | struct ww_acquire_ctx ww; | |
87 | ||
88 | xe_vm_lock(m->eng->vm, &ww, 0, false); | |
89 | xe_bo_unpin(m->pt_bo); | |
90 | if (m->cleared_bo) | |
91 | xe_bo_unpin(m->cleared_bo); | |
92 | xe_vm_unlock(m->eng->vm, &ww); | |
93 | ||
94 | dma_fence_put(m->fence); | |
95 | if (m->cleared_bo) | |
96 | xe_bo_put(m->cleared_bo); | |
97 | xe_bo_put(m->pt_bo); | |
98 | drm_suballoc_manager_fini(&m->vm_update_sa); | |
99 | mutex_destroy(&m->job_mutex); | |
100 | xe_vm_close_and_put(m->eng->vm); | |
101 | xe_engine_put(m->eng); | |
102 | } | |
103 | ||
104 | static u64 xe_migrate_vm_addr(u64 slot, u32 level) | |
105 | { | |
106 | XE_BUG_ON(slot >= NUM_PT_SLOTS); | |
107 | ||
108 | /* First slot is reserved for mapping of PT bo and bb, start from 1 */ | |
109 | return (slot + 1ULL) << xe_pt_shift(level + 1); | |
110 | } | |
111 | ||
112 | static u64 xe_migrate_vram_ofs(u64 addr) | |
113 | { | |
114 | return addr + (256ULL << xe_pt_shift(2)); | |
115 | } | |
116 | ||
117 | /* | |
118 | * For flat CCS clearing we need a cleared chunk of memory to copy from, | |
119 | * since the CCS clearing mode of XY_FAST_COLOR_BLT appears to be buggy | |
120 | * (it clears on only 14 bytes in each chunk of 16). | |
121 | * If clearing the main surface one can use the part of the main surface | |
122 | * already cleared, but for clearing as part of copying non-compressed | |
123 | * data out of system memory, we don't readily have a cleared part of | |
124 | * VRAM to copy from, so create one to use for that case. | |
125 | */ | |
126 | static int xe_migrate_create_cleared_bo(struct xe_migrate *m, struct xe_vm *vm) | |
127 | { | |
128 | struct xe_gt *gt = m->gt; | |
129 | struct xe_device *xe = vm->xe; | |
130 | size_t cleared_size; | |
131 | u64 vram_addr; | |
132 | bool is_vram; | |
133 | ||
134 | if (!xe_device_has_flat_ccs(xe)) | |
135 | return 0; | |
136 | ||
137 | cleared_size = xe_device_ccs_bytes(xe, MAX_PREEMPTDISABLE_TRANSFER); | |
138 | cleared_size = PAGE_ALIGN(cleared_size); | |
139 | m->cleared_bo = xe_bo_create_pin_map(xe, gt, vm, cleared_size, | |
140 | ttm_bo_type_kernel, | |
141 | XE_BO_CREATE_VRAM_IF_DGFX(gt) | | |
142 | XE_BO_CREATE_PINNED_BIT); | |
143 | if (IS_ERR(m->cleared_bo)) | |
144 | return PTR_ERR(m->cleared_bo); | |
145 | ||
146 | xe_map_memset(xe, &m->cleared_bo->vmap, 0, 0x00, cleared_size); | |
147 | vram_addr = xe_bo_addr(m->cleared_bo, 0, GEN8_PAGE_SIZE, &is_vram); | |
148 | XE_BUG_ON(!is_vram); | |
149 | m->cleared_vram_ofs = xe_migrate_vram_ofs(vram_addr); | |
150 | ||
151 | return 0; | |
152 | } | |
153 | ||
154 | static int xe_migrate_prepare_vm(struct xe_gt *gt, struct xe_migrate *m, | |
155 | struct xe_vm *vm) | |
156 | { | |
157 | u8 id = gt->info.id; | |
158 | u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; | |
159 | u32 map_ofs, level, i; | |
160 | struct xe_device *xe = gt_to_xe(m->gt); | |
161 | struct xe_bo *bo, *batch = gt->kernel_bb_pool.bo; | |
162 | u64 entry; | |
163 | int ret; | |
164 | ||
165 | /* Can't bump NUM_PT_SLOTS too high */ | |
166 | BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/GEN8_PAGE_SIZE); | |
167 | /* Must be a multiple of 64K to support all platforms */ | |
168 | BUILD_BUG_ON(NUM_PT_SLOTS * GEN8_PAGE_SIZE % SZ_64K); | |
169 | /* And one slot reserved for the 4KiB page table updates */ | |
170 | BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); | |
171 | ||
172 | /* Need to be sure everything fits in the first PT, or create more */ | |
173 | XE_BUG_ON(m->batch_base_ofs + batch->size >= SZ_2M); | |
174 | ||
175 | bo = xe_bo_create_pin_map(vm->xe, m->gt, vm, | |
176 | num_entries * GEN8_PAGE_SIZE, | |
177 | ttm_bo_type_kernel, | |
178 | XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | | |
179 | XE_BO_CREATE_PINNED_BIT); | |
180 | if (IS_ERR(bo)) | |
181 | return PTR_ERR(bo); | |
182 | ||
183 | ret = xe_migrate_create_cleared_bo(m, vm); | |
184 | if (ret) { | |
185 | xe_bo_put(bo); | |
186 | return ret; | |
187 | } | |
188 | ||
189 | entry = gen8_pde_encode(bo, bo->size - GEN8_PAGE_SIZE, XE_CACHE_WB); | |
190 | xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); | |
191 | ||
192 | map_ofs = (num_entries - num_level) * GEN8_PAGE_SIZE; | |
193 | ||
194 | /* Map the entire BO in our level 0 pt */ | |
195 | for (i = 0, level = 0; i < num_entries; level++) { | |
196 | entry = gen8_pte_encode(NULL, bo, i * GEN8_PAGE_SIZE, | |
197 | XE_CACHE_WB, 0, 0); | |
198 | ||
199 | xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); | |
200 | ||
201 | if (vm->flags & XE_VM_FLAGS_64K) | |
202 | i += 16; | |
203 | else | |
204 | i += 1; | |
205 | } | |
206 | ||
207 | if (!IS_DGFX(xe)) { | |
208 | XE_BUG_ON(xe->info.supports_usm); | |
209 | ||
210 | /* Write out batch too */ | |
211 | m->batch_base_ofs = NUM_PT_SLOTS * GEN8_PAGE_SIZE; | |
212 | for (i = 0; i < batch->size; | |
213 | i += vm->flags & XE_VM_FLAGS_64K ? GEN8_64K_PAGE_SIZE : | |
214 | GEN8_PAGE_SIZE) { | |
215 | entry = gen8_pte_encode(NULL, batch, i, | |
216 | XE_CACHE_WB, 0, 0); | |
217 | ||
218 | xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, | |
219 | entry); | |
220 | level++; | |
221 | } | |
222 | } else { | |
223 | bool is_lmem; | |
224 | u64 batch_addr = xe_bo_addr(batch, 0, GEN8_PAGE_SIZE, &is_lmem); | |
225 | ||
226 | m->batch_base_ofs = xe_migrate_vram_ofs(batch_addr); | |
227 | ||
228 | if (xe->info.supports_usm) { | |
229 | batch = gt->usm.bb_pool.bo; | |
230 | batch_addr = xe_bo_addr(batch, 0, GEN8_PAGE_SIZE, | |
231 | &is_lmem); | |
232 | m->usm_batch_base_ofs = xe_migrate_vram_ofs(batch_addr); | |
233 | } | |
234 | } | |
235 | ||
236 | for (level = 1; level < num_level; level++) { | |
237 | u32 flags = 0; | |
238 | ||
239 | if (vm->flags & XE_VM_FLAGS_64K && level == 1) | |
240 | flags = GEN12_PDE_64K; | |
241 | ||
242 | entry = gen8_pde_encode(bo, map_ofs + (level - 1) * | |
243 | GEN8_PAGE_SIZE, XE_CACHE_WB); | |
244 | xe_map_wr(xe, &bo->vmap, map_ofs + GEN8_PAGE_SIZE * level, u64, | |
245 | entry | flags); | |
246 | } | |
247 | ||
248 | /* Write PDE's that point to our BO. */ | |
249 | for (i = 0; i < num_entries - num_level; i++) { | |
250 | entry = gen8_pde_encode(bo, i * GEN8_PAGE_SIZE, | |
251 | XE_CACHE_WB); | |
252 | ||
253 | xe_map_wr(xe, &bo->vmap, map_ofs + GEN8_PAGE_SIZE + | |
254 | (i + 1) * 8, u64, entry); | |
255 | } | |
256 | ||
257 | /* Identity map the entire vram at 256GiB offset */ | |
258 | if (IS_DGFX(xe)) { | |
259 | u64 pos, ofs, flags; | |
260 | ||
261 | level = 2; | |
262 | ofs = map_ofs + GEN8_PAGE_SIZE * level + 256 * 8; | |
263 | flags = GEN8_PAGE_RW | GEN8_PAGE_PRESENT | PPAT_CACHED | | |
264 | GEN12_PPGTT_PTE_LM | GEN8_PDPE_PS_1G; | |
265 | ||
266 | /* | |
267 | * Use 1GB pages, it shouldn't matter the physical amount of | |
268 | * vram is less, when we don't access it. | |
269 | */ | |
270 | for (pos = 0; pos < xe->mem.vram.size; pos += SZ_1G, ofs += 8) | |
271 | xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); | |
272 | } | |
273 | ||
274 | /* | |
275 | * Example layout created above, with root level = 3: | |
276 | * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's | |
277 | * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's | |
278 | * [PT9...PT28]: Userspace PT's for VM_BIND, 4 KiB PTE's | |
279 | * [PT29 = PDE 0] [PT30 = PDE 1] [PT31 = PDE 2] | |
280 | * | |
281 | * This makes the lowest part of the VM point to the pagetables. | |
282 | * Hence the lowest 2M in the vm should point to itself, with a few writes | |
283 | * and flushes, other parts of the VM can be used either for copying and | |
284 | * clearing. | |
285 | * | |
286 | * For performance, the kernel reserves PDE's, so about 20 are left | |
287 | * for async VM updates. | |
288 | * | |
289 | * To make it easier to work, each scratch PT is put in slot (1 + PT #) | |
290 | * everywhere, this allows lockless updates to scratch pages by using | |
291 | * the different addresses in VM. | |
292 | */ | |
293 | #define NUM_VMUSA_UNIT_PER_PAGE 32 | |
294 | #define VM_SA_UPDATE_UNIT_SIZE (GEN8_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) | |
295 | #define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) | |
296 | drm_suballoc_manager_init(&m->vm_update_sa, | |
297 | (map_ofs / GEN8_PAGE_SIZE - NUM_KERNEL_PDE) * | |
298 | NUM_VMUSA_UNIT_PER_PAGE, 0); | |
299 | ||
300 | m->pt_bo = bo; | |
301 | return 0; | |
302 | } | |
303 | ||
e9d285ff TH |
304 | /** |
305 | * xe_migrate_init() - Initialize a migrate context | |
306 | * @gt: Back-pointer to the gt we're initializing for. | |
307 | * | |
308 | * Return: Pointer to a migrate context on success. Error pointer on error. | |
309 | */ | |
dd08ebf6 MB |
310 | struct xe_migrate *xe_migrate_init(struct xe_gt *gt) |
311 | { | |
312 | struct xe_device *xe = gt_to_xe(gt); | |
313 | struct xe_migrate *m; | |
314 | struct xe_vm *vm; | |
315 | struct ww_acquire_ctx ww; | |
316 | int err; | |
317 | ||
318 | XE_BUG_ON(xe_gt_is_media_type(gt)); | |
319 | ||
320 | m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); | |
321 | if (!m) | |
322 | return ERR_PTR(-ENOMEM); | |
323 | ||
324 | m->gt = gt; | |
325 | ||
326 | /* Special layout, prepared below.. */ | |
327 | vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | | |
328 | XE_VM_FLAG_SET_GT_ID(gt)); | |
329 | if (IS_ERR(vm)) | |
330 | return ERR_CAST(vm); | |
331 | ||
332 | xe_vm_lock(vm, &ww, 0, false); | |
333 | err = xe_migrate_prepare_vm(gt, m, vm); | |
334 | xe_vm_unlock(vm, &ww); | |
335 | if (err) { | |
336 | xe_vm_close_and_put(vm); | |
337 | return ERR_PTR(err); | |
338 | } | |
339 | ||
340 | if (xe->info.supports_usm) { | |
341 | struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, | |
342 | XE_ENGINE_CLASS_COPY, | |
343 | gt->usm.reserved_bcs_instance, | |
344 | false); | |
345 | if (!hwe) | |
346 | return ERR_PTR(-EINVAL); | |
347 | ||
348 | m->eng = xe_engine_create(xe, vm, | |
349 | BIT(hwe->logical_instance), 1, | |
350 | hwe, ENGINE_FLAG_KERNEL); | |
351 | } else { | |
352 | m->eng = xe_engine_create_class(xe, gt, vm, | |
353 | XE_ENGINE_CLASS_COPY, | |
354 | ENGINE_FLAG_KERNEL); | |
355 | } | |
356 | if (IS_ERR(m->eng)) { | |
357 | xe_vm_close_and_put(vm); | |
358 | return ERR_CAST(m->eng); | |
359 | } | |
360 | ||
361 | mutex_init(&m->job_mutex); | |
362 | ||
363 | err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); | |
364 | if (err) | |
365 | return ERR_PTR(err); | |
366 | ||
367 | return m; | |
368 | } | |
369 | ||
370 | static void emit_arb_clear(struct xe_bb *bb) | |
371 | { | |
372 | /* 1 dword */ | |
373 | bb->cs[bb->len++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; | |
374 | } | |
375 | ||
376 | static u64 xe_migrate_res_sizes(struct xe_res_cursor *cur) | |
377 | { | |
378 | /* | |
379 | * For VRAM we use identity mapped pages so we are limited to current | |
380 | * cursor size. For system we program the pages ourselves so we have no | |
381 | * such limitation. | |
382 | */ | |
383 | return min_t(u64, MAX_PREEMPTDISABLE_TRANSFER, | |
384 | mem_type_is_vram(cur->mem_type) ? cur->size : | |
385 | cur->remaining); | |
386 | } | |
387 | ||
388 | static u32 pte_update_size(struct xe_migrate *m, | |
389 | bool is_vram, | |
390 | struct xe_res_cursor *cur, | |
391 | u64 *L0, u64 *L0_ofs, u32 *L0_pt, | |
392 | u32 cmd_size, u32 pt_ofs, u32 avail_pts) | |
393 | { | |
394 | u32 cmds = 0; | |
395 | ||
396 | *L0_pt = pt_ofs; | |
397 | if (!is_vram) { | |
398 | /* Clip L0 to available size */ | |
399 | u64 size = min(*L0, (u64)avail_pts * SZ_2M); | |
400 | u64 num_4k_pages = DIV_ROUND_UP(size, GEN8_PAGE_SIZE); | |
401 | ||
402 | *L0 = size; | |
403 | *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); | |
404 | ||
405 | /* MI_STORE_DATA_IMM */ | |
406 | cmds += 3 * DIV_ROUND_UP(num_4k_pages, 0x1ff); | |
407 | ||
408 | /* PDE qwords */ | |
409 | cmds += num_4k_pages * 2; | |
410 | ||
411 | /* Each chunk has a single blit command */ | |
412 | cmds += cmd_size; | |
413 | } else { | |
414 | /* Offset into identity map. */ | |
415 | *L0_ofs = xe_migrate_vram_ofs(cur->start); | |
416 | cmds += cmd_size; | |
417 | } | |
418 | ||
419 | return cmds; | |
420 | } | |
421 | ||
422 | static void emit_pte(struct xe_migrate *m, | |
423 | struct xe_bb *bb, u32 at_pt, | |
424 | bool is_vram, | |
425 | struct xe_res_cursor *cur, | |
426 | u32 size, struct xe_bo *bo) | |
427 | { | |
428 | u32 ptes; | |
429 | u64 ofs = at_pt * GEN8_PAGE_SIZE; | |
430 | u64 cur_ofs; | |
431 | ||
432 | /* | |
433 | * FIXME: Emitting VRAM PTEs to L0 PTs is forbidden. Currently | |
434 | * we're only emitting VRAM PTEs during sanity tests, so when | |
435 | * that's moved to a Kunit test, we should condition VRAM PTEs | |
436 | * on running tests. | |
437 | */ | |
438 | ||
439 | ptes = DIV_ROUND_UP(size, GEN8_PAGE_SIZE); | |
440 | ||
441 | while (ptes) { | |
442 | u32 chunk = min(0x1ffU, ptes); | |
443 | ||
444 | bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(21) | | |
445 | (chunk * 2 + 1); | |
446 | bb->cs[bb->len++] = ofs; | |
447 | bb->cs[bb->len++] = 0; | |
448 | ||
449 | cur_ofs = ofs; | |
450 | ofs += chunk * 8; | |
451 | ptes -= chunk; | |
452 | ||
453 | while (chunk--) { | |
454 | u64 addr; | |
455 | ||
e89b384c | 456 | addr = xe_res_dma(cur) & PAGE_MASK; |
dd08ebf6 | 457 | if (is_vram) { |
dd08ebf6 MB |
458 | /* Is this a 64K PTE entry? */ |
459 | if ((m->eng->vm->flags & XE_VM_FLAGS_64K) && | |
460 | !(cur_ofs & (16 * 8 - 1))) { | |
461 | XE_WARN_ON(!IS_ALIGNED(addr, SZ_64K)); | |
462 | addr |= GEN12_PTE_PS64; | |
463 | } | |
464 | ||
465 | addr |= GEN12_PPGTT_PTE_LM; | |
dd08ebf6 MB |
466 | } |
467 | addr |= PPAT_CACHED | GEN8_PAGE_PRESENT | GEN8_PAGE_RW; | |
468 | bb->cs[bb->len++] = lower_32_bits(addr); | |
469 | bb->cs[bb->len++] = upper_32_bits(addr); | |
470 | ||
e89b384c | 471 | xe_res_next(cur, min(size, (u32)PAGE_SIZE)); |
dd08ebf6 MB |
472 | cur_ofs += 8; |
473 | } | |
474 | } | |
475 | } | |
476 | ||
477 | #define EMIT_COPY_CCS_DW 5 | |
478 | static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, | |
479 | u64 dst_ofs, bool dst_is_indirect, | |
480 | u64 src_ofs, bool src_is_indirect, | |
481 | u32 size) | |
482 | { | |
483 | u32 *cs = bb->cs + bb->len; | |
484 | u32 num_ccs_blks; | |
485 | u32 mocs = xe_mocs_index_to_value(gt->mocs.uc_index); | |
486 | ||
487 | num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), | |
488 | NUM_CCS_BYTES_PER_BLOCK); | |
489 | XE_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER); | |
490 | *cs++ = XY_CTRL_SURF_COPY_BLT | | |
491 | (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | | |
492 | (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | | |
493 | ((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT; | |
494 | *cs++ = lower_32_bits(src_ofs); | |
495 | *cs++ = upper_32_bits(src_ofs) | | |
496 | FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs); | |
497 | *cs++ = lower_32_bits(dst_ofs); | |
498 | *cs++ = upper_32_bits(dst_ofs) | | |
499 | FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs); | |
500 | ||
501 | bb->len = cs - bb->cs; | |
502 | } | |
503 | ||
504 | #define EMIT_COPY_DW 10 | |
505 | static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, | |
506 | u64 src_ofs, u64 dst_ofs, unsigned int size, | |
507 | unsigned pitch) | |
508 | { | |
509 | XE_BUG_ON(size / pitch > S16_MAX); | |
510 | XE_BUG_ON(pitch / 4 > S16_MAX); | |
511 | XE_BUG_ON(pitch > U16_MAX); | |
512 | ||
513 | bb->cs[bb->len++] = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); | |
514 | bb->cs[bb->len++] = BLT_DEPTH_32 | pitch; | |
515 | bb->cs[bb->len++] = 0; | |
516 | bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; | |
517 | bb->cs[bb->len++] = lower_32_bits(dst_ofs); | |
518 | bb->cs[bb->len++] = upper_32_bits(dst_ofs); | |
519 | bb->cs[bb->len++] = 0; | |
520 | bb->cs[bb->len++] = pitch; | |
521 | bb->cs[bb->len++] = lower_32_bits(src_ofs); | |
522 | bb->cs[bb->len++] = upper_32_bits(src_ofs); | |
523 | } | |
524 | ||
525 | static int job_add_deps(struct xe_sched_job *job, struct dma_resv *resv, | |
526 | enum dma_resv_usage usage) | |
527 | { | |
528 | return drm_sched_job_add_resv_dependencies(&job->drm, resv, usage); | |
529 | } | |
530 | ||
531 | static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) | |
532 | { | |
533 | return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; | |
534 | } | |
535 | ||
536 | static u32 xe_migrate_ccs_copy(struct xe_migrate *m, | |
537 | struct xe_bb *bb, | |
538 | u64 src_ofs, bool src_is_vram, | |
539 | u64 dst_ofs, bool dst_is_vram, u32 dst_size, | |
540 | u64 ccs_ofs, bool copy_ccs) | |
541 | { | |
542 | struct xe_gt *gt = m->gt; | |
543 | u32 flush_flags = 0; | |
544 | ||
545 | if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_vram) { | |
546 | /* | |
547 | * If the bo doesn't have any CCS metadata attached, we still | |
548 | * need to clear it for security reasons. | |
549 | */ | |
550 | emit_copy_ccs(gt, bb, dst_ofs, true, m->cleared_vram_ofs, false, | |
551 | dst_size); | |
552 | flush_flags = MI_FLUSH_DW_CCS; | |
553 | } else if (copy_ccs) { | |
554 | if (!src_is_vram) | |
555 | src_ofs = ccs_ofs; | |
556 | else if (!dst_is_vram) | |
557 | dst_ofs = ccs_ofs; | |
558 | ||
559 | /* | |
560 | * At the moment, we don't support copying CCS metadata from | |
561 | * system to system. | |
562 | */ | |
563 | XE_BUG_ON(!src_is_vram && !dst_is_vram); | |
564 | ||
565 | emit_copy_ccs(gt, bb, dst_ofs, dst_is_vram, src_ofs, | |
566 | src_is_vram, dst_size); | |
567 | if (dst_is_vram) | |
568 | flush_flags = MI_FLUSH_DW_CCS; | |
569 | } | |
570 | ||
571 | return flush_flags; | |
572 | } | |
573 | ||
e9d285ff TH |
574 | /** |
575 | * xe_migrate_copy() - Copy content of TTM resources. | |
576 | * @m: The migration context. | |
577 | * @bo: The buffer object @src is currently bound to. | |
578 | * @src: The source TTM resource. | |
579 | * @dst: The dst TTM resource. | |
580 | * | |
581 | * Copies the contents of @src to @dst: On flat CCS devices, | |
582 | * the CCS metadata is copied as well if needed, or if not present, | |
583 | * the CCS metadata of @dst is cleared for security reasons. | |
584 | * It's currently not possible to copy between two system resources, | |
585 | * since that would require two TTM page-vectors. | |
586 | * TODO: Eliminate the @bo argument and supply two TTM page-vectors. | |
587 | * | |
588 | * Return: Pointer to a dma_fence representing the last copy batch, or | |
589 | * an error pointer on failure. If there is a failure, any copy operation | |
590 | * started by the function call has been synced. | |
591 | */ | |
dd08ebf6 MB |
592 | struct dma_fence *xe_migrate_copy(struct xe_migrate *m, |
593 | struct xe_bo *bo, | |
594 | struct ttm_resource *src, | |
595 | struct ttm_resource *dst) | |
596 | { | |
597 | struct xe_gt *gt = m->gt; | |
598 | struct xe_device *xe = gt_to_xe(gt); | |
599 | struct dma_fence *fence = NULL; | |
600 | u64 size = bo->size; | |
601 | struct xe_res_cursor src_it, dst_it, ccs_it; | |
602 | u64 src_L0_ofs, dst_L0_ofs; | |
603 | u32 src_L0_pt, dst_L0_pt; | |
604 | u64 src_L0, dst_L0; | |
605 | int pass = 0; | |
606 | int err; | |
607 | bool src_is_vram = mem_type_is_vram(src->mem_type); | |
608 | bool dst_is_vram = mem_type_is_vram(dst->mem_type); | |
609 | bool copy_ccs = xe_device_has_flat_ccs(xe) && xe_bo_needs_ccs_pages(bo); | |
610 | bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); | |
611 | ||
612 | if (!src_is_vram) | |
e89b384c | 613 | xe_res_first_sg(xe_bo_get_sg(bo), 0, size, &src_it); |
dd08ebf6 | 614 | else |
e89b384c | 615 | xe_res_first(src, 0, size, &src_it); |
dd08ebf6 | 616 | if (!dst_is_vram) |
e89b384c | 617 | xe_res_first_sg(xe_bo_get_sg(bo), 0, size, &dst_it); |
dd08ebf6 | 618 | else |
e89b384c | 619 | xe_res_first(dst, 0, size, &dst_it); |
dd08ebf6 MB |
620 | |
621 | if (copy_system_ccs) | |
622 | xe_res_first_sg(xe_bo_get_sg(bo), xe_bo_ccs_pages_start(bo), | |
623 | PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), | |
624 | &ccs_it); | |
625 | ||
626 | while (size) { | |
627 | u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ | |
628 | struct xe_sched_job *job; | |
629 | struct xe_bb *bb; | |
630 | u32 flush_flags; | |
631 | u32 update_idx; | |
632 | u64 ccs_ofs, ccs_size; | |
633 | u32 ccs_pt; | |
634 | bool usm = xe->info.supports_usm; | |
635 | ||
636 | src_L0 = xe_migrate_res_sizes(&src_it); | |
637 | dst_L0 = xe_migrate_res_sizes(&dst_it); | |
638 | ||
639 | drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", | |
640 | pass++, src_L0, dst_L0); | |
641 | ||
642 | src_L0 = min(src_L0, dst_L0); | |
643 | ||
644 | batch_size += pte_update_size(m, src_is_vram, &src_it, &src_L0, | |
645 | &src_L0_ofs, &src_L0_pt, 0, 0, | |
646 | NUM_PT_PER_BLIT); | |
647 | ||
648 | batch_size += pte_update_size(m, dst_is_vram, &dst_it, &src_L0, | |
649 | &dst_L0_ofs, &dst_L0_pt, 0, | |
650 | NUM_PT_PER_BLIT, NUM_PT_PER_BLIT); | |
651 | ||
652 | if (copy_system_ccs) { | |
653 | ccs_size = xe_device_ccs_bytes(xe, src_L0); | |
654 | batch_size += pte_update_size(m, false, &ccs_it, &ccs_size, | |
655 | &ccs_ofs, &ccs_pt, 0, | |
656 | 2 * NUM_PT_PER_BLIT, | |
657 | NUM_PT_PER_BLIT); | |
658 | } | |
659 | ||
660 | /* Add copy commands size here */ | |
661 | batch_size += EMIT_COPY_DW + | |
662 | (xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0); | |
663 | ||
664 | bb = xe_bb_new(gt, batch_size, usm); | |
665 | if (IS_ERR(bb)) { | |
666 | err = PTR_ERR(bb); | |
667 | goto err_sync; | |
668 | } | |
669 | ||
670 | /* Preemption is enabled again by the ring ops. */ | |
671 | if (!src_is_vram || !dst_is_vram) | |
672 | emit_arb_clear(bb); | |
673 | ||
674 | if (!src_is_vram) | |
675 | emit_pte(m, bb, src_L0_pt, src_is_vram, &src_it, src_L0, | |
676 | bo); | |
677 | else | |
678 | xe_res_next(&src_it, src_L0); | |
679 | ||
680 | if (!dst_is_vram) | |
681 | emit_pte(m, bb, dst_L0_pt, dst_is_vram, &dst_it, src_L0, | |
682 | bo); | |
683 | else | |
684 | xe_res_next(&dst_it, src_L0); | |
685 | ||
686 | if (copy_system_ccs) | |
687 | emit_pte(m, bb, ccs_pt, false, &ccs_it, ccs_size, bo); | |
688 | ||
689 | bb->cs[bb->len++] = MI_BATCH_BUFFER_END; | |
690 | update_idx = bb->len; | |
691 | ||
692 | emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, GEN8_PAGE_SIZE); | |
693 | flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_vram, | |
694 | dst_L0_ofs, dst_is_vram, | |
695 | src_L0, ccs_ofs, copy_ccs); | |
696 | ||
697 | mutex_lock(&m->job_mutex); | |
698 | job = xe_bb_create_migration_job(m->eng, bb, | |
699 | xe_migrate_batch_base(m, usm), | |
700 | update_idx); | |
701 | if (IS_ERR(job)) { | |
702 | err = PTR_ERR(job); | |
703 | goto err; | |
704 | } | |
705 | ||
706 | xe_sched_job_add_migrate_flush(job, flush_flags); | |
707 | if (!fence) { | |
708 | err = job_add_deps(job, bo->ttm.base.resv, | |
709 | DMA_RESV_USAGE_BOOKKEEP); | |
710 | if (err) | |
711 | goto err_job; | |
712 | } | |
713 | ||
714 | xe_sched_job_arm(job); | |
715 | dma_fence_put(fence); | |
716 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
717 | xe_sched_job_push(job); | |
718 | ||
719 | dma_fence_put(m->fence); | |
720 | m->fence = dma_fence_get(fence); | |
721 | ||
722 | mutex_unlock(&m->job_mutex); | |
723 | ||
724 | xe_bb_free(bb, fence); | |
725 | size -= src_L0; | |
726 | continue; | |
727 | ||
728 | err_job: | |
729 | xe_sched_job_put(job); | |
730 | err: | |
731 | mutex_unlock(&m->job_mutex); | |
732 | xe_bb_free(bb, NULL); | |
733 | ||
734 | err_sync: | |
e9d285ff | 735 | /* Sync partial copy if any. FIXME: under job_mutex? */ |
dd08ebf6 MB |
736 | if (fence) { |
737 | dma_fence_wait(fence, false); | |
738 | dma_fence_put(fence); | |
739 | } | |
740 | ||
741 | return ERR_PTR(err); | |
742 | } | |
743 | ||
744 | return fence; | |
745 | } | |
746 | ||
747 | static int emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, | |
748 | u32 size, u32 pitch, u32 value, bool is_vram) | |
749 | { | |
750 | u32 *cs = bb->cs + bb->len; | |
751 | u32 len = XY_FAST_COLOR_BLT_DW; | |
752 | u32 mocs = xe_mocs_index_to_value(gt->mocs.uc_index); | |
753 | ||
754 | if (GRAPHICS_VERx100(gt->xe) < 1250) | |
755 | len = 11; | |
756 | ||
757 | *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | | |
758 | (len - 2); | |
759 | *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | | |
760 | (pitch - 1); | |
761 | *cs++ = 0; | |
762 | *cs++ = (size / pitch) << 16 | pitch / 4; | |
763 | *cs++ = lower_32_bits(src_ofs); | |
764 | *cs++ = upper_32_bits(src_ofs); | |
765 | *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; | |
766 | *cs++ = value; | |
767 | *cs++ = 0; | |
768 | *cs++ = 0; | |
769 | *cs++ = 0; | |
770 | ||
771 | if (len > 11) { | |
772 | *cs++ = 0; | |
773 | *cs++ = 0; | |
774 | *cs++ = 0; | |
775 | *cs++ = 0; | |
776 | *cs++ = 0; | |
777 | } | |
778 | ||
779 | XE_BUG_ON(cs - bb->cs != len + bb->len); | |
780 | bb->len += len; | |
781 | ||
782 | return 0; | |
783 | } | |
784 | ||
e9d285ff TH |
785 | /** |
786 | * xe_migrate_clear() - Copy content of TTM resources. | |
787 | * @m: The migration context. | |
788 | * @bo: The buffer object @dst is currently bound to. | |
789 | * @dst: The dst TTM resource to be cleared. | |
790 | * @value: Clear value. | |
791 | * | |
792 | * Clear the contents of @dst. On flat CCS devices, | |
793 | * the CCS metadata is cleared to zero as well on VRAM destionations. | |
794 | * TODO: Eliminate the @bo argument. | |
795 | * | |
796 | * Return: Pointer to a dma_fence representing the last clear batch, or | |
797 | * an error pointer on failure. If there is a failure, any clear operation | |
798 | * started by the function call has been synced. | |
799 | */ | |
dd08ebf6 MB |
800 | struct dma_fence *xe_migrate_clear(struct xe_migrate *m, |
801 | struct xe_bo *bo, | |
802 | struct ttm_resource *dst, | |
803 | u32 value) | |
804 | { | |
805 | bool clear_vram = mem_type_is_vram(dst->mem_type); | |
806 | struct xe_gt *gt = m->gt; | |
807 | struct xe_device *xe = gt_to_xe(gt); | |
808 | struct dma_fence *fence = NULL; | |
809 | u64 size = bo->size; | |
810 | struct xe_res_cursor src_it; | |
811 | struct ttm_resource *src = dst; | |
812 | int err; | |
813 | int pass = 0; | |
814 | ||
815 | if (!clear_vram) | |
816 | xe_res_first_sg(xe_bo_get_sg(bo), 0, bo->size, &src_it); | |
817 | else | |
818 | xe_res_first(src, 0, bo->size, &src_it); | |
819 | ||
820 | while (size) { | |
821 | u64 clear_L0_ofs; | |
822 | u32 clear_L0_pt; | |
823 | u32 flush_flags = 0; | |
824 | u64 clear_L0; | |
825 | struct xe_sched_job *job; | |
826 | struct xe_bb *bb; | |
827 | u32 batch_size, update_idx; | |
828 | bool usm = xe->info.supports_usm; | |
829 | ||
830 | clear_L0 = xe_migrate_res_sizes(&src_it); | |
831 | drm_dbg(&xe->drm, "Pass %u, size: %llu\n", pass++, clear_L0); | |
832 | ||
833 | /* Calculate final sizes and batch size.. */ | |
834 | batch_size = 2 + | |
835 | pte_update_size(m, clear_vram, &src_it, | |
836 | &clear_L0, &clear_L0_ofs, &clear_L0_pt, | |
837 | XY_FAST_COLOR_BLT_DW, 0, NUM_PT_PER_BLIT); | |
838 | if (xe_device_has_flat_ccs(xe) && clear_vram) | |
839 | batch_size += EMIT_COPY_CCS_DW; | |
840 | ||
841 | /* Clear commands */ | |
842 | ||
843 | if (WARN_ON_ONCE(!clear_L0)) | |
844 | break; | |
845 | ||
846 | bb = xe_bb_new(gt, batch_size, usm); | |
847 | if (IS_ERR(bb)) { | |
848 | err = PTR_ERR(bb); | |
849 | goto err_sync; | |
850 | } | |
851 | ||
852 | size -= clear_L0; | |
853 | ||
854 | /* TODO: Add dependencies here */ | |
855 | ||
856 | /* Preemption is enabled again by the ring ops. */ | |
857 | if (!clear_vram) { | |
858 | emit_arb_clear(bb); | |
859 | emit_pte(m, bb, clear_L0_pt, clear_vram, &src_it, clear_L0, | |
860 | bo); | |
861 | } else { | |
862 | xe_res_next(&src_it, clear_L0); | |
863 | } | |
864 | bb->cs[bb->len++] = MI_BATCH_BUFFER_END; | |
865 | update_idx = bb->len; | |
866 | ||
867 | emit_clear(gt, bb, clear_L0_ofs, clear_L0, GEN8_PAGE_SIZE, | |
868 | value, clear_vram); | |
869 | if (xe_device_has_flat_ccs(xe) && clear_vram) { | |
870 | emit_copy_ccs(gt, bb, clear_L0_ofs, true, | |
871 | m->cleared_vram_ofs, false, clear_L0); | |
872 | flush_flags = MI_FLUSH_DW_CCS; | |
873 | } | |
874 | ||
875 | mutex_lock(&m->job_mutex); | |
876 | job = xe_bb_create_migration_job(m->eng, bb, | |
877 | xe_migrate_batch_base(m, usm), | |
878 | update_idx); | |
879 | if (IS_ERR(job)) { | |
880 | err = PTR_ERR(job); | |
881 | goto err; | |
882 | } | |
883 | ||
884 | xe_sched_job_add_migrate_flush(job, flush_flags); | |
885 | ||
886 | xe_sched_job_arm(job); | |
887 | dma_fence_put(fence); | |
888 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
889 | xe_sched_job_push(job); | |
890 | ||
891 | dma_fence_put(m->fence); | |
892 | m->fence = dma_fence_get(fence); | |
893 | ||
894 | mutex_unlock(&m->job_mutex); | |
895 | ||
896 | xe_bb_free(bb, fence); | |
897 | continue; | |
898 | ||
899 | err: | |
900 | mutex_unlock(&m->job_mutex); | |
901 | xe_bb_free(bb, NULL); | |
902 | err_sync: | |
e9d285ff | 903 | /* Sync partial copies if any. FIXME: job_mutex? */ |
dd08ebf6 MB |
904 | if (fence) { |
905 | dma_fence_wait(m->fence, false); | |
906 | dma_fence_put(fence); | |
907 | } | |
908 | ||
909 | return ERR_PTR(err); | |
910 | } | |
911 | ||
912 | return fence; | |
913 | } | |
914 | ||
915 | static void write_pgtable(struct xe_gt *gt, struct xe_bb *bb, u64 ppgtt_ofs, | |
916 | const struct xe_vm_pgtable_update *update, | |
917 | struct xe_migrate_pt_update *pt_update) | |
918 | { | |
919 | const struct xe_migrate_pt_update_ops *ops = pt_update->ops; | |
920 | u32 chunk; | |
921 | u32 ofs = update->ofs, size = update->qwords; | |
922 | ||
923 | /* | |
924 | * If we have 512 entries (max), we would populate it ourselves, | |
925 | * and update the PDE above it to the new pointer. | |
926 | * The only time this can only happen if we have to update the top | |
927 | * PDE. This requires a BO that is almost vm->size big. | |
928 | * | |
929 | * This shouldn't be possible in practice.. might change when 16K | |
930 | * pages are used. Hence the BUG_ON. | |
931 | */ | |
932 | XE_BUG_ON(update->qwords > 0x1ff); | |
933 | if (!ppgtt_ofs) { | |
934 | bool is_lmem; | |
935 | ||
936 | ppgtt_ofs = xe_migrate_vram_ofs(xe_bo_addr(update->pt_bo, 0, | |
937 | GEN8_PAGE_SIZE, | |
938 | &is_lmem)); | |
939 | XE_BUG_ON(!is_lmem); | |
940 | } | |
941 | ||
942 | do { | |
943 | u64 addr = ppgtt_ofs + ofs * 8; | |
944 | chunk = min(update->qwords, 0x1ffU); | |
945 | ||
946 | /* Ensure populatefn can do memset64 by aligning bb->cs */ | |
947 | if (!(bb->len & 1)) | |
948 | bb->cs[bb->len++] = MI_NOOP; | |
949 | ||
950 | bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(21) | | |
951 | (chunk * 2 + 1); | |
952 | bb->cs[bb->len++] = lower_32_bits(addr); | |
953 | bb->cs[bb->len++] = upper_32_bits(addr); | |
954 | ops->populate(pt_update, gt, NULL, bb->cs + bb->len, ofs, chunk, | |
955 | update); | |
956 | ||
957 | bb->len += chunk * 2; | |
958 | ofs += chunk; | |
959 | size -= chunk; | |
960 | } while (size); | |
961 | } | |
962 | ||
963 | struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) | |
964 | { | |
965 | return xe_vm_get(m->eng->vm); | |
966 | } | |
967 | ||
968 | static struct dma_fence * | |
969 | xe_migrate_update_pgtables_cpu(struct xe_migrate *m, | |
970 | struct xe_vm *vm, struct xe_bo *bo, | |
971 | const struct xe_vm_pgtable_update *updates, | |
972 | u32 num_updates, bool wait_vm, | |
973 | struct xe_migrate_pt_update *pt_update) | |
974 | { | |
975 | const struct xe_migrate_pt_update_ops *ops = pt_update->ops; | |
976 | struct dma_fence *fence; | |
977 | int err; | |
978 | u32 i; | |
979 | ||
980 | /* Wait on BO moves for 10 ms, then fall back to GPU job */ | |
981 | if (bo) { | |
982 | long wait; | |
983 | ||
984 | wait = dma_resv_wait_timeout(bo->ttm.base.resv, | |
985 | DMA_RESV_USAGE_KERNEL, | |
986 | true, HZ / 100); | |
987 | if (wait <= 0) | |
988 | return ERR_PTR(-ETIME); | |
989 | } | |
990 | if (wait_vm) { | |
991 | long wait; | |
992 | ||
993 | wait = dma_resv_wait_timeout(&vm->resv, | |
994 | DMA_RESV_USAGE_BOOKKEEP, | |
995 | true, HZ / 100); | |
996 | if (wait <= 0) | |
997 | return ERR_PTR(-ETIME); | |
998 | } | |
999 | ||
1000 | if (ops->pre_commit) { | |
1001 | err = ops->pre_commit(pt_update); | |
1002 | if (err) | |
1003 | return ERR_PTR(err); | |
1004 | } | |
1005 | for (i = 0; i < num_updates; i++) { | |
1006 | const struct xe_vm_pgtable_update *update = &updates[i]; | |
1007 | ||
1008 | ops->populate(pt_update, m->gt, &update->pt_bo->vmap, NULL, | |
1009 | update->ofs, update->qwords, update); | |
1010 | } | |
1011 | ||
1012 | trace_xe_vm_cpu_bind(vm); | |
1013 | xe_device_wmb(vm->xe); | |
1014 | ||
1015 | fence = dma_fence_get_stub(); | |
1016 | ||
1017 | return fence; | |
1018 | } | |
1019 | ||
1020 | static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs) | |
1021 | { | |
1022 | int i; | |
1023 | ||
1024 | for (i = 0; i < num_syncs; i++) { | |
1025 | struct dma_fence *fence = syncs[i].fence; | |
1026 | ||
1027 | if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, | |
1028 | &fence->flags)) | |
1029 | return false; | |
1030 | } | |
1031 | ||
1032 | return true; | |
1033 | } | |
1034 | ||
1035 | static bool engine_is_idle(struct xe_engine *e) | |
1036 | { | |
1037 | return !e || e->lrc[0].fence_ctx.next_seqno == 1 || | |
1038 | xe_lrc_seqno(&e->lrc[0]) == e->lrc[0].fence_ctx.next_seqno; | |
1039 | } | |
1040 | ||
e9d285ff TH |
1041 | /** |
1042 | * xe_migrate_update_pgtables() - Pipelined page-table update | |
1043 | * @m: The migrate context. | |
1044 | * @vm: The vm we'll be updating. | |
1045 | * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr. | |
1046 | * @eng: The engine to be used for the update or NULL if the default | |
1047 | * migration engine is to be used. | |
1048 | * @updates: An array of update descriptors. | |
1049 | * @num_updates: Number of descriptors in @updates. | |
1050 | * @syncs: Array of xe_sync_entry to await before updating. Note that waits | |
1051 | * will block the engine timeline. | |
1052 | * @num_syncs: Number of entries in @syncs. | |
1053 | * @pt_update: Pointer to a struct xe_migrate_pt_update, which contains | |
1054 | * pointers to callback functions and, if subclassed, private arguments to | |
1055 | * those. | |
1056 | * | |
1057 | * Perform a pipelined page-table update. The update descriptors are typically | |
1058 | * built under the same lock critical section as a call to this function. If | |
1059 | * using the default engine for the updates, they will be performed in the | |
1060 | * order they grab the job_mutex. If different engines are used, external | |
1061 | * synchronization is needed for overlapping updates to maintain page-table | |
1062 | * consistency. Note that the meaing of "overlapping" is that the updates | |
1063 | * touch the same page-table, which might be a higher-level page-directory. | |
1064 | * If no pipelining is needed, then updates may be performed by the cpu. | |
1065 | * | |
1066 | * Return: A dma_fence that, when signaled, indicates the update completion. | |
1067 | */ | |
dd08ebf6 MB |
1068 | struct dma_fence * |
1069 | xe_migrate_update_pgtables(struct xe_migrate *m, | |
1070 | struct xe_vm *vm, | |
1071 | struct xe_bo *bo, | |
1072 | struct xe_engine *eng, | |
1073 | const struct xe_vm_pgtable_update *updates, | |
1074 | u32 num_updates, | |
1075 | struct xe_sync_entry *syncs, u32 num_syncs, | |
1076 | struct xe_migrate_pt_update *pt_update) | |
1077 | { | |
1078 | const struct xe_migrate_pt_update_ops *ops = pt_update->ops; | |
1079 | struct xe_gt *gt = m->gt; | |
1080 | struct xe_device *xe = gt_to_xe(gt); | |
1081 | struct xe_sched_job *job; | |
1082 | struct dma_fence *fence; | |
1083 | struct drm_suballoc *sa_bo = NULL; | |
1084 | struct xe_vma *vma = pt_update->vma; | |
1085 | struct xe_bb *bb; | |
1086 | u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0; | |
1087 | u64 addr; | |
1088 | int err = 0; | |
1089 | bool usm = !eng && xe->info.supports_usm; | |
1090 | bool first_munmap_rebind = vma && vma->first_munmap_rebind; | |
1091 | ||
1092 | /* Use the CPU if no in syncs and engine is idle */ | |
1093 | if (no_in_syncs(syncs, num_syncs) && engine_is_idle(eng)) { | |
1094 | fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates, | |
1095 | num_updates, | |
1096 | first_munmap_rebind, | |
1097 | pt_update); | |
1098 | if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN)) | |
1099 | return fence; | |
1100 | } | |
1101 | ||
1102 | /* fixed + PTE entries */ | |
1103 | if (IS_DGFX(xe)) | |
1104 | batch_size = 2; | |
1105 | else | |
1106 | batch_size = 6 + num_updates * 2; | |
1107 | ||
1108 | for (i = 0; i < num_updates; i++) { | |
1109 | u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, 0x1ff); | |
1110 | ||
1111 | /* align noop + MI_STORE_DATA_IMM cmd prefix */ | |
1112 | batch_size += 4 * num_cmds + updates[i].qwords * 2; | |
1113 | } | |
1114 | ||
1115 | /* | |
1116 | * XXX: Create temp bo to copy from, if batch_size becomes too big? | |
1117 | * | |
1118 | * Worst case: Sum(2 * (each lower level page size) + (top level page size)) | |
1119 | * Should be reasonably bound.. | |
1120 | */ | |
1121 | XE_BUG_ON(batch_size >= SZ_128K); | |
1122 | ||
1123 | bb = xe_bb_new(gt, batch_size, !eng && xe->info.supports_usm); | |
1124 | if (IS_ERR(bb)) | |
1125 | return ERR_CAST(bb); | |
1126 | ||
1127 | /* For sysmem PTE's, need to map them in our hole.. */ | |
1128 | if (!IS_DGFX(xe)) { | |
1129 | ppgtt_ofs = NUM_KERNEL_PDE - 1; | |
1130 | if (eng) { | |
1131 | XE_BUG_ON(num_updates > NUM_VMUSA_WRITES_PER_UNIT); | |
1132 | ||
1133 | sa_bo = drm_suballoc_new(&m->vm_update_sa, 1, | |
1134 | GFP_KERNEL, true, 0); | |
1135 | if (IS_ERR(sa_bo)) { | |
1136 | err = PTR_ERR(sa_bo); | |
1137 | goto err; | |
1138 | } | |
1139 | ||
1140 | ppgtt_ofs = NUM_KERNEL_PDE + | |
1141 | (drm_suballoc_soffset(sa_bo) / | |
1142 | NUM_VMUSA_UNIT_PER_PAGE); | |
1143 | page_ofs = (drm_suballoc_soffset(sa_bo) % | |
1144 | NUM_VMUSA_UNIT_PER_PAGE) * | |
1145 | VM_SA_UPDATE_UNIT_SIZE; | |
1146 | } | |
1147 | ||
1148 | /* Preemption is enabled again by the ring ops. */ | |
1149 | emit_arb_clear(bb); | |
1150 | ||
1151 | /* Map our PT's to gtt */ | |
1152 | bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(21) | | |
1153 | (num_updates * 2 + 1); | |
1154 | bb->cs[bb->len++] = ppgtt_ofs * GEN8_PAGE_SIZE + page_ofs; | |
1155 | bb->cs[bb->len++] = 0; /* upper_32_bits */ | |
1156 | ||
1157 | for (i = 0; i < num_updates; i++) { | |
1158 | struct xe_bo *pt_bo = updates[i].pt_bo; | |
1159 | ||
1160 | BUG_ON(pt_bo->size != SZ_4K); | |
1161 | ||
1162 | addr = gen8_pte_encode(NULL, pt_bo, 0, XE_CACHE_WB, | |
1163 | 0, 0); | |
1164 | bb->cs[bb->len++] = lower_32_bits(addr); | |
1165 | bb->cs[bb->len++] = upper_32_bits(addr); | |
1166 | } | |
1167 | ||
1168 | bb->cs[bb->len++] = MI_BATCH_BUFFER_END; | |
1169 | update_idx = bb->len; | |
1170 | ||
1171 | addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + | |
1172 | (page_ofs / sizeof(u64)) * GEN8_PAGE_SIZE; | |
1173 | for (i = 0; i < num_updates; i++) | |
1174 | write_pgtable(m->gt, bb, addr + i * GEN8_PAGE_SIZE, | |
1175 | &updates[i], pt_update); | |
1176 | } else { | |
1177 | /* phys pages, no preamble required */ | |
1178 | bb->cs[bb->len++] = MI_BATCH_BUFFER_END; | |
1179 | update_idx = bb->len; | |
1180 | ||
1181 | /* Preemption is enabled again by the ring ops. */ | |
1182 | emit_arb_clear(bb); | |
1183 | for (i = 0; i < num_updates; i++) | |
1184 | write_pgtable(m->gt, bb, 0, &updates[i], pt_update); | |
1185 | } | |
1186 | ||
1187 | if (!eng) | |
1188 | mutex_lock(&m->job_mutex); | |
1189 | ||
1190 | job = xe_bb_create_migration_job(eng ?: m->eng, bb, | |
1191 | xe_migrate_batch_base(m, usm), | |
1192 | update_idx); | |
1193 | if (IS_ERR(job)) { | |
1194 | err = PTR_ERR(job); | |
1195 | goto err_bb; | |
1196 | } | |
1197 | ||
1198 | /* Wait on BO move */ | |
1199 | if (bo) { | |
1200 | err = job_add_deps(job, bo->ttm.base.resv, | |
1201 | DMA_RESV_USAGE_KERNEL); | |
1202 | if (err) | |
1203 | goto err_job; | |
1204 | } | |
1205 | ||
1206 | /* | |
1207 | * Munmap style VM unbind, need to wait for all jobs to be complete / | |
1208 | * trigger preempts before moving forward | |
1209 | */ | |
1210 | if (first_munmap_rebind) { | |
1211 | err = job_add_deps(job, &vm->resv, | |
1212 | DMA_RESV_USAGE_BOOKKEEP); | |
1213 | if (err) | |
1214 | goto err_job; | |
1215 | } | |
1216 | ||
1217 | for (i = 0; !err && i < num_syncs; i++) | |
1218 | err = xe_sync_entry_add_deps(&syncs[i], job); | |
1219 | ||
1220 | if (err) | |
1221 | goto err_job; | |
1222 | ||
1223 | if (ops->pre_commit) { | |
1224 | err = ops->pre_commit(pt_update); | |
1225 | if (err) | |
1226 | goto err_job; | |
1227 | } | |
1228 | xe_sched_job_arm(job); | |
1229 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
1230 | xe_sched_job_push(job); | |
1231 | ||
1232 | if (!eng) | |
1233 | mutex_unlock(&m->job_mutex); | |
1234 | ||
1235 | xe_bb_free(bb, fence); | |
1236 | drm_suballoc_free(sa_bo, fence); | |
1237 | ||
1238 | return fence; | |
1239 | ||
1240 | err_job: | |
1241 | xe_sched_job_put(job); | |
1242 | err_bb: | |
1243 | if (!eng) | |
1244 | mutex_unlock(&m->job_mutex); | |
1245 | xe_bb_free(bb, NULL); | |
1246 | err: | |
1247 | drm_suballoc_free(sa_bo, NULL); | |
1248 | return ERR_PTR(err); | |
1249 | } | |
1250 | ||
e9d285ff TH |
1251 | /** |
1252 | * xe_migrate_wait() - Complete all operations using the xe_migrate context | |
1253 | * @m: Migrate context to wait for. | |
1254 | * | |
1255 | * Waits until the GPU no longer uses the migrate context's default engine | |
1256 | * or its page-table objects. FIXME: What about separate page-table update | |
1257 | * engines? | |
1258 | */ | |
dd08ebf6 MB |
1259 | void xe_migrate_wait(struct xe_migrate *m) |
1260 | { | |
1261 | if (m->fence) | |
1262 | dma_fence_wait(m->fence, false); | |
1263 | } | |
1264 | ||
1265 | #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) | |
1266 | #include "tests/xe_migrate.c" | |
1267 | #endif |