Commit | Line | Data |
---|---|---|
038ecc50 TH |
1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /************************************************************************** | |
3 | * | |
4 | * Copyright © 2018 VMware, Inc., Palo Alto, CA., USA | |
5 | * All Rights Reserved. | |
6 | * | |
7 | * Permission is hereby granted, free of charge, to any person obtaining a | |
8 | * copy of this software and associated documentation files (the | |
9 | * "Software"), to deal in the Software without restriction, including | |
10 | * without limitation the rights to use, copy, modify, merge, publish, | |
11 | * distribute, sub license, and/or sell copies of the Software, and to | |
12 | * permit persons to whom the Software is furnished to do so, subject to | |
13 | * the following conditions: | |
14 | * | |
15 | * The above copyright notice and this permission notice (including the | |
16 | * next paragraph) shall be included in all copies or substantial portions | |
17 | * of the Software. | |
18 | * | |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
26 | * | |
27 | **************************************************************************/ | |
28 | #include <linux/slab.h> | |
29 | #include "vmwgfx_validation.h" | |
30 | #include "vmwgfx_drv.h" | |
31 | ||
32 | /** | |
33 | * struct vmw_validation_bo_node - Buffer object validation metadata. | |
34 | * @base: Metadata used for TTM reservation- and validation. | |
35 | * @hash: A hash entry used for the duplicate detection hash table. | |
b7468b15 TH |
36 | * @coherent_count: If switching backup buffers, number of new coherent |
37 | * resources that will have this buffer as a backup buffer. | |
038ecc50 TH |
38 | * @as_mob: Validate as mob. |
39 | * @cpu_blit: Validate for cpu blit access. | |
40 | * | |
41 | * Bit fields are used since these structures are allocated and freed in | |
42 | * large numbers and space conservation is desired. | |
43 | */ | |
44 | struct vmw_validation_bo_node { | |
45 | struct ttm_validate_buffer base; | |
46 | struct drm_hash_item hash; | |
b7468b15 | 47 | unsigned int coherent_count; |
038ecc50 TH |
48 | u32 as_mob : 1; |
49 | u32 cpu_blit : 1; | |
50 | }; | |
51 | ||
52 | /** | |
53 | * struct vmw_validation_res_node - Resource validation metadata. | |
54 | * @head: List head for the resource validation list. | |
55 | * @hash: A hash entry used for the duplicate detection hash table. | |
56 | * @res: Reference counted resource pointer. | |
57 | * @new_backup: Non ref-counted pointer to new backup buffer to be assigned | |
58 | * to a resource. | |
59 | * @new_backup_offset: Offset into the new backup mob for resources that can | |
60 | * share MOBs. | |
61 | * @no_buffer_needed: Kernel does not need to allocate a MOB during validation, | |
62 | * the command stream provides a mob bind operation. | |
63 | * @switching_backup: The validation process is switching backup MOB. | |
64 | * @first_usage: True iff the resource has been seen only once in the current | |
65 | * validation batch. | |
66 | * @reserved: Whether the resource is currently reserved by this process. | |
67 | * @private: Optionally additional memory for caller-private data. | |
68 | * | |
69 | * Bit fields are used since these structures are allocated and freed in | |
70 | * large numbers and space conservation is desired. | |
71 | */ | |
72 | struct vmw_validation_res_node { | |
73 | struct list_head head; | |
74 | struct drm_hash_item hash; | |
75 | struct vmw_resource *res; | |
76 | struct vmw_buffer_object *new_backup; | |
77 | unsigned long new_backup_offset; | |
78 | u32 no_buffer_needed : 1; | |
79 | u32 switching_backup : 1; | |
80 | u32 first_usage : 1; | |
81 | u32 reserved : 1; | |
a9f58c45 TH |
82 | u32 dirty : 1; |
83 | u32 dirty_set : 1; | |
038ecc50 TH |
84 | unsigned long private[0]; |
85 | }; | |
86 | ||
fc18afcf TH |
87 | /** |
88 | * vmw_validation_mem_alloc - Allocate kernel memory from the validation | |
89 | * context based allocator | |
90 | * @ctx: The validation context | |
91 | * @size: The number of bytes to allocated. | |
92 | * | |
93 | * The memory allocated may not exceed PAGE_SIZE, and the returned | |
94 | * address is aligned to sizeof(long). All memory allocated this way is | |
95 | * reclaimed after validation when calling any of the exported functions: | |
96 | * vmw_validation_unref_lists() | |
97 | * vmw_validation_revert() | |
98 | * vmw_validation_done() | |
99 | * | |
100 | * Return: Pointer to the allocated memory on success. NULL on failure. | |
101 | */ | |
64ad2abf TH |
102 | void *vmw_validation_mem_alloc(struct vmw_validation_context *ctx, |
103 | unsigned int size) | |
fc18afcf TH |
104 | { |
105 | void *addr; | |
106 | ||
64ad2abf | 107 | size = vmw_validation_align(size); |
fc18afcf TH |
108 | if (size > PAGE_SIZE) |
109 | return NULL; | |
110 | ||
111 | if (ctx->mem_size_left < size) { | |
fd567467 | 112 | struct page *page; |
fc18afcf | 113 | |
fd567467 TH |
114 | if (ctx->vm && ctx->vm_size_left < PAGE_SIZE) { |
115 | int ret = ctx->vm->reserve_mem(ctx->vm, ctx->vm->gran); | |
116 | ||
117 | if (ret) | |
118 | return NULL; | |
119 | ||
120 | ctx->vm_size_left += ctx->vm->gran; | |
121 | ctx->total_mem += ctx->vm->gran; | |
122 | } | |
123 | ||
124 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); | |
fc18afcf TH |
125 | if (!page) |
126 | return NULL; | |
127 | ||
fd567467 TH |
128 | if (ctx->vm) |
129 | ctx->vm_size_left -= PAGE_SIZE; | |
130 | ||
fc18afcf TH |
131 | list_add_tail(&page->lru, &ctx->page_list); |
132 | ctx->page_address = page_address(page); | |
133 | ctx->mem_size_left = PAGE_SIZE; | |
134 | } | |
135 | ||
136 | addr = (void *) (ctx->page_address + (PAGE_SIZE - ctx->mem_size_left)); | |
137 | ctx->mem_size_left -= size; | |
138 | ||
139 | return addr; | |
140 | } | |
141 | ||
142 | /** | |
143 | * vmw_validation_mem_free - Free all memory allocated using | |
144 | * vmw_validation_mem_alloc() | |
145 | * @ctx: The validation context | |
146 | * | |
147 | * All memory previously allocated for this context using | |
148 | * vmw_validation_mem_alloc() is freed. | |
149 | */ | |
150 | static void vmw_validation_mem_free(struct vmw_validation_context *ctx) | |
151 | { | |
152 | struct page *entry, *next; | |
153 | ||
154 | list_for_each_entry_safe(entry, next, &ctx->page_list, lru) { | |
155 | list_del_init(&entry->lru); | |
156 | __free_page(entry); | |
157 | } | |
158 | ||
159 | ctx->mem_size_left = 0; | |
fd567467 TH |
160 | if (ctx->vm && ctx->total_mem) { |
161 | ctx->vm->unreserve_mem(ctx->vm, ctx->total_mem); | |
162 | ctx->total_mem = 0; | |
163 | ctx->vm_size_left = 0; | |
164 | } | |
fc18afcf TH |
165 | } |
166 | ||
038ecc50 TH |
167 | /** |
168 | * vmw_validation_find_bo_dup - Find a duplicate buffer object entry in the | |
169 | * validation context's lists. | |
170 | * @ctx: The validation context to search. | |
171 | * @vbo: The buffer object to search for. | |
172 | * | |
173 | * Return: Pointer to the struct vmw_validation_bo_node referencing the | |
174 | * duplicate, or NULL if none found. | |
175 | */ | |
176 | static struct vmw_validation_bo_node * | |
177 | vmw_validation_find_bo_dup(struct vmw_validation_context *ctx, | |
178 | struct vmw_buffer_object *vbo) | |
179 | { | |
180 | struct vmw_validation_bo_node *bo_node = NULL; | |
181 | ||
182 | if (!ctx->merge_dups) | |
183 | return NULL; | |
184 | ||
185 | if (ctx->ht) { | |
186 | struct drm_hash_item *hash; | |
187 | ||
188 | if (!drm_ht_find_item(ctx->ht, (unsigned long) vbo, &hash)) | |
189 | bo_node = container_of(hash, typeof(*bo_node), hash); | |
190 | } else { | |
191 | struct vmw_validation_bo_node *entry; | |
192 | ||
193 | list_for_each_entry(entry, &ctx->bo_list, base.head) { | |
194 | if (entry->base.bo == &vbo->base) { | |
195 | bo_node = entry; | |
196 | break; | |
197 | } | |
198 | } | |
199 | } | |
200 | ||
201 | return bo_node; | |
202 | } | |
203 | ||
204 | /** | |
205 | * vmw_validation_find_res_dup - Find a duplicate resource entry in the | |
206 | * validation context's lists. | |
207 | * @ctx: The validation context to search. | |
208 | * @vbo: The buffer object to search for. | |
209 | * | |
210 | * Return: Pointer to the struct vmw_validation_bo_node referencing the | |
211 | * duplicate, or NULL if none found. | |
212 | */ | |
213 | static struct vmw_validation_res_node * | |
214 | vmw_validation_find_res_dup(struct vmw_validation_context *ctx, | |
215 | struct vmw_resource *res) | |
216 | { | |
217 | struct vmw_validation_res_node *res_node = NULL; | |
218 | ||
219 | if (!ctx->merge_dups) | |
220 | return NULL; | |
221 | ||
222 | if (ctx->ht) { | |
223 | struct drm_hash_item *hash; | |
224 | ||
225 | if (!drm_ht_find_item(ctx->ht, (unsigned long) res, &hash)) | |
226 | res_node = container_of(hash, typeof(*res_node), hash); | |
227 | } else { | |
228 | struct vmw_validation_res_node *entry; | |
229 | ||
230 | list_for_each_entry(entry, &ctx->resource_ctx_list, head) { | |
231 | if (entry->res == res) { | |
232 | res_node = entry; | |
233 | goto out; | |
234 | } | |
235 | } | |
236 | ||
237 | list_for_each_entry(entry, &ctx->resource_list, head) { | |
238 | if (entry->res == res) { | |
239 | res_node = entry; | |
240 | break; | |
241 | } | |
242 | } | |
243 | ||
244 | } | |
245 | out: | |
246 | return res_node; | |
247 | } | |
248 | ||
249 | /** | |
250 | * vmw_validation_add_bo - Add a buffer object to the validation context. | |
251 | * @ctx: The validation context. | |
252 | * @vbo: The buffer object. | |
253 | * @as_mob: Validate as mob, otherwise suitable for GMR operations. | |
254 | * @cpu_blit: Validate in a page-mappable location. | |
255 | * | |
256 | * Return: Zero on success, negative error code otherwise. | |
257 | */ | |
258 | int vmw_validation_add_bo(struct vmw_validation_context *ctx, | |
259 | struct vmw_buffer_object *vbo, | |
260 | bool as_mob, | |
261 | bool cpu_blit) | |
262 | { | |
263 | struct vmw_validation_bo_node *bo_node; | |
264 | ||
265 | bo_node = vmw_validation_find_bo_dup(ctx, vbo); | |
266 | if (bo_node) { | |
267 | if (bo_node->as_mob != as_mob || | |
268 | bo_node->cpu_blit != cpu_blit) { | |
269 | DRM_ERROR("Inconsistent buffer usage.\n"); | |
270 | return -EINVAL; | |
271 | } | |
272 | } else { | |
273 | struct ttm_validate_buffer *val_buf; | |
274 | int ret; | |
275 | ||
fc18afcf | 276 | bo_node = vmw_validation_mem_alloc(ctx, sizeof(*bo_node)); |
038ecc50 TH |
277 | if (!bo_node) |
278 | return -ENOMEM; | |
279 | ||
280 | if (ctx->ht) { | |
281 | bo_node->hash.key = (unsigned long) vbo; | |
282 | ret = drm_ht_insert_item(ctx->ht, &bo_node->hash); | |
283 | if (ret) { | |
284 | DRM_ERROR("Failed to initialize a buffer " | |
285 | "validation entry.\n"); | |
038ecc50 TH |
286 | return ret; |
287 | } | |
288 | } | |
289 | val_buf = &bo_node->base; | |
64ad2abf TH |
290 | val_buf->bo = ttm_bo_get_unless_zero(&vbo->base); |
291 | if (!val_buf->bo) | |
292 | return -ESRCH; | |
a9f34c70 | 293 | val_buf->num_shared = 0; |
038ecc50 TH |
294 | list_add_tail(&val_buf->head, &ctx->bo_list); |
295 | bo_node->as_mob = as_mob; | |
296 | bo_node->cpu_blit = cpu_blit; | |
297 | } | |
298 | ||
299 | return 0; | |
300 | } | |
301 | ||
302 | /** | |
303 | * vmw_validation_add_resource - Add a resource to the validation context. | |
304 | * @ctx: The validation context. | |
305 | * @res: The resource. | |
306 | * @priv_size: Size of private, additional metadata. | |
a9f58c45 | 307 | * @dirty: Whether to change dirty status. |
038ecc50 TH |
308 | * @p_node: Output pointer of additional metadata address. |
309 | * @first_usage: Whether this was the first time this resource was seen. | |
310 | * | |
311 | * Return: Zero on success, negative error code otherwise. | |
312 | */ | |
313 | int vmw_validation_add_resource(struct vmw_validation_context *ctx, | |
314 | struct vmw_resource *res, | |
315 | size_t priv_size, | |
a9f58c45 | 316 | u32 dirty, |
038ecc50 TH |
317 | void **p_node, |
318 | bool *first_usage) | |
319 | { | |
320 | struct vmw_validation_res_node *node; | |
321 | int ret; | |
322 | ||
323 | node = vmw_validation_find_res_dup(ctx, res); | |
324 | if (node) { | |
325 | node->first_usage = 0; | |
326 | goto out_fill; | |
327 | } | |
328 | ||
fc18afcf | 329 | node = vmw_validation_mem_alloc(ctx, sizeof(*node) + priv_size); |
038ecc50 | 330 | if (!node) { |
5724f899 | 331 | VMW_DEBUG_USER("Failed to allocate a resource validation entry.\n"); |
038ecc50 TH |
332 | return -ENOMEM; |
333 | } | |
334 | ||
335 | if (ctx->ht) { | |
336 | node->hash.key = (unsigned long) res; | |
337 | ret = drm_ht_insert_item(ctx->ht, &node->hash); | |
338 | if (ret) { | |
339 | DRM_ERROR("Failed to initialize a resource validation " | |
340 | "entry.\n"); | |
038ecc50 TH |
341 | return ret; |
342 | } | |
343 | } | |
64ad2abf TH |
344 | node->res = vmw_resource_reference_unless_doomed(res); |
345 | if (!node->res) | |
346 | return -ESRCH; | |
347 | ||
038ecc50 TH |
348 | node->first_usage = 1; |
349 | if (!res->dev_priv->has_mob) { | |
350 | list_add_tail(&node->head, &ctx->resource_list); | |
351 | } else { | |
352 | switch (vmw_res_type(res)) { | |
353 | case vmw_res_context: | |
354 | case vmw_res_dx_context: | |
355 | list_add(&node->head, &ctx->resource_ctx_list); | |
356 | break; | |
357 | case vmw_res_cotable: | |
358 | list_add_tail(&node->head, &ctx->resource_ctx_list); | |
359 | break; | |
360 | default: | |
361 | list_add_tail(&node->head, &ctx->resource_list); | |
362 | break; | |
363 | } | |
364 | } | |
365 | ||
366 | out_fill: | |
a9f58c45 TH |
367 | if (dirty) { |
368 | node->dirty_set = 1; | |
369 | /* Overwriting previous information here is intentional! */ | |
370 | node->dirty = (dirty & VMW_RES_DIRTY_SET) ? 1 : 0; | |
371 | } | |
038ecc50 TH |
372 | if (first_usage) |
373 | *first_usage = node->first_usage; | |
374 | if (p_node) | |
375 | *p_node = &node->private; | |
376 | ||
377 | return 0; | |
378 | } | |
379 | ||
a9f58c45 TH |
380 | /** |
381 | * vmw_validation_res_set_dirty - Register a resource dirty set or clear during | |
382 | * validation. | |
383 | * @ctx: The validation context. | |
384 | * @val_private: The additional meta-data pointer returned when the | |
385 | * resource was registered with the validation context. Used to identify | |
386 | * the resource. | |
387 | * @dirty: Dirty information VMW_RES_DIRTY_XX | |
388 | */ | |
389 | void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx, | |
390 | void *val_private, u32 dirty) | |
391 | { | |
392 | struct vmw_validation_res_node *val; | |
393 | ||
394 | if (!dirty) | |
395 | return; | |
396 | ||
397 | val = container_of(val_private, typeof(*val), private); | |
398 | val->dirty_set = 1; | |
399 | /* Overwriting previous information here is intentional! */ | |
400 | val->dirty = (dirty & VMW_RES_DIRTY_SET) ? 1 : 0; | |
401 | } | |
402 | ||
038ecc50 TH |
403 | /** |
404 | * vmw_validation_res_switch_backup - Register a backup MOB switch during | |
405 | * validation. | |
406 | * @ctx: The validation context. | |
407 | * @val_private: The additional meta-data pointer returned when the | |
408 | * resource was registered with the validation context. Used to identify | |
409 | * the resource. | |
410 | * @vbo: The new backup buffer object MOB. This buffer object needs to have | |
411 | * already been registered with the validation context. | |
412 | * @backup_offset: Offset into the new backup MOB. | |
413 | */ | |
414 | void vmw_validation_res_switch_backup(struct vmw_validation_context *ctx, | |
415 | void *val_private, | |
416 | struct vmw_buffer_object *vbo, | |
417 | unsigned long backup_offset) | |
418 | { | |
419 | struct vmw_validation_res_node *val; | |
420 | ||
421 | val = container_of(val_private, typeof(*val), private); | |
422 | ||
423 | val->switching_backup = 1; | |
424 | if (val->first_usage) | |
425 | val->no_buffer_needed = 1; | |
426 | ||
427 | val->new_backup = vbo; | |
428 | val->new_backup_offset = backup_offset; | |
429 | } | |
430 | ||
431 | /** | |
432 | * vmw_validation_res_reserve - Reserve all resources registered with this | |
433 | * validation context. | |
434 | * @ctx: The validation context. | |
435 | * @intr: Use interruptible waits when possible. | |
436 | * | |
437 | * Return: Zero on success, -ERESTARTSYS if interrupted. Negative error | |
438 | * code on failure. | |
439 | */ | |
440 | int vmw_validation_res_reserve(struct vmw_validation_context *ctx, | |
441 | bool intr) | |
442 | { | |
443 | struct vmw_validation_res_node *val; | |
444 | int ret = 0; | |
445 | ||
446 | list_splice_init(&ctx->resource_ctx_list, &ctx->resource_list); | |
447 | ||
448 | list_for_each_entry(val, &ctx->resource_list, head) { | |
449 | struct vmw_resource *res = val->res; | |
450 | ||
451 | ret = vmw_resource_reserve(res, intr, val->no_buffer_needed); | |
452 | if (ret) | |
453 | goto out_unreserve; | |
454 | ||
455 | val->reserved = 1; | |
456 | if (res->backup) { | |
457 | struct vmw_buffer_object *vbo = res->backup; | |
458 | ||
459 | ret = vmw_validation_add_bo | |
460 | (ctx, vbo, vmw_resource_needs_backup(res), | |
461 | false); | |
462 | if (ret) | |
463 | goto out_unreserve; | |
464 | } | |
b7468b15 TH |
465 | |
466 | if (val->switching_backup && val->new_backup && | |
467 | res->coherent) { | |
468 | struct vmw_validation_bo_node *bo_node = | |
469 | vmw_validation_find_bo_dup(ctx, | |
470 | val->new_backup); | |
471 | ||
472 | if (WARN_ON(!bo_node)) { | |
473 | ret = -EINVAL; | |
474 | goto out_unreserve; | |
475 | } | |
476 | bo_node->coherent_count++; | |
477 | } | |
038ecc50 TH |
478 | } |
479 | ||
480 | return 0; | |
481 | ||
482 | out_unreserve: | |
483 | vmw_validation_res_unreserve(ctx, true); | |
484 | return ret; | |
485 | } | |
486 | ||
487 | /** | |
488 | * vmw_validation_res_unreserve - Unreserve all reserved resources | |
489 | * registered with this validation context. | |
490 | * @ctx: The validation context. | |
491 | * @backoff: Whether this is a backoff- of a commit-type operation. This | |
492 | * is used to determine whether to switch backup MOBs or not. | |
493 | */ | |
494 | void vmw_validation_res_unreserve(struct vmw_validation_context *ctx, | |
495 | bool backoff) | |
496 | { | |
497 | struct vmw_validation_res_node *val; | |
498 | ||
499 | list_splice_init(&ctx->resource_ctx_list, &ctx->resource_list); | |
a9f58c45 TH |
500 | if (backoff) |
501 | list_for_each_entry(val, &ctx->resource_list, head) { | |
502 | if (val->reserved) | |
503 | vmw_resource_unreserve(val->res, | |
504 | false, false, false, | |
505 | NULL, 0); | |
506 | } | |
507 | else | |
508 | list_for_each_entry(val, &ctx->resource_list, head) { | |
509 | if (val->reserved) | |
510 | vmw_resource_unreserve(val->res, | |
511 | val->dirty_set, | |
512 | val->dirty, | |
513 | val->switching_backup, | |
514 | val->new_backup, | |
515 | val->new_backup_offset); | |
516 | } | |
038ecc50 TH |
517 | } |
518 | ||
519 | /** | |
520 | * vmw_validation_bo_validate_single - Validate a single buffer object. | |
521 | * @bo: The TTM buffer object base. | |
522 | * @interruptible: Whether to perform waits interruptible if possible. | |
523 | * @validate_as_mob: Whether to validate in MOB memory. | |
524 | * | |
525 | * Return: Zero on success, -ERESTARTSYS if interrupted. Negative error | |
526 | * code on failure. | |
527 | */ | |
528 | int vmw_validation_bo_validate_single(struct ttm_buffer_object *bo, | |
529 | bool interruptible, | |
530 | bool validate_as_mob) | |
531 | { | |
532 | struct vmw_buffer_object *vbo = | |
533 | container_of(bo, struct vmw_buffer_object, base); | |
534 | struct ttm_operation_ctx ctx = { | |
535 | .interruptible = interruptible, | |
536 | .no_wait_gpu = false | |
537 | }; | |
538 | int ret; | |
539 | ||
7fb03cc3 CK |
540 | if (atomic_read(&vbo->cpu_writers)) |
541 | return -EBUSY; | |
542 | ||
fbe86ca5 | 543 | if (vbo->base.pin_count > 0) |
038ecc50 TH |
544 | return 0; |
545 | ||
546 | if (validate_as_mob) | |
547 | return ttm_bo_validate(bo, &vmw_mob_placement, &ctx); | |
548 | ||
549 | /** | |
550 | * Put BO in VRAM if there is space, otherwise as a GMR. | |
551 | * If there is no space in VRAM and GMR ids are all used up, | |
552 | * start evicting GMRs to make room. If the DMA buffer can't be | |
553 | * used as a GMR, this will return -ENOMEM. | |
554 | */ | |
555 | ||
556 | ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx); | |
557 | if (ret == 0 || ret == -ERESTARTSYS) | |
558 | return ret; | |
559 | ||
560 | /** | |
561 | * If that failed, try VRAM again, this time evicting | |
562 | * previous contents. | |
563 | */ | |
564 | ||
565 | ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx); | |
566 | return ret; | |
567 | } | |
568 | ||
569 | /** | |
570 | * vmw_validation_bo_validate - Validate all buffer objects registered with | |
571 | * the validation context. | |
572 | * @ctx: The validation context. | |
573 | * @intr: Whether to perform waits interruptible if possible. | |
574 | * | |
575 | * Return: Zero on success, -ERESTARTSYS if interrupted, | |
576 | * negative error code on failure. | |
577 | */ | |
578 | int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr) | |
579 | { | |
580 | struct vmw_validation_bo_node *entry; | |
581 | int ret; | |
582 | ||
583 | list_for_each_entry(entry, &ctx->bo_list, base.head) { | |
b7468b15 TH |
584 | struct vmw_buffer_object *vbo = |
585 | container_of(entry->base.bo, typeof(*vbo), base); | |
586 | ||
038ecc50 TH |
587 | if (entry->cpu_blit) { |
588 | struct ttm_operation_ctx ctx = { | |
589 | .interruptible = intr, | |
590 | .no_wait_gpu = false | |
591 | }; | |
592 | ||
593 | ret = ttm_bo_validate(entry->base.bo, | |
594 | &vmw_nonfixed_placement, &ctx); | |
595 | } else { | |
596 | ret = vmw_validation_bo_validate_single | |
597 | (entry->base.bo, intr, entry->as_mob); | |
598 | } | |
599 | if (ret) | |
600 | return ret; | |
b7468b15 TH |
601 | |
602 | /* | |
603 | * Rather than having the resource code allocating the bo | |
604 | * dirty tracker in resource_unreserve() where we can't fail, | |
605 | * Do it here when validating the buffer object. | |
606 | */ | |
607 | if (entry->coherent_count) { | |
608 | unsigned int coherent_count = entry->coherent_count; | |
609 | ||
610 | while (coherent_count) { | |
611 | ret = vmw_bo_dirty_add(vbo); | |
612 | if (ret) | |
613 | return ret; | |
614 | ||
615 | coherent_count--; | |
616 | } | |
617 | entry->coherent_count -= coherent_count; | |
618 | } | |
619 | ||
620 | if (vbo->dirty) | |
621 | vmw_bo_dirty_scan(vbo); | |
038ecc50 TH |
622 | } |
623 | return 0; | |
624 | } | |
625 | ||
626 | /** | |
627 | * vmw_validation_res_validate - Validate all resources registered with the | |
628 | * validation context. | |
629 | * @ctx: The validation context. | |
630 | * @intr: Whether to perform waits interruptible if possible. | |
631 | * | |
632 | * Before this function is called, all resource backup buffers must have | |
633 | * been validated. | |
634 | * | |
635 | * Return: Zero on success, -ERESTARTSYS if interrupted, | |
636 | * negative error code on failure. | |
637 | */ | |
638 | int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr) | |
639 | { | |
640 | struct vmw_validation_res_node *val; | |
641 | int ret; | |
642 | ||
643 | list_for_each_entry(val, &ctx->resource_list, head) { | |
644 | struct vmw_resource *res = val->res; | |
645 | struct vmw_buffer_object *backup = res->backup; | |
646 | ||
fb80edb0 TH |
647 | ret = vmw_resource_validate(res, intr, val->dirty_set && |
648 | val->dirty); | |
038ecc50 TH |
649 | if (ret) { |
650 | if (ret != -ERESTARTSYS) | |
651 | DRM_ERROR("Failed to validate resource.\n"); | |
652 | return ret; | |
653 | } | |
654 | ||
655 | /* Check if the resource switched backup buffer */ | |
656 | if (backup && res->backup && (backup != res->backup)) { | |
657 | struct vmw_buffer_object *vbo = res->backup; | |
658 | ||
659 | ret = vmw_validation_add_bo | |
660 | (ctx, vbo, vmw_resource_needs_backup(res), | |
661 | false); | |
662 | if (ret) | |
663 | return ret; | |
664 | } | |
665 | } | |
666 | return 0; | |
667 | } | |
668 | ||
669 | /** | |
670 | * vmw_validation_drop_ht - Reset the hash table used for duplicate finding | |
671 | * and unregister it from this validation context. | |
672 | * @ctx: The validation context. | |
673 | * | |
674 | * The hash table used for duplicate finding is an expensive resource and | |
675 | * may be protected by mutexes that may cause deadlocks during resource | |
676 | * unreferencing if held. After resource- and buffer object registering, | |
677 | * there is no longer any use for this hash table, so allow freeing it | |
678 | * either to shorten any mutex locking time, or before resources- and | |
679 | * buffer objects are freed during validation context cleanup. | |
680 | */ | |
681 | void vmw_validation_drop_ht(struct vmw_validation_context *ctx) | |
682 | { | |
683 | struct vmw_validation_bo_node *entry; | |
684 | struct vmw_validation_res_node *val; | |
685 | ||
686 | if (!ctx->ht) | |
687 | return; | |
688 | ||
689 | list_for_each_entry(entry, &ctx->bo_list, base.head) | |
690 | (void) drm_ht_remove_item(ctx->ht, &entry->hash); | |
691 | ||
692 | list_for_each_entry(val, &ctx->resource_list, head) | |
693 | (void) drm_ht_remove_item(ctx->ht, &val->hash); | |
694 | ||
695 | list_for_each_entry(val, &ctx->resource_ctx_list, head) | |
696 | (void) drm_ht_remove_item(ctx->ht, &val->hash); | |
697 | ||
698 | ctx->ht = NULL; | |
699 | } | |
700 | ||
701 | /** | |
702 | * vmw_validation_unref_lists - Unregister previously registered buffer | |
703 | * object and resources. | |
704 | * @ctx: The validation context. | |
705 | * | |
706 | * Note that this function may cause buffer object- and resource destructors | |
707 | * to be invoked. | |
708 | */ | |
709 | void vmw_validation_unref_lists(struct vmw_validation_context *ctx) | |
710 | { | |
fc18afcf TH |
711 | struct vmw_validation_bo_node *entry; |
712 | struct vmw_validation_res_node *val; | |
038ecc50 | 713 | |
6034d9d4 TZ |
714 | list_for_each_entry(entry, &ctx->bo_list, base.head) { |
715 | ttm_bo_put(entry->base.bo); | |
716 | entry->base.bo = NULL; | |
717 | } | |
038ecc50 TH |
718 | |
719 | list_splice_init(&ctx->resource_ctx_list, &ctx->resource_list); | |
fc18afcf | 720 | list_for_each_entry(val, &ctx->resource_list, head) |
038ecc50 | 721 | vmw_resource_unreference(&val->res); |
038ecc50 | 722 | |
fc18afcf TH |
723 | /* |
724 | * No need to detach each list entry since they are all freed with | |
725 | * vmw_validation_free_mem. Just make the inaccessible. | |
726 | */ | |
727 | INIT_LIST_HEAD(&ctx->bo_list); | |
728 | INIT_LIST_HEAD(&ctx->resource_list); | |
729 | ||
730 | vmw_validation_mem_free(ctx); | |
038ecc50 TH |
731 | } |
732 | ||
733 | /** | |
734 | * vmw_validation_prepare - Prepare a validation context for command | |
735 | * submission. | |
736 | * @ctx: The validation context. | |
737 | * @mutex: The mutex used to protect resource reservation. | |
738 | * @intr: Whether to perform waits interruptible if possible. | |
739 | * | |
740 | * Note that the single reservation mutex @mutex is an unfortunate | |
741 | * construct. Ideally resource reservation should be moved to per-resource | |
742 | * ww_mutexes. | |
743 | * If this functions doesn't return Zero to indicate success, all resources | |
744 | * are left unreserved but still referenced. | |
745 | * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code | |
746 | * on error. | |
747 | */ | |
748 | int vmw_validation_prepare(struct vmw_validation_context *ctx, | |
749 | struct mutex *mutex, | |
750 | bool intr) | |
751 | { | |
752 | int ret = 0; | |
753 | ||
754 | if (mutex) { | |
755 | if (intr) | |
756 | ret = mutex_lock_interruptible(mutex); | |
757 | else | |
758 | mutex_lock(mutex); | |
759 | if (ret) | |
760 | return -ERESTARTSYS; | |
761 | } | |
762 | ||
763 | ctx->res_mutex = mutex; | |
764 | ret = vmw_validation_res_reserve(ctx, intr); | |
765 | if (ret) | |
766 | goto out_no_res_reserve; | |
767 | ||
768 | ret = vmw_validation_bo_reserve(ctx, intr); | |
769 | if (ret) | |
770 | goto out_no_bo_reserve; | |
771 | ||
772 | ret = vmw_validation_bo_validate(ctx, intr); | |
773 | if (ret) | |
774 | goto out_no_validate; | |
775 | ||
776 | ret = vmw_validation_res_validate(ctx, intr); | |
777 | if (ret) | |
778 | goto out_no_validate; | |
779 | ||
780 | return 0; | |
781 | ||
782 | out_no_validate: | |
783 | vmw_validation_bo_backoff(ctx); | |
784 | out_no_bo_reserve: | |
785 | vmw_validation_res_unreserve(ctx, true); | |
786 | out_no_res_reserve: | |
787 | if (mutex) | |
788 | mutex_unlock(mutex); | |
789 | ||
790 | return ret; | |
791 | } | |
792 | ||
793 | /** | |
794 | * vmw_validation_revert - Revert validation actions if command submission | |
795 | * failed. | |
796 | * | |
797 | * @ctx: The validation context. | |
798 | * | |
799 | * The caller still needs to unref resources after a call to this function. | |
800 | */ | |
801 | void vmw_validation_revert(struct vmw_validation_context *ctx) | |
802 | { | |
803 | vmw_validation_bo_backoff(ctx); | |
804 | vmw_validation_res_unreserve(ctx, true); | |
805 | if (ctx->res_mutex) | |
806 | mutex_unlock(ctx->res_mutex); | |
fc18afcf | 807 | vmw_validation_unref_lists(ctx); |
038ecc50 TH |
808 | } |
809 | ||
810 | /** | |
811 | * vmw_validation_cone - Commit validation actions after command submission | |
812 | * success. | |
813 | * @ctx: The validation context. | |
814 | * @fence: Fence with which to fence all buffer objects taking part in the | |
815 | * command submission. | |
816 | * | |
817 | * The caller does NOT need to unref resources after a call to this function. | |
818 | */ | |
819 | void vmw_validation_done(struct vmw_validation_context *ctx, | |
820 | struct vmw_fence_obj *fence) | |
821 | { | |
822 | vmw_validation_bo_fence(ctx, fence); | |
823 | vmw_validation_res_unreserve(ctx, false); | |
824 | if (ctx->res_mutex) | |
825 | mutex_unlock(ctx->res_mutex); | |
826 | vmw_validation_unref_lists(ctx); | |
827 | } | |
64ad2abf TH |
828 | |
829 | /** | |
830 | * vmw_validation_preload_bo - Preload the validation memory allocator for a | |
831 | * call to vmw_validation_add_bo(). | |
832 | * @ctx: Pointer to the validation context. | |
833 | * | |
834 | * Iff this function returns successfully, the next call to | |
835 | * vmw_validation_add_bo() is guaranteed not to sleep. An error is not fatal | |
836 | * but voids the guarantee. | |
837 | * | |
838 | * Returns: Zero if successful, %-EINVAL otherwise. | |
839 | */ | |
840 | int vmw_validation_preload_bo(struct vmw_validation_context *ctx) | |
841 | { | |
842 | unsigned int size = sizeof(struct vmw_validation_bo_node); | |
843 | ||
844 | if (!vmw_validation_mem_alloc(ctx, size)) | |
845 | return -ENOMEM; | |
846 | ||
847 | ctx->mem_size_left += size; | |
848 | return 0; | |
849 | } | |
850 | ||
851 | /** | |
852 | * vmw_validation_preload_res - Preload the validation memory allocator for a | |
853 | * call to vmw_validation_add_res(). | |
854 | * @ctx: Pointer to the validation context. | |
855 | * @size: Size of the validation node extra data. See below. | |
856 | * | |
857 | * Iff this function returns successfully, the next call to | |
858 | * vmw_validation_add_res() with the same or smaller @size is guaranteed not to | |
859 | * sleep. An error is not fatal but voids the guarantee. | |
860 | * | |
861 | * Returns: Zero if successful, %-EINVAL otherwise. | |
862 | */ | |
863 | int vmw_validation_preload_res(struct vmw_validation_context *ctx, | |
864 | unsigned int size) | |
865 | { | |
866 | size = vmw_validation_align(sizeof(struct vmw_validation_res_node) + | |
867 | size) + | |
868 | vmw_validation_align(sizeof(struct vmw_validation_bo_node)); | |
869 | if (!vmw_validation_mem_alloc(ctx, size)) | |
870 | return -ENOMEM; | |
871 | ||
872 | ctx->mem_size_left += size; | |
873 | return 0; | |
874 | } | |
b7468b15 TH |
875 | |
876 | /** | |
877 | * vmw_validation_bo_backoff - Unreserve buffer objects registered with a | |
878 | * validation context | |
879 | * @ctx: The validation context | |
880 | * | |
881 | * This function unreserves the buffer objects previously reserved using | |
882 | * vmw_validation_bo_reserve. It's typically used as part of an error path | |
883 | */ | |
884 | void vmw_validation_bo_backoff(struct vmw_validation_context *ctx) | |
885 | { | |
886 | struct vmw_validation_bo_node *entry; | |
887 | ||
888 | /* | |
889 | * Switching coherent resource backup buffers failed. | |
890 | * Release corresponding buffer object dirty trackers. | |
891 | */ | |
892 | list_for_each_entry(entry, &ctx->bo_list, base.head) { | |
893 | if (entry->coherent_count) { | |
894 | unsigned int coherent_count = entry->coherent_count; | |
895 | struct vmw_buffer_object *vbo = | |
896 | container_of(entry->base.bo, typeof(*vbo), | |
897 | base); | |
898 | ||
899 | while (coherent_count--) | |
900 | vmw_bo_dirty_release(vbo); | |
901 | } | |
902 | } | |
903 | ||
904 | ttm_eu_backoff_reservation(&ctx->ticket, &ctx->bo_list); | |
905 | } |