Commit | Line | Data |
---|---|---|
63cf4cad TH |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2021 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/dma-fence.h> | |
7 | #include <linux/slab.h> | |
8 | ||
a3185f91 | 9 | #include <drm/ttm/ttm_bo.h> |
63cf4cad TH |
10 | |
11 | #include "i915_deps.h" | |
12 | ||
13 | /** | |
14 | * DOC: Set of utilities to dynamically collect dependencies into a | |
15 | * structure which is fed into the GT migration code. | |
16 | * | |
17 | * Once we can do async unbinding, this is also needed to coalesce | |
18 | * the migration fence with the unbind fences if these are coalesced | |
19 | * post-migration. | |
20 | * | |
21 | * While collecting the individual dependencies, we store the refcounted | |
22 | * struct dma_fence pointers in a realloc-managed pointer array, since | |
23 | * that can be easily fed into a dma_fence_array. Other options are | |
24 | * available, like for example an xarray for similarity with drm/sched. | |
25 | * Can be changed easily if needed. | |
26 | * | |
27 | * A struct i915_deps need to be initialized using i915_deps_init(). | |
28 | * If i915_deps_add_dependency() or i915_deps_add_resv() return an | |
29 | * error code they will internally call i915_deps_fini(), which frees | |
30 | * all internal references and allocations. | |
31 | */ | |
32 | ||
33 | /* Min number of fence pointers in the array when an allocation occurs. */ | |
34 | #define I915_DEPS_MIN_ALLOC_CHUNK 8U | |
35 | ||
36 | static void i915_deps_reset_fences(struct i915_deps *deps) | |
37 | { | |
38 | if (deps->fences != &deps->single) | |
39 | kfree(deps->fences); | |
40 | deps->num_deps = 0; | |
41 | deps->fences_size = 1; | |
42 | deps->fences = &deps->single; | |
43 | } | |
44 | ||
45 | /** | |
46 | * i915_deps_init - Initialize an i915_deps structure | |
47 | * @deps: Pointer to the i915_deps structure to initialize. | |
48 | * @gfp: The allocation mode for subsequenst allocations. | |
49 | */ | |
50 | void i915_deps_init(struct i915_deps *deps, gfp_t gfp) | |
51 | { | |
52 | deps->fences = NULL; | |
53 | deps->gfp = gfp; | |
54 | i915_deps_reset_fences(deps); | |
55 | } | |
56 | ||
57 | /** | |
58 | * i915_deps_fini - Finalize an i915_deps structure | |
59 | * @deps: Pointer to the i915_deps structure to finalize. | |
60 | * | |
61 | * This function drops all fence references taken, conditionally frees and | |
62 | * then resets the fences array. | |
63 | */ | |
64 | void i915_deps_fini(struct i915_deps *deps) | |
65 | { | |
66 | unsigned int i; | |
67 | ||
68 | for (i = 0; i < deps->num_deps; ++i) | |
69 | dma_fence_put(deps->fences[i]); | |
70 | ||
71 | if (deps->fences != &deps->single) | |
72 | kfree(deps->fences); | |
73 | } | |
74 | ||
75 | static int i915_deps_grow(struct i915_deps *deps, struct dma_fence *fence, | |
76 | const struct ttm_operation_ctx *ctx) | |
77 | { | |
78 | int ret; | |
79 | ||
80 | if (deps->num_deps >= deps->fences_size) { | |
81 | unsigned int new_size = 2 * deps->fences_size; | |
82 | struct dma_fence **new_fences; | |
83 | ||
84 | new_size = max(new_size, I915_DEPS_MIN_ALLOC_CHUNK); | |
85 | new_fences = kmalloc_array(new_size, sizeof(*new_fences), deps->gfp); | |
86 | if (!new_fences) | |
87 | goto sync; | |
88 | ||
89 | memcpy(new_fences, deps->fences, | |
90 | deps->fences_size * sizeof(*new_fences)); | |
91 | swap(new_fences, deps->fences); | |
92 | if (new_fences != &deps->single) | |
93 | kfree(new_fences); | |
94 | deps->fences_size = new_size; | |
95 | } | |
96 | deps->fences[deps->num_deps++] = dma_fence_get(fence); | |
97 | return 0; | |
98 | ||
99 | sync: | |
100 | if (ctx->no_wait_gpu && !dma_fence_is_signaled(fence)) { | |
101 | ret = -EBUSY; | |
102 | goto unref; | |
103 | } | |
104 | ||
105 | ret = dma_fence_wait(fence, ctx->interruptible); | |
106 | if (ret) | |
107 | goto unref; | |
108 | ||
109 | ret = fence->error; | |
110 | if (ret) | |
111 | goto unref; | |
112 | ||
113 | return 0; | |
114 | ||
115 | unref: | |
116 | i915_deps_fini(deps); | |
117 | return ret; | |
118 | } | |
119 | ||
120 | /** | |
121 | * i915_deps_sync - Wait for all the fences in the dependency collection | |
122 | * @deps: Pointer to the i915_deps structure the fences of which to wait for. | |
123 | * @ctx: Pointer to a struct ttm_operation_ctx indicating how the waits | |
124 | * should be performed. | |
125 | * | |
126 | * This function waits for fences in the dependency collection. If it | |
127 | * encounters an error during the wait or a fence error, the wait for | |
128 | * further fences is aborted and the error returned. | |
129 | * | |
130 | * Return: Zero if successful, Negative error code on error. | |
131 | */ | |
132 | int i915_deps_sync(const struct i915_deps *deps, const struct ttm_operation_ctx *ctx) | |
133 | { | |
134 | struct dma_fence **fences = deps->fences; | |
135 | unsigned int i; | |
136 | int ret = 0; | |
137 | ||
138 | for (i = 0; i < deps->num_deps; ++i, ++fences) { | |
139 | if (ctx->no_wait_gpu && !dma_fence_is_signaled(*fences)) { | |
140 | ret = -EBUSY; | |
141 | break; | |
142 | } | |
143 | ||
144 | ret = dma_fence_wait(*fences, ctx->interruptible); | |
145 | if (!ret) | |
146 | ret = (*fences)->error; | |
147 | if (ret) | |
148 | break; | |
149 | } | |
150 | ||
151 | return ret; | |
152 | } | |
153 | ||
154 | /** | |
155 | * i915_deps_add_dependency - Add a fence to the dependency collection | |
156 | * @deps: Pointer to the i915_deps structure a fence is to be added to. | |
157 | * @fence: The fence to add. | |
158 | * @ctx: Pointer to a struct ttm_operation_ctx indicating how waits are to | |
159 | * be performed if waiting. | |
160 | * | |
161 | * Adds a fence to the dependency collection, and takes a reference on it. | |
162 | * If the fence context is not zero and there was a later fence from the | |
163 | * same fence context already added, then the fence is not added to the | |
164 | * dependency collection. If the fence context is not zero and there was | |
165 | * an earlier fence already added, then the fence will replace the older | |
166 | * fence from the same context and the reference on the earlier fence will | |
167 | * be dropped. | |
168 | * If there is a failure to allocate memory to accommodate the new fence to | |
169 | * be added, the new fence will instead be waited for and an error may | |
170 | * be returned; depending on the value of @ctx, or if there was a fence | |
171 | * error. If an error was returned, the dependency collection will be | |
172 | * finalized and all fence reference dropped. | |
173 | * | |
174 | * Return: 0 if success. Negative error code on error. | |
175 | */ | |
176 | int i915_deps_add_dependency(struct i915_deps *deps, | |
177 | struct dma_fence *fence, | |
178 | const struct ttm_operation_ctx *ctx) | |
179 | { | |
180 | unsigned int i; | |
181 | int ret; | |
182 | ||
183 | if (!fence) | |
184 | return 0; | |
185 | ||
186 | if (dma_fence_is_signaled(fence)) { | |
187 | ret = fence->error; | |
188 | if (ret) | |
189 | i915_deps_fini(deps); | |
190 | return ret; | |
191 | } | |
192 | ||
193 | for (i = 0; i < deps->num_deps; ++i) { | |
194 | struct dma_fence *entry = deps->fences[i]; | |
195 | ||
196 | if (!entry->context || entry->context != fence->context) | |
197 | continue; | |
198 | ||
199 | if (dma_fence_is_later(fence, entry)) { | |
200 | dma_fence_put(entry); | |
201 | deps->fences[i] = dma_fence_get(fence); | |
202 | } | |
203 | ||
204 | return 0; | |
205 | } | |
206 | ||
207 | return i915_deps_grow(deps, fence, ctx); | |
208 | } | |
209 | ||
210 | /** | |
211 | * i915_deps_add_resv - Add the fences of a reservation object to a dependency | |
212 | * collection. | |
213 | * @deps: Pointer to the i915_deps structure a fence is to be added to. | |
214 | * @resv: The reservation object, then fences of which to add. | |
215 | * @ctx: Pointer to a struct ttm_operation_ctx indicating how waits are to | |
216 | * be performed if waiting. | |
217 | * | |
218 | * Calls i915_deps_add_depencency() on the indicated fences of @resv. | |
219 | * | |
220 | * Return: Zero on success. Negative error code on error. | |
221 | */ | |
222 | int i915_deps_add_resv(struct i915_deps *deps, struct dma_resv *resv, | |
223 | const struct ttm_operation_ctx *ctx) | |
224 | { | |
225 | struct dma_resv_iter iter; | |
226 | struct dma_fence *fence; | |
227 | ||
228 | dma_resv_assert_held(resv); | |
17b048d4 | 229 | dma_resv_for_each_fence(&iter, resv, dma_resv_usage_rw(true), fence) { |
63cf4cad TH |
230 | int ret = i915_deps_add_dependency(deps, fence, ctx); |
231 | ||
232 | if (ret) | |
233 | return ret; | |
234 | } | |
235 | ||
236 | return 0; | |
237 | } |