Commit | Line | Data |
---|---|---|
6501aa4e MA |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2019 Intel Corporation | |
4 | */ | |
6501aa4e | 5 | |
6da4a2c4 | 6 | #include "i915_drv.h" |
963ad128 | 7 | #include "gt/intel_context.h" |
554e330c | 8 | #include "gt/intel_engine_pm.h" |
1d455f8d | 9 | #include "i915_gem_client_blt.h" |
6501aa4e | 10 | #include "i915_gem_object_blt.h" |
6501aa4e MA |
11 | |
12 | struct i915_sleeve { | |
13 | struct i915_vma *vma; | |
14 | struct drm_i915_gem_object *obj; | |
15 | struct sg_table *pages; | |
16 | struct i915_page_sizes page_sizes; | |
17 | }; | |
18 | ||
19 | static int vma_set_pages(struct i915_vma *vma) | |
20 | { | |
21 | struct i915_sleeve *sleeve = vma->private; | |
22 | ||
23 | vma->pages = sleeve->pages; | |
24 | vma->page_sizes = sleeve->page_sizes; | |
25 | ||
26 | return 0; | |
27 | } | |
28 | ||
29 | static void vma_clear_pages(struct i915_vma *vma) | |
30 | { | |
31 | GEM_BUG_ON(!vma->pages); | |
32 | vma->pages = NULL; | |
33 | } | |
34 | ||
cd0452aa CW |
35 | static void vma_bind(struct i915_address_space *vm, |
36 | struct i915_vm_pt_stash *stash, | |
37 | struct i915_vma *vma, | |
38 | enum i915_cache_level cache_level, | |
39 | u32 flags) | |
6501aa4e | 40 | { |
cd0452aa | 41 | vm->vma_ops.bind_vma(vm, stash, vma, cache_level, flags); |
6501aa4e MA |
42 | } |
43 | ||
12b07256 | 44 | static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma) |
6501aa4e | 45 | { |
12b07256 | 46 | vm->vma_ops.unbind_vma(vm, vma); |
6501aa4e MA |
47 | } |
48 | ||
49 | static const struct i915_vma_ops proxy_vma_ops = { | |
50 | .set_pages = vma_set_pages, | |
51 | .clear_pages = vma_clear_pages, | |
52 | .bind_vma = vma_bind, | |
53 | .unbind_vma = vma_unbind, | |
54 | }; | |
55 | ||
56 | static struct i915_sleeve *create_sleeve(struct i915_address_space *vm, | |
57 | struct drm_i915_gem_object *obj, | |
58 | struct sg_table *pages, | |
59 | struct i915_page_sizes *page_sizes) | |
60 | { | |
61 | struct i915_sleeve *sleeve; | |
62 | struct i915_vma *vma; | |
63 | int err; | |
64 | ||
65 | sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL); | |
66 | if (!sleeve) | |
67 | return ERR_PTR(-ENOMEM); | |
68 | ||
69 | vma = i915_vma_instance(obj, vm, NULL); | |
70 | if (IS_ERR(vma)) { | |
71 | err = PTR_ERR(vma); | |
72 | goto err_free; | |
73 | } | |
74 | ||
75 | vma->private = sleeve; | |
76 | vma->ops = &proxy_vma_ops; | |
77 | ||
78 | sleeve->vma = vma; | |
6501aa4e MA |
79 | sleeve->pages = pages; |
80 | sleeve->page_sizes = *page_sizes; | |
81 | ||
82 | return sleeve; | |
83 | ||
84 | err_free: | |
85 | kfree(sleeve); | |
86 | return ERR_PTR(err); | |
87 | } | |
88 | ||
89 | static void destroy_sleeve(struct i915_sleeve *sleeve) | |
90 | { | |
6501aa4e MA |
91 | kfree(sleeve); |
92 | } | |
93 | ||
94 | struct clear_pages_work { | |
95 | struct dma_fence dma; | |
96 | struct dma_fence_cb cb; | |
97 | struct i915_sw_fence wait; | |
98 | struct work_struct work; | |
99 | struct irq_work irq_work; | |
100 | struct i915_sleeve *sleeve; | |
101 | struct intel_context *ce; | |
102 | u32 value; | |
103 | }; | |
104 | ||
105 | static const char *clear_pages_work_driver_name(struct dma_fence *fence) | |
106 | { | |
107 | return DRIVER_NAME; | |
108 | } | |
109 | ||
110 | static const char *clear_pages_work_timeline_name(struct dma_fence *fence) | |
111 | { | |
112 | return "clear"; | |
113 | } | |
114 | ||
115 | static void clear_pages_work_release(struct dma_fence *fence) | |
116 | { | |
117 | struct clear_pages_work *w = container_of(fence, typeof(*w), dma); | |
118 | ||
119 | destroy_sleeve(w->sleeve); | |
120 | ||
121 | i915_sw_fence_fini(&w->wait); | |
122 | ||
123 | BUILD_BUG_ON(offsetof(typeof(*w), dma)); | |
124 | dma_fence_free(&w->dma); | |
125 | } | |
126 | ||
127 | static const struct dma_fence_ops clear_pages_work_ops = { | |
128 | .get_driver_name = clear_pages_work_driver_name, | |
129 | .get_timeline_name = clear_pages_work_timeline_name, | |
130 | .release = clear_pages_work_release, | |
131 | }; | |
132 | ||
133 | static void clear_pages_signal_irq_worker(struct irq_work *work) | |
134 | { | |
135 | struct clear_pages_work *w = container_of(work, typeof(*w), irq_work); | |
136 | ||
137 | dma_fence_signal(&w->dma); | |
138 | dma_fence_put(&w->dma); | |
139 | } | |
140 | ||
141 | static void clear_pages_dma_fence_cb(struct dma_fence *fence, | |
142 | struct dma_fence_cb *cb) | |
143 | { | |
144 | struct clear_pages_work *w = container_of(cb, typeof(*w), cb); | |
145 | ||
146 | if (fence->error) | |
147 | dma_fence_set_error(&w->dma, fence->error); | |
148 | ||
149 | /* | |
150 | * Push the signalling of the fence into yet another worker to avoid | |
151 | * the nightmare locking around the fence spinlock. | |
152 | */ | |
153 | irq_work_queue(&w->irq_work); | |
154 | } | |
155 | ||
156 | static void clear_pages_worker(struct work_struct *work) | |
157 | { | |
158 | struct clear_pages_work *w = container_of(work, typeof(*w), work); | |
b2dbf8d9 | 159 | struct drm_i915_gem_object *obj = w->sleeve->vma->obj; |
6501aa4e MA |
160 | struct i915_vma *vma = w->sleeve->vma; |
161 | struct i915_request *rq; | |
554e330c | 162 | struct i915_vma *batch; |
6501aa4e MA |
163 | int err = w->dma.error; |
164 | ||
165 | if (unlikely(err)) | |
166 | goto out_signal; | |
167 | ||
168 | if (obj->cache_dirty) { | |
6501aa4e MA |
169 | if (i915_gem_object_has_struct_page(obj)) |
170 | drm_clflush_sg(w->sleeve->pages); | |
171 | obj->cache_dirty = false; | |
172 | } | |
871918df CW |
173 | obj->read_domains = I915_GEM_GPU_DOMAINS; |
174 | obj->write_domain = 0; | |
6501aa4e | 175 | |
6501aa4e MA |
176 | err = i915_vma_pin(vma, 0, 0, PIN_USER); |
177 | if (unlikely(err)) | |
7e805762 | 178 | goto out_signal; |
6501aa4e | 179 | |
554e330c MA |
180 | batch = intel_emit_vma_fill_blt(w->ce, vma, w->value); |
181 | if (IS_ERR(batch)) { | |
182 | err = PTR_ERR(batch); | |
183 | goto out_unpin; | |
184 | } | |
185 | ||
963ad128 | 186 | rq = intel_context_create_request(w->ce); |
6501aa4e MA |
187 | if (IS_ERR(rq)) { |
188 | err = PTR_ERR(rq); | |
554e330c | 189 | goto out_batch; |
6501aa4e MA |
190 | } |
191 | ||
192 | /* There's no way the fence has signalled */ | |
193 | if (dma_fence_add_callback(&rq->fence, &w->cb, | |
194 | clear_pages_dma_fence_cb)) | |
195 | GEM_BUG_ON(1); | |
196 | ||
554e330c MA |
197 | err = intel_emit_vma_mark_active(batch, rq); |
198 | if (unlikely(err)) | |
199 | goto out_request; | |
200 | ||
6501aa4e MA |
201 | if (w->ce->engine->emit_init_breadcrumb) { |
202 | err = w->ce->engine->emit_init_breadcrumb(rq); | |
203 | if (unlikely(err)) | |
204 | goto out_request; | |
205 | } | |
206 | ||
b2dbf8d9 CW |
207 | /* |
208 | * w->dma is already exported via (vma|obj)->resv we need only | |
209 | * keep track of the GPU activity within this vma/request, and | |
210 | * propagate the signal from the request to w->dma. | |
211 | */ | |
2850748e | 212 | err = __i915_vma_move_to_active(vma, rq); |
6501aa4e MA |
213 | if (err) |
214 | goto out_request; | |
215 | ||
554e330c MA |
216 | err = w->ce->engine->emit_bb_start(rq, |
217 | batch->node.start, batch->node.size, | |
218 | 0); | |
6501aa4e MA |
219 | out_request: |
220 | if (unlikely(err)) { | |
36e191f0 | 221 | i915_request_set_error_once(rq, err); |
6501aa4e MA |
222 | err = 0; |
223 | } | |
224 | ||
225 | i915_request_add(rq); | |
554e330c MA |
226 | out_batch: |
227 | intel_emit_vma_release(w->ce, batch); | |
6501aa4e MA |
228 | out_unpin: |
229 | i915_vma_unpin(vma); | |
6501aa4e MA |
230 | out_signal: |
231 | if (unlikely(err)) { | |
232 | dma_fence_set_error(&w->dma, err); | |
233 | dma_fence_signal(&w->dma); | |
234 | dma_fence_put(&w->dma); | |
235 | } | |
236 | } | |
237 | ||
238 | static int __i915_sw_fence_call | |
239 | clear_pages_work_notify(struct i915_sw_fence *fence, | |
240 | enum i915_sw_fence_notify state) | |
241 | { | |
242 | struct clear_pages_work *w = container_of(fence, typeof(*w), wait); | |
243 | ||
244 | switch (state) { | |
245 | case FENCE_COMPLETE: | |
246 | schedule_work(&w->work); | |
247 | break; | |
248 | ||
249 | case FENCE_FREE: | |
250 | dma_fence_put(&w->dma); | |
251 | break; | |
252 | } | |
253 | ||
254 | return NOTIFY_DONE; | |
255 | } | |
256 | ||
257 | static DEFINE_SPINLOCK(fence_lock); | |
258 | ||
259 | /* XXX: better name please */ | |
260 | int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj, | |
261 | struct intel_context *ce, | |
262 | struct sg_table *pages, | |
263 | struct i915_page_sizes *page_sizes, | |
264 | u32 value) | |
265 | { | |
6501aa4e MA |
266 | struct clear_pages_work *work; |
267 | struct i915_sleeve *sleeve; | |
268 | int err; | |
269 | ||
f5d974f9 | 270 | sleeve = create_sleeve(ce->vm, obj, pages, page_sizes); |
6501aa4e MA |
271 | if (IS_ERR(sleeve)) |
272 | return PTR_ERR(sleeve); | |
273 | ||
274 | work = kmalloc(sizeof(*work), GFP_KERNEL); | |
275 | if (!work) { | |
276 | destroy_sleeve(sleeve); | |
277 | return -ENOMEM; | |
278 | } | |
279 | ||
280 | work->value = value; | |
281 | work->sleeve = sleeve; | |
282 | work->ce = ce; | |
283 | ||
284 | INIT_WORK(&work->work, clear_pages_worker); | |
285 | ||
286 | init_irq_work(&work->irq_work, clear_pages_signal_irq_worker); | |
287 | ||
cc337560 | 288 | dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0); |
6501aa4e MA |
289 | i915_sw_fence_init(&work->wait, clear_pages_work_notify); |
290 | ||
291 | i915_gem_object_lock(obj); | |
292 | err = i915_sw_fence_await_reservation(&work->wait, | |
16dc224f | 293 | obj->base.resv, NULL, true, 0, |
6501aa4e MA |
294 | I915_FENCE_GFP); |
295 | if (err < 0) { | |
296 | dma_fence_set_error(&work->dma, err); | |
297 | } else { | |
52791eee | 298 | dma_resv_add_excl_fence(obj->base.resv, &work->dma); |
6501aa4e MA |
299 | err = 0; |
300 | } | |
301 | i915_gem_object_unlock(obj); | |
302 | ||
303 | dma_fence_get(&work->dma); | |
304 | i915_sw_fence_commit(&work->wait); | |
305 | ||
306 | return err; | |
307 | } | |
308 | ||
309 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
310 | #include "selftests/i915_gem_client_blt.c" | |
311 | #endif |