Commit | Line | Data |
---|---|---|
b414fcd5 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2014-2016 Intel Corporation | |
5 | */ | |
6 | ||
f17b8980 | 7 | #include <linux/anon_inodes.h> |
b414fcd5 | 8 | #include <linux/mman.h> |
cc662126 | 9 | #include <linux/pfn_t.h> |
b414fcd5 CW |
10 | #include <linux/sizes.h> |
11 | ||
cb823ed9 | 12 | #include "gt/intel_gt.h" |
66101975 | 13 | #include "gt/intel_gt_requests.h" |
cb823ed9 | 14 | |
b414fcd5 CW |
15 | #include "i915_drv.h" |
16 | #include "i915_gem_gtt.h" | |
17 | #include "i915_gem_ioctls.h" | |
18 | #include "i915_gem_object.h" | |
cc662126 | 19 | #include "i915_gem_mman.h" |
a09d9a80 | 20 | #include "i915_trace.h" |
126d5de3 | 21 | #include "i915_user_extensions.h" |
b414fcd5 | 22 | #include "i915_vma.h" |
b414fcd5 CW |
23 | |
24 | static inline bool | |
25 | __vma_matches(struct vm_area_struct *vma, struct file *filp, | |
26 | unsigned long addr, unsigned long size) | |
27 | { | |
28 | if (vma->vm_file != filp) | |
29 | return false; | |
30 | ||
31 | return vma->vm_start == addr && | |
32 | (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); | |
33 | } | |
34 | ||
35 | /** | |
36 | * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address | |
37 | * it is mapped to. | |
38 | * @dev: drm device | |
39 | * @data: ioctl data blob | |
40 | * @file: drm file | |
41 | * | |
42 | * While the mapping holds a reference on the contents of the object, it doesn't | |
43 | * imply a ref on the object itself. | |
44 | * | |
45 | * IMPORTANT: | |
46 | * | |
47 | * DRM driver writers who look a this function as an example for how to do GEM | |
48 | * mmap support, please don't implement mmap support like here. The modern way | |
49 | * to implement DRM mmap support is with an mmap offset ioctl (like | |
50 | * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. | |
51 | * That way debug tooling like valgrind will understand what's going on, hiding | |
52 | * the mmap call in a driver private ioctl will break that. The i915 driver only | |
53 | * does cpu mmaps this way because we didn't know better. | |
54 | */ | |
55 | int | |
56 | i915_gem_mmap_ioctl(struct drm_device *dev, void *data, | |
57 | struct drm_file *file) | |
58 | { | |
59 | struct drm_i915_gem_mmap *args = data; | |
60 | struct drm_i915_gem_object *obj; | |
61 | unsigned long addr; | |
62 | ||
63 | if (args->flags & ~(I915_MMAP_WC)) | |
64 | return -EINVAL; | |
65 | ||
66 | if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) | |
67 | return -ENODEV; | |
68 | ||
69 | obj = i915_gem_object_lookup(file, args->handle); | |
70 | if (!obj) | |
71 | return -ENOENT; | |
72 | ||
73 | /* prime objects have no backing filp to GEM mmap | |
74 | * pages from. | |
75 | */ | |
76 | if (!obj->base.filp) { | |
77 | addr = -ENXIO; | |
78 | goto err; | |
79 | } | |
80 | ||
81 | if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { | |
82 | addr = -EINVAL; | |
83 | goto err; | |
84 | } | |
85 | ||
86 | addr = vm_mmap(obj->base.filp, 0, args->size, | |
87 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
88 | args->offset); | |
89 | if (IS_ERR_VALUE(addr)) | |
90 | goto err; | |
91 | ||
92 | if (args->flags & I915_MMAP_WC) { | |
93 | struct mm_struct *mm = current->mm; | |
94 | struct vm_area_struct *vma; | |
95 | ||
d8ed45c5 | 96 | if (mmap_write_lock_killable(mm)) { |
b414fcd5 CW |
97 | addr = -EINTR; |
98 | goto err; | |
99 | } | |
100 | vma = find_vma(mm, addr); | |
101 | if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) | |
102 | vma->vm_page_prot = | |
103 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
104 | else | |
105 | addr = -ENOMEM; | |
d8ed45c5 | 106 | mmap_write_unlock(mm); |
b414fcd5 CW |
107 | if (IS_ERR_VALUE(addr)) |
108 | goto err; | |
b414fcd5 CW |
109 | } |
110 | i915_gem_object_put(obj); | |
111 | ||
112 | args->addr_ptr = (u64)addr; | |
113 | return 0; | |
114 | ||
115 | err: | |
116 | i915_gem_object_put(obj); | |
117 | return addr; | |
118 | } | |
119 | ||
120 | static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) | |
121 | { | |
122 | return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; | |
123 | } | |
124 | ||
125 | /** | |
126 | * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps | |
127 | * | |
128 | * A history of the GTT mmap interface: | |
129 | * | |
130 | * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to | |
131 | * aligned and suitable for fencing, and still fit into the available | |
132 | * mappable space left by the pinned display objects. A classic problem | |
133 | * we called the page-fault-of-doom where we would ping-pong between | |
134 | * two objects that could not fit inside the GTT and so the memcpy | |
135 | * would page one object in at the expense of the other between every | |
136 | * single byte. | |
137 | * | |
138 | * 1 - Objects can be any size, and have any compatible fencing (X Y, or none | |
139 | * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the | |
140 | * object is too large for the available space (or simply too large | |
141 | * for the mappable aperture!), a view is created instead and faulted | |
142 | * into userspace. (This view is aligned and sized appropriately for | |
143 | * fenced access.) | |
144 | * | |
145 | * 2 - Recognise WC as a separate cache domain so that we can flush the | |
146 | * delayed writes via GTT before performing direct access via WC. | |
147 | * | |
148 | * 3 - Remove implicit set-domain(GTT) and synchronisation on initial | |
149 | * pagefault; swapin remains transparent. | |
150 | * | |
cc662126 AJ |
151 | * 4 - Support multiple fault handlers per object depending on object's |
152 | * backing storage (a.k.a. MMAP_OFFSET). | |
153 | * | |
b414fcd5 CW |
154 | * Restrictions: |
155 | * | |
156 | * * snoopable objects cannot be accessed via the GTT. It can cause machine | |
157 | * hangs on some architectures, corruption on others. An attempt to service | |
158 | * a GTT page fault from a snoopable object will generate a SIGBUS. | |
159 | * | |
160 | * * the object must be able to fit into RAM (physical memory, though no | |
161 | * limited to the mappable aperture). | |
162 | * | |
163 | * | |
164 | * Caveats: | |
165 | * | |
166 | * * a new GTT page fault will synchronize rendering from the GPU and flush | |
167 | * all data to system memory. Subsequent access will not be synchronized. | |
168 | * | |
169 | * * all mappings are revoked on runtime device suspend. | |
170 | * | |
171 | * * there are only 8, 16 or 32 fence registers to share between all users | |
172 | * (older machines require fence register for display and blitter access | |
173 | * as well). Contention of the fence registers will cause the previous users | |
174 | * to be unmapped and any new access will generate new page faults. | |
175 | * | |
176 | * * running out of memory while servicing a fault may generate a SIGBUS, | |
177 | * rather than the expected SIGSEGV. | |
178 | */ | |
179 | int i915_gem_mmap_gtt_version(void) | |
180 | { | |
cc662126 | 181 | return 4; |
b414fcd5 CW |
182 | } |
183 | ||
184 | static inline struct i915_ggtt_view | |
185 | compute_partial_view(const struct drm_i915_gem_object *obj, | |
186 | pgoff_t page_offset, | |
187 | unsigned int chunk) | |
188 | { | |
189 | struct i915_ggtt_view view; | |
190 | ||
191 | if (i915_gem_object_is_tiled(obj)) | |
192 | chunk = roundup(chunk, tile_row_pages(obj)); | |
193 | ||
194 | view.type = I915_GGTT_VIEW_PARTIAL; | |
195 | view.partial.offset = rounddown(page_offset, chunk); | |
196 | view.partial.size = | |
197 | min_t(unsigned int, chunk, | |
198 | (obj->base.size >> PAGE_SHIFT) - view.partial.offset); | |
199 | ||
200 | /* If the partial covers the entire object, just create a normal VMA. */ | |
201 | if (chunk >= obj->base.size >> PAGE_SHIFT) | |
202 | view.type = I915_GGTT_VIEW_NORMAL; | |
203 | ||
204 | return view; | |
205 | } | |
206 | ||
cc662126 AJ |
207 | static vm_fault_t i915_error_to_vmf_fault(int err) |
208 | { | |
209 | switch (err) { | |
210 | default: | |
211 | WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err); | |
df561f66 | 212 | fallthrough; |
cc662126 AJ |
213 | case -EIO: /* shmemfs failure from swap device */ |
214 | case -EFAULT: /* purged object */ | |
215 | case -ENODEV: /* bad object, how did you get here! */ | |
4e598fad | 216 | case -ENXIO: /* unable to access backing store (on device) */ |
cc662126 AJ |
217 | return VM_FAULT_SIGBUS; |
218 | ||
cc662126 AJ |
219 | case -ENOMEM: /* our allocation failure */ |
220 | return VM_FAULT_OOM; | |
221 | ||
222 | case 0: | |
223 | case -EAGAIN: | |
552e01f6 | 224 | case -ENOSPC: /* transient failure to evict? */ |
cc662126 AJ |
225 | case -ERESTARTSYS: |
226 | case -EINTR: | |
227 | case -EBUSY: | |
228 | /* | |
229 | * EBUSY is ok: this just means that another thread | |
230 | * already did the job. | |
231 | */ | |
232 | return VM_FAULT_NOPAGE; | |
233 | } | |
234 | } | |
235 | ||
236 | static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) | |
237 | { | |
238 | struct vm_area_struct *area = vmf->vma; | |
239 | struct i915_mmap_offset *mmo = area->vm_private_data; | |
240 | struct drm_i915_gem_object *obj = mmo->obj; | |
4e598fad | 241 | resource_size_t iomap; |
cc662126 AJ |
242 | int err; |
243 | ||
cc662126 | 244 | /* Sanity check that we allow writing into this object */ |
1764b992 AJ |
245 | if (unlikely(i915_gem_object_is_readonly(obj) && |
246 | area->vm_flags & VM_WRITE)) | |
247 | return VM_FAULT_SIGBUS; | |
cc662126 AJ |
248 | |
249 | err = i915_gem_object_pin_pages(obj); | |
250 | if (err) | |
1764b992 | 251 | goto out; |
cc662126 | 252 | |
4e598fad AJ |
253 | iomap = -1; |
254 | if (!i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE)) { | |
255 | iomap = obj->mm.region->iomap.base; | |
256 | iomap -= obj->mm.region->region.start; | |
257 | } | |
258 | ||
cc662126 | 259 | /* PTEs are revoked in obj->ops->put_pages() */ |
4e598fad AJ |
260 | err = remap_io_sg(area, |
261 | area->vm_start, area->vm_end - area->vm_start, | |
262 | obj->mm.pages->sgl, iomap); | |
cc662126 | 263 | |
1764b992 | 264 | if (area->vm_flags & VM_WRITE) { |
cc662126 | 265 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); |
cc662126 AJ |
266 | obj->mm.dirty = true; |
267 | } | |
268 | ||
269 | i915_gem_object_unpin_pages(obj); | |
270 | ||
1764b992 AJ |
271 | out: |
272 | return i915_error_to_vmf_fault(err); | |
cc662126 AJ |
273 | } |
274 | ||
275 | static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) | |
b414fcd5 CW |
276 | { |
277 | #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) | |
278 | struct vm_area_struct *area = vmf->vma; | |
cc662126 AJ |
279 | struct i915_mmap_offset *mmo = area->vm_private_data; |
280 | struct drm_i915_gem_object *obj = mmo->obj; | |
b414fcd5 CW |
281 | struct drm_device *dev = obj->base.dev; |
282 | struct drm_i915_private *i915 = to_i915(dev); | |
d858d569 | 283 | struct intel_runtime_pm *rpm = &i915->runtime_pm; |
b414fcd5 CW |
284 | struct i915_ggtt *ggtt = &i915->ggtt; |
285 | bool write = area->vm_flags & VM_WRITE; | |
3c0ffa27 | 286 | struct i915_gem_ww_ctx ww; |
b414fcd5 CW |
287 | intel_wakeref_t wakeref; |
288 | struct i915_vma *vma; | |
289 | pgoff_t page_offset; | |
290 | int srcu; | |
291 | int ret; | |
292 | ||
b414fcd5 CW |
293 | /* We don't use vmf->pgoff since that has the fake offset */ |
294 | page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; | |
295 | ||
296 | trace_i915_gem_object_fault(obj, page_offset, true, write); | |
297 | ||
3c0ffa27 ML |
298 | wakeref = intel_runtime_pm_get(rpm); |
299 | ||
300 | i915_gem_ww_ctx_init(&ww, true); | |
301 | retry: | |
302 | ret = i915_gem_object_lock(obj, &ww); | |
b414fcd5 | 303 | if (ret) |
3c0ffa27 | 304 | goto err_rpm; |
b414fcd5 | 305 | |
3c0ffa27 ML |
306 | /* Sanity check that we allow writing into this object */ |
307 | if (i915_gem_object_is_readonly(obj) && write) { | |
308 | ret = -EFAULT; | |
309 | goto err_rpm; | |
310 | } | |
b414fcd5 | 311 | |
3c0ffa27 | 312 | ret = i915_gem_object_pin_pages(obj); |
eebab60f | 313 | if (ret) |
b414fcd5 | 314 | goto err_rpm; |
b414fcd5 | 315 | |
3c0ffa27 ML |
316 | ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu); |
317 | if (ret) | |
318 | goto err_pages; | |
319 | ||
b414fcd5 | 320 | /* Now pin it into the GTT as needed */ |
3c0ffa27 ML |
321 | vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0, |
322 | PIN_MAPPABLE | | |
323 | PIN_NONBLOCK /* NOWARN */ | | |
324 | PIN_NOEVICT); | |
325 | if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { | |
b414fcd5 CW |
326 | /* Use a partial view if it is bigger than available space */ |
327 | struct i915_ggtt_view view = | |
328 | compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); | |
329 | unsigned int flags; | |
330 | ||
6846895f | 331 | flags = PIN_MAPPABLE | PIN_NOSEARCH; |
b414fcd5 CW |
332 | if (view.type == I915_GGTT_VIEW_NORMAL) |
333 | flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ | |
334 | ||
335 | /* | |
336 | * Userspace is now writing through an untracked VMA, abandon | |
337 | * all hope that the hardware is able to track future writes. | |
338 | */ | |
b414fcd5 | 339 | |
3c0ffa27 ML |
340 | vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); |
341 | if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { | |
b414fcd5 CW |
342 | flags = PIN_MAPPABLE; |
343 | view.type = I915_GGTT_VIEW_PARTIAL; | |
3c0ffa27 | 344 | vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); |
b414fcd5 | 345 | } |
8f9fb61c CW |
346 | |
347 | /* The entire mappable GGTT is pinned? Unexpected! */ | |
348 | GEM_BUG_ON(vma == ERR_PTR(-ENOSPC)); | |
b414fcd5 CW |
349 | } |
350 | if (IS_ERR(vma)) { | |
351 | ret = PTR_ERR(vma); | |
2850748e CW |
352 | goto err_reset; |
353 | } | |
354 | ||
355 | /* Access to snoopable pages through the GTT is incoherent. */ | |
356 | if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) { | |
357 | ret = -EFAULT; | |
358 | goto err_unpin; | |
b414fcd5 CW |
359 | } |
360 | ||
361 | ret = i915_vma_pin_fence(vma); | |
362 | if (ret) | |
363 | goto err_unpin; | |
364 | ||
365 | /* Finally, remap it using the new GTT offset */ | |
366 | ret = remap_io_mapping(area, | |
367 | area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), | |
368 | (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, | |
369 | min_t(u64, vma->size, area->vm_end - area->vm_start), | |
370 | &ggtt->iomap); | |
371 | if (ret) | |
372 | goto err_fence; | |
373 | ||
d858d569 | 374 | assert_rpm_wakelock_held(rpm); |
b7d151ba CW |
375 | |
376 | /* Mark as being mmapped into userspace for later revocation */ | |
377 | mutex_lock(&i915->ggtt.vm.mutex); | |
b414fcd5 | 378 | if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) |
0cf289bd | 379 | list_add(&obj->userfault_link, &i915->ggtt.userfault_list); |
b7d151ba CW |
380 | mutex_unlock(&i915->ggtt.vm.mutex); |
381 | ||
cc662126 AJ |
382 | /* Track the mmo associated with the fenced vma */ |
383 | vma->mmo = mmo; | |
384 | ||
babaab2f | 385 | if (IS_ACTIVE(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)) |
0cf289bd | 386 | intel_wakeref_auto(&i915->ggtt.userfault_wakeref, |
b414fcd5 | 387 | msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
b414fcd5 | 388 | |
5028851c CW |
389 | if (write) { |
390 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
391 | i915_vma_set_ggtt_write(vma); | |
392 | obj->mm.dirty = true; | |
393 | } | |
b414fcd5 CW |
394 | |
395 | err_fence: | |
396 | i915_vma_unpin_fence(vma); | |
397 | err_unpin: | |
398 | __i915_vma_unpin(vma); | |
b414fcd5 | 399 | err_reset: |
cb823ed9 | 400 | intel_gt_reset_unlock(ggtt->vm.gt, srcu); |
3c0ffa27 ML |
401 | err_pages: |
402 | i915_gem_object_unpin_pages(obj); | |
b414fcd5 | 403 | err_rpm: |
3c0ffa27 ML |
404 | if (ret == -EDEADLK) { |
405 | ret = i915_gem_ww_ctx_backoff(&ww); | |
406 | if (!ret) | |
407 | goto retry; | |
408 | } | |
409 | i915_gem_ww_ctx_fini(&ww); | |
d858d569 | 410 | intel_runtime_pm_put(rpm, wakeref); |
cc662126 | 411 | return i915_error_to_vmf_fault(ret); |
b414fcd5 CW |
412 | } |
413 | ||
9f909e21 CW |
414 | static int |
415 | vm_access(struct vm_area_struct *area, unsigned long addr, | |
416 | void *buf, int len, int write) | |
417 | { | |
418 | struct i915_mmap_offset *mmo = area->vm_private_data; | |
419 | struct drm_i915_gem_object *obj = mmo->obj; | |
420 | void *vaddr; | |
421 | ||
422 | if (i915_gem_object_is_readonly(obj) && write) | |
423 | return -EACCES; | |
424 | ||
425 | addr -= area->vm_start; | |
426 | if (addr >= obj->base.size) | |
427 | return -EINVAL; | |
428 | ||
429 | /* As this is primarily for debugging, let's focus on simplicity */ | |
430 | vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); | |
431 | if (IS_ERR(vaddr)) | |
432 | return PTR_ERR(vaddr); | |
433 | ||
434 | if (write) { | |
435 | memcpy(vaddr + addr, buf, len); | |
436 | __i915_gem_object_flush_map(obj, addr, len); | |
437 | } else { | |
438 | memcpy(buf, vaddr + addr, len); | |
439 | } | |
440 | ||
441 | i915_gem_object_unpin_map(obj); | |
442 | ||
443 | return len; | |
444 | } | |
445 | ||
cc662126 | 446 | void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
b414fcd5 CW |
447 | { |
448 | struct i915_vma *vma; | |
449 | ||
450 | GEM_BUG_ON(!obj->userfault_count); | |
451 | ||
b414fcd5 | 452 | for_each_ggtt_vma(vma, obj) |
cc662126 AJ |
453 | i915_vma_revoke_mmap(vma); |
454 | ||
455 | GEM_BUG_ON(obj->userfault_count); | |
b414fcd5 CW |
456 | } |
457 | ||
cc662126 | 458 | /* |
b414fcd5 CW |
459 | * It is vital that we remove the page mapping if we have mapped a tiled |
460 | * object through the GTT and then lose the fence register due to | |
461 | * resource pressure. Similarly if the object has been moved out of the | |
462 | * aperture, than pages mapped into userspace must be revoked. Removing the | |
463 | * mapping will then trigger a page fault on the next user access, allowing | |
cc662126 | 464 | * fixup by vm_fault_gtt(). |
b414fcd5 | 465 | */ |
cb2baf42 | 466 | void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
b414fcd5 CW |
467 | { |
468 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
469 | intel_wakeref_t wakeref; | |
470 | ||
cc662126 AJ |
471 | /* |
472 | * Serialisation between user GTT access and our code depends upon | |
b414fcd5 CW |
473 | * revoking the CPU's PTE whilst the mutex is held. The next user |
474 | * pagefault then has to wait until we release the mutex. | |
475 | * | |
476 | * Note that RPM complicates somewhat by adding an additional | |
477 | * requirement that operations to the GGTT be made holding the RPM | |
478 | * wakeref. | |
479 | */ | |
d858d569 | 480 | wakeref = intel_runtime_pm_get(&i915->runtime_pm); |
b7d151ba | 481 | mutex_lock(&i915->ggtt.vm.mutex); |
b414fcd5 CW |
482 | |
483 | if (!obj->userfault_count) | |
484 | goto out; | |
485 | ||
cc662126 | 486 | __i915_gem_object_release_mmap_gtt(obj); |
b414fcd5 | 487 | |
cc662126 AJ |
488 | /* |
489 | * Ensure that the CPU's PTE are revoked and there are not outstanding | |
b414fcd5 CW |
490 | * memory transactions from userspace before we return. The TLB |
491 | * flushing implied above by changing the PTE above *should* be | |
492 | * sufficient, an extra barrier here just provides us with a bit | |
493 | * of paranoid documentation about our requirement to serialise | |
494 | * memory writes before touching registers / GSM. | |
495 | */ | |
496 | wmb(); | |
497 | ||
498 | out: | |
b7d151ba | 499 | mutex_unlock(&i915->ggtt.vm.mutex); |
d858d569 | 500 | intel_runtime_pm_put(&i915->runtime_pm, wakeref); |
b414fcd5 CW |
501 | } |
502 | ||
cc662126 AJ |
503 | void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) |
504 | { | |
78655598 | 505 | struct i915_mmap_offset *mmo, *mn; |
cc662126 AJ |
506 | |
507 | spin_lock(&obj->mmo.lock); | |
78655598 CW |
508 | rbtree_postorder_for_each_entry_safe(mmo, mn, |
509 | &obj->mmo.offsets, offset) { | |
cc662126 AJ |
510 | /* |
511 | * vma_node_unmap for GTT mmaps handled already in | |
512 | * __i915_gem_object_release_mmap_gtt | |
513 | */ | |
514 | if (mmo->mmap_type == I915_MMAP_TYPE_GTT) | |
515 | continue; | |
516 | ||
517 | spin_unlock(&obj->mmo.lock); | |
518 | drm_vma_node_unmap(&mmo->vma_node, | |
519 | obj->base.dev->anon_inode->i_mapping); | |
520 | spin_lock(&obj->mmo.lock); | |
521 | } | |
522 | spin_unlock(&obj->mmo.lock); | |
523 | } | |
524 | ||
78655598 CW |
525 | static struct i915_mmap_offset * |
526 | lookup_mmo(struct drm_i915_gem_object *obj, | |
527 | enum i915_mmap_type mmap_type) | |
528 | { | |
529 | struct rb_node *rb; | |
530 | ||
531 | spin_lock(&obj->mmo.lock); | |
532 | rb = obj->mmo.offsets.rb_node; | |
533 | while (rb) { | |
534 | struct i915_mmap_offset *mmo = | |
535 | rb_entry(rb, typeof(*mmo), offset); | |
536 | ||
537 | if (mmo->mmap_type == mmap_type) { | |
538 | spin_unlock(&obj->mmo.lock); | |
539 | return mmo; | |
540 | } | |
541 | ||
542 | if (mmo->mmap_type < mmap_type) | |
543 | rb = rb->rb_right; | |
544 | else | |
545 | rb = rb->rb_left; | |
546 | } | |
547 | spin_unlock(&obj->mmo.lock); | |
548 | ||
549 | return NULL; | |
550 | } | |
551 | ||
552 | static struct i915_mmap_offset * | |
553 | insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo) | |
554 | { | |
555 | struct rb_node *rb, **p; | |
556 | ||
557 | spin_lock(&obj->mmo.lock); | |
558 | rb = NULL; | |
559 | p = &obj->mmo.offsets.rb_node; | |
560 | while (*p) { | |
561 | struct i915_mmap_offset *pos; | |
562 | ||
563 | rb = *p; | |
564 | pos = rb_entry(rb, typeof(*pos), offset); | |
565 | ||
566 | if (pos->mmap_type == mmo->mmap_type) { | |
567 | spin_unlock(&obj->mmo.lock); | |
568 | drm_vma_offset_remove(obj->base.dev->vma_offset_manager, | |
569 | &mmo->vma_node); | |
570 | kfree(mmo); | |
571 | return pos; | |
572 | } | |
573 | ||
574 | if (pos->mmap_type < mmo->mmap_type) | |
575 | p = &rb->rb_right; | |
576 | else | |
577 | p = &rb->rb_left; | |
578 | } | |
579 | rb_link_node(&mmo->offset, rb, p); | |
580 | rb_insert_color(&mmo->offset, &obj->mmo.offsets); | |
581 | spin_unlock(&obj->mmo.lock); | |
582 | ||
583 | return mmo; | |
584 | } | |
585 | ||
cc662126 AJ |
586 | static struct i915_mmap_offset * |
587 | mmap_offset_attach(struct drm_i915_gem_object *obj, | |
588 | enum i915_mmap_type mmap_type, | |
589 | struct drm_file *file) | |
b414fcd5 CW |
590 | { |
591 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
cc662126 | 592 | struct i915_mmap_offset *mmo; |
b414fcd5 CW |
593 | int err; |
594 | ||
78655598 CW |
595 | mmo = lookup_mmo(obj, mmap_type); |
596 | if (mmo) | |
597 | goto out; | |
598 | ||
cc662126 AJ |
599 | mmo = kmalloc(sizeof(*mmo), GFP_KERNEL); |
600 | if (!mmo) | |
601 | return ERR_PTR(-ENOMEM); | |
602 | ||
603 | mmo->obj = obj; | |
cc662126 AJ |
604 | mmo->mmap_type = mmap_type; |
605 | drm_vma_node_reset(&mmo->vma_node); | |
606 | ||
78655598 CW |
607 | err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, |
608 | &mmo->vma_node, obj->base.size / PAGE_SIZE); | |
b414fcd5 | 609 | if (likely(!err)) |
78655598 | 610 | goto insert; |
b414fcd5 CW |
611 | |
612 | /* Attempt to reap some mmap space from dead objects */ | |
cc662126 | 613 | err = intel_gt_retire_requests_timeout(&i915->gt, MAX_SCHEDULE_TIMEOUT); |
789ed955 | 614 | if (err) |
cc662126 | 615 | goto err; |
b414fcd5 | 616 | |
789ed955 | 617 | i915_gem_drain_freed_objects(i915); |
78655598 CW |
618 | err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, |
619 | &mmo->vma_node, obj->base.size / PAGE_SIZE); | |
cc662126 AJ |
620 | if (err) |
621 | goto err; | |
622 | ||
78655598 CW |
623 | insert: |
624 | mmo = insert_mmo(obj, mmo); | |
625 | GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo); | |
cc662126 AJ |
626 | out: |
627 | if (file) | |
628 | drm_vma_node_allow(&mmo->vma_node, file); | |
cc662126 AJ |
629 | return mmo; |
630 | ||
631 | err: | |
632 | kfree(mmo); | |
633 | return ERR_PTR(err); | |
b414fcd5 CW |
634 | } |
635 | ||
cc662126 AJ |
636 | static int |
637 | __assign_mmap_offset(struct drm_file *file, | |
638 | u32 handle, | |
639 | enum i915_mmap_type mmap_type, | |
640 | u64 *offset) | |
b414fcd5 CW |
641 | { |
642 | struct drm_i915_gem_object *obj; | |
cc662126 AJ |
643 | struct i915_mmap_offset *mmo; |
644 | int err; | |
bee0a70a | 645 | |
b414fcd5 CW |
646 | obj = i915_gem_object_lookup(file, handle); |
647 | if (!obj) | |
648 | return -ENOENT; | |
649 | ||
f6c26b55 | 650 | if (i915_gem_object_never_mmap(obj)) { |
cc662126 | 651 | err = -ENODEV; |
a4311745 CW |
652 | goto out; |
653 | } | |
654 | ||
cc662126 | 655 | if (mmap_type != I915_MMAP_TYPE_GTT && |
4e598fad AJ |
656 | !i915_gem_object_type_has(obj, |
657 | I915_GEM_OBJECT_HAS_STRUCT_PAGE | | |
658 | I915_GEM_OBJECT_HAS_IOMEM)) { | |
cc662126 AJ |
659 | err = -ENODEV; |
660 | goto out; | |
661 | } | |
662 | ||
663 | mmo = mmap_offset_attach(obj, mmap_type, file); | |
664 | if (IS_ERR(mmo)) { | |
665 | err = PTR_ERR(mmo); | |
666 | goto out; | |
667 | } | |
b414fcd5 | 668 | |
cc662126 AJ |
669 | *offset = drm_vma_node_offset_addr(&mmo->vma_node); |
670 | err = 0; | |
a4311745 | 671 | out: |
b414fcd5 | 672 | i915_gem_object_put(obj); |
cc662126 AJ |
673 | return err; |
674 | } | |
675 | ||
676 | int | |
677 | i915_gem_dumb_mmap_offset(struct drm_file *file, | |
678 | struct drm_device *dev, | |
679 | u32 handle, | |
680 | u64 *offset) | |
681 | { | |
682 | enum i915_mmap_type mmap_type; | |
683 | ||
684 | if (boot_cpu_has(X86_FEATURE_PAT)) | |
685 | mmap_type = I915_MMAP_TYPE_WC; | |
686 | else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt)) | |
687 | return -ENODEV; | |
688 | else | |
689 | mmap_type = I915_MMAP_TYPE_GTT; | |
690 | ||
691 | return __assign_mmap_offset(file, handle, mmap_type, offset); | |
b414fcd5 CW |
692 | } |
693 | ||
694 | /** | |
cc662126 | 695 | * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing |
b414fcd5 CW |
696 | * @dev: DRM device |
697 | * @data: GTT mapping ioctl data | |
698 | * @file: GEM object info | |
699 | * | |
700 | * Simply returns the fake offset to userspace so it can mmap it. | |
701 | * The mmap call will end up in drm_gem_mmap(), which will set things | |
702 | * up so we can get faults in the handler above. | |
703 | * | |
704 | * The fault handler will take care of binding the object into the GTT | |
705 | * (since it may have been evicted to make room for something), allocating | |
706 | * a fence register, and mapping the appropriate aperture address into | |
707 | * userspace. | |
708 | */ | |
709 | int | |
cc662126 AJ |
710 | i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, |
711 | struct drm_file *file) | |
b414fcd5 | 712 | { |
cc662126 AJ |
713 | struct drm_i915_private *i915 = to_i915(dev); |
714 | struct drm_i915_gem_mmap_offset *args = data; | |
715 | enum i915_mmap_type type; | |
126d5de3 | 716 | int err; |
cc662126 | 717 | |
8d65859a CW |
718 | /* |
719 | * Historically we failed to check args.pad and args.offset | |
720 | * and so we cannot use those fields for user input and we cannot | |
721 | * add -EINVAL for them as the ABI is fixed, i.e. old userspace | |
722 | * may be feeding in garbage in those fields. | |
723 | * | |
724 | * if (args->pad) return -EINVAL; is verbotten! | |
725 | */ | |
726 | ||
126d5de3 CW |
727 | err = i915_user_extensions(u64_to_user_ptr(args->extensions), |
728 | NULL, 0, NULL); | |
729 | if (err) | |
730 | return err; | |
cc662126 AJ |
731 | |
732 | switch (args->flags) { | |
733 | case I915_MMAP_OFFSET_GTT: | |
734 | if (!i915_ggtt_has_aperture(&i915->ggtt)) | |
735 | return -ENODEV; | |
736 | type = I915_MMAP_TYPE_GTT; | |
737 | break; | |
738 | ||
739 | case I915_MMAP_OFFSET_WC: | |
740 | if (!boot_cpu_has(X86_FEATURE_PAT)) | |
741 | return -ENODEV; | |
742 | type = I915_MMAP_TYPE_WC; | |
743 | break; | |
744 | ||
745 | case I915_MMAP_OFFSET_WB: | |
746 | type = I915_MMAP_TYPE_WB; | |
747 | break; | |
748 | ||
749 | case I915_MMAP_OFFSET_UC: | |
750 | if (!boot_cpu_has(X86_FEATURE_PAT)) | |
751 | return -ENODEV; | |
752 | type = I915_MMAP_TYPE_UC; | |
753 | break; | |
754 | ||
755 | default: | |
756 | return -EINVAL; | |
757 | } | |
b414fcd5 | 758 | |
cc662126 AJ |
759 | return __assign_mmap_offset(file, args->handle, type, &args->offset); |
760 | } | |
761 | ||
762 | static void vm_open(struct vm_area_struct *vma) | |
763 | { | |
764 | struct i915_mmap_offset *mmo = vma->vm_private_data; | |
765 | struct drm_i915_gem_object *obj = mmo->obj; | |
766 | ||
767 | GEM_BUG_ON(!obj); | |
768 | i915_gem_object_get(obj); | |
769 | } | |
770 | ||
771 | static void vm_close(struct vm_area_struct *vma) | |
772 | { | |
773 | struct i915_mmap_offset *mmo = vma->vm_private_data; | |
774 | struct drm_i915_gem_object *obj = mmo->obj; | |
775 | ||
776 | GEM_BUG_ON(!obj); | |
777 | i915_gem_object_put(obj); | |
778 | } | |
779 | ||
780 | static const struct vm_operations_struct vm_ops_gtt = { | |
781 | .fault = vm_fault_gtt, | |
9f909e21 | 782 | .access = vm_access, |
cc662126 AJ |
783 | .open = vm_open, |
784 | .close = vm_close, | |
785 | }; | |
786 | ||
787 | static const struct vm_operations_struct vm_ops_cpu = { | |
788 | .fault = vm_fault_cpu, | |
9f909e21 | 789 | .access = vm_access, |
cc662126 AJ |
790 | .open = vm_open, |
791 | .close = vm_close, | |
792 | }; | |
793 | ||
f17b8980 CW |
794 | static int singleton_release(struct inode *inode, struct file *file) |
795 | { | |
796 | struct drm_i915_private *i915 = file->private_data; | |
797 | ||
798 | cmpxchg(&i915->gem.mmap_singleton, file, NULL); | |
799 | drm_dev_put(&i915->drm); | |
800 | ||
801 | return 0; | |
802 | } | |
803 | ||
804 | static const struct file_operations singleton_fops = { | |
805 | .owner = THIS_MODULE, | |
806 | .release = singleton_release, | |
807 | }; | |
808 | ||
809 | static struct file *mmap_singleton(struct drm_i915_private *i915) | |
810 | { | |
811 | struct file *file; | |
812 | ||
813 | rcu_read_lock(); | |
4aea5a9e | 814 | file = READ_ONCE(i915->gem.mmap_singleton); |
f17b8980 CW |
815 | if (file && !get_file_rcu(file)) |
816 | file = NULL; | |
817 | rcu_read_unlock(); | |
818 | if (file) | |
819 | return file; | |
820 | ||
821 | file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR); | |
822 | if (IS_ERR(file)) | |
823 | return file; | |
824 | ||
825 | /* Everyone shares a single global address space */ | |
826 | file->f_mapping = i915->drm.anon_inode->i_mapping; | |
827 | ||
828 | smp_store_mb(i915->gem.mmap_singleton, file); | |
829 | drm_dev_get(&i915->drm); | |
830 | ||
831 | return file; | |
832 | } | |
833 | ||
cc662126 AJ |
834 | /* |
835 | * This overcomes the limitation in drm_gem_mmap's assignment of a | |
836 | * drm_gem_object as the vma->vm_private_data. Since we need to | |
837 | * be able to resolve multiple mmap offsets which could be tied | |
838 | * to a single gem object. | |
839 | */ | |
840 | int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) | |
841 | { | |
842 | struct drm_vma_offset_node *node; | |
843 | struct drm_file *priv = filp->private_data; | |
844 | struct drm_device *dev = priv->minor->dev; | |
280d14a6 | 845 | struct drm_i915_gem_object *obj = NULL; |
cc662126 | 846 | struct i915_mmap_offset *mmo = NULL; |
f17b8980 | 847 | struct file *anon; |
cc662126 AJ |
848 | |
849 | if (drm_dev_is_unplugged(dev)) | |
850 | return -ENODEV; | |
851 | ||
280d14a6 | 852 | rcu_read_lock(); |
cc662126 AJ |
853 | drm_vma_offset_lock_lookup(dev->vma_offset_manager); |
854 | node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, | |
855 | vma->vm_pgoff, | |
856 | vma_pages(vma)); | |
280d14a6 | 857 | if (node && drm_vma_node_is_allowed(node, priv)) { |
cc662126 AJ |
858 | /* |
859 | * Skip 0-refcnted objects as it is in the process of being | |
860 | * destroyed and will be invalid when the vma manager lock | |
861 | * is released. | |
862 | */ | |
280d14a6 CW |
863 | mmo = container_of(node, struct i915_mmap_offset, vma_node); |
864 | obj = i915_gem_object_get_rcu(mmo->obj); | |
cc662126 AJ |
865 | } |
866 | drm_vma_offset_unlock_lookup(dev->vma_offset_manager); | |
280d14a6 | 867 | rcu_read_unlock(); |
cc662126 | 868 | if (!obj) |
280d14a6 | 869 | return node ? -EACCES : -EINVAL; |
cc662126 | 870 | |
280d14a6 | 871 | if (i915_gem_object_is_readonly(obj)) { |
cc662126 | 872 | if (vma->vm_flags & VM_WRITE) { |
280d14a6 | 873 | i915_gem_object_put(obj); |
cc662126 AJ |
874 | return -EINVAL; |
875 | } | |
876 | vma->vm_flags &= ~VM_MAYWRITE; | |
877 | } | |
878 | ||
280d14a6 | 879 | anon = mmap_singleton(to_i915(dev)); |
f17b8980 | 880 | if (IS_ERR(anon)) { |
280d14a6 | 881 | i915_gem_object_put(obj); |
f17b8980 CW |
882 | return PTR_ERR(anon); |
883 | } | |
884 | ||
cc662126 AJ |
885 | vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; |
886 | vma->vm_private_data = mmo; | |
887 | ||
f17b8980 CW |
888 | /* |
889 | * We keep the ref on mmo->obj, not vm_file, but we require | |
890 | * vma->vm_file->f_mapping, see vma_link(), for later revocation. | |
891 | * Our userspace is accustomed to having per-file resource cleanup | |
892 | * (i.e. contexts, objects and requests) on their close(fd), which | |
893 | * requires avoiding extraneous references to their filp, hence why | |
894 | * we prefer to use an anonymous file for their mmaps. | |
895 | */ | |
295992fb CK |
896 | vma_set_file(vma, anon); |
897 | /* Drop the initial creation reference, the vma is now holding one. */ | |
898 | fput(anon); | |
f17b8980 | 899 | |
cc662126 AJ |
900 | switch (mmo->mmap_type) { |
901 | case I915_MMAP_TYPE_WC: | |
902 | vma->vm_page_prot = | |
903 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
904 | vma->vm_ops = &vm_ops_cpu; | |
905 | break; | |
906 | ||
907 | case I915_MMAP_TYPE_WB: | |
908 | vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); | |
909 | vma->vm_ops = &vm_ops_cpu; | |
910 | break; | |
911 | ||
912 | case I915_MMAP_TYPE_UC: | |
913 | vma->vm_page_prot = | |
914 | pgprot_noncached(vm_get_page_prot(vma->vm_flags)); | |
915 | vma->vm_ops = &vm_ops_cpu; | |
916 | break; | |
917 | ||
918 | case I915_MMAP_TYPE_GTT: | |
919 | vma->vm_page_prot = | |
920 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
921 | vma->vm_ops = &vm_ops_gtt; | |
922 | break; | |
923 | } | |
924 | vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); | |
925 | ||
926 | return 0; | |
b414fcd5 CW |
927 | } |
928 | ||
929 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
930 | #include "selftests/i915_gem_mman.c" | |
931 | #endif |