Commit | Line | Data |
---|---|---|
b414fcd5 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2014-2016 Intel Corporation | |
5 | */ | |
6 | ||
f17b8980 | 7 | #include <linux/anon_inodes.h> |
b414fcd5 | 8 | #include <linux/mman.h> |
cc662126 | 9 | #include <linux/pfn_t.h> |
b414fcd5 CW |
10 | #include <linux/sizes.h> |
11 | ||
cb823ed9 | 12 | #include "gt/intel_gt.h" |
66101975 | 13 | #include "gt/intel_gt_requests.h" |
cb823ed9 | 14 | |
b414fcd5 CW |
15 | #include "i915_drv.h" |
16 | #include "i915_gem_gtt.h" | |
17 | #include "i915_gem_ioctls.h" | |
18 | #include "i915_gem_object.h" | |
cc662126 | 19 | #include "i915_gem_mman.h" |
a09d9a80 | 20 | #include "i915_trace.h" |
126d5de3 | 21 | #include "i915_user_extensions.h" |
b414fcd5 | 22 | #include "i915_vma.h" |
b414fcd5 CW |
23 | |
24 | static inline bool | |
25 | __vma_matches(struct vm_area_struct *vma, struct file *filp, | |
26 | unsigned long addr, unsigned long size) | |
27 | { | |
28 | if (vma->vm_file != filp) | |
29 | return false; | |
30 | ||
31 | return vma->vm_start == addr && | |
32 | (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); | |
33 | } | |
34 | ||
35 | /** | |
36 | * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address | |
37 | * it is mapped to. | |
38 | * @dev: drm device | |
39 | * @data: ioctl data blob | |
40 | * @file: drm file | |
41 | * | |
42 | * While the mapping holds a reference on the contents of the object, it doesn't | |
43 | * imply a ref on the object itself. | |
44 | * | |
45 | * IMPORTANT: | |
46 | * | |
47 | * DRM driver writers who look a this function as an example for how to do GEM | |
48 | * mmap support, please don't implement mmap support like here. The modern way | |
49 | * to implement DRM mmap support is with an mmap offset ioctl (like | |
50 | * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. | |
51 | * That way debug tooling like valgrind will understand what's going on, hiding | |
52 | * the mmap call in a driver private ioctl will break that. The i915 driver only | |
53 | * does cpu mmaps this way because we didn't know better. | |
54 | */ | |
55 | int | |
56 | i915_gem_mmap_ioctl(struct drm_device *dev, void *data, | |
57 | struct drm_file *file) | |
58 | { | |
59 | struct drm_i915_gem_mmap *args = data; | |
60 | struct drm_i915_gem_object *obj; | |
61 | unsigned long addr; | |
62 | ||
63 | if (args->flags & ~(I915_MMAP_WC)) | |
64 | return -EINVAL; | |
65 | ||
66 | if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT)) | |
67 | return -ENODEV; | |
68 | ||
69 | obj = i915_gem_object_lookup(file, args->handle); | |
70 | if (!obj) | |
71 | return -ENOENT; | |
72 | ||
73 | /* prime objects have no backing filp to GEM mmap | |
74 | * pages from. | |
75 | */ | |
76 | if (!obj->base.filp) { | |
77 | addr = -ENXIO; | |
78 | goto err; | |
79 | } | |
80 | ||
81 | if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { | |
82 | addr = -EINVAL; | |
83 | goto err; | |
84 | } | |
85 | ||
86 | addr = vm_mmap(obj->base.filp, 0, args->size, | |
87 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
88 | args->offset); | |
89 | if (IS_ERR_VALUE(addr)) | |
90 | goto err; | |
91 | ||
92 | if (args->flags & I915_MMAP_WC) { | |
93 | struct mm_struct *mm = current->mm; | |
94 | struct vm_area_struct *vma; | |
95 | ||
d8ed45c5 | 96 | if (mmap_write_lock_killable(mm)) { |
b414fcd5 CW |
97 | addr = -EINTR; |
98 | goto err; | |
99 | } | |
100 | vma = find_vma(mm, addr); | |
101 | if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) | |
102 | vma->vm_page_prot = | |
103 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
104 | else | |
105 | addr = -ENOMEM; | |
d8ed45c5 | 106 | mmap_write_unlock(mm); |
b414fcd5 CW |
107 | if (IS_ERR_VALUE(addr)) |
108 | goto err; | |
b414fcd5 CW |
109 | } |
110 | i915_gem_object_put(obj); | |
111 | ||
112 | args->addr_ptr = (u64)addr; | |
113 | return 0; | |
114 | ||
115 | err: | |
116 | i915_gem_object_put(obj); | |
117 | return addr; | |
118 | } | |
119 | ||
120 | static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) | |
121 | { | |
122 | return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; | |
123 | } | |
124 | ||
125 | /** | |
126 | * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps | |
127 | * | |
128 | * A history of the GTT mmap interface: | |
129 | * | |
130 | * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to | |
131 | * aligned and suitable for fencing, and still fit into the available | |
132 | * mappable space left by the pinned display objects. A classic problem | |
133 | * we called the page-fault-of-doom where we would ping-pong between | |
134 | * two objects that could not fit inside the GTT and so the memcpy | |
135 | * would page one object in at the expense of the other between every | |
136 | * single byte. | |
137 | * | |
138 | * 1 - Objects can be any size, and have any compatible fencing (X Y, or none | |
139 | * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the | |
140 | * object is too large for the available space (or simply too large | |
141 | * for the mappable aperture!), a view is created instead and faulted | |
142 | * into userspace. (This view is aligned and sized appropriately for | |
143 | * fenced access.) | |
144 | * | |
145 | * 2 - Recognise WC as a separate cache domain so that we can flush the | |
146 | * delayed writes via GTT before performing direct access via WC. | |
147 | * | |
148 | * 3 - Remove implicit set-domain(GTT) and synchronisation on initial | |
149 | * pagefault; swapin remains transparent. | |
150 | * | |
cc662126 AJ |
151 | * 4 - Support multiple fault handlers per object depending on object's |
152 | * backing storage (a.k.a. MMAP_OFFSET). | |
153 | * | |
b414fcd5 CW |
154 | * Restrictions: |
155 | * | |
156 | * * snoopable objects cannot be accessed via the GTT. It can cause machine | |
157 | * hangs on some architectures, corruption on others. An attempt to service | |
158 | * a GTT page fault from a snoopable object will generate a SIGBUS. | |
159 | * | |
160 | * * the object must be able to fit into RAM (physical memory, though no | |
161 | * limited to the mappable aperture). | |
162 | * | |
163 | * | |
164 | * Caveats: | |
165 | * | |
166 | * * a new GTT page fault will synchronize rendering from the GPU and flush | |
167 | * all data to system memory. Subsequent access will not be synchronized. | |
168 | * | |
169 | * * all mappings are revoked on runtime device suspend. | |
170 | * | |
171 | * * there are only 8, 16 or 32 fence registers to share between all users | |
172 | * (older machines require fence register for display and blitter access | |
173 | * as well). Contention of the fence registers will cause the previous users | |
174 | * to be unmapped and any new access will generate new page faults. | |
175 | * | |
176 | * * running out of memory while servicing a fault may generate a SIGBUS, | |
177 | * rather than the expected SIGSEGV. | |
178 | */ | |
179 | int i915_gem_mmap_gtt_version(void) | |
180 | { | |
cc662126 | 181 | return 4; |
b414fcd5 CW |
182 | } |
183 | ||
184 | static inline struct i915_ggtt_view | |
185 | compute_partial_view(const struct drm_i915_gem_object *obj, | |
186 | pgoff_t page_offset, | |
187 | unsigned int chunk) | |
188 | { | |
189 | struct i915_ggtt_view view; | |
190 | ||
191 | if (i915_gem_object_is_tiled(obj)) | |
192 | chunk = roundup(chunk, tile_row_pages(obj)); | |
193 | ||
194 | view.type = I915_GGTT_VIEW_PARTIAL; | |
195 | view.partial.offset = rounddown(page_offset, chunk); | |
196 | view.partial.size = | |
197 | min_t(unsigned int, chunk, | |
198 | (obj->base.size >> PAGE_SHIFT) - view.partial.offset); | |
199 | ||
200 | /* If the partial covers the entire object, just create a normal VMA. */ | |
201 | if (chunk >= obj->base.size >> PAGE_SHIFT) | |
202 | view.type = I915_GGTT_VIEW_NORMAL; | |
203 | ||
204 | return view; | |
205 | } | |
206 | ||
cc662126 AJ |
207 | static vm_fault_t i915_error_to_vmf_fault(int err) |
208 | { | |
209 | switch (err) { | |
210 | default: | |
211 | WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err); | |
212 | /* fallthrough */ | |
213 | case -EIO: /* shmemfs failure from swap device */ | |
214 | case -EFAULT: /* purged object */ | |
215 | case -ENODEV: /* bad object, how did you get here! */ | |
4e598fad | 216 | case -ENXIO: /* unable to access backing store (on device) */ |
cc662126 AJ |
217 | return VM_FAULT_SIGBUS; |
218 | ||
219 | case -ENOSPC: /* shmemfs allocation failure */ | |
220 | case -ENOMEM: /* our allocation failure */ | |
221 | return VM_FAULT_OOM; | |
222 | ||
223 | case 0: | |
224 | case -EAGAIN: | |
225 | case -ERESTARTSYS: | |
226 | case -EINTR: | |
227 | case -EBUSY: | |
228 | /* | |
229 | * EBUSY is ok: this just means that another thread | |
230 | * already did the job. | |
231 | */ | |
232 | return VM_FAULT_NOPAGE; | |
233 | } | |
234 | } | |
235 | ||
236 | static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) | |
237 | { | |
238 | struct vm_area_struct *area = vmf->vma; | |
239 | struct i915_mmap_offset *mmo = area->vm_private_data; | |
240 | struct drm_i915_gem_object *obj = mmo->obj; | |
4e598fad | 241 | resource_size_t iomap; |
cc662126 AJ |
242 | int err; |
243 | ||
cc662126 | 244 | /* Sanity check that we allow writing into this object */ |
1764b992 AJ |
245 | if (unlikely(i915_gem_object_is_readonly(obj) && |
246 | area->vm_flags & VM_WRITE)) | |
247 | return VM_FAULT_SIGBUS; | |
cc662126 AJ |
248 | |
249 | err = i915_gem_object_pin_pages(obj); | |
250 | if (err) | |
1764b992 | 251 | goto out; |
cc662126 | 252 | |
4e598fad AJ |
253 | iomap = -1; |
254 | if (!i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE)) { | |
255 | iomap = obj->mm.region->iomap.base; | |
256 | iomap -= obj->mm.region->region.start; | |
257 | } | |
258 | ||
cc662126 | 259 | /* PTEs are revoked in obj->ops->put_pages() */ |
4e598fad AJ |
260 | err = remap_io_sg(area, |
261 | area->vm_start, area->vm_end - area->vm_start, | |
262 | obj->mm.pages->sgl, iomap); | |
cc662126 | 263 | |
1764b992 | 264 | if (area->vm_flags & VM_WRITE) { |
cc662126 | 265 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); |
cc662126 AJ |
266 | obj->mm.dirty = true; |
267 | } | |
268 | ||
269 | i915_gem_object_unpin_pages(obj); | |
270 | ||
1764b992 AJ |
271 | out: |
272 | return i915_error_to_vmf_fault(err); | |
cc662126 AJ |
273 | } |
274 | ||
275 | static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) | |
b414fcd5 CW |
276 | { |
277 | #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) | |
278 | struct vm_area_struct *area = vmf->vma; | |
cc662126 AJ |
279 | struct i915_mmap_offset *mmo = area->vm_private_data; |
280 | struct drm_i915_gem_object *obj = mmo->obj; | |
b414fcd5 CW |
281 | struct drm_device *dev = obj->base.dev; |
282 | struct drm_i915_private *i915 = to_i915(dev); | |
d858d569 | 283 | struct intel_runtime_pm *rpm = &i915->runtime_pm; |
b414fcd5 CW |
284 | struct i915_ggtt *ggtt = &i915->ggtt; |
285 | bool write = area->vm_flags & VM_WRITE; | |
286 | intel_wakeref_t wakeref; | |
287 | struct i915_vma *vma; | |
288 | pgoff_t page_offset; | |
289 | int srcu; | |
290 | int ret; | |
291 | ||
292 | /* Sanity check that we allow writing into this object */ | |
293 | if (i915_gem_object_is_readonly(obj) && write) | |
294 | return VM_FAULT_SIGBUS; | |
295 | ||
296 | /* We don't use vmf->pgoff since that has the fake offset */ | |
297 | page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; | |
298 | ||
299 | trace_i915_gem_object_fault(obj, page_offset, true, write); | |
300 | ||
301 | ret = i915_gem_object_pin_pages(obj); | |
302 | if (ret) | |
303 | goto err; | |
304 | ||
d858d569 | 305 | wakeref = intel_runtime_pm_get(rpm); |
b414fcd5 | 306 | |
eebab60f CW |
307 | ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu); |
308 | if (ret) | |
b414fcd5 | 309 | goto err_rpm; |
b414fcd5 | 310 | |
b414fcd5 CW |
311 | /* Now pin it into the GTT as needed */ |
312 | vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, | |
313 | PIN_MAPPABLE | | |
6846895f | 314 | PIN_NONBLOCK /* NOWARN */ | |
ebfdf5cd | 315 | PIN_NOEVICT); |
b414fcd5 CW |
316 | if (IS_ERR(vma)) { |
317 | /* Use a partial view if it is bigger than available space */ | |
318 | struct i915_ggtt_view view = | |
319 | compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); | |
320 | unsigned int flags; | |
321 | ||
6846895f | 322 | flags = PIN_MAPPABLE | PIN_NOSEARCH; |
b414fcd5 CW |
323 | if (view.type == I915_GGTT_VIEW_NORMAL) |
324 | flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ | |
325 | ||
326 | /* | |
327 | * Userspace is now writing through an untracked VMA, abandon | |
328 | * all hope that the hardware is able to track future writes. | |
329 | */ | |
b414fcd5 CW |
330 | |
331 | vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); | |
6846895f | 332 | if (IS_ERR(vma)) { |
b414fcd5 CW |
333 | flags = PIN_MAPPABLE; |
334 | view.type = I915_GGTT_VIEW_PARTIAL; | |
335 | vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags); | |
336 | } | |
8f9fb61c CW |
337 | |
338 | /* The entire mappable GGTT is pinned? Unexpected! */ | |
339 | GEM_BUG_ON(vma == ERR_PTR(-ENOSPC)); | |
b414fcd5 CW |
340 | } |
341 | if (IS_ERR(vma)) { | |
342 | ret = PTR_ERR(vma); | |
2850748e CW |
343 | goto err_reset; |
344 | } | |
345 | ||
346 | /* Access to snoopable pages through the GTT is incoherent. */ | |
347 | if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) { | |
348 | ret = -EFAULT; | |
349 | goto err_unpin; | |
b414fcd5 CW |
350 | } |
351 | ||
352 | ret = i915_vma_pin_fence(vma); | |
353 | if (ret) | |
354 | goto err_unpin; | |
355 | ||
356 | /* Finally, remap it using the new GTT offset */ | |
357 | ret = remap_io_mapping(area, | |
358 | area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT), | |
359 | (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT, | |
360 | min_t(u64, vma->size, area->vm_end - area->vm_start), | |
361 | &ggtt->iomap); | |
362 | if (ret) | |
363 | goto err_fence; | |
364 | ||
d858d569 | 365 | assert_rpm_wakelock_held(rpm); |
b7d151ba CW |
366 | |
367 | /* Mark as being mmapped into userspace for later revocation */ | |
368 | mutex_lock(&i915->ggtt.vm.mutex); | |
b414fcd5 | 369 | if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) |
0cf289bd | 370 | list_add(&obj->userfault_link, &i915->ggtt.userfault_list); |
b7d151ba CW |
371 | mutex_unlock(&i915->ggtt.vm.mutex); |
372 | ||
cc662126 AJ |
373 | /* Track the mmo associated with the fenced vma */ |
374 | vma->mmo = mmo; | |
375 | ||
babaab2f | 376 | if (IS_ACTIVE(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)) |
0cf289bd | 377 | intel_wakeref_auto(&i915->ggtt.userfault_wakeref, |
b414fcd5 | 378 | msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); |
b414fcd5 | 379 | |
5028851c CW |
380 | if (write) { |
381 | GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); | |
382 | i915_vma_set_ggtt_write(vma); | |
383 | obj->mm.dirty = true; | |
384 | } | |
b414fcd5 CW |
385 | |
386 | err_fence: | |
387 | i915_vma_unpin_fence(vma); | |
388 | err_unpin: | |
389 | __i915_vma_unpin(vma); | |
b414fcd5 | 390 | err_reset: |
cb823ed9 | 391 | intel_gt_reset_unlock(ggtt->vm.gt, srcu); |
b414fcd5 | 392 | err_rpm: |
d858d569 | 393 | intel_runtime_pm_put(rpm, wakeref); |
b414fcd5 CW |
394 | i915_gem_object_unpin_pages(obj); |
395 | err: | |
cc662126 | 396 | return i915_error_to_vmf_fault(ret); |
b414fcd5 CW |
397 | } |
398 | ||
9f909e21 CW |
399 | static int |
400 | vm_access(struct vm_area_struct *area, unsigned long addr, | |
401 | void *buf, int len, int write) | |
402 | { | |
403 | struct i915_mmap_offset *mmo = area->vm_private_data; | |
404 | struct drm_i915_gem_object *obj = mmo->obj; | |
405 | void *vaddr; | |
406 | ||
407 | if (i915_gem_object_is_readonly(obj) && write) | |
408 | return -EACCES; | |
409 | ||
410 | addr -= area->vm_start; | |
411 | if (addr >= obj->base.size) | |
412 | return -EINVAL; | |
413 | ||
414 | /* As this is primarily for debugging, let's focus on simplicity */ | |
415 | vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); | |
416 | if (IS_ERR(vaddr)) | |
417 | return PTR_ERR(vaddr); | |
418 | ||
419 | if (write) { | |
420 | memcpy(vaddr + addr, buf, len); | |
421 | __i915_gem_object_flush_map(obj, addr, len); | |
422 | } else { | |
423 | memcpy(buf, vaddr + addr, len); | |
424 | } | |
425 | ||
426 | i915_gem_object_unpin_map(obj); | |
427 | ||
428 | return len; | |
429 | } | |
430 | ||
cc662126 | 431 | void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
b414fcd5 CW |
432 | { |
433 | struct i915_vma *vma; | |
434 | ||
435 | GEM_BUG_ON(!obj->userfault_count); | |
436 | ||
b414fcd5 | 437 | for_each_ggtt_vma(vma, obj) |
cc662126 AJ |
438 | i915_vma_revoke_mmap(vma); |
439 | ||
440 | GEM_BUG_ON(obj->userfault_count); | |
b414fcd5 CW |
441 | } |
442 | ||
cc662126 | 443 | /* |
b414fcd5 CW |
444 | * It is vital that we remove the page mapping if we have mapped a tiled |
445 | * object through the GTT and then lose the fence register due to | |
446 | * resource pressure. Similarly if the object has been moved out of the | |
447 | * aperture, than pages mapped into userspace must be revoked. Removing the | |
448 | * mapping will then trigger a page fault on the next user access, allowing | |
cc662126 | 449 | * fixup by vm_fault_gtt(). |
b414fcd5 | 450 | */ |
cc662126 | 451 | static void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) |
b414fcd5 CW |
452 | { |
453 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
454 | intel_wakeref_t wakeref; | |
455 | ||
cc662126 AJ |
456 | /* |
457 | * Serialisation between user GTT access and our code depends upon | |
b414fcd5 CW |
458 | * revoking the CPU's PTE whilst the mutex is held. The next user |
459 | * pagefault then has to wait until we release the mutex. | |
460 | * | |
461 | * Note that RPM complicates somewhat by adding an additional | |
462 | * requirement that operations to the GGTT be made holding the RPM | |
463 | * wakeref. | |
464 | */ | |
d858d569 | 465 | wakeref = intel_runtime_pm_get(&i915->runtime_pm); |
b7d151ba | 466 | mutex_lock(&i915->ggtt.vm.mutex); |
b414fcd5 CW |
467 | |
468 | if (!obj->userfault_count) | |
469 | goto out; | |
470 | ||
cc662126 | 471 | __i915_gem_object_release_mmap_gtt(obj); |
b414fcd5 | 472 | |
cc662126 AJ |
473 | /* |
474 | * Ensure that the CPU's PTE are revoked and there are not outstanding | |
b414fcd5 CW |
475 | * memory transactions from userspace before we return. The TLB |
476 | * flushing implied above by changing the PTE above *should* be | |
477 | * sufficient, an extra barrier here just provides us with a bit | |
478 | * of paranoid documentation about our requirement to serialise | |
479 | * memory writes before touching registers / GSM. | |
480 | */ | |
481 | wmb(); | |
482 | ||
483 | out: | |
b7d151ba | 484 | mutex_unlock(&i915->ggtt.vm.mutex); |
d858d569 | 485 | intel_runtime_pm_put(&i915->runtime_pm, wakeref); |
b414fcd5 CW |
486 | } |
487 | ||
cc662126 AJ |
488 | void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) |
489 | { | |
78655598 | 490 | struct i915_mmap_offset *mmo, *mn; |
cc662126 AJ |
491 | |
492 | spin_lock(&obj->mmo.lock); | |
78655598 CW |
493 | rbtree_postorder_for_each_entry_safe(mmo, mn, |
494 | &obj->mmo.offsets, offset) { | |
cc662126 AJ |
495 | /* |
496 | * vma_node_unmap for GTT mmaps handled already in | |
497 | * __i915_gem_object_release_mmap_gtt | |
498 | */ | |
499 | if (mmo->mmap_type == I915_MMAP_TYPE_GTT) | |
500 | continue; | |
501 | ||
502 | spin_unlock(&obj->mmo.lock); | |
503 | drm_vma_node_unmap(&mmo->vma_node, | |
504 | obj->base.dev->anon_inode->i_mapping); | |
505 | spin_lock(&obj->mmo.lock); | |
506 | } | |
507 | spin_unlock(&obj->mmo.lock); | |
508 | } | |
509 | ||
510 | /** | |
511 | * i915_gem_object_release_mmap - remove physical page mappings | |
512 | * @obj: obj in question | |
513 | * | |
514 | * Preserve the reservation of the mmapping with the DRM core code, but | |
515 | * relinquish ownership of the pages back to the system. | |
516 | */ | |
517 | void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj) | |
518 | { | |
519 | i915_gem_object_release_mmap_gtt(obj); | |
520 | i915_gem_object_release_mmap_offset(obj); | |
521 | } | |
522 | ||
78655598 CW |
523 | static struct i915_mmap_offset * |
524 | lookup_mmo(struct drm_i915_gem_object *obj, | |
525 | enum i915_mmap_type mmap_type) | |
526 | { | |
527 | struct rb_node *rb; | |
528 | ||
529 | spin_lock(&obj->mmo.lock); | |
530 | rb = obj->mmo.offsets.rb_node; | |
531 | while (rb) { | |
532 | struct i915_mmap_offset *mmo = | |
533 | rb_entry(rb, typeof(*mmo), offset); | |
534 | ||
535 | if (mmo->mmap_type == mmap_type) { | |
536 | spin_unlock(&obj->mmo.lock); | |
537 | return mmo; | |
538 | } | |
539 | ||
540 | if (mmo->mmap_type < mmap_type) | |
541 | rb = rb->rb_right; | |
542 | else | |
543 | rb = rb->rb_left; | |
544 | } | |
545 | spin_unlock(&obj->mmo.lock); | |
546 | ||
547 | return NULL; | |
548 | } | |
549 | ||
550 | static struct i915_mmap_offset * | |
551 | insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo) | |
552 | { | |
553 | struct rb_node *rb, **p; | |
554 | ||
555 | spin_lock(&obj->mmo.lock); | |
556 | rb = NULL; | |
557 | p = &obj->mmo.offsets.rb_node; | |
558 | while (*p) { | |
559 | struct i915_mmap_offset *pos; | |
560 | ||
561 | rb = *p; | |
562 | pos = rb_entry(rb, typeof(*pos), offset); | |
563 | ||
564 | if (pos->mmap_type == mmo->mmap_type) { | |
565 | spin_unlock(&obj->mmo.lock); | |
566 | drm_vma_offset_remove(obj->base.dev->vma_offset_manager, | |
567 | &mmo->vma_node); | |
568 | kfree(mmo); | |
569 | return pos; | |
570 | } | |
571 | ||
572 | if (pos->mmap_type < mmo->mmap_type) | |
573 | p = &rb->rb_right; | |
574 | else | |
575 | p = &rb->rb_left; | |
576 | } | |
577 | rb_link_node(&mmo->offset, rb, p); | |
578 | rb_insert_color(&mmo->offset, &obj->mmo.offsets); | |
579 | spin_unlock(&obj->mmo.lock); | |
580 | ||
581 | return mmo; | |
582 | } | |
583 | ||
cc662126 AJ |
584 | static struct i915_mmap_offset * |
585 | mmap_offset_attach(struct drm_i915_gem_object *obj, | |
586 | enum i915_mmap_type mmap_type, | |
587 | struct drm_file *file) | |
b414fcd5 CW |
588 | { |
589 | struct drm_i915_private *i915 = to_i915(obj->base.dev); | |
cc662126 | 590 | struct i915_mmap_offset *mmo; |
b414fcd5 CW |
591 | int err; |
592 | ||
78655598 CW |
593 | mmo = lookup_mmo(obj, mmap_type); |
594 | if (mmo) | |
595 | goto out; | |
596 | ||
cc662126 AJ |
597 | mmo = kmalloc(sizeof(*mmo), GFP_KERNEL); |
598 | if (!mmo) | |
599 | return ERR_PTR(-ENOMEM); | |
600 | ||
601 | mmo->obj = obj; | |
cc662126 AJ |
602 | mmo->mmap_type = mmap_type; |
603 | drm_vma_node_reset(&mmo->vma_node); | |
604 | ||
78655598 CW |
605 | err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, |
606 | &mmo->vma_node, obj->base.size / PAGE_SIZE); | |
b414fcd5 | 607 | if (likely(!err)) |
78655598 | 608 | goto insert; |
b414fcd5 CW |
609 | |
610 | /* Attempt to reap some mmap space from dead objects */ | |
cc662126 | 611 | err = intel_gt_retire_requests_timeout(&i915->gt, MAX_SCHEDULE_TIMEOUT); |
789ed955 | 612 | if (err) |
cc662126 | 613 | goto err; |
b414fcd5 | 614 | |
789ed955 | 615 | i915_gem_drain_freed_objects(i915); |
78655598 CW |
616 | err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, |
617 | &mmo->vma_node, obj->base.size / PAGE_SIZE); | |
cc662126 AJ |
618 | if (err) |
619 | goto err; | |
620 | ||
78655598 CW |
621 | insert: |
622 | mmo = insert_mmo(obj, mmo); | |
623 | GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo); | |
cc662126 AJ |
624 | out: |
625 | if (file) | |
626 | drm_vma_node_allow(&mmo->vma_node, file); | |
cc662126 AJ |
627 | return mmo; |
628 | ||
629 | err: | |
630 | kfree(mmo); | |
631 | return ERR_PTR(err); | |
b414fcd5 CW |
632 | } |
633 | ||
cc662126 AJ |
634 | static int |
635 | __assign_mmap_offset(struct drm_file *file, | |
636 | u32 handle, | |
637 | enum i915_mmap_type mmap_type, | |
638 | u64 *offset) | |
b414fcd5 CW |
639 | { |
640 | struct drm_i915_gem_object *obj; | |
cc662126 AJ |
641 | struct i915_mmap_offset *mmo; |
642 | int err; | |
bee0a70a | 643 | |
b414fcd5 CW |
644 | obj = i915_gem_object_lookup(file, handle); |
645 | if (!obj) | |
646 | return -ENOENT; | |
647 | ||
f6c26b55 | 648 | if (i915_gem_object_never_mmap(obj)) { |
cc662126 | 649 | err = -ENODEV; |
a4311745 CW |
650 | goto out; |
651 | } | |
652 | ||
cc662126 | 653 | if (mmap_type != I915_MMAP_TYPE_GTT && |
4e598fad AJ |
654 | !i915_gem_object_type_has(obj, |
655 | I915_GEM_OBJECT_HAS_STRUCT_PAGE | | |
656 | I915_GEM_OBJECT_HAS_IOMEM)) { | |
cc662126 AJ |
657 | err = -ENODEV; |
658 | goto out; | |
659 | } | |
660 | ||
661 | mmo = mmap_offset_attach(obj, mmap_type, file); | |
662 | if (IS_ERR(mmo)) { | |
663 | err = PTR_ERR(mmo); | |
664 | goto out; | |
665 | } | |
b414fcd5 | 666 | |
cc662126 AJ |
667 | *offset = drm_vma_node_offset_addr(&mmo->vma_node); |
668 | err = 0; | |
a4311745 | 669 | out: |
b414fcd5 | 670 | i915_gem_object_put(obj); |
cc662126 AJ |
671 | return err; |
672 | } | |
673 | ||
674 | int | |
675 | i915_gem_dumb_mmap_offset(struct drm_file *file, | |
676 | struct drm_device *dev, | |
677 | u32 handle, | |
678 | u64 *offset) | |
679 | { | |
680 | enum i915_mmap_type mmap_type; | |
681 | ||
682 | if (boot_cpu_has(X86_FEATURE_PAT)) | |
683 | mmap_type = I915_MMAP_TYPE_WC; | |
684 | else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt)) | |
685 | return -ENODEV; | |
686 | else | |
687 | mmap_type = I915_MMAP_TYPE_GTT; | |
688 | ||
689 | return __assign_mmap_offset(file, handle, mmap_type, offset); | |
b414fcd5 CW |
690 | } |
691 | ||
692 | /** | |
cc662126 | 693 | * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing |
b414fcd5 CW |
694 | * @dev: DRM device |
695 | * @data: GTT mapping ioctl data | |
696 | * @file: GEM object info | |
697 | * | |
698 | * Simply returns the fake offset to userspace so it can mmap it. | |
699 | * The mmap call will end up in drm_gem_mmap(), which will set things | |
700 | * up so we can get faults in the handler above. | |
701 | * | |
702 | * The fault handler will take care of binding the object into the GTT | |
703 | * (since it may have been evicted to make room for something), allocating | |
704 | * a fence register, and mapping the appropriate aperture address into | |
705 | * userspace. | |
706 | */ | |
707 | int | |
cc662126 AJ |
708 | i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, |
709 | struct drm_file *file) | |
b414fcd5 | 710 | { |
cc662126 AJ |
711 | struct drm_i915_private *i915 = to_i915(dev); |
712 | struct drm_i915_gem_mmap_offset *args = data; | |
713 | enum i915_mmap_type type; | |
126d5de3 | 714 | int err; |
cc662126 | 715 | |
8d65859a CW |
716 | /* |
717 | * Historically we failed to check args.pad and args.offset | |
718 | * and so we cannot use those fields for user input and we cannot | |
719 | * add -EINVAL for them as the ABI is fixed, i.e. old userspace | |
720 | * may be feeding in garbage in those fields. | |
721 | * | |
722 | * if (args->pad) return -EINVAL; is verbotten! | |
723 | */ | |
724 | ||
126d5de3 CW |
725 | err = i915_user_extensions(u64_to_user_ptr(args->extensions), |
726 | NULL, 0, NULL); | |
727 | if (err) | |
728 | return err; | |
cc662126 AJ |
729 | |
730 | switch (args->flags) { | |
731 | case I915_MMAP_OFFSET_GTT: | |
732 | if (!i915_ggtt_has_aperture(&i915->ggtt)) | |
733 | return -ENODEV; | |
734 | type = I915_MMAP_TYPE_GTT; | |
735 | break; | |
736 | ||
737 | case I915_MMAP_OFFSET_WC: | |
738 | if (!boot_cpu_has(X86_FEATURE_PAT)) | |
739 | return -ENODEV; | |
740 | type = I915_MMAP_TYPE_WC; | |
741 | break; | |
742 | ||
743 | case I915_MMAP_OFFSET_WB: | |
744 | type = I915_MMAP_TYPE_WB; | |
745 | break; | |
746 | ||
747 | case I915_MMAP_OFFSET_UC: | |
748 | if (!boot_cpu_has(X86_FEATURE_PAT)) | |
749 | return -ENODEV; | |
750 | type = I915_MMAP_TYPE_UC; | |
751 | break; | |
752 | ||
753 | default: | |
754 | return -EINVAL; | |
755 | } | |
b414fcd5 | 756 | |
cc662126 AJ |
757 | return __assign_mmap_offset(file, args->handle, type, &args->offset); |
758 | } | |
759 | ||
760 | static void vm_open(struct vm_area_struct *vma) | |
761 | { | |
762 | struct i915_mmap_offset *mmo = vma->vm_private_data; | |
763 | struct drm_i915_gem_object *obj = mmo->obj; | |
764 | ||
765 | GEM_BUG_ON(!obj); | |
766 | i915_gem_object_get(obj); | |
767 | } | |
768 | ||
769 | static void vm_close(struct vm_area_struct *vma) | |
770 | { | |
771 | struct i915_mmap_offset *mmo = vma->vm_private_data; | |
772 | struct drm_i915_gem_object *obj = mmo->obj; | |
773 | ||
774 | GEM_BUG_ON(!obj); | |
775 | i915_gem_object_put(obj); | |
776 | } | |
777 | ||
778 | static const struct vm_operations_struct vm_ops_gtt = { | |
779 | .fault = vm_fault_gtt, | |
9f909e21 | 780 | .access = vm_access, |
cc662126 AJ |
781 | .open = vm_open, |
782 | .close = vm_close, | |
783 | }; | |
784 | ||
785 | static const struct vm_operations_struct vm_ops_cpu = { | |
786 | .fault = vm_fault_cpu, | |
9f909e21 | 787 | .access = vm_access, |
cc662126 AJ |
788 | .open = vm_open, |
789 | .close = vm_close, | |
790 | }; | |
791 | ||
f17b8980 CW |
792 | static int singleton_release(struct inode *inode, struct file *file) |
793 | { | |
794 | struct drm_i915_private *i915 = file->private_data; | |
795 | ||
796 | cmpxchg(&i915->gem.mmap_singleton, file, NULL); | |
797 | drm_dev_put(&i915->drm); | |
798 | ||
799 | return 0; | |
800 | } | |
801 | ||
802 | static const struct file_operations singleton_fops = { | |
803 | .owner = THIS_MODULE, | |
804 | .release = singleton_release, | |
805 | }; | |
806 | ||
807 | static struct file *mmap_singleton(struct drm_i915_private *i915) | |
808 | { | |
809 | struct file *file; | |
810 | ||
811 | rcu_read_lock(); | |
4aea5a9e | 812 | file = READ_ONCE(i915->gem.mmap_singleton); |
f17b8980 CW |
813 | if (file && !get_file_rcu(file)) |
814 | file = NULL; | |
815 | rcu_read_unlock(); | |
816 | if (file) | |
817 | return file; | |
818 | ||
819 | file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR); | |
820 | if (IS_ERR(file)) | |
821 | return file; | |
822 | ||
823 | /* Everyone shares a single global address space */ | |
824 | file->f_mapping = i915->drm.anon_inode->i_mapping; | |
825 | ||
826 | smp_store_mb(i915->gem.mmap_singleton, file); | |
827 | drm_dev_get(&i915->drm); | |
828 | ||
829 | return file; | |
830 | } | |
831 | ||
cc662126 AJ |
832 | /* |
833 | * This overcomes the limitation in drm_gem_mmap's assignment of a | |
834 | * drm_gem_object as the vma->vm_private_data. Since we need to | |
835 | * be able to resolve multiple mmap offsets which could be tied | |
836 | * to a single gem object. | |
837 | */ | |
838 | int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) | |
839 | { | |
840 | struct drm_vma_offset_node *node; | |
841 | struct drm_file *priv = filp->private_data; | |
842 | struct drm_device *dev = priv->minor->dev; | |
280d14a6 | 843 | struct drm_i915_gem_object *obj = NULL; |
cc662126 | 844 | struct i915_mmap_offset *mmo = NULL; |
f17b8980 | 845 | struct file *anon; |
cc662126 AJ |
846 | |
847 | if (drm_dev_is_unplugged(dev)) | |
848 | return -ENODEV; | |
849 | ||
280d14a6 | 850 | rcu_read_lock(); |
cc662126 AJ |
851 | drm_vma_offset_lock_lookup(dev->vma_offset_manager); |
852 | node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager, | |
853 | vma->vm_pgoff, | |
854 | vma_pages(vma)); | |
280d14a6 | 855 | if (node && drm_vma_node_is_allowed(node, priv)) { |
cc662126 AJ |
856 | /* |
857 | * Skip 0-refcnted objects as it is in the process of being | |
858 | * destroyed and will be invalid when the vma manager lock | |
859 | * is released. | |
860 | */ | |
280d14a6 CW |
861 | mmo = container_of(node, struct i915_mmap_offset, vma_node); |
862 | obj = i915_gem_object_get_rcu(mmo->obj); | |
cc662126 AJ |
863 | } |
864 | drm_vma_offset_unlock_lookup(dev->vma_offset_manager); | |
280d14a6 | 865 | rcu_read_unlock(); |
cc662126 | 866 | if (!obj) |
280d14a6 | 867 | return node ? -EACCES : -EINVAL; |
cc662126 | 868 | |
280d14a6 | 869 | if (i915_gem_object_is_readonly(obj)) { |
cc662126 | 870 | if (vma->vm_flags & VM_WRITE) { |
280d14a6 | 871 | i915_gem_object_put(obj); |
cc662126 AJ |
872 | return -EINVAL; |
873 | } | |
874 | vma->vm_flags &= ~VM_MAYWRITE; | |
875 | } | |
876 | ||
280d14a6 | 877 | anon = mmap_singleton(to_i915(dev)); |
f17b8980 | 878 | if (IS_ERR(anon)) { |
280d14a6 | 879 | i915_gem_object_put(obj); |
f17b8980 CW |
880 | return PTR_ERR(anon); |
881 | } | |
882 | ||
cc662126 AJ |
883 | vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; |
884 | vma->vm_private_data = mmo; | |
885 | ||
f17b8980 CW |
886 | /* |
887 | * We keep the ref on mmo->obj, not vm_file, but we require | |
888 | * vma->vm_file->f_mapping, see vma_link(), for later revocation. | |
889 | * Our userspace is accustomed to having per-file resource cleanup | |
890 | * (i.e. contexts, objects and requests) on their close(fd), which | |
891 | * requires avoiding extraneous references to their filp, hence why | |
892 | * we prefer to use an anonymous file for their mmaps. | |
893 | */ | |
894 | fput(vma->vm_file); | |
895 | vma->vm_file = anon; | |
896 | ||
cc662126 AJ |
897 | switch (mmo->mmap_type) { |
898 | case I915_MMAP_TYPE_WC: | |
899 | vma->vm_page_prot = | |
900 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
901 | vma->vm_ops = &vm_ops_cpu; | |
902 | break; | |
903 | ||
904 | case I915_MMAP_TYPE_WB: | |
905 | vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); | |
906 | vma->vm_ops = &vm_ops_cpu; | |
907 | break; | |
908 | ||
909 | case I915_MMAP_TYPE_UC: | |
910 | vma->vm_page_prot = | |
911 | pgprot_noncached(vm_get_page_prot(vma->vm_flags)); | |
912 | vma->vm_ops = &vm_ops_cpu; | |
913 | break; | |
914 | ||
915 | case I915_MMAP_TYPE_GTT: | |
916 | vma->vm_page_prot = | |
917 | pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); | |
918 | vma->vm_ops = &vm_ops_gtt; | |
919 | break; | |
920 | } | |
921 | vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); | |
922 | ||
923 | return 0; | |
b414fcd5 CW |
924 | } |
925 | ||
926 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
927 | #include "selftests/i915_gem_mman.c" | |
928 | #endif |