Merge remote-tracking branch 'asoc/topic/core' into asoc-next
[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_gem_userptr.c
CommitLineData
5cc9ed4b
CW
1/*
2 * Copyright © 2012-2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
b588c92b
ML
25#include <drm/drmP.h>
26#include <drm/i915_drm.h>
5cc9ed4b
CW
27#include "i915_drv.h"
28#include "i915_trace.h"
29#include "intel_drv.h"
30#include <linux/mmu_context.h>
31#include <linux/mmu_notifier.h>
32#include <linux/mempolicy.h>
33#include <linux/swap.h>
34
ad46cb53
CW
35struct i915_mm_struct {
36 struct mm_struct *mm;
37 struct drm_device *dev;
38 struct i915_mmu_notifier *mn;
39 struct hlist_node node;
40 struct kref kref;
41 struct work_struct work;
42};
43
5cc9ed4b
CW
44#if defined(CONFIG_MMU_NOTIFIER)
45#include <linux/interval_tree.h>
46
47struct i915_mmu_notifier {
48 spinlock_t lock;
49 struct hlist_node node;
50 struct mmu_notifier mn;
51 struct rb_root objects;
ec8b0dd5 52 struct list_head linear;
5cc9ed4b 53 unsigned long serial;
ec8b0dd5 54 bool has_linear;
5cc9ed4b
CW
55};
56
57struct i915_mmu_object {
ad46cb53 58 struct i915_mmu_notifier *mn;
5cc9ed4b 59 struct interval_tree_node it;
ec8b0dd5 60 struct list_head link;
5cc9ed4b 61 struct drm_i915_gem_object *obj;
ec8b0dd5 62 bool is_linear;
5cc9ed4b
CW
63};
64
ec8b0dd5
CW
65static unsigned long cancel_userptr(struct drm_i915_gem_object *obj)
66{
67 struct drm_device *dev = obj->base.dev;
68 unsigned long end;
69
70 mutex_lock(&dev->struct_mutex);
71 /* Cancel any active worker and force us to re-evaluate gup */
72 obj->userptr.work = NULL;
73
74 if (obj->pages != NULL) {
75 struct drm_i915_private *dev_priv = to_i915(dev);
76 struct i915_vma *vma, *tmp;
77 bool was_interruptible;
78
79 was_interruptible = dev_priv->mm.interruptible;
80 dev_priv->mm.interruptible = false;
81
82 list_for_each_entry_safe(vma, tmp, &obj->vma_list, vma_link) {
83 int ret = i915_vma_unbind(vma);
84 WARN_ON(ret && ret != -EIO);
85 }
86 WARN_ON(i915_gem_object_put_pages(obj));
87
88 dev_priv->mm.interruptible = was_interruptible;
89 }
90
91 end = obj->userptr.ptr + obj->base.size;
92
93 drm_gem_object_unreference(&obj->base);
94 mutex_unlock(&dev->struct_mutex);
95
96 return end;
97}
98
48777767
CW
99static void *invalidate_range__linear(struct i915_mmu_notifier *mn,
100 struct mm_struct *mm,
101 unsigned long start,
102 unsigned long end)
ec8b0dd5 103{
ad46cb53 104 struct i915_mmu_object *mo;
ec8b0dd5
CW
105 unsigned long serial;
106
107restart:
108 serial = mn->serial;
ad46cb53 109 list_for_each_entry(mo, &mn->linear, link) {
ec8b0dd5
CW
110 struct drm_i915_gem_object *obj;
111
ad46cb53 112 if (mo->it.last < start || mo->it.start > end)
ec8b0dd5
CW
113 continue;
114
ad46cb53 115 obj = mo->obj;
460822b0
MW
116
117 if (!kref_get_unless_zero(&obj->base.refcount))
118 continue;
119
ec8b0dd5
CW
120 spin_unlock(&mn->lock);
121
122 cancel_userptr(obj);
123
124 spin_lock(&mn->lock);
125 if (serial != mn->serial)
126 goto restart;
127 }
128
48777767 129 return NULL;
ec8b0dd5
CW
130}
131
5cc9ed4b
CW
132static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
133 struct mm_struct *mm,
134 unsigned long start,
135 unsigned long end)
136{
137 struct i915_mmu_notifier *mn = container_of(_mn, struct i915_mmu_notifier, mn);
138 struct interval_tree_node *it = NULL;
ec8b0dd5 139 unsigned long next = start;
5cc9ed4b
CW
140 unsigned long serial = 0;
141
142 end--; /* interval ranges are inclusive, but invalidate range is exclusive */
ec8b0dd5 143 while (next < end) {
48777767 144 struct drm_i915_gem_object *obj = NULL;
5cc9ed4b 145
5cc9ed4b 146 spin_lock(&mn->lock);
ec8b0dd5 147 if (mn->has_linear)
48777767
CW
148 it = invalidate_range__linear(mn, mm, start, end);
149 else if (serial == mn->serial)
ec8b0dd5 150 it = interval_tree_iter_next(it, next, end);
5cc9ed4b
CW
151 else
152 it = interval_tree_iter_first(&mn->objects, start, end);
153 if (it != NULL) {
154 obj = container_of(it, struct i915_mmu_object, it)->obj;
460822b0
MW
155
156 /* The mmu_object is released late when destroying the
157 * GEM object so it is entirely possible to gain a
158 * reference on an object in the process of being freed
159 * since our serialisation is via the spinlock and not
160 * the struct_mutex - and consequently use it after it
161 * is freed and then double free it.
162 */
163 if (!kref_get_unless_zero(&obj->base.refcount)) {
164 spin_unlock(&mn->lock);
165 serial = 0;
166 continue;
167 }
168
5cc9ed4b
CW
169 serial = mn->serial;
170 }
171 spin_unlock(&mn->lock);
172 if (obj == NULL)
173 return;
174
ec8b0dd5 175 next = cancel_userptr(obj);
5cc9ed4b
CW
176 }
177}
178
179static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
180 .invalidate_range_start = i915_gem_userptr_mn_invalidate_range_start,
181};
182
183static struct i915_mmu_notifier *
ad46cb53 184i915_mmu_notifier_create(struct mm_struct *mm)
5cc9ed4b 185{
ad46cb53 186 struct i915_mmu_notifier *mn;
5cc9ed4b
CW
187 int ret;
188
ad46cb53
CW
189 mn = kmalloc(sizeof(*mn), GFP_KERNEL);
190 if (mn == NULL)
5cc9ed4b
CW
191 return ERR_PTR(-ENOMEM);
192
ad46cb53
CW
193 spin_lock_init(&mn->lock);
194 mn->mn.ops = &i915_gem_userptr_notifier;
195 mn->objects = RB_ROOT;
196 mn->serial = 1;
197 INIT_LIST_HEAD(&mn->linear);
198 mn->has_linear = false;
199
200 /* Protected by mmap_sem (write-lock) */
201 ret = __mmu_notifier_register(&mn->mn, mm);
5cc9ed4b 202 if (ret) {
ad46cb53 203 kfree(mn);
5cc9ed4b
CW
204 return ERR_PTR(ret);
205 }
206
ad46cb53 207 return mn;
5cc9ed4b
CW
208}
209
ad46cb53 210static void __i915_mmu_notifier_update_serial(struct i915_mmu_notifier *mn)
5cc9ed4b 211{
ad46cb53
CW
212 if (++mn->serial == 0)
213 mn->serial = 1;
5cc9ed4b
CW
214}
215
216static int
ad46cb53
CW
217i915_mmu_notifier_add(struct drm_device *dev,
218 struct i915_mmu_notifier *mn,
219 struct i915_mmu_object *mo)
5cc9ed4b
CW
220{
221 struct interval_tree_node *it;
281400ff 222 int ret = 0;
5cc9ed4b 223
281400ff
CW
224 /* By this point we have already done a lot of expensive setup that
225 * we do not want to repeat just because the caller (e.g. X) has a
226 * signal pending (and partly because of that expensive setup, X
227 * using an interrupt timer is likely to get stuck in an EINTR loop).
228 */
229 mutex_lock(&dev->struct_mutex);
5cc9ed4b
CW
230
231 /* Make sure we drop the final active reference (and thereby
232 * remove the objects from the interval tree) before we do
233 * the check for overlapping objects.
234 */
ad46cb53 235 i915_gem_retire_requests(dev);
5cc9ed4b 236
ad46cb53
CW
237 spin_lock(&mn->lock);
238 it = interval_tree_iter_first(&mn->objects,
239 mo->it.start, mo->it.last);
5cc9ed4b
CW
240 if (it) {
241 struct drm_i915_gem_object *obj;
242
243 /* We only need to check the first object in the range as it
244 * either has cancelled gup work queued and we need to
245 * return back to the user to give time for the gup-workers
246 * to flush their object references upon which the object will
247 * be removed from the interval-tree, or the the range is
248 * still in use by another client and the overlap is invalid.
ec8b0dd5
CW
249 *
250 * If we do have an overlap, we cannot use the interval tree
251 * for fast range invalidation.
5cc9ed4b
CW
252 */
253
254 obj = container_of(it, struct i915_mmu_object, it)->obj;
ec8b0dd5 255 if (!obj->userptr.workers)
ad46cb53 256 mn->has_linear = mo->is_linear = true;
ec8b0dd5
CW
257 else
258 ret = -EAGAIN;
259 } else
ad46cb53 260 interval_tree_insert(&mo->it, &mn->objects);
ec8b0dd5
CW
261
262 if (ret == 0) {
ad46cb53
CW
263 list_add(&mo->link, &mn->linear);
264 __i915_mmu_notifier_update_serial(mn);
5cc9ed4b 265 }
ad46cb53
CW
266 spin_unlock(&mn->lock);
267 mutex_unlock(&dev->struct_mutex);
5cc9ed4b
CW
268
269 return ret;
270}
271
ad46cb53
CW
272static bool i915_mmu_notifier_has_linear(struct i915_mmu_notifier *mn)
273{
274 struct i915_mmu_object *mo;
275
276 list_for_each_entry(mo, &mn->linear, link)
277 if (mo->is_linear)
278 return true;
279
280 return false;
281}
282
283static void
284i915_mmu_notifier_del(struct i915_mmu_notifier *mn,
285 struct i915_mmu_object *mo)
286{
287 spin_lock(&mn->lock);
288 list_del(&mo->link);
289 if (mo->is_linear)
290 mn->has_linear = i915_mmu_notifier_has_linear(mn);
291 else
292 interval_tree_remove(&mo->it, &mn->objects);
293 __i915_mmu_notifier_update_serial(mn);
294 spin_unlock(&mn->lock);
295}
296
5cc9ed4b
CW
297static void
298i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
299{
ad46cb53 300 struct i915_mmu_object *mo;
5cc9ed4b 301
ad46cb53
CW
302 mo = obj->userptr.mmu_object;
303 if (mo == NULL)
5cc9ed4b
CW
304 return;
305
ad46cb53
CW
306 i915_mmu_notifier_del(mo->mn, mo);
307 kfree(mo);
308
309 obj->userptr.mmu_object = NULL;
310}
311
312static struct i915_mmu_notifier *
313i915_mmu_notifier_find(struct i915_mm_struct *mm)
314{
e9681366
CW
315 struct i915_mmu_notifier *mn = mm->mn;
316
317 mn = mm->mn;
318 if (mn)
319 return mn;
320
321 down_write(&mm->mm->mmap_sem);
322 mutex_lock(&to_i915(mm->dev)->mm_lock);
323 if ((mn = mm->mn) == NULL) {
324 mn = i915_mmu_notifier_create(mm->mm);
325 if (!IS_ERR(mn))
326 mm->mn = mn;
ad46cb53 327 }
e9681366
CW
328 mutex_unlock(&to_i915(mm->dev)->mm_lock);
329 up_write(&mm->mm->mmap_sem);
330
331 return mn;
5cc9ed4b
CW
332}
333
334static int
335i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
336 unsigned flags)
337{
ad46cb53
CW
338 struct i915_mmu_notifier *mn;
339 struct i915_mmu_object *mo;
5cc9ed4b
CW
340 int ret;
341
342 if (flags & I915_USERPTR_UNSYNCHRONIZED)
343 return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
344
ad46cb53
CW
345 if (WARN_ON(obj->userptr.mm == NULL))
346 return -EINVAL;
5cc9ed4b 347
ad46cb53
CW
348 mn = i915_mmu_notifier_find(obj->userptr.mm);
349 if (IS_ERR(mn))
350 return PTR_ERR(mn);
5cc9ed4b 351
ad46cb53
CW
352 mo = kzalloc(sizeof(*mo), GFP_KERNEL);
353 if (mo == NULL)
354 return -ENOMEM;
5cc9ed4b 355
ad46cb53
CW
356 mo->mn = mn;
357 mo->it.start = obj->userptr.ptr;
358 mo->it.last = mo->it.start + obj->base.size - 1;
359 mo->obj = obj;
5cc9ed4b 360
ad46cb53
CW
361 ret = i915_mmu_notifier_add(obj->base.dev, mn, mo);
362 if (ret) {
363 kfree(mo);
364 return ret;
365 }
366
367 obj->userptr.mmu_object = mo;
5cc9ed4b 368 return 0;
ad46cb53
CW
369}
370
371static void
372i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
373 struct mm_struct *mm)
374{
375 if (mn == NULL)
376 return;
5cc9ed4b 377
ad46cb53 378 mmu_notifier_unregister(&mn->mn, mm);
5cc9ed4b 379 kfree(mn);
5cc9ed4b
CW
380}
381
382#else
383
384static void
385i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
386{
387}
388
389static int
390i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
391 unsigned flags)
392{
393 if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0)
394 return -ENODEV;
395
396 if (!capable(CAP_SYS_ADMIN))
397 return -EPERM;
398
399 return 0;
400}
ad46cb53
CW
401
402static void
403i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
404 struct mm_struct *mm)
405{
406}
407
5cc9ed4b
CW
408#endif
409
ad46cb53
CW
410static struct i915_mm_struct *
411__i915_mm_struct_find(struct drm_i915_private *dev_priv, struct mm_struct *real)
412{
413 struct i915_mm_struct *mm;
414
415 /* Protected by dev_priv->mm_lock */
416 hash_for_each_possible(dev_priv->mm_structs, mm, node, (unsigned long)real)
417 if (mm->mm == real)
418 return mm;
419
420 return NULL;
421}
422
423static int
424i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)
425{
426 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
427 struct i915_mm_struct *mm;
428 int ret = 0;
429
430 /* During release of the GEM object we hold the struct_mutex. This
431 * precludes us from calling mmput() at that time as that may be
432 * the last reference and so call exit_mmap(). exit_mmap() will
433 * attempt to reap the vma, and if we were holding a GTT mmap
434 * would then call drm_gem_vm_close() and attempt to reacquire
435 * the struct mutex. So in order to avoid that recursion, we have
436 * to defer releasing the mm reference until after we drop the
437 * struct_mutex, i.e. we need to schedule a worker to do the clean
438 * up.
439 */
440 mutex_lock(&dev_priv->mm_lock);
441 mm = __i915_mm_struct_find(dev_priv, current->mm);
442 if (mm == NULL) {
443 mm = kmalloc(sizeof(*mm), GFP_KERNEL);
444 if (mm == NULL) {
445 ret = -ENOMEM;
446 goto out;
447 }
448
449 kref_init(&mm->kref);
450 mm->dev = obj->base.dev;
451
452 mm->mm = current->mm;
453 atomic_inc(&current->mm->mm_count);
454
455 mm->mn = NULL;
456
457 /* Protected by dev_priv->mm_lock */
458 hash_add(dev_priv->mm_structs,
459 &mm->node, (unsigned long)mm->mm);
460 } else
461 kref_get(&mm->kref);
462
463 obj->userptr.mm = mm;
464out:
465 mutex_unlock(&dev_priv->mm_lock);
466 return ret;
467}
468
469static void
470__i915_mm_struct_free__worker(struct work_struct *work)
471{
472 struct i915_mm_struct *mm = container_of(work, typeof(*mm), work);
473 i915_mmu_notifier_free(mm->mn, mm->mm);
474 mmdrop(mm->mm);
475 kfree(mm);
476}
477
478static void
479__i915_mm_struct_free(struct kref *kref)
480{
481 struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref);
482
483 /* Protected by dev_priv->mm_lock */
484 hash_del(&mm->node);
485 mutex_unlock(&to_i915(mm->dev)->mm_lock);
486
487 INIT_WORK(&mm->work, __i915_mm_struct_free__worker);
488 schedule_work(&mm->work);
489}
490
491static void
492i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)
493{
494 if (obj->userptr.mm == NULL)
495 return;
496
497 kref_put_mutex(&obj->userptr.mm->kref,
498 __i915_mm_struct_free,
499 &to_i915(obj->base.dev)->mm_lock);
500 obj->userptr.mm = NULL;
501}
502
5cc9ed4b
CW
503struct get_pages_work {
504 struct work_struct work;
505 struct drm_i915_gem_object *obj;
506 struct task_struct *task;
507};
508
5cc9ed4b
CW
509#if IS_ENABLED(CONFIG_SWIOTLB)
510#define swiotlb_active() swiotlb_nr_tbl()
511#else
512#define swiotlb_active() 0
513#endif
514
515static int
516st_set_pages(struct sg_table **st, struct page **pvec, int num_pages)
517{
518 struct scatterlist *sg;
519 int ret, n;
520
521 *st = kmalloc(sizeof(**st), GFP_KERNEL);
522 if (*st == NULL)
523 return -ENOMEM;
524
525 if (swiotlb_active()) {
526 ret = sg_alloc_table(*st, num_pages, GFP_KERNEL);
527 if (ret)
528 goto err;
529
530 for_each_sg((*st)->sgl, sg, num_pages, n)
531 sg_set_page(sg, pvec[n], PAGE_SIZE, 0);
532 } else {
533 ret = sg_alloc_table_from_pages(*st, pvec, num_pages,
534 0, num_pages << PAGE_SHIFT,
535 GFP_KERNEL);
536 if (ret)
537 goto err;
538 }
539
540 return 0;
541
542err:
543 kfree(*st);
544 *st = NULL;
545 return ret;
546}
547
e2273302
ID
548static int
549__i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
550 struct page **pvec, int num_pages)
551{
552 int ret;
553
554 ret = st_set_pages(&obj->pages, pvec, num_pages);
555 if (ret)
556 return ret;
557
558 ret = i915_gem_gtt_prepare_object(obj);
559 if (ret) {
560 sg_free_table(obj->pages);
561 kfree(obj->pages);
562 obj->pages = NULL;
563 }
564
565 return ret;
566}
567
5cc9ed4b
CW
568static void
569__i915_gem_userptr_get_pages_worker(struct work_struct *_work)
570{
571 struct get_pages_work *work = container_of(_work, typeof(*work), work);
572 struct drm_i915_gem_object *obj = work->obj;
573 struct drm_device *dev = obj->base.dev;
574 const int num_pages = obj->base.size >> PAGE_SHIFT;
575 struct page **pvec;
576 int pinned, ret;
577
578 ret = -ENOMEM;
579 pinned = 0;
580
581 pvec = kmalloc(num_pages*sizeof(struct page *),
582 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
583 if (pvec == NULL)
584 pvec = drm_malloc_ab(num_pages, sizeof(struct page *));
585 if (pvec != NULL) {
ad46cb53 586 struct mm_struct *mm = obj->userptr.mm->mm;
5cc9ed4b
CW
587
588 down_read(&mm->mmap_sem);
589 while (pinned < num_pages) {
590 ret = get_user_pages(work->task, mm,
591 obj->userptr.ptr + pinned * PAGE_SIZE,
592 num_pages - pinned,
593 !obj->userptr.read_only, 0,
594 pvec + pinned, NULL);
595 if (ret < 0)
596 break;
597
598 pinned += ret;
599 }
600 up_read(&mm->mmap_sem);
601 }
602
603 mutex_lock(&dev->struct_mutex);
604 if (obj->userptr.work != &work->work) {
605 ret = 0;
606 } else if (pinned == num_pages) {
e2273302 607 ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
5cc9ed4b
CW
608 if (ret == 0) {
609 list_add_tail(&obj->global_list, &to_i915(dev)->mm.unbound_list);
e2273302
ID
610 obj->get_page.sg = obj->pages->sgl;
611 obj->get_page.last = 0;
612
5cc9ed4b
CW
613 pinned = 0;
614 }
615 }
616
617 obj->userptr.work = ERR_PTR(ret);
618 obj->userptr.workers--;
619 drm_gem_object_unreference(&obj->base);
620 mutex_unlock(&dev->struct_mutex);
621
622 release_pages(pvec, pinned, 0);
623 drm_free_large(pvec);
624
625 put_task_struct(work->task);
626 kfree(work);
627}
628
629static int
630i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
631{
632 const int num_pages = obj->base.size >> PAGE_SHIFT;
633 struct page **pvec;
634 int pinned, ret;
635
636 /* If userspace should engineer that these pages are replaced in
637 * the vma between us binding this page into the GTT and completion
638 * of rendering... Their loss. If they change the mapping of their
639 * pages they need to create a new bo to point to the new vma.
640 *
641 * However, that still leaves open the possibility of the vma
642 * being copied upon fork. Which falls under the same userspace
643 * synchronisation issue as a regular bo, except that this time
644 * the process may not be expecting that a particular piece of
645 * memory is tied to the GPU.
646 *
647 * Fortunately, we can hook into the mmu_notifier in order to
648 * discard the page references prior to anything nasty happening
649 * to the vma (discard or cloning) which should prevent the more
650 * egregious cases from causing harm.
651 */
652
653 pvec = NULL;
654 pinned = 0;
ad46cb53 655 if (obj->userptr.mm->mm == current->mm) {
5cc9ed4b
CW
656 pvec = kmalloc(num_pages*sizeof(struct page *),
657 GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
658 if (pvec == NULL) {
659 pvec = drm_malloc_ab(num_pages, sizeof(struct page *));
660 if (pvec == NULL)
661 return -ENOMEM;
662 }
663
664 pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,
665 !obj->userptr.read_only, pvec);
666 }
667 if (pinned < num_pages) {
668 if (pinned < 0) {
669 ret = pinned;
670 pinned = 0;
671 } else {
672 /* Spawn a worker so that we can acquire the
673 * user pages without holding our mutex. Access
674 * to the user pages requires mmap_sem, and we have
675 * a strict lock ordering of mmap_sem, struct_mutex -
676 * we already hold struct_mutex here and so cannot
677 * call gup without encountering a lock inversion.
678 *
679 * Userspace will keep on repeating the operation
680 * (thanks to EAGAIN) until either we hit the fast
681 * path or the worker completes. If the worker is
682 * cancelled or superseded, the task is still run
683 * but the results ignored. (This leads to
684 * complications that we may have a stray object
685 * refcount that we need to be wary of when
686 * checking for existing objects during creation.)
687 * If the worker encounters an error, it reports
688 * that error back to this function through
689 * obj->userptr.work = ERR_PTR.
690 */
691 ret = -EAGAIN;
692 if (obj->userptr.work == NULL &&
693 obj->userptr.workers < I915_GEM_USERPTR_MAX_WORKERS) {
694 struct get_pages_work *work;
695
696 work = kmalloc(sizeof(*work), GFP_KERNEL);
697 if (work != NULL) {
698 obj->userptr.work = &work->work;
699 obj->userptr.workers++;
700
701 work->obj = obj;
702 drm_gem_object_reference(&obj->base);
703
704 work->task = current;
705 get_task_struct(work->task);
706
707 INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
708 schedule_work(&work->work);
709 } else
710 ret = -ENOMEM;
711 } else {
712 if (IS_ERR(obj->userptr.work)) {
713 ret = PTR_ERR(obj->userptr.work);
714 obj->userptr.work = NULL;
715 }
716 }
717 }
718 } else {
e2273302 719 ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
5cc9ed4b
CW
720 if (ret == 0) {
721 obj->userptr.work = NULL;
722 pinned = 0;
723 }
724 }
725
726 release_pages(pvec, pinned, 0);
727 drm_free_large(pvec);
728 return ret;
729}
730
731static void
732i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
733{
c479f438 734 struct sg_page_iter sg_iter;
5cc9ed4b
CW
735
736 BUG_ON(obj->userptr.work != NULL);
737
738 if (obj->madv != I915_MADV_WILLNEED)
739 obj->dirty = 0;
740
e2273302
ID
741 i915_gem_gtt_finish_object(obj);
742
c479f438
TU
743 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
744 struct page *page = sg_page_iter_page(&sg_iter);
5cc9ed4b
CW
745
746 if (obj->dirty)
747 set_page_dirty(page);
748
749 mark_page_accessed(page);
750 page_cache_release(page);
751 }
752 obj->dirty = 0;
753
754 sg_free_table(obj->pages);
755 kfree(obj->pages);
756}
757
758static void
759i915_gem_userptr_release(struct drm_i915_gem_object *obj)
760{
761 i915_gem_userptr_release__mmu_notifier(obj);
ad46cb53 762 i915_gem_userptr_release__mm_struct(obj);
5cc9ed4b
CW
763}
764
765static int
766i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
767{
ad46cb53 768 if (obj->userptr.mmu_object)
5cc9ed4b
CW
769 return 0;
770
771 return i915_gem_userptr_init__mmu_notifier(obj, 0);
772}
773
774static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
775 .dmabuf_export = i915_gem_userptr_dmabuf_export,
776 .get_pages = i915_gem_userptr_get_pages,
777 .put_pages = i915_gem_userptr_put_pages,
778 .release = i915_gem_userptr_release,
779};
780
781/**
782 * Creates a new mm object that wraps some normal memory from the process
783 * context - user memory.
784 *
785 * We impose several restrictions upon the memory being mapped
786 * into the GPU.
787 * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
ec8b0dd5 788 * 2. It must be normal system memory, not a pointer into another map of IO
5cc9ed4b 789 * space (e.g. it must not be a GTT mmapping of another object).
ec8b0dd5 790 * 3. We only allow a bo as large as we could in theory map into the GTT,
5cc9ed4b 791 * that is we limit the size to the total size of the GTT.
ec8b0dd5 792 * 4. The bo is marked as being snoopable. The backing pages are left
5cc9ed4b
CW
793 * accessible directly by the CPU, but reads and writes by the GPU may
794 * incur the cost of a snoop (unless you have an LLC architecture).
795 *
796 * Synchronisation between multiple users and the GPU is left to userspace
797 * through the normal set-domain-ioctl. The kernel will enforce that the
798 * GPU relinquishes the VMA before it is returned back to the system
799 * i.e. upon free(), munmap() or process termination. However, the userspace
800 * malloc() library may not immediately relinquish the VMA after free() and
801 * instead reuse it whilst the GPU is still reading and writing to the VMA.
802 * Caveat emptor.
803 *
804 * Also note, that the object created here is not currently a "first class"
805 * object, in that several ioctls are banned. These are the CPU access
806 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
807 * direct access via your pointer rather than use those ioctls.
808 *
809 * If you think this is a good interface to use to pass GPU memory between
810 * drivers, please use dma-buf instead. In fact, wherever possible use
811 * dma-buf instead.
812 */
813int
814i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
815{
816 struct drm_i915_private *dev_priv = dev->dev_private;
817 struct drm_i915_gem_userptr *args = data;
818 struct drm_i915_gem_object *obj;
819 int ret;
820 u32 handle;
821
822 if (args->flags & ~(I915_USERPTR_READ_ONLY |
823 I915_USERPTR_UNSYNCHRONIZED))
824 return -EINVAL;
825
826 if (offset_in_page(args->user_ptr | args->user_size))
827 return -EINVAL;
828
829 if (args->user_size > dev_priv->gtt.base.total)
830 return -E2BIG;
831
832 if (!access_ok(args->flags & I915_USERPTR_READ_ONLY ? VERIFY_READ : VERIFY_WRITE,
833 (char __user *)(unsigned long)args->user_ptr, args->user_size))
834 return -EFAULT;
835
836 if (args->flags & I915_USERPTR_READ_ONLY) {
837 /* On almost all of the current hw, we cannot tell the GPU that a
838 * page is readonly, so this is just a placeholder in the uAPI.
839 */
840 return -ENODEV;
841 }
842
5cc9ed4b
CW
843 obj = i915_gem_object_alloc(dev);
844 if (obj == NULL)
845 return -ENOMEM;
846
847 drm_gem_private_object_init(dev, &obj->base, args->user_size);
848 i915_gem_object_init(obj, &i915_gem_userptr_ops);
849 obj->cache_level = I915_CACHE_LLC;
850 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
851 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
852
853 obj->userptr.ptr = args->user_ptr;
854 obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY);
855
856 /* And keep a pointer to the current->mm for resolving the user pages
857 * at binding. This means that we need to hook into the mmu_notifier
858 * in order to detect if the mmu is destroyed.
859 */
ad46cb53
CW
860 ret = i915_gem_userptr_init__mm_struct(obj);
861 if (ret == 0)
5cc9ed4b
CW
862 ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);
863 if (ret == 0)
864 ret = drm_gem_handle_create(file, &obj->base, &handle);
865
866 /* drop reference from allocate - handle holds it now */
867 drm_gem_object_unreference_unlocked(&obj->base);
868 if (ret)
869 return ret;
870
871 args->handle = handle;
872 return 0;
873}
874
875int
876i915_gem_init_userptr(struct drm_device *dev)
877{
5cc9ed4b 878 struct drm_i915_private *dev_priv = to_i915(dev);
ad46cb53
CW
879 mutex_init(&dev_priv->mm_lock);
880 hash_init(dev_priv->mm_structs);
5cc9ed4b
CW
881 return 0;
882}