drm/i915: Promote i915->mm.obj_lock to be irqsafe

The intent is to be able to update the mm.lists from inside an irqsoff
section (e.g. from a softirq rcu workqueue), ergo we need to make the
i915->mm.obj_lock irqsafe.

v2: can_discard_pages() ensures we are shrinkable
v3: Beware shadowing of 'flags'

Fixes: 3b4fa9640ccd ("drm/i915: Track the purgeable objects on a separate eviction list")
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=110869
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Matthew Auld <matthew.william.auld@gmail.com>
Reviewed-by: Matthew Auld <matthew.william.auld@gmail.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190610145430.17717-1-chris@chris-wilson.co.uk

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index e5deae62681f61d983bdbf33633dae9ecefe05d1..31929220b90f9c88436b057ff514d17ac8b63bf5 100644 (file)
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -475,15 +475,20 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
        }
        mutex_unlock(&i915->ggtt.vm.mutex);
 
-       if (i915_gem_object_is_shrinkable(obj) &&
-           obj->mm.madv == I915_MADV_WILLNEED) {
-               struct list_head *list;
-
-               spin_lock(&i915->mm.obj_lock);
-               list = obj->bind_count ?
-                       &i915->mm.bound_list : &i915->mm.unbound_list;
-               list_move_tail(&obj->mm.link, list);
-               spin_unlock(&i915->mm.obj_lock);
+       if (i915_gem_object_is_shrinkable(obj)) {
+               unsigned long flags;
+
+               spin_lock_irqsave(&i915->mm.obj_lock, flags);
+
+               if (obj->mm.madv == I915_MADV_WILLNEED) {
+                       struct list_head *list;
+
+                       list = obj->bind_count ?
+                               &i915->mm.bound_list : &i915->mm.unbound_list;
+                       list_move_tail(&obj->mm.link, list);
+               }
+
+               spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
        }
 }
 

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index a0bc8f7ab7807b70be762f6610641faec59a2c17..d02a1aff20584b3d5db92d4cb7f79f772cfad3dc 100644 (file)
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -207,9 +207,11 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
                 */
                if (i915_gem_object_has_pages(obj) &&
                    i915_gem_object_is_shrinkable(obj)) {
-                       spin_lock(&i915->mm.obj_lock);
+                       unsigned long flags;
+
+                       spin_lock_irqsave(&i915->mm.obj_lock, flags);
                        list_del_init(&obj->mm.link);
-                       spin_unlock(&i915->mm.obj_lock);
+                       spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
                }
 
                mutex_unlock(&i915->drm.struct_mutex);
@@ -330,9 +332,11 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
                obj->mm.madv = I915_MADV_DONTNEED;
 
                if (i915_gem_object_has_pages(obj)) {
-                       spin_lock(&i915->mm.obj_lock);
+                       unsigned long flags;
+
+                       spin_lock_irqsave(&i915->mm.obj_lock, flags);
                        list_move_tail(&obj->mm.link, &i915->mm.purge_list);
-                       spin_unlock(&i915->mm.obj_lock);
+                       spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
                }
        }
 

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
index 7e64fd6bc19bd574d271a4a313ba2bf0fa9f3ed6..7ff907d6d0c6f2e4f2cddb6b0d42b11573dda084 100644 (file)
--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
@@ -57,11 +57,15 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
        GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
 
        if (i915_gem_object_is_shrinkable(obj)) {
-               spin_lock(&i915->mm.obj_lock);
+               unsigned long flags;
+
+               spin_lock_irqsave(&i915->mm.obj_lock, flags);
+
                i915->mm.shrink_count++;
                i915->mm.shrink_memory += obj->base.size;
                list_add(&obj->mm.link, &i915->mm.unbound_list);
-               spin_unlock(&i915->mm.obj_lock);
+
+               spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
        }
 }
 
@@ -151,11 +155,15 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
                return pages;
 
        if (i915_gem_object_is_shrinkable(obj)) {
-               spin_lock(&i915->mm.obj_lock);
+               unsigned long flags;
+
+               spin_lock_irqsave(&i915->mm.obj_lock, flags);
+
                list_del(&obj->mm.link);
                i915->mm.shrink_count--;
                i915->mm.shrink_memory -= obj->base.size;
-               spin_unlock(&i915->mm.obj_lock);
+
+               spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
        }
 
        if (obj->mm.mapping) {

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
index d71e630c6fb80a03f86af75ede5897e79491b1f1..70a4c9d3c098250ad67de639f4a7258844a967f2 100644 (file)
--- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
@@ -138,7 +138,7 @@ unsigned long
 i915_gem_shrink(struct drm_i915_private *i915,
                unsigned long target,
                unsigned long *nr_scanned,
-               unsigned flags)
+               unsigned int shrink)
 {
        const struct {
                struct list_head *list;
@@ -154,7 +154,7 @@ i915_gem_shrink(struct drm_i915_private *i915,
        unsigned long scanned = 0;
        bool unlock;
 
-       if (!shrinker_lock(i915, flags, &unlock))
+       if (!shrinker_lock(i915, shrink, &unlock))
                return 0;
 
        /*
@@ -166,12 +166,12 @@ i915_gem_shrink(struct drm_i915_private *i915,
         * We don't care about errors here; if we cannot wait upon the GPU,
         * we will free as much as we can and hope to get a second chance.
         */
-       if (flags & I915_SHRINK_ACTIVE)
+       if (shrink & I915_SHRINK_ACTIVE)
                i915_gem_wait_for_idle(i915,
                                       I915_WAIT_LOCKED,
                                       MAX_SCHEDULE_TIMEOUT);
 
-       trace_i915_gem_shrink(i915, target, flags);
+       trace_i915_gem_shrink(i915, target, shrink);
        i915_retire_requests(i915);
 
        /*
@@ -179,10 +179,10 @@ i915_gem_shrink(struct drm_i915_private *i915,
         * device just to recover a little memory. If absolutely necessary,
         * we will force the wake during oom-notifier.
         */
-       if (flags & I915_SHRINK_BOUND) {
+       if (shrink & I915_SHRINK_BOUND) {
                wakeref = intel_runtime_pm_get_if_in_use(i915);
                if (!wakeref)
-                       flags &= ~I915_SHRINK_BOUND;
+                       shrink &= ~I915_SHRINK_BOUND;
        }
 
        /*
@@ -207,8 +207,9 @@ i915_gem_shrink(struct drm_i915_private *i915,
        for (phase = phases; phase->list; phase++) {
                struct list_head still_in_list;
                struct drm_i915_gem_object *obj;
+               unsigned long flags;
 
-               if ((flags & phase->bit) == 0)
+               if ((shrink & phase->bit) == 0)
                        continue;
 
                INIT_LIST_HEAD(&still_in_list);
@@ -220,50 +221,50 @@ i915_gem_shrink(struct drm_i915_private *i915,
                 * to be able to shrink their pages, so they remain on
                 * the unbound/bound list until actually freed.
                 */
-               spin_lock(&i915->mm.obj_lock);
+               spin_lock_irqsave(&i915->mm.obj_lock, flags);
                while (count < target &&
                       (obj = list_first_entry_or_null(phase->list,
                                                       typeof(*obj),
                                                       mm.link))) {
                        list_move_tail(&obj->mm.link, &still_in_list);
 
-                       if (flags & I915_SHRINK_VMAPS &&
+                       if (shrink & I915_SHRINK_VMAPS &&
                            !is_vmalloc_addr(obj->mm.mapping))
                                continue;
 
-                       if (!(flags & I915_SHRINK_ACTIVE) &&
+                       if (!(shrink & I915_SHRINK_ACTIVE) &&
                            (i915_gem_object_is_active(obj) ||
                             i915_gem_object_is_framebuffer(obj)))
                                continue;
 
-                       if (!(flags & I915_SHRINK_BOUND) &&
+                       if (!(shrink & I915_SHRINK_BOUND) &&
                            READ_ONCE(obj->bind_count))
                                continue;
 
                        if (!can_release_pages(obj))
                                continue;
 
-                       spin_unlock(&i915->mm.obj_lock);
+                       spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 
                        if (unsafe_drop_pages(obj)) {
                                /* May arrive from get_pages on another bo */
                                mutex_lock_nested(&obj->mm.lock,
                                                  I915_MM_SHRINKER);
                                if (!i915_gem_object_has_pages(obj)) {
-                                       try_to_writeback(obj, flags);
+                                       try_to_writeback(obj, shrink);
                                        count += obj->base.size >> PAGE_SHIFT;
                                }
                                mutex_unlock(&obj->mm.lock);
                        }
                        scanned += obj->base.size >> PAGE_SHIFT;
 
-                       spin_lock(&i915->mm.obj_lock);
+                       spin_lock_irqsave(&i915->mm.obj_lock, flags);
                }
                list_splice_tail(&still_in_list, phase->list);
-               spin_unlock(&i915->mm.obj_lock);
+               spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
        }
 
-       if (flags & I915_SHRINK_BOUND)
+       if (shrink & I915_SHRINK_BOUND)
                intel_runtime_pm_put(i915, wakeref);
 
        i915_retire_requests(i915);
@@ -379,6 +380,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
        struct drm_i915_gem_object *obj;
        unsigned long unevictable, bound, unbound, freed_pages;
        intel_wakeref_t wakeref;
+       unsigned long flags;
 
        freed_pages = 0;
        with_intel_runtime_pm(i915, wakeref)
@@ -392,7 +394,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
         * being pointed to by hardware.
         */
        unbound = bound = unevictable = 0;
-       spin_lock(&i915->mm.obj_lock);
+       spin_lock_irqsave(&i915->mm.obj_lock, flags);
        list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) {
                if (!can_release_pages(obj))
                        unevictable += obj->base.size >> PAGE_SHIFT;
@@ -405,7 +407,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
                else
                        bound += obj->base.size >> PAGE_SHIFT;
        }
-       spin_unlock(&i915->mm.obj_lock);
+       spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 
        if (freed_pages || unbound || bound)
                pr_info("Purging GPU memory, %lu pages freed, "

diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index 432037e5d0a5184065bbe1049a3b30f0d3a7fa35..8f619ad2fab89cb17e7ecc354cd618d4e2387120 100644 (file)
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -613,6 +613,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
        struct drm_i915_gem_object *obj;
        struct drm_mm_node *stolen;
        struct i915_vma *vma;
+       unsigned long flags;
        int ret;
 
        if (!drm_mm_initialized(&dev_priv->mm.stolen))
@@ -689,10 +690,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
        list_move_tail(&vma->vm_link, &ggtt->vm.bound_list);
        mutex_unlock(&ggtt->vm.mutex);
 
-       spin_lock(&dev_priv->mm.obj_lock);
+       spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
        GEM_BUG_ON(i915_gem_object_is_shrinkable(obj));
        obj->bind_count++;
-       spin_unlock(&dev_priv->mm.obj_lock);
+       spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
 
        return obj;
 

diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index f212241a275896d197c4f31ee6afc8a74bef3d18..2e79f0e4c5afee06a6c4de059c7794e4fca980eb 100644 (file)
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -269,6 +269,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
        struct drm_i915_gem_object *obj;
        u64 total_obj_size, total_gtt_size;
        unsigned long total, count, n;
+       unsigned long flags;
        int ret;
 
        total = READ_ONCE(dev_priv->mm.shrink_count);
@@ -282,7 +283,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 
        total_obj_size = total_gtt_size = count = 0;
 
-       spin_lock(&dev_priv->mm.obj_lock);
+       spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
        list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) {
                if (count == total)
                        break;
@@ -305,7 +306,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
                objects[count++] = obj;
                total_obj_size += obj->base.size;
        }
-       spin_unlock(&dev_priv->mm.obj_lock);
+       spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
 
        sort(objects, count, sizeof(*objects), obj_rank_by_stolen, NULL);
 
@@ -457,6 +458,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
        u64 size, mapped_size, purgeable_size, dpy_size, huge_size;
        struct drm_i915_gem_object *obj;
        unsigned int page_sizes = 0;
+       unsigned long flags;
        char buf[80];
        int ret;
 
@@ -469,7 +471,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
        purgeable_size = purgeable_count = 0;
        huge_size = huge_count = 0;
 
-       spin_lock(&dev_priv->mm.obj_lock);
+       spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
        list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) {
                size += obj->base.size;
                ++count;
@@ -518,7 +520,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
                        page_sizes |= obj->mm.page_sizes.sg;
                }
        }
-       spin_unlock(&dev_priv->mm.obj_lock);
+       spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
 
        seq_printf(m, "%u bound objects, %llu bytes\n",
                   count, size);

diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 9f2e213c6046fe8a7d260cbac8de67fc539c8bd1..e980c1ee3dcf790bdbd2c30af3a2efeb6b6e948d 100644 (file)
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1138,7 +1138,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                struct list_head *list;
 
                if (i915_gem_object_is_shrinkable(obj)) {
-                       spin_lock(&i915->mm.obj_lock);
+                       unsigned long flags;
+
+                       spin_lock_irqsave(&i915->mm.obj_lock, flags);
+
                        if (obj->mm.madv != I915_MADV_WILLNEED)
                                list = &i915->mm.purge_list;
                        else if (obj->bind_count)
@@ -1146,7 +1149,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                        else
                                list = &i915->mm.unbound_list;
                        list_move_tail(&obj->mm.link, list);
-                       spin_unlock(&i915->mm.obj_lock);
+
+                       spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
                }
        }
 

diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index f6ac8394da77ae07e4c83959538f46d21680f8aa..80050f6a089388608a8edae99ae64bb0f4f2cbf1 100644 (file)
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -80,11 +80,14 @@ static void vma_print_allocator(struct i915_vma *vma, const char *reason)
 static void obj_bump_mru(struct drm_i915_gem_object *obj)
 {
        struct drm_i915_private *i915 = to_i915(obj->base.dev);
+       unsigned long flags;
+
+       spin_lock_irqsave(&i915->mm.obj_lock, flags);
 
-       spin_lock(&i915->mm.obj_lock);
        if (obj->bind_count)
                list_move_tail(&obj->mm.link, &i915->mm.bound_list);
-       spin_unlock(&i915->mm.obj_lock);
+
+       spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 
        obj->mm.dirty = true; /* be paranoid  */
 }
@@ -678,8 +681,9 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 
        if (vma->obj) {
                struct drm_i915_gem_object *obj = vma->obj;
+               unsigned long flags;
 
-               spin_lock(&dev_priv->mm.obj_lock);
+               spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
 
                if (i915_gem_object_is_shrinkable(obj))
                        list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
@@ -687,7 +691,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
                obj->bind_count++;
                assert_bind_count(obj);
 
-               spin_unlock(&dev_priv->mm.obj_lock);
+               spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
        }
 
        return 0;
@@ -721,8 +725,9 @@ i915_vma_remove(struct i915_vma *vma)
         */
        if (vma->obj) {
                struct drm_i915_gem_object *obj = vma->obj;
+               unsigned long flags;
 
-               spin_lock(&i915->mm.obj_lock);
+               spin_lock_irqsave(&i915->mm.obj_lock, flags);
 
                GEM_BUG_ON(obj->bind_count == 0);
                if (--obj->bind_count == 0 &&
@@ -730,7 +735,7 @@ i915_vma_remove(struct i915_vma *vma)
                    obj->mm.madv == I915_MADV_WILLNEED)
                        list_move_tail(&obj->mm.link, &i915->mm.unbound_list);
 
-               spin_unlock(&i915->mm.obj_lock);
+               spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 
                /*
                 * And finally now the object is completely decoupled from this