drivers/gpu/drm/i915/intel_wakeref.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/wait_bit.h>
   8
   9 #include "intel_runtime_pm.h"
  10 #include "intel_wakeref.h"
  11 #include "i915_drv.h"
  12
  13 static void rpm_get(struct intel_wakeref *wf)
  14 {
  15         wf->wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm);
  16 }
  17
  18 static void rpm_put(struct intel_wakeref *wf)
  19 {
  20         intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref);
  21
  22         intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
  23         INTEL_WAKEREF_BUG_ON(!wakeref);
  24 }
  25
  26 int __intel_wakeref_get_first(struct intel_wakeref *wf)
  27 {
  28         /*
  29          * Treat get/put as different subclasses, as we may need to run
  30          * the put callback from under the shrinker and do not want to
  31          * cross-contanimate that callback with any extra work performed
  32          * upon acquiring the wakeref.
  33          */
  34         mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
  35         if (!atomic_read(&wf->count)) {
  36                 int err;
  37
  38                 rpm_get(wf);
  39
  40                 err = wf->ops->get(wf);
  41                 if (unlikely(err)) {
  42                         rpm_put(wf);
  43                         mutex_unlock(&wf->mutex);
  44                         return err;
  45                 }
  46
  47                 smp_mb__before_atomic(); /* release wf->count */
  48         }
  49         atomic_inc(&wf->count);
  50         mutex_unlock(&wf->mutex);
  51
  52         INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
  53         return 0;
  54 }
  55
  56 static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
  57 {
  58         INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
  59         if (unlikely(!atomic_dec_and_test(&wf->count)))
  60                 goto unlock;
  61
  62         /* ops->put() must reschedule its own release on error/deferral */
  63         if (likely(!wf->ops->put(wf))) {
  64                 rpm_put(wf);
  65                 wake_up_var(&wf->wakeref);
  66         }
  67
  68 unlock:
  69         mutex_unlock(&wf->mutex);
  70 }
  71
  72 void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
  73 {
  74         INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));
  75
  76         /* Assume we are not in process context and so cannot sleep. */
  77         if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
  78                 mod_delayed_work(wf->i915->unordered_wq, &wf->work,
  79                                  FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));
  80                 return;
  81         }
  82
  83         ____intel_wakeref_put_last(wf);
  84 }
  85
  86 static void __intel_wakeref_put_work(struct work_struct *wrk)
  87 {
  88         struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);
  89
  90         if (atomic_add_unless(&wf->count, -1, 1))
  91                 return;
  92
  93         mutex_lock(&wf->mutex);
  94         ____intel_wakeref_put_last(wf);
  95 }
  96
  97 void __intel_wakeref_init(struct intel_wakeref *wf,
  98                           struct drm_i915_private *i915,
  99                           const struct intel_wakeref_ops *ops,
 100                           struct intel_wakeref_lockclass *key)
 101 {
 102         wf->i915 = i915;
 103         wf->ops = ops;
 104
 105         __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
 106         atomic_set(&wf->count, 0);
 107         wf->wakeref = 0;
 108
 109         INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);
 110         lockdep_init_map(&wf->work.work.lockdep_map,
 111                          "wakeref.work", &key->work, 0);
 112 }
 113
 114 int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
 115 {
 116         int err;
 117
 118         might_sleep();
 119
 120         err = wait_var_event_killable(&wf->wakeref,
 121                                       !intel_wakeref_is_active(wf));
 122         if (err)
 123                 return err;
 124
 125         intel_wakeref_unlock_wait(wf);
 126         return 0;
 127 }
 128
 129 static void wakeref_auto_timeout(struct timer_list *t)
 130 {
 131         struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
 132         intel_wakeref_t wakeref;
 133         unsigned long flags;
 134
 135         if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
 136                 return;
 137
 138         wakeref = fetch_and_zero(&wf->wakeref);
 139         spin_unlock_irqrestore(&wf->lock, flags);
 140
 141         intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);
 142 }
 143
 144 void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
 145                              struct drm_i915_private *i915)
 146 {
 147         spin_lock_init(&wf->lock);
 148         timer_setup(&wf->timer, wakeref_auto_timeout, 0);
 149         refcount_set(&wf->count, 0);
 150         wf->wakeref = 0;
 151         wf->i915 = i915;
 152 }
 153
 154 void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
 155 {
 156         unsigned long flags;
 157
 158         if (!timeout) {
 159                 if (del_timer_sync(&wf->timer))
 160                         wakeref_auto_timeout(&wf->timer);
 161                 return;
 162         }
 163
 164         /* Our mission is that we only extend an already active wakeref */
 165         assert_rpm_wakelock_held(&wf->i915->runtime_pm);
 166
 167         if (!refcount_inc_not_zero(&wf->count)) {
 168                 spin_lock_irqsave(&wf->lock, flags);
 169                 if (!refcount_inc_not_zero(&wf->count)) {
 170                         INTEL_WAKEREF_BUG_ON(wf->wakeref);
 171                         wf->wakeref =
 172                                 intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm);
 173                         refcount_set(&wf->count, 1);
 174                 }
 175                 spin_unlock_irqrestore(&wf->lock, flags);
 176         }
 177
 178         /*
 179          * If we extend a pending timer, we will only get a single timer
 180          * callback and so need to cancel the local inc by running the
 181          * elided callback to keep the wf->count balanced.
 182          */
 183         if (mod_timer(&wf->timer, jiffies + timeout))
 184                 wakeref_auto_timeout(&wf->timer);
 185 }
 186
 187 void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
 188 {
 189         intel_wakeref_auto(wf, 0);
 190         INTEL_WAKEREF_BUG_ON(wf->wakeref);
 191 }