Commit | Line | Data |
---|---|---|
d91e6578 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2019 Intel Corporation | |
5 | */ | |
6 | ||
c7302f20 CW |
7 | #include <linux/wait_bit.h> |
8 | ||
58a111f0 | 9 | #include "intel_runtime_pm.h" |
fb993aa7 | 10 | #include "intel_wakeref.h" |
8d208a5e | 11 | #include "i915_drv.h" |
d91e6578 | 12 | |
c7302f20 | 13 | int __intel_wakeref_get_first(struct intel_wakeref *wf) |
d91e6578 | 14 | { |
4cd64e9d CW |
15 | intel_wakeref_t wakeref; |
16 | int ret = 0; | |
17 | ||
18 | wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm); | |
d91e6578 CW |
19 | /* |
20 | * Treat get/put as different subclasses, as we may need to run | |
21 | * the put callback from under the shrinker and do not want to | |
22 | * cross-contanimate that callback with any extra work performed | |
23 | * upon acquiring the wakeref. | |
24 | */ | |
25 | mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING); | |
d91e6578 | 26 | |
4cd64e9d CW |
27 | if (!atomic_read(&wf->count)) { |
28 | INTEL_WAKEREF_BUG_ON(wf->wakeref); | |
29 | wf->wakeref = wakeref; | |
30 | wakeref = 0; | |
31 | ||
32 | ret = wf->ops->get(wf); | |
33 | if (ret) { | |
34 | wakeref = xchg(&wf->wakeref, 0); | |
35 | wake_up_var(&wf->wakeref); | |
36 | goto unlock; | |
d91e6578 CW |
37 | } |
38 | ||
39 | smp_mb__before_atomic(); /* release wf->count */ | |
40 | } | |
4cd64e9d | 41 | |
d91e6578 | 42 | atomic_inc(&wf->count); |
4cd64e9d CW |
43 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
44 | ||
45 | unlock: | |
d91e6578 | 46 | mutex_unlock(&wf->mutex); |
4cd64e9d CW |
47 | if (unlikely(wakeref)) |
48 | intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); | |
d91e6578 | 49 | |
4cd64e9d | 50 | return ret; |
d91e6578 CW |
51 | } |
52 | ||
c7302f20 | 53 | static void ____intel_wakeref_put_last(struct intel_wakeref *wf) |
d91e6578 | 54 | { |
4cd64e9d CW |
55 | intel_wakeref_t wakeref = 0; |
56 | ||
07779a76 CW |
57 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
58 | if (unlikely(!atomic_dec_and_test(&wf->count))) | |
c7302f20 CW |
59 | goto unlock; |
60 | ||
a79ca656 | 61 | /* ops->put() must reschedule its own release on error/deferral */ |
c7302f20 | 62 | if (likely(!wf->ops->put(wf))) { |
4cd64e9d CW |
63 | INTEL_WAKEREF_BUG_ON(!wf->wakeref); |
64 | wakeref = xchg(&wf->wakeref, 0); | |
c7302f20 | 65 | wake_up_var(&wf->wakeref); |
c7302f20 | 66 | } |
d91e6578 | 67 | |
c7302f20 | 68 | unlock: |
d91e6578 | 69 | mutex_unlock(&wf->mutex); |
4cd64e9d CW |
70 | if (wakeref) |
71 | intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); | |
c7302f20 CW |
72 | } |
73 | ||
07779a76 | 74 | void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) |
c7302f20 | 75 | { |
e9037e7f | 76 | INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work)); |
d91e6578 | 77 | |
c7302f20 | 78 | /* Assume we are not in process context and so cannot sleep. */ |
07779a76 | 79 | if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { |
848a4e5c | 80 | mod_delayed_work(wf->i915->unordered_wq, &wf->work, |
e9037e7f | 81 | FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); |
c7302f20 CW |
82 | return; |
83 | } | |
84 | ||
85 | ____intel_wakeref_put_last(wf); | |
d91e6578 CW |
86 | } |
87 | ||
c7302f20 | 88 | static void __intel_wakeref_put_work(struct work_struct *wrk) |
d91e6578 | 89 | { |
e9037e7f | 90 | struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work); |
c7302f20 CW |
91 | |
92 | if (atomic_add_unless(&wf->count, -1, 1)) | |
93 | return; | |
94 | ||
95 | mutex_lock(&wf->mutex); | |
96 | ____intel_wakeref_put_last(wf); | |
97 | } | |
98 | ||
99 | void __intel_wakeref_init(struct intel_wakeref *wf, | |
8d208a5e | 100 | struct drm_i915_private *i915, |
c7302f20 | 101 | const struct intel_wakeref_ops *ops, |
5e4e06e4 AH |
102 | struct intel_wakeref_lockclass *key, |
103 | const char *name) | |
c7302f20 | 104 | { |
8d208a5e | 105 | wf->i915 = i915; |
c7302f20 CW |
106 | wf->ops = ops; |
107 | ||
cdd280b1 | 108 | __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex); |
d91e6578 | 109 | atomic_set(&wf->count, 0); |
7ee280a7 | 110 | wf->wakeref = 0; |
c7302f20 | 111 | |
e9037e7f CW |
112 | INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work); |
113 | lockdep_init_map(&wf->work.work.lockdep_map, | |
114 | "wakeref.work", &key->work, 0); | |
5e4e06e4 AH |
115 | |
116 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_WAKEREF) | |
117 | ref_tracker_dir_init(&wf->debug, INTEL_REFTRACK_DEAD_COUNT, name); | |
118 | #endif | |
c7302f20 CW |
119 | } |
120 | ||
121 | int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) | |
122 | { | |
f4ba0707 CW |
123 | int err; |
124 | ||
125 | might_sleep(); | |
126 | ||
127 | err = wait_var_event_killable(&wf->wakeref, | |
128 | !intel_wakeref_is_active(wf)); | |
129 | if (err) | |
130 | return err; | |
131 | ||
132 | intel_wakeref_unlock_wait(wf); | |
133 | return 0; | |
d91e6578 | 134 | } |
b27e35ae CW |
135 | |
136 | static void wakeref_auto_timeout(struct timer_list *t) | |
137 | { | |
138 | struct intel_wakeref_auto *wf = from_timer(wf, t, timer); | |
139 | intel_wakeref_t wakeref; | |
140 | unsigned long flags; | |
141 | ||
142 | if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags)) | |
143 | return; | |
144 | ||
145 | wakeref = fetch_and_zero(&wf->wakeref); | |
146 | spin_unlock_irqrestore(&wf->lock, flags); | |
147 | ||
8d208a5e | 148 | intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref); |
b27e35ae CW |
149 | } |
150 | ||
151 | void intel_wakeref_auto_init(struct intel_wakeref_auto *wf, | |
8d208a5e | 152 | struct drm_i915_private *i915) |
b27e35ae CW |
153 | { |
154 | spin_lock_init(&wf->lock); | |
155 | timer_setup(&wf->timer, wakeref_auto_timeout, 0); | |
156 | refcount_set(&wf->count, 0); | |
157 | wf->wakeref = 0; | |
8d208a5e | 158 | wf->i915 = i915; |
b27e35ae CW |
159 | } |
160 | ||
161 | void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout) | |
162 | { | |
163 | unsigned long flags; | |
164 | ||
165 | if (!timeout) { | |
166 | if (del_timer_sync(&wf->timer)) | |
167 | wakeref_auto_timeout(&wf->timer); | |
168 | return; | |
169 | } | |
170 | ||
171 | /* Our mission is that we only extend an already active wakeref */ | |
8d208a5e | 172 | assert_rpm_wakelock_held(&wf->i915->runtime_pm); |
b27e35ae CW |
173 | |
174 | if (!refcount_inc_not_zero(&wf->count)) { | |
175 | spin_lock_irqsave(&wf->lock, flags); | |
0c1f8457 | 176 | if (!refcount_inc_not_zero(&wf->count)) { |
fb993aa7 | 177 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
8d208a5e LC |
178 | wf->wakeref = |
179 | intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm); | |
0c1f8457 | 180 | refcount_set(&wf->count, 1); |
b27e35ae | 181 | } |
b27e35ae CW |
182 | spin_unlock_irqrestore(&wf->lock, flags); |
183 | } | |
184 | ||
185 | /* | |
186 | * If we extend a pending timer, we will only get a single timer | |
187 | * callback and so need to cancel the local inc by running the | |
188 | * elided callback to keep the wf->count balanced. | |
189 | */ | |
190 | if (mod_timer(&wf->timer, jiffies + timeout)) | |
191 | wakeref_auto_timeout(&wf->timer); | |
192 | } | |
193 | ||
194 | void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf) | |
195 | { | |
196 | intel_wakeref_auto(wf, 0); | |
fb993aa7 | 197 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
b27e35ae | 198 | } |
b49e894c AH |
199 | |
200 | void intel_ref_tracker_show(struct ref_tracker_dir *dir, | |
201 | struct drm_printer *p) | |
202 | { | |
203 | const size_t buf_size = PAGE_SIZE; | |
204 | char *buf, *sb, *se; | |
205 | size_t count; | |
206 | ||
207 | buf = kmalloc(buf_size, GFP_NOWAIT); | |
208 | if (!buf) | |
209 | return; | |
210 | ||
211 | count = ref_tracker_dir_snprint(dir, buf, buf_size); | |
212 | if (!count) | |
213 | goto free; | |
214 | /* printk does not like big buffers, so we split it */ | |
215 | for (sb = buf; *sb; sb = se + 1) { | |
216 | se = strchrnul(sb, '\n'); | |
217 | drm_printf(p, "%.*s", (int)(se - sb + 1), sb); | |
218 | if (!*se) | |
219 | break; | |
220 | } | |
221 | if (count >= buf_size) | |
222 | drm_printf(p, "\n...dropped %zd extra bytes of leak report.\n", | |
223 | count + 1 - buf_size); | |
224 | free: | |
225 | kfree(buf); | |
226 | } |