Commit | Line | Data |
---|---|---|
d91e6578 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2019 Intel Corporation | |
5 | */ | |
6 | ||
c7302f20 CW |
7 | #include <linux/wait_bit.h> |
8 | ||
58a111f0 | 9 | #include "intel_runtime_pm.h" |
fb993aa7 | 10 | #include "intel_wakeref.h" |
d91e6578 | 11 | |
c7302f20 | 12 | static void rpm_get(struct intel_wakeref *wf) |
7ee280a7 | 13 | { |
c7302f20 | 14 | wf->wakeref = intel_runtime_pm_get(wf->rpm); |
7ee280a7 CW |
15 | } |
16 | ||
c7302f20 | 17 | static void rpm_put(struct intel_wakeref *wf) |
7ee280a7 CW |
18 | { |
19 | intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref); | |
20 | ||
c7302f20 | 21 | intel_runtime_pm_put(wf->rpm, wakeref); |
fb993aa7 | 22 | INTEL_WAKEREF_BUG_ON(!wakeref); |
7ee280a7 CW |
23 | } |
24 | ||
c7302f20 | 25 | int __intel_wakeref_get_first(struct intel_wakeref *wf) |
d91e6578 CW |
26 | { |
27 | /* | |
28 | * Treat get/put as different subclasses, as we may need to run | |
29 | * the put callback from under the shrinker and do not want to | |
30 | * cross-contanimate that callback with any extra work performed | |
31 | * upon acquiring the wakeref. | |
32 | */ | |
33 | mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING); | |
34 | if (!atomic_read(&wf->count)) { | |
35 | int err; | |
36 | ||
c7302f20 | 37 | rpm_get(wf); |
d91e6578 | 38 | |
c7302f20 | 39 | err = wf->ops->get(wf); |
d91e6578 | 40 | if (unlikely(err)) { |
c7302f20 | 41 | rpm_put(wf); |
d91e6578 CW |
42 | mutex_unlock(&wf->mutex); |
43 | return err; | |
44 | } | |
45 | ||
46 | smp_mb__before_atomic(); /* release wf->count */ | |
47 | } | |
48 | atomic_inc(&wf->count); | |
49 | mutex_unlock(&wf->mutex); | |
50 | ||
fb993aa7 | 51 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
d91e6578 CW |
52 | return 0; |
53 | } | |
54 | ||
c7302f20 | 55 | static void ____intel_wakeref_put_last(struct intel_wakeref *wf) |
d91e6578 | 56 | { |
07779a76 CW |
57 | INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); |
58 | if (unlikely(!atomic_dec_and_test(&wf->count))) | |
c7302f20 CW |
59 | goto unlock; |
60 | ||
a79ca656 | 61 | /* ops->put() must reschedule its own release on error/deferral */ |
c7302f20 CW |
62 | if (likely(!wf->ops->put(wf))) { |
63 | rpm_put(wf); | |
64 | wake_up_var(&wf->wakeref); | |
c7302f20 | 65 | } |
d91e6578 | 66 | |
c7302f20 | 67 | unlock: |
d91e6578 | 68 | mutex_unlock(&wf->mutex); |
c7302f20 CW |
69 | } |
70 | ||
07779a76 | 71 | void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) |
c7302f20 CW |
72 | { |
73 | INTEL_WAKEREF_BUG_ON(work_pending(&wf->work)); | |
d91e6578 | 74 | |
c7302f20 | 75 | /* Assume we are not in process context and so cannot sleep. */ |
07779a76 | 76 | if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { |
c7302f20 CW |
77 | schedule_work(&wf->work); |
78 | return; | |
79 | } | |
80 | ||
81 | ____intel_wakeref_put_last(wf); | |
d91e6578 CW |
82 | } |
83 | ||
c7302f20 | 84 | static void __intel_wakeref_put_work(struct work_struct *wrk) |
d91e6578 | 85 | { |
c7302f20 CW |
86 | struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work); |
87 | ||
88 | if (atomic_add_unless(&wf->count, -1, 1)) | |
89 | return; | |
90 | ||
91 | mutex_lock(&wf->mutex); | |
92 | ____intel_wakeref_put_last(wf); | |
93 | } | |
94 | ||
95 | void __intel_wakeref_init(struct intel_wakeref *wf, | |
96 | struct intel_runtime_pm *rpm, | |
97 | const struct intel_wakeref_ops *ops, | |
98 | struct lock_class_key *key) | |
99 | { | |
100 | wf->rpm = rpm; | |
101 | wf->ops = ops; | |
102 | ||
d91e6578 CW |
103 | __mutex_init(&wf->mutex, "wakeref", key); |
104 | atomic_set(&wf->count, 0); | |
7ee280a7 | 105 | wf->wakeref = 0; |
c7302f20 CW |
106 | |
107 | INIT_WORK(&wf->work, __intel_wakeref_put_work); | |
108 | } | |
109 | ||
110 | int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) | |
111 | { | |
f4ba0707 CW |
112 | int err; |
113 | ||
114 | might_sleep(); | |
115 | ||
116 | err = wait_var_event_killable(&wf->wakeref, | |
117 | !intel_wakeref_is_active(wf)); | |
118 | if (err) | |
119 | return err; | |
120 | ||
121 | intel_wakeref_unlock_wait(wf); | |
122 | return 0; | |
d91e6578 | 123 | } |
b27e35ae CW |
124 | |
125 | static void wakeref_auto_timeout(struct timer_list *t) | |
126 | { | |
127 | struct intel_wakeref_auto *wf = from_timer(wf, t, timer); | |
128 | intel_wakeref_t wakeref; | |
129 | unsigned long flags; | |
130 | ||
131 | if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags)) | |
132 | return; | |
133 | ||
134 | wakeref = fetch_and_zero(&wf->wakeref); | |
135 | spin_unlock_irqrestore(&wf->lock, flags); | |
136 | ||
58a111f0 | 137 | intel_runtime_pm_put(wf->rpm, wakeref); |
b27e35ae CW |
138 | } |
139 | ||
140 | void intel_wakeref_auto_init(struct intel_wakeref_auto *wf, | |
58a111f0 | 141 | struct intel_runtime_pm *rpm) |
b27e35ae CW |
142 | { |
143 | spin_lock_init(&wf->lock); | |
144 | timer_setup(&wf->timer, wakeref_auto_timeout, 0); | |
145 | refcount_set(&wf->count, 0); | |
146 | wf->wakeref = 0; | |
58a111f0 | 147 | wf->rpm = rpm; |
b27e35ae CW |
148 | } |
149 | ||
150 | void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout) | |
151 | { | |
152 | unsigned long flags; | |
153 | ||
154 | if (!timeout) { | |
155 | if (del_timer_sync(&wf->timer)) | |
156 | wakeref_auto_timeout(&wf->timer); | |
157 | return; | |
158 | } | |
159 | ||
160 | /* Our mission is that we only extend an already active wakeref */ | |
58a111f0 | 161 | assert_rpm_wakelock_held(wf->rpm); |
b27e35ae CW |
162 | |
163 | if (!refcount_inc_not_zero(&wf->count)) { | |
164 | spin_lock_irqsave(&wf->lock, flags); | |
0c1f8457 | 165 | if (!refcount_inc_not_zero(&wf->count)) { |
fb993aa7 | 166 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
58a111f0 | 167 | wf->wakeref = intel_runtime_pm_get_if_in_use(wf->rpm); |
0c1f8457 | 168 | refcount_set(&wf->count, 1); |
b27e35ae | 169 | } |
b27e35ae CW |
170 | spin_unlock_irqrestore(&wf->lock, flags); |
171 | } | |
172 | ||
173 | /* | |
174 | * If we extend a pending timer, we will only get a single timer | |
175 | * callback and so need to cancel the local inc by running the | |
176 | * elided callback to keep the wf->count balanced. | |
177 | */ | |
178 | if (mod_timer(&wf->timer, jiffies + timeout)) | |
179 | wakeref_auto_timeout(&wf->timer); | |
180 | } | |
181 | ||
182 | void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf) | |
183 | { | |
184 | intel_wakeref_auto(wf, 0); | |
fb993aa7 | 185 | INTEL_WAKEREF_BUG_ON(wf->wakeref); |
b27e35ae | 186 | } |