Commit | Line | Data |
---|---|---|
5de363b6 | 1 | // SPDX-License-Identifier: GPL-2.0 |
b02c999a RW |
2 | /* |
3 | * drivers/base/power/domain_governor.c - Governors for device PM domains. | |
4 | * | |
5 | * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp. | |
b02c999a | 6 | */ |
b02c999a RW |
7 | #include <linux/kernel.h> |
8 | #include <linux/pm_domain.h> | |
9 | #include <linux/pm_qos.h> | |
221e9b58 | 10 | #include <linux/hrtimer.h> |
e9499968 UH |
11 | #include <linux/cpuidle.h> |
12 | #include <linux/cpumask.h> | |
13 | #include <linux/ktime.h> | |
b02c999a | 14 | |
a5bef810 RW |
15 | static int dev_update_qos_constraint(struct device *dev, void *data) |
16 | { | |
17 | s64 *constraint_ns_p = data; | |
704d2ce6 | 18 | s64 constraint_ns; |
a5bef810 | 19 | |
704d2ce6 | 20 | if (dev->power.subsys_data && dev->power.subsys_data->domain_data) { |
66d29d80 UH |
21 | struct gpd_timing_data *td = dev_gpd_data(dev)->td; |
22 | ||
704d2ce6 RW |
23 | /* |
24 | * Only take suspend-time QoS constraints of devices into | |
25 | * account, because constraints updated after the device has | |
26 | * been suspended are not guaranteed to be taken into account | |
27 | * anyway. In order for them to take effect, the device has to | |
28 | * be resumed and suspended again. | |
29 | */ | |
66d29d80 UH |
30 | constraint_ns = td ? td->effective_constraint_ns : |
31 | PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; | |
704d2ce6 RW |
32 | } else { |
33 | /* | |
34 | * The child is not in a domain and there's no info on its | |
35 | * suspend/resume latencies, so assume them to be negligible and | |
36 | * take its current PM QoS constraint (that's the only thing | |
37 | * known at this point anyway). | |
38 | */ | |
2a79ea5e | 39 | constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY); |
0759e80b | 40 | constraint_ns *= NSEC_PER_USEC; |
a5bef810 | 41 | } |
704d2ce6 | 42 | |
0759e80b | 43 | if (constraint_ns < *constraint_ns_p) |
a5bef810 RW |
44 | *constraint_ns_p = constraint_ns; |
45 | ||
46 | return 0; | |
47 | } | |
48 | ||
b02c999a | 49 | /** |
9df3921e | 50 | * default_suspend_ok - Default PM domain governor routine to suspend devices. |
b02c999a RW |
51 | * @dev: Device to check. |
52 | */ | |
9df3921e | 53 | static bool default_suspend_ok(struct device *dev) |
b02c999a | 54 | { |
66d29d80 | 55 | struct gpd_timing_data *td = dev_gpd_data(dev)->td; |
6ff7bb0d | 56 | unsigned long flags; |
a5bef810 | 57 | s64 constraint_ns; |
b02c999a RW |
58 | |
59 | dev_dbg(dev, "%s()\n", __func__); | |
60 | ||
6ff7bb0d RW |
61 | spin_lock_irqsave(&dev->power.lock, flags); |
62 | ||
63 | if (!td->constraint_changed) { | |
9df3921e | 64 | bool ret = td->cached_suspend_ok; |
6ff7bb0d RW |
65 | |
66 | spin_unlock_irqrestore(&dev->power.lock, flags); | |
67 | return ret; | |
68 | } | |
69 | td->constraint_changed = false; | |
9df3921e | 70 | td->cached_suspend_ok = false; |
0759e80b | 71 | td->effective_constraint_ns = 0; |
8262331e | 72 | constraint_ns = __dev_pm_qos_resume_latency(dev); |
6ff7bb0d RW |
73 | |
74 | spin_unlock_irqrestore(&dev->power.lock, flags); | |
75 | ||
0759e80b | 76 | if (constraint_ns == 0) |
a5bef810 | 77 | return false; |
b02c999a | 78 | |
a5bef810 RW |
79 | constraint_ns *= NSEC_PER_USEC; |
80 | /* | |
81 | * We can walk the children without any additional locking, because | |
6ff7bb0d RW |
82 | * they all have been suspended at this point and their |
83 | * effective_constraint_ns fields won't be modified in parallel with us. | |
a5bef810 RW |
84 | */ |
85 | if (!dev->power.ignore_children) | |
86 | device_for_each_child(dev, &constraint_ns, | |
87 | dev_update_qos_constraint); | |
88 | ||
0759e80b | 89 | if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) { |
704d2ce6 | 90 | /* "No restriction", so the device is allowed to suspend. */ |
0759e80b | 91 | td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS; |
704d2ce6 | 92 | td->cached_suspend_ok = true; |
0759e80b | 93 | } else if (constraint_ns == 0) { |
704d2ce6 RW |
94 | /* |
95 | * This triggers if one of the children that don't belong to a | |
0759e80b RW |
96 | * domain has a zero PM QoS constraint and it's better not to |
97 | * suspend then. effective_constraint_ns is zero already and | |
98 | * cached_suspend_ok is false, so bail out. | |
704d2ce6 RW |
99 | */ |
100 | return false; | |
101 | } else { | |
2b1d88cd UH |
102 | constraint_ns -= td->suspend_latency_ns + |
103 | td->resume_latency_ns; | |
704d2ce6 | 104 | /* |
0759e80b RW |
105 | * effective_constraint_ns is zero already and cached_suspend_ok |
106 | * is false, so if the computed value is not positive, return | |
107 | * right away. | |
704d2ce6 RW |
108 | */ |
109 | if (constraint_ns <= 0) | |
a5bef810 | 110 | return false; |
704d2ce6 RW |
111 | |
112 | td->effective_constraint_ns = constraint_ns; | |
113 | td->cached_suspend_ok = true; | |
a5bef810 | 114 | } |
a98f1b78 | 115 | |
a5bef810 RW |
116 | /* |
117 | * The children have been suspended already, so we don't need to take | |
9df3921e | 118 | * their suspend latencies into account here. |
a5bef810 | 119 | */ |
9df3921e | 120 | return td->cached_suspend_ok; |
b02c999a RW |
121 | } |
122 | ||
c79aa080 LI |
123 | static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now) |
124 | { | |
125 | ktime_t domain_wakeup = KTIME_MAX; | |
126 | ktime_t next_wakeup; | |
127 | struct pm_domain_data *pdd; | |
128 | struct gpd_link *link; | |
129 | ||
130 | if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY)) | |
131 | return; | |
132 | ||
133 | /* | |
134 | * Devices that have a predictable wakeup pattern, may specify | |
135 | * their next wakeup. Let's find the next wakeup from all the | |
136 | * devices attached to this domain and from all the sub-domains. | |
137 | * It is possible that component's a next wakeup may have become | |
138 | * stale when we read that here. We will ignore to ensure the domain | |
139 | * is able to enter its optimal idle state. | |
140 | */ | |
141 | list_for_each_entry(pdd, &genpd->dev_list, list_node) { | |
9c74f2ac | 142 | next_wakeup = to_gpd_data(pdd)->td->next_wakeup; |
c79aa080 LI |
143 | if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) |
144 | if (ktime_before(next_wakeup, domain_wakeup)) | |
145 | domain_wakeup = next_wakeup; | |
146 | } | |
147 | ||
148 | list_for_each_entry(link, &genpd->parent_links, parent_node) { | |
f38d1a6d UH |
149 | struct genpd_governor_data *cgd = link->child->gd; |
150 | ||
151 | next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX; | |
c79aa080 LI |
152 | if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now)) |
153 | if (ktime_before(next_wakeup, domain_wakeup)) | |
154 | domain_wakeup = next_wakeup; | |
155 | } | |
156 | ||
f38d1a6d | 157 | genpd->gd->next_wakeup = domain_wakeup; |
c79aa080 LI |
158 | } |
159 | ||
160 | static bool next_wakeup_allows_state(struct generic_pm_domain *genpd, | |
161 | unsigned int state, ktime_t now) | |
162 | { | |
f38d1a6d | 163 | ktime_t domain_wakeup = genpd->gd->next_wakeup; |
c79aa080 LI |
164 | s64 idle_time_ns, min_sleep_ns; |
165 | ||
166 | min_sleep_ns = genpd->states[state].power_off_latency_ns + | |
167 | genpd->states[state].residency_ns; | |
168 | ||
169 | idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); | |
170 | ||
171 | return idle_time_ns >= min_sleep_ns; | |
172 | } | |
173 | ||
fc5cbf0c AH |
174 | static bool __default_power_down_ok(struct dev_pm_domain *pd, |
175 | unsigned int state) | |
221e9b58 RW |
176 | { |
177 | struct generic_pm_domain *genpd = pd_to_genpd(pd); | |
178 | struct gpd_link *link; | |
179 | struct pm_domain_data *pdd; | |
b723b0eb | 180 | s64 min_off_time_ns; |
221e9b58 | 181 | s64 off_on_time_ns; |
221e9b58 | 182 | |
fc5cbf0c AH |
183 | off_on_time_ns = genpd->states[state].power_off_latency_ns + |
184 | genpd->states[state].power_on_latency_ns; | |
6ff7bb0d | 185 | |
b723b0eb | 186 | min_off_time_ns = -1; |
221e9b58 RW |
187 | /* |
188 | * Check if subdomains can be off for enough time. | |
189 | * | |
190 | * All subdomains have been powered off already at this point. | |
191 | */ | |
8d87ae48 | 192 | list_for_each_entry(link, &genpd->parent_links, parent_node) { |
f38d1a6d UH |
193 | struct genpd_governor_data *cgd = link->child->gd; |
194 | ||
195 | s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1; | |
221e9b58 RW |
196 | |
197 | if (sd_max_off_ns < 0) | |
198 | continue; | |
199 | ||
221e9b58 RW |
200 | /* |
201 | * Check if the subdomain is allowed to be off long enough for | |
202 | * the current domain to turn off and on (that's how much time | |
203 | * it will have to wait worst case). | |
204 | */ | |
205 | if (sd_max_off_ns <= off_on_time_ns) | |
206 | return false; | |
b723b0eb RW |
207 | |
208 | if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0) | |
209 | min_off_time_ns = sd_max_off_ns; | |
221e9b58 RW |
210 | } |
211 | ||
212 | /* | |
213 | * Check if the devices in the domain can be off enough time. | |
214 | */ | |
221e9b58 RW |
215 | list_for_each_entry(pdd, &genpd->dev_list, list_node) { |
216 | struct gpd_timing_data *td; | |
dd8683e9 | 217 | s64 constraint_ns; |
221e9b58 | 218 | |
dd8683e9 RW |
219 | /* |
220 | * Check if the device is allowed to be off long enough for the | |
221 | * domain to turn off and on (that's how much time it will | |
222 | * have to wait worst case). | |
223 | */ | |
66d29d80 | 224 | td = to_gpd_data(pdd)->td; |
dd8683e9 | 225 | constraint_ns = td->effective_constraint_ns; |
704d2ce6 | 226 | /* |
0759e80b RW |
227 | * Zero means "no suspend at all" and this runs only when all |
228 | * devices in the domain are suspended, so it must be positive. | |
704d2ce6 | 229 | */ |
0759e80b | 230 | if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) |
dd8683e9 | 231 | continue; |
221e9b58 | 232 | |
dd8683e9 RW |
233 | if (constraint_ns <= off_on_time_ns) |
234 | return false; | |
235 | ||
b723b0eb RW |
236 | if (min_off_time_ns > constraint_ns || min_off_time_ns < 0) |
237 | min_off_time_ns = constraint_ns; | |
221e9b58 RW |
238 | } |
239 | ||
240 | /* | |
dd8683e9 RW |
241 | * If the computed minimum device off time is negative, there are no |
242 | * latency constraints, so the domain can spend arbitrary time in the | |
243 | * "off" state. | |
221e9b58 | 244 | */ |
b723b0eb | 245 | if (min_off_time_ns < 0) |
dd8683e9 | 246 | return true; |
221e9b58 RW |
247 | |
248 | /* | |
b723b0eb RW |
249 | * The difference between the computed minimum subdomain or device off |
250 | * time and the time needed to turn the domain on is the maximum | |
251 | * theoretical time this domain can spend in the "off" state. | |
221e9b58 | 252 | */ |
f38d1a6d | 253 | genpd->gd->max_off_time_ns = min_off_time_ns - |
fc5cbf0c | 254 | genpd->states[state].power_on_latency_ns; |
221e9b58 RW |
255 | return true; |
256 | } | |
257 | ||
268cd2ed | 258 | /** |
c79aa080 | 259 | * _default_power_down_ok - Default generic PM domain power off governor routine. |
268cd2ed | 260 | * @pd: PM domain to check. |
763663c9 | 261 | * @now: current ktime. |
268cd2ed KK |
262 | * |
263 | * This routine must be executed under the PM domain's lock. | |
264 | */ | |
c79aa080 | 265 | static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now) |
fc5cbf0c AH |
266 | { |
267 | struct generic_pm_domain *genpd = pd_to_genpd(pd); | |
f38d1a6d | 268 | struct genpd_governor_data *gd = genpd->gd; |
c79aa080 | 269 | int state_idx = genpd->state_count - 1; |
fc5cbf0c AH |
270 | struct gpd_link *link; |
271 | ||
c79aa080 LI |
272 | /* |
273 | * Find the next wakeup from devices that can determine their own wakeup | |
274 | * to find when the domain would wakeup and do it for every device down | |
275 | * the hierarchy. It is not worth while to sleep if the state's residency | |
276 | * cannot be met. | |
277 | */ | |
278 | update_domain_next_wakeup(genpd, now); | |
f38d1a6d | 279 | if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) { |
c79aa080 LI |
280 | /* Let's find out the deepest domain idle state, the devices prefer */ |
281 | while (state_idx >= 0) { | |
282 | if (next_wakeup_allows_state(genpd, state_idx, now)) { | |
f38d1a6d | 283 | gd->max_off_time_changed = true; |
c79aa080 LI |
284 | break; |
285 | } | |
286 | state_idx--; | |
287 | } | |
288 | ||
289 | if (state_idx < 0) { | |
290 | state_idx = 0; | |
f38d1a6d | 291 | gd->cached_power_down_ok = false; |
c79aa080 LI |
292 | goto done; |
293 | } | |
294 | } | |
295 | ||
f38d1a6d UH |
296 | if (!gd->max_off_time_changed) { |
297 | genpd->state_idx = gd->cached_power_down_state_idx; | |
298 | return gd->cached_power_down_ok; | |
e9499968 | 299 | } |
fc5cbf0c AH |
300 | |
301 | /* | |
8d87ae48 | 302 | * We have to invalidate the cached results for the parents, so |
fc5cbf0c | 303 | * use the observation that default_power_down_ok() is not |
8d87ae48 | 304 | * going to be called for any parent until this instance |
fc5cbf0c AH |
305 | * returns. |
306 | */ | |
f38d1a6d UH |
307 | list_for_each_entry(link, &genpd->child_links, child_node) { |
308 | struct genpd_governor_data *pgd = link->parent->gd; | |
309 | ||
310 | if (pgd) | |
311 | pgd->max_off_time_changed = true; | |
312 | } | |
fc5cbf0c | 313 | |
f38d1a6d UH |
314 | gd->max_off_time_ns = -1; |
315 | gd->max_off_time_changed = false; | |
316 | gd->cached_power_down_ok = true; | |
fc5cbf0c | 317 | |
c79aa080 LI |
318 | /* |
319 | * Find a state to power down to, starting from the state | |
320 | * determined by the next wakeup. | |
321 | */ | |
322 | while (!__default_power_down_ok(pd, state_idx)) { | |
323 | if (state_idx == 0) { | |
f38d1a6d | 324 | gd->cached_power_down_ok = false; |
fc5cbf0c AH |
325 | break; |
326 | } | |
c79aa080 | 327 | state_idx--; |
fc5cbf0c AH |
328 | } |
329 | ||
c79aa080 LI |
330 | done: |
331 | genpd->state_idx = state_idx; | |
f38d1a6d UH |
332 | gd->cached_power_down_state_idx = genpd->state_idx; |
333 | return gd->cached_power_down_ok; | |
fc5cbf0c AH |
334 | } |
335 | ||
c79aa080 LI |
336 | static bool default_power_down_ok(struct dev_pm_domain *pd) |
337 | { | |
338 | return _default_power_down_ok(pd, ktime_get()); | |
339 | } | |
340 | ||
e9499968 UH |
341 | #ifdef CONFIG_CPU_IDLE |
342 | static bool cpu_power_down_ok(struct dev_pm_domain *pd) | |
343 | { | |
344 | struct generic_pm_domain *genpd = pd_to_genpd(pd); | |
345 | struct cpuidle_device *dev; | |
346 | ktime_t domain_wakeup, next_hrtimer; | |
c79aa080 | 347 | ktime_t now = ktime_get(); |
e9499968 UH |
348 | s64 idle_duration_ns; |
349 | int cpu, i; | |
350 | ||
351 | /* Validate dev PM QoS constraints. */ | |
c79aa080 | 352 | if (!_default_power_down_ok(pd, now)) |
e9499968 UH |
353 | return false; |
354 | ||
355 | if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) | |
356 | return true; | |
357 | ||
358 | /* | |
359 | * Find the next wakeup for any of the online CPUs within the PM domain | |
360 | * and its subdomains. Note, we only need the genpd->cpus, as it already | |
361 | * contains a mask of all CPUs from subdomains. | |
362 | */ | |
363 | domain_wakeup = ktime_set(KTIME_SEC_MAX, 0); | |
364 | for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) { | |
365 | dev = per_cpu(cpuidle_devices, cpu); | |
366 | if (dev) { | |
367 | next_hrtimer = READ_ONCE(dev->next_hrtimer); | |
368 | if (ktime_before(next_hrtimer, domain_wakeup)) | |
369 | domain_wakeup = next_hrtimer; | |
370 | } | |
371 | } | |
372 | ||
373 | /* The minimum idle duration is from now - until the next wakeup. */ | |
c79aa080 | 374 | idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now)); |
e9499968 UH |
375 | if (idle_duration_ns <= 0) |
376 | return false; | |
377 | ||
1498c503 MS |
378 | /* Store the next domain_wakeup to allow consumers to use it. */ |
379 | genpd->gd->next_hrtimer = domain_wakeup; | |
380 | ||
e9499968 UH |
381 | /* |
382 | * Find the deepest idle state that has its residency value satisfied | |
383 | * and by also taking into account the power off latency for the state. | |
384 | * Start at the state picked by the dev PM QoS constraint validation. | |
385 | */ | |
386 | i = genpd->state_idx; | |
387 | do { | |
388 | if (idle_duration_ns >= (genpd->states[i].residency_ns + | |
389 | genpd->states[i].power_off_latency_ns)) { | |
390 | genpd->state_idx = i; | |
391 | return true; | |
392 | } | |
393 | } while (--i >= 0); | |
394 | ||
395 | return false; | |
396 | } | |
397 | ||
398 | struct dev_power_governor pm_domain_cpu_gov = { | |
399 | .suspend_ok = default_suspend_ok, | |
400 | .power_down_ok = cpu_power_down_ok, | |
401 | }; | |
402 | #endif | |
403 | ||
e59a8db8 | 404 | struct dev_power_governor simple_qos_governor = { |
9df3921e | 405 | .suspend_ok = default_suspend_ok, |
e59a8db8 RW |
406 | .power_down_ok = default_power_down_ok, |
407 | }; | |
408 | ||
925b44a2 MB |
409 | /** |
410 | * pm_genpd_gov_always_on - A governor implementing an always-on policy | |
411 | */ | |
412 | struct dev_power_governor pm_domain_always_on_gov = { | |
9df3921e | 413 | .suspend_ok = default_suspend_ok, |
925b44a2 | 414 | }; |