Commit | Line | Data |
---|---|---|
c0796298 | 1 | #ifdef CONFIG_SMP |
23127296 | 2 | #include "sched-pelt.h" |
c0796298 | 3 | |
23127296 VG |
4 | int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); |
5 | int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); | |
6 | int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); | |
371bf427 | 7 | int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); |
3727e0e1 | 8 | int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); |
c0796298 | 9 | |
d4dbc991 VG |
10 | #ifdef CONFIG_SCHED_HW_PRESSURE |
11 | int update_hw_load_avg(u64 now, struct rq *rq, u64 capacity); | |
76504793 | 12 | |
d4dbc991 | 13 | static inline u64 hw_load_avg(struct rq *rq) |
76504793 | 14 | { |
d4dbc991 | 15 | return READ_ONCE(rq->avg_hw.load_avg); |
76504793 TG |
16 | } |
17 | #else | |
18 | static inline int | |
d4dbc991 | 19 | update_hw_load_avg(u64 now, struct rq *rq, u64 capacity) |
76504793 TG |
20 | { |
21 | return 0; | |
22 | } | |
23 | ||
d4dbc991 | 24 | static inline u64 hw_load_avg(struct rq *rq) |
76504793 TG |
25 | { |
26 | return 0; | |
27 | } | |
28 | #endif | |
29 | ||
11d4afd4 | 30 | #ifdef CONFIG_HAVE_SCHED_AVG_IRQ |
91c27493 VG |
31 | int update_irq_load_avg(struct rq *rq, u64 running); |
32 | #else | |
33 | static inline int | |
34 | update_irq_load_avg(struct rq *rq, u64 running) | |
35 | { | |
36 | return 0; | |
37 | } | |
38 | #endif | |
39 | ||
98b0d890 VG |
40 | #define PELT_MIN_DIVIDER (LOAD_AVG_MAX - 1024) |
41 | ||
87e867b4 VG |
42 | static inline u32 get_pelt_divider(struct sched_avg *avg) |
43 | { | |
98b0d890 | 44 | return PELT_MIN_DIVIDER + avg->period_contrib; |
87e867b4 VG |
45 | } |
46 | ||
c0796298 VG |
47 | static inline void cfs_se_util_change(struct sched_avg *avg) |
48 | { | |
49 | unsigned int enqueued; | |
50 | ||
51 | if (!sched_feat(UTIL_EST)) | |
52 | return; | |
53 | ||
68d7a190 | 54 | /* Avoid store if the flag has been already reset */ |
11137d38 | 55 | enqueued = avg->util_est; |
c0796298 VG |
56 | if (!(enqueued & UTIL_AVG_UNCHANGED)) |
57 | return; | |
58 | ||
59 | /* Reset flag to report util_avg has been updated */ | |
60 | enqueued &= ~UTIL_AVG_UNCHANGED; | |
11137d38 | 61 | WRITE_ONCE(avg->util_est, enqueued); |
c0796298 VG |
62 | } |
63 | ||
e2f3e35f VD |
64 | static inline u64 rq_clock_pelt(struct rq *rq) |
65 | { | |
66 | lockdep_assert_rq_held(rq); | |
67 | assert_clock_updated(rq); | |
68 | ||
69 | return rq->clock_pelt - rq->lost_idle_time; | |
70 | } | |
71 | ||
72 | /* The rq is idle, we can sync to clock_task */ | |
73 | static inline void _update_idle_rq_clock_pelt(struct rq *rq) | |
74 | { | |
75 | rq->clock_pelt = rq_clock_task(rq); | |
76 | ||
77 | u64_u32_store(rq->clock_idle, rq_clock(rq)); | |
78 | /* Paired with smp_rmb in migrate_se_pelt_lag() */ | |
79 | smp_wmb(); | |
80 | u64_u32_store(rq->clock_pelt_idle, rq_clock_pelt(rq)); | |
81 | } | |
82 | ||
23127296 VG |
83 | /* |
84 | * The clock_pelt scales the time to reflect the effective amount of | |
85 | * computation done during the running delta time but then sync back to | |
86 | * clock_task when rq is idle. | |
87 | * | |
88 | * | |
89 | * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 | |
90 | * @ max capacity ------******---------------******--------------- | |
91 | * @ half capacity ------************---------************--------- | |
92 | * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 | |
93 | * | |
94 | */ | |
95 | static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) | |
96 | { | |
97 | if (unlikely(is_idle_task(rq->curr))) { | |
e2f3e35f | 98 | _update_idle_rq_clock_pelt(rq); |
23127296 VG |
99 | return; |
100 | } | |
101 | ||
102 | /* | |
103 | * When a rq runs at a lower compute capacity, it will need | |
104 | * more time to do the same amount of work than at max | |
105 | * capacity. In order to be invariant, we scale the delta to | |
106 | * reflect how much work has been really done. | |
107 | * Running longer results in stealing idle time that will | |
108 | * disturb the load signal compared to max capacity. This | |
109 | * stolen idle time will be automatically reflected when the | |
110 | * rq will be idle and the clock will be synced with | |
111 | * rq_clock_task. | |
112 | */ | |
113 | ||
114 | /* | |
115 | * Scale the elapsed time to reflect the real amount of | |
116 | * computation | |
117 | */ | |
8ec59c0f | 118 | delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); |
23127296 VG |
119 | delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); |
120 | ||
121 | rq->clock_pelt += delta; | |
122 | } | |
123 | ||
124 | /* | |
125 | * When rq becomes idle, we have to check if it has lost idle time | |
126 | * because it was fully busy. A rq is fully used when the /Sum util_sum | |
127 | * is greater or equal to: | |
128 | * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; | |
129 | * For optimization and computing rounding purpose, we don't take into account | |
130 | * the position in the current window (period_contrib) and we use the higher | |
131 | * bound of util_sum to decide. | |
132 | */ | |
133 | static inline void update_idle_rq_clock_pelt(struct rq *rq) | |
134 | { | |
135 | u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; | |
136 | u32 util_sum = rq->cfs.avg.util_sum; | |
137 | util_sum += rq->avg_rt.util_sum; | |
138 | util_sum += rq->avg_dl.util_sum; | |
139 | ||
140 | /* | |
141 | * Reflecting stolen time makes sense only if the idle | |
142 | * phase would be present at max capacity. As soon as the | |
143 | * utilization of a rq has reached the maximum value, it is | |
3b03706f | 144 | * considered as an always running rq without idle time to |
23127296 VG |
145 | * steal. This potential idle time is considered as lost in |
146 | * this case. We keep track of this lost idle time compare to | |
147 | * rq's clock_task. | |
148 | */ | |
149 | if (util_sum >= divider) | |
150 | rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; | |
e2f3e35f VD |
151 | |
152 | _update_idle_rq_clock_pelt(rq); | |
23127296 VG |
153 | } |
154 | ||
e2f3e35f VD |
155 | #ifdef CONFIG_CFS_BANDWIDTH |
156 | static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) | |
23127296 | 157 | { |
e2f3e35f | 158 | u64 throttled; |
23127296 | 159 | |
e2f3e35f VD |
160 | if (unlikely(cfs_rq->throttle_count)) |
161 | throttled = U64_MAX; | |
162 | else | |
163 | throttled = cfs_rq->throttled_clock_pelt_time; | |
164 | ||
165 | u64_u32_store(cfs_rq->throttled_pelt_idle, throttled); | |
23127296 VG |
166 | } |
167 | ||
23127296 VG |
168 | /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ |
169 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) | |
170 | { | |
171 | if (unlikely(cfs_rq->throttle_count)) | |
64eaf507 | 172 | return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time; |
23127296 | 173 | |
64eaf507 | 174 | return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time; |
23127296 VG |
175 | } |
176 | #else | |
e2f3e35f | 177 | static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } |
23127296 VG |
178 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) |
179 | { | |
180 | return rq_clock_pelt(rq_of(cfs_rq)); | |
181 | } | |
182 | #endif | |
183 | ||
c0796298 VG |
184 | #else |
185 | ||
186 | static inline int | |
187 | update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) | |
188 | { | |
189 | return 0; | |
190 | } | |
191 | ||
371bf427 VG |
192 | static inline int |
193 | update_rt_rq_load_avg(u64 now, struct rq *rq, int running) | |
194 | { | |
195 | return 0; | |
196 | } | |
197 | ||
3727e0e1 VG |
198 | static inline int |
199 | update_dl_rq_load_avg(u64 now, struct rq *rq, int running) | |
200 | { | |
201 | return 0; | |
202 | } | |
91c27493 | 203 | |
76504793 | 204 | static inline int |
d4dbc991 | 205 | update_hw_load_avg(u64 now, struct rq *rq, u64 capacity) |
76504793 TG |
206 | { |
207 | return 0; | |
208 | } | |
209 | ||
d4dbc991 | 210 | static inline u64 hw_load_avg(struct rq *rq) |
76504793 TG |
211 | { |
212 | return 0; | |
213 | } | |
214 | ||
91c27493 VG |
215 | static inline int |
216 | update_irq_load_avg(struct rq *rq, u64 running) | |
217 | { | |
218 | return 0; | |
219 | } | |
23127296 VG |
220 | |
221 | static inline u64 rq_clock_pelt(struct rq *rq) | |
222 | { | |
223 | return rq_clock_task(rq); | |
224 | } | |
225 | ||
226 | static inline void | |
227 | update_rq_clock_pelt(struct rq *rq, s64 delta) { } | |
228 | ||
229 | static inline void | |
230 | update_idle_rq_clock_pelt(struct rq *rq) { } | |
231 | ||
e2f3e35f | 232 | static inline void update_idle_cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) { } |
c0796298 VG |
233 | #endif |
234 | ||
235 |