Commit | Line | Data |
---|---|---|
c0796298 | 1 | #ifdef CONFIG_SMP |
23127296 | 2 | #include "sched-pelt.h" |
c0796298 | 3 | |
23127296 VG |
4 | int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); |
5 | int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); | |
6 | int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); | |
371bf427 | 7 | int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); |
3727e0e1 | 8 | int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); |
c0796298 | 9 | |
76504793 TG |
10 | #ifdef CONFIG_SCHED_THERMAL_PRESSURE |
11 | int update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity); | |
12 | ||
13 | static inline u64 thermal_load_avg(struct rq *rq) | |
14 | { | |
15 | return READ_ONCE(rq->avg_thermal.load_avg); | |
16 | } | |
17 | #else | |
18 | static inline int | |
19 | update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) | |
20 | { | |
21 | return 0; | |
22 | } | |
23 | ||
24 | static inline u64 thermal_load_avg(struct rq *rq) | |
25 | { | |
26 | return 0; | |
27 | } | |
28 | #endif | |
29 | ||
11d4afd4 | 30 | #ifdef CONFIG_HAVE_SCHED_AVG_IRQ |
91c27493 VG |
31 | int update_irq_load_avg(struct rq *rq, u64 running); |
32 | #else | |
33 | static inline int | |
34 | update_irq_load_avg(struct rq *rq, u64 running) | |
35 | { | |
36 | return 0; | |
37 | } | |
38 | #endif | |
39 | ||
87e867b4 VG |
40 | static inline u32 get_pelt_divider(struct sched_avg *avg) |
41 | { | |
42 | return LOAD_AVG_MAX - 1024 + avg->period_contrib; | |
43 | } | |
44 | ||
c0796298 VG |
45 | /* |
46 | * When a task is dequeued, its estimated utilization should not be update if | |
47 | * its util_avg has not been updated at least once. | |
48 | * This flag is used to synchronize util_avg updates with util_est updates. | |
49 | * We map this information into the LSB bit of the utilization saved at | |
50 | * dequeue time (i.e. util_est.dequeued). | |
51 | */ | |
52 | #define UTIL_AVG_UNCHANGED 0x1 | |
53 | ||
54 | static inline void cfs_se_util_change(struct sched_avg *avg) | |
55 | { | |
56 | unsigned int enqueued; | |
57 | ||
58 | if (!sched_feat(UTIL_EST)) | |
59 | return; | |
60 | ||
61 | /* Avoid store if the flag has been already set */ | |
62 | enqueued = avg->util_est.enqueued; | |
63 | if (!(enqueued & UTIL_AVG_UNCHANGED)) | |
64 | return; | |
65 | ||
66 | /* Reset flag to report util_avg has been updated */ | |
67 | enqueued &= ~UTIL_AVG_UNCHANGED; | |
68 | WRITE_ONCE(avg->util_est.enqueued, enqueued); | |
69 | } | |
70 | ||
23127296 VG |
71 | /* |
72 | * The clock_pelt scales the time to reflect the effective amount of | |
73 | * computation done during the running delta time but then sync back to | |
74 | * clock_task when rq is idle. | |
75 | * | |
76 | * | |
77 | * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 | |
78 | * @ max capacity ------******---------------******--------------- | |
79 | * @ half capacity ------************---------************--------- | |
80 | * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 | |
81 | * | |
82 | */ | |
83 | static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) | |
84 | { | |
85 | if (unlikely(is_idle_task(rq->curr))) { | |
86 | /* The rq is idle, we can sync to clock_task */ | |
87 | rq->clock_pelt = rq_clock_task(rq); | |
88 | return; | |
89 | } | |
90 | ||
91 | /* | |
92 | * When a rq runs at a lower compute capacity, it will need | |
93 | * more time to do the same amount of work than at max | |
94 | * capacity. In order to be invariant, we scale the delta to | |
95 | * reflect how much work has been really done. | |
96 | * Running longer results in stealing idle time that will | |
97 | * disturb the load signal compared to max capacity. This | |
98 | * stolen idle time will be automatically reflected when the | |
99 | * rq will be idle and the clock will be synced with | |
100 | * rq_clock_task. | |
101 | */ | |
102 | ||
103 | /* | |
104 | * Scale the elapsed time to reflect the real amount of | |
105 | * computation | |
106 | */ | |
8ec59c0f | 107 | delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); |
23127296 VG |
108 | delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); |
109 | ||
110 | rq->clock_pelt += delta; | |
111 | } | |
112 | ||
113 | /* | |
114 | * When rq becomes idle, we have to check if it has lost idle time | |
115 | * because it was fully busy. A rq is fully used when the /Sum util_sum | |
116 | * is greater or equal to: | |
117 | * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; | |
118 | * For optimization and computing rounding purpose, we don't take into account | |
119 | * the position in the current window (period_contrib) and we use the higher | |
120 | * bound of util_sum to decide. | |
121 | */ | |
122 | static inline void update_idle_rq_clock_pelt(struct rq *rq) | |
123 | { | |
124 | u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; | |
125 | u32 util_sum = rq->cfs.avg.util_sum; | |
126 | util_sum += rq->avg_rt.util_sum; | |
127 | util_sum += rq->avg_dl.util_sum; | |
128 | ||
129 | /* | |
130 | * Reflecting stolen time makes sense only if the idle | |
131 | * phase would be present at max capacity. As soon as the | |
132 | * utilization of a rq has reached the maximum value, it is | |
3b03706f | 133 | * considered as an always running rq without idle time to |
23127296 VG |
134 | * steal. This potential idle time is considered as lost in |
135 | * this case. We keep track of this lost idle time compare to | |
136 | * rq's clock_task. | |
137 | */ | |
138 | if (util_sum >= divider) | |
139 | rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; | |
140 | } | |
141 | ||
142 | static inline u64 rq_clock_pelt(struct rq *rq) | |
143 | { | |
144 | lockdep_assert_held(&rq->lock); | |
145 | assert_clock_updated(rq); | |
146 | ||
147 | return rq->clock_pelt - rq->lost_idle_time; | |
148 | } | |
149 | ||
150 | #ifdef CONFIG_CFS_BANDWIDTH | |
151 | /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ | |
152 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) | |
153 | { | |
154 | if (unlikely(cfs_rq->throttle_count)) | |
155 | return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; | |
156 | ||
157 | return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; | |
158 | } | |
159 | #else | |
160 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) | |
161 | { | |
162 | return rq_clock_pelt(rq_of(cfs_rq)); | |
163 | } | |
164 | #endif | |
165 | ||
c0796298 VG |
166 | #else |
167 | ||
168 | static inline int | |
169 | update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) | |
170 | { | |
171 | return 0; | |
172 | } | |
173 | ||
371bf427 VG |
174 | static inline int |
175 | update_rt_rq_load_avg(u64 now, struct rq *rq, int running) | |
176 | { | |
177 | return 0; | |
178 | } | |
179 | ||
3727e0e1 VG |
180 | static inline int |
181 | update_dl_rq_load_avg(u64 now, struct rq *rq, int running) | |
182 | { | |
183 | return 0; | |
184 | } | |
91c27493 | 185 | |
76504793 TG |
186 | static inline int |
187 | update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity) | |
188 | { | |
189 | return 0; | |
190 | } | |
191 | ||
192 | static inline u64 thermal_load_avg(struct rq *rq) | |
193 | { | |
194 | return 0; | |
195 | } | |
196 | ||
91c27493 VG |
197 | static inline int |
198 | update_irq_load_avg(struct rq *rq, u64 running) | |
199 | { | |
200 | return 0; | |
201 | } | |
23127296 VG |
202 | |
203 | static inline u64 rq_clock_pelt(struct rq *rq) | |
204 | { | |
205 | return rq_clock_task(rq); | |
206 | } | |
207 | ||
208 | static inline void | |
209 | update_rq_clock_pelt(struct rq *rq, s64 delta) { } | |
210 | ||
211 | static inline void | |
212 | update_idle_rq_clock_pelt(struct rq *rq) { } | |
213 | ||
c0796298 VG |
214 | #endif |
215 | ||
216 |