Commit | Line | Data |
---|---|---|
c0796298 | 1 | #ifdef CONFIG_SMP |
23127296 | 2 | #include "sched-pelt.h" |
c0796298 | 3 | |
23127296 VG |
4 | int __update_load_avg_blocked_se(u64 now, struct sched_entity *se); |
5 | int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se); | |
6 | int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq); | |
371bf427 | 7 | int update_rt_rq_load_avg(u64 now, struct rq *rq, int running); |
3727e0e1 | 8 | int update_dl_rq_load_avg(u64 now, struct rq *rq, int running); |
c0796298 | 9 | |
11d4afd4 | 10 | #ifdef CONFIG_HAVE_SCHED_AVG_IRQ |
91c27493 VG |
11 | int update_irq_load_avg(struct rq *rq, u64 running); |
12 | #else | |
13 | static inline int | |
14 | update_irq_load_avg(struct rq *rq, u64 running) | |
15 | { | |
16 | return 0; | |
17 | } | |
18 | #endif | |
19 | ||
c0796298 VG |
20 | /* |
21 | * When a task is dequeued, its estimated utilization should not be update if | |
22 | * its util_avg has not been updated at least once. | |
23 | * This flag is used to synchronize util_avg updates with util_est updates. | |
24 | * We map this information into the LSB bit of the utilization saved at | |
25 | * dequeue time (i.e. util_est.dequeued). | |
26 | */ | |
27 | #define UTIL_AVG_UNCHANGED 0x1 | |
28 | ||
29 | static inline void cfs_se_util_change(struct sched_avg *avg) | |
30 | { | |
31 | unsigned int enqueued; | |
32 | ||
33 | if (!sched_feat(UTIL_EST)) | |
34 | return; | |
35 | ||
36 | /* Avoid store if the flag has been already set */ | |
37 | enqueued = avg->util_est.enqueued; | |
38 | if (!(enqueued & UTIL_AVG_UNCHANGED)) | |
39 | return; | |
40 | ||
41 | /* Reset flag to report util_avg has been updated */ | |
42 | enqueued &= ~UTIL_AVG_UNCHANGED; | |
43 | WRITE_ONCE(avg->util_est.enqueued, enqueued); | |
44 | } | |
45 | ||
23127296 VG |
46 | /* |
47 | * The clock_pelt scales the time to reflect the effective amount of | |
48 | * computation done during the running delta time but then sync back to | |
49 | * clock_task when rq is idle. | |
50 | * | |
51 | * | |
52 | * absolute time | 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15|16 | |
53 | * @ max capacity ------******---------------******--------------- | |
54 | * @ half capacity ------************---------************--------- | |
55 | * clock pelt | 1| 2| 3| 4| 7| 8| 9| 10| 11|14|15|16 | |
56 | * | |
57 | */ | |
58 | static inline void update_rq_clock_pelt(struct rq *rq, s64 delta) | |
59 | { | |
60 | if (unlikely(is_idle_task(rq->curr))) { | |
61 | /* The rq is idle, we can sync to clock_task */ | |
62 | rq->clock_pelt = rq_clock_task(rq); | |
63 | return; | |
64 | } | |
65 | ||
66 | /* | |
67 | * When a rq runs at a lower compute capacity, it will need | |
68 | * more time to do the same amount of work than at max | |
69 | * capacity. In order to be invariant, we scale the delta to | |
70 | * reflect how much work has been really done. | |
71 | * Running longer results in stealing idle time that will | |
72 | * disturb the load signal compared to max capacity. This | |
73 | * stolen idle time will be automatically reflected when the | |
74 | * rq will be idle and the clock will be synced with | |
75 | * rq_clock_task. | |
76 | */ | |
77 | ||
78 | /* | |
79 | * Scale the elapsed time to reflect the real amount of | |
80 | * computation | |
81 | */ | |
8ec59c0f | 82 | delta = cap_scale(delta, arch_scale_cpu_capacity(cpu_of(rq))); |
23127296 VG |
83 | delta = cap_scale(delta, arch_scale_freq_capacity(cpu_of(rq))); |
84 | ||
85 | rq->clock_pelt += delta; | |
86 | } | |
87 | ||
88 | /* | |
89 | * When rq becomes idle, we have to check if it has lost idle time | |
90 | * because it was fully busy. A rq is fully used when the /Sum util_sum | |
91 | * is greater or equal to: | |
92 | * (LOAD_AVG_MAX - 1024 + rq->cfs.avg.period_contrib) << SCHED_CAPACITY_SHIFT; | |
93 | * For optimization and computing rounding purpose, we don't take into account | |
94 | * the position in the current window (period_contrib) and we use the higher | |
95 | * bound of util_sum to decide. | |
96 | */ | |
97 | static inline void update_idle_rq_clock_pelt(struct rq *rq) | |
98 | { | |
99 | u32 divider = ((LOAD_AVG_MAX - 1024) << SCHED_CAPACITY_SHIFT) - LOAD_AVG_MAX; | |
100 | u32 util_sum = rq->cfs.avg.util_sum; | |
101 | util_sum += rq->avg_rt.util_sum; | |
102 | util_sum += rq->avg_dl.util_sum; | |
103 | ||
104 | /* | |
105 | * Reflecting stolen time makes sense only if the idle | |
106 | * phase would be present at max capacity. As soon as the | |
107 | * utilization of a rq has reached the maximum value, it is | |
108 | * considered as an always runnig rq without idle time to | |
109 | * steal. This potential idle time is considered as lost in | |
110 | * this case. We keep track of this lost idle time compare to | |
111 | * rq's clock_task. | |
112 | */ | |
113 | if (util_sum >= divider) | |
114 | rq->lost_idle_time += rq_clock_task(rq) - rq->clock_pelt; | |
115 | } | |
116 | ||
117 | static inline u64 rq_clock_pelt(struct rq *rq) | |
118 | { | |
119 | lockdep_assert_held(&rq->lock); | |
120 | assert_clock_updated(rq); | |
121 | ||
122 | return rq->clock_pelt - rq->lost_idle_time; | |
123 | } | |
124 | ||
125 | #ifdef CONFIG_CFS_BANDWIDTH | |
126 | /* rq->task_clock normalized against any time this cfs_rq has spent throttled */ | |
127 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) | |
128 | { | |
129 | if (unlikely(cfs_rq->throttle_count)) | |
130 | return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time; | |
131 | ||
132 | return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time; | |
133 | } | |
134 | #else | |
135 | static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq) | |
136 | { | |
137 | return rq_clock_pelt(rq_of(cfs_rq)); | |
138 | } | |
139 | #endif | |
140 | ||
c0796298 VG |
141 | #else |
142 | ||
143 | static inline int | |
144 | update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) | |
145 | { | |
146 | return 0; | |
147 | } | |
148 | ||
371bf427 VG |
149 | static inline int |
150 | update_rt_rq_load_avg(u64 now, struct rq *rq, int running) | |
151 | { | |
152 | return 0; | |
153 | } | |
154 | ||
3727e0e1 VG |
155 | static inline int |
156 | update_dl_rq_load_avg(u64 now, struct rq *rq, int running) | |
157 | { | |
158 | return 0; | |
159 | } | |
91c27493 VG |
160 | |
161 | static inline int | |
162 | update_irq_load_avg(struct rq *rq, u64 running) | |
163 | { | |
164 | return 0; | |
165 | } | |
23127296 VG |
166 | |
167 | static inline u64 rq_clock_pelt(struct rq *rq) | |
168 | { | |
169 | return rq_clock_task(rq); | |
170 | } | |
171 | ||
172 | static inline void | |
173 | update_rq_clock_pelt(struct rq *rq, s64 delta) { } | |
174 | ||
175 | static inline void | |
176 | update_idle_rq_clock_pelt(struct rq *rq) { } | |
177 | ||
c0796298 VG |
178 | #endif |
179 | ||
180 |