Commit | Line | Data |
---|---|---|
112f38a4 | 1 | /* |
32fea568 IM |
2 | * sched_clock.c: Generic sched_clock() support, to extend low level |
3 | * hardware time counters to full 64-bit ns values. | |
112f38a4 RK |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | */ | |
9 | #include <linux/clocksource.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/jiffies.h> | |
a08ca5d1 | 12 | #include <linux/ktime.h> |
112f38a4 | 13 | #include <linux/kernel.h> |
a42c3629 | 14 | #include <linux/moduleparam.h> |
112f38a4 | 15 | #include <linux/sched.h> |
f153d017 | 16 | #include <linux/syscore_ops.h> |
a08ca5d1 | 17 | #include <linux/hrtimer.h> |
38ff87f7 | 18 | #include <linux/sched_clock.h> |
85c3d2dd | 19 | #include <linux/seqlock.h> |
e7e3ff1b | 20 | #include <linux/bitops.h> |
112f38a4 | 21 | |
cf7c9c17 | 22 | /** |
32fea568 | 23 | * struct clock_read_data - data required to read from sched_clock() |
cf7c9c17 | 24 | * |
32fea568 IM |
25 | * @epoch_ns: sched_clock() value at last update |
26 | * @epoch_cyc: Clock cycle value at last update. | |
cf7c9c17 | 27 | * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit |
32fea568 IM |
28 | * clocks. |
29 | * @read_sched_clock: Current clock source (or dummy source when suspended). | |
30 | * @mult: Multipler for scaled math conversion. | |
31 | * @shift: Shift value for scaled math conversion. | |
cf7c9c17 DT |
32 | * |
33 | * Care must be taken when updating this structure; it is read by | |
13dbeb38 | 34 | * some very hot code paths. It occupies <=40 bytes and, when combined |
cf7c9c17 DT |
35 | * with the seqcount used to synchronize access, comfortably fits into |
36 | * a 64 byte cache line. | |
37 | */ | |
38 | struct clock_read_data { | |
2f0778af | 39 | u64 epoch_ns; |
e7e3ff1b | 40 | u64 epoch_cyc; |
cf7c9c17 DT |
41 | u64 sched_clock_mask; |
42 | u64 (*read_sched_clock)(void); | |
2f0778af MZ |
43 | u32 mult; |
44 | u32 shift; | |
45 | }; | |
46 | ||
cf7c9c17 | 47 | /** |
32fea568 | 48 | * struct clock_data - all data needed for sched_clock() (including |
cf7c9c17 DT |
49 | * registration of a new clock source) |
50 | * | |
1809bfa4 DT |
51 | * @seq: Sequence counter for protecting updates. The lowest |
52 | * bit is the index for @read_data. | |
cf7c9c17 | 53 | * @read_data: Data required to read from sched_clock. |
32fea568 IM |
54 | * @wrap_kt: Duration for which clock can run before wrapping. |
55 | * @rate: Tick rate of the registered clock. | |
56 | * @actual_read_sched_clock: Registered hardware level clock read function. | |
cf7c9c17 DT |
57 | * |
58 | * The ordering of this structure has been chosen to optimize cache | |
32fea568 IM |
59 | * performance. In particular 'seq' and 'read_data[0]' (combined) should fit |
60 | * into a single 64-byte cache line. | |
cf7c9c17 DT |
61 | */ |
62 | struct clock_data { | |
32fea568 IM |
63 | seqcount_t seq; |
64 | struct clock_read_data read_data[2]; | |
65 | ktime_t wrap_kt; | |
66 | unsigned long rate; | |
67 | ||
13dbeb38 | 68 | u64 (*actual_read_sched_clock)(void); |
cf7c9c17 DT |
69 | }; |
70 | ||
a08ca5d1 | 71 | static struct hrtimer sched_clock_timer; |
a42c3629 RK |
72 | static int irqtime = -1; |
73 | ||
74 | core_param(irqtime, irqtime, int, 0400); | |
2f0778af | 75 | |
e7e3ff1b | 76 | static u64 notrace jiffy_sched_clock_read(void) |
2f0778af | 77 | { |
e7e3ff1b SB |
78 | /* |
79 | * We don't need to use get_jiffies_64 on 32-bit arches here | |
80 | * because we register with BITS_PER_LONG | |
81 | */ | |
82 | return (u64)(jiffies - INITIAL_JIFFIES); | |
83 | } | |
84 | ||
cf7c9c17 | 85 | static struct clock_data cd ____cacheline_aligned = { |
1809bfa4 DT |
86 | .read_data[0] = { .mult = NSEC_PER_SEC / HZ, |
87 | .read_sched_clock = jiffy_sched_clock_read, }, | |
13dbeb38 | 88 | .actual_read_sched_clock = jiffy_sched_clock_read, |
cf7c9c17 | 89 | }; |
2f0778af | 90 | |
cea15092 | 91 | static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) |
2f0778af MZ |
92 | { |
93 | return (cyc * mult) >> shift; | |
94 | } | |
95 | ||
b4042cea | 96 | unsigned long long notrace sched_clock(void) |
2f0778af | 97 | { |
8710e914 | 98 | u64 cyc, res; |
85c3d2dd | 99 | unsigned long seq; |
1809bfa4 | 100 | struct clock_read_data *rd; |
336ae118 | 101 | |
2f0778af | 102 | do { |
1809bfa4 DT |
103 | seq = raw_read_seqcount(&cd.seq); |
104 | rd = cd.read_data + (seq & 1); | |
8710e914 | 105 | |
13dbeb38 DT |
106 | cyc = (rd->read_sched_clock() - rd->epoch_cyc) & |
107 | rd->sched_clock_mask; | |
108 | res = rd->epoch_ns + cyc_to_ns(cyc, rd->mult, rd->shift); | |
85c3d2dd | 109 | } while (read_seqcount_retry(&cd.seq, seq)); |
2f0778af | 110 | |
8710e914 | 111 | return res; |
2f0778af MZ |
112 | } |
113 | ||
1809bfa4 DT |
114 | /* |
115 | * Updating the data required to read the clock. | |
116 | * | |
32fea568 | 117 | * sched_clock() will never observe mis-matched data even if called from |
1809bfa4 | 118 | * an NMI. We do this by maintaining an odd/even copy of the data and |
32fea568 IM |
119 | * steering sched_clock() to one or the other using a sequence counter. |
120 | * In order to preserve the data cache profile of sched_clock() as much | |
1809bfa4 DT |
121 | * as possible the system reverts back to the even copy when the update |
122 | * completes; the odd copy is used *only* during an update. | |
123 | */ | |
124 | static void update_clock_read_data(struct clock_read_data *rd) | |
125 | { | |
126 | /* update the backup (odd) copy with the new data */ | |
127 | cd.read_data[1] = *rd; | |
128 | ||
129 | /* steer readers towards the odd copy */ | |
130 | raw_write_seqcount_latch(&cd.seq); | |
131 | ||
132 | /* now its safe for us to update the normal (even) copy */ | |
133 | cd.read_data[0] = *rd; | |
134 | ||
135 | /* switch readers back to the even copy */ | |
136 | raw_write_seqcount_latch(&cd.seq); | |
137 | } | |
138 | ||
2f0778af | 139 | /* |
32fea568 | 140 | * Atomically update the sched_clock() epoch. |
2f0778af | 141 | */ |
9fee69a8 | 142 | static void update_sched_clock(void) |
2f0778af | 143 | { |
e7e3ff1b | 144 | u64 cyc; |
2f0778af | 145 | u64 ns; |
1809bfa4 DT |
146 | struct clock_read_data rd; |
147 | ||
148 | rd = cd.read_data[0]; | |
2f0778af | 149 | |
13dbeb38 | 150 | cyc = cd.actual_read_sched_clock(); |
32fea568 | 151 | ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift); |
1809bfa4 DT |
152 | |
153 | rd.epoch_ns = ns; | |
154 | rd.epoch_cyc = cyc; | |
155 | ||
156 | update_clock_read_data(&rd); | |
2f0778af | 157 | } |
112f38a4 | 158 | |
a08ca5d1 | 159 | static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) |
112f38a4 | 160 | { |
2f0778af | 161 | update_sched_clock(); |
a08ca5d1 | 162 | hrtimer_forward_now(hrt, cd.wrap_kt); |
32fea568 | 163 | |
a08ca5d1 | 164 | return HRTIMER_RESTART; |
112f38a4 RK |
165 | } |
166 | ||
32fea568 IM |
167 | void __init |
168 | sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) | |
112f38a4 | 169 | { |
5ae8aabe SB |
170 | u64 res, wrap, new_mask, new_epoch, cyc, ns; |
171 | u32 new_mult, new_shift; | |
a08ca5d1 | 172 | unsigned long r; |
112f38a4 | 173 | char r_unit; |
1809bfa4 | 174 | struct clock_read_data rd; |
112f38a4 | 175 | |
c115739d RH |
176 | if (cd.rate > rate) |
177 | return; | |
178 | ||
2f0778af | 179 | WARN_ON(!irqs_disabled()); |
112f38a4 | 180 | |
32fea568 | 181 | /* Calculate the mult/shift to convert counter ticks to ns. */ |
5ae8aabe SB |
182 | clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); |
183 | ||
184 | new_mask = CLOCKSOURCE_MASK(bits); | |
8710e914 | 185 | cd.rate = rate; |
5ae8aabe | 186 | |
32fea568 | 187 | /* Calculate how many nanosecs until we risk wrapping */ |
fb82fe2f | 188 | wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL); |
8710e914 | 189 | cd.wrap_kt = ns_to_ktime(wrap); |
5ae8aabe | 190 | |
1809bfa4 DT |
191 | rd = cd.read_data[0]; |
192 | ||
32fea568 | 193 | /* Update epoch for new counter and update 'epoch_ns' from old counter*/ |
5ae8aabe | 194 | new_epoch = read(); |
13dbeb38 | 195 | cyc = cd.actual_read_sched_clock(); |
32fea568 | 196 | ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift); |
13dbeb38 | 197 | cd.actual_read_sched_clock = read; |
5ae8aabe | 198 | |
32fea568 IM |
199 | rd.read_sched_clock = read; |
200 | rd.sched_clock_mask = new_mask; | |
201 | rd.mult = new_mult; | |
202 | rd.shift = new_shift; | |
203 | rd.epoch_cyc = new_epoch; | |
204 | rd.epoch_ns = ns; | |
205 | ||
1809bfa4 | 206 | update_clock_read_data(&rd); |
112f38a4 RK |
207 | |
208 | r = rate; | |
209 | if (r >= 4000000) { | |
210 | r /= 1000000; | |
211 | r_unit = 'M'; | |
32fea568 IM |
212 | } else { |
213 | if (r >= 1000) { | |
214 | r /= 1000; | |
215 | r_unit = 'k'; | |
216 | } else { | |
217 | r_unit = ' '; | |
218 | } | |
219 | } | |
220 | ||
221 | /* Calculate the ns resolution of this counter */ | |
5ae8aabe SB |
222 | res = cyc_to_ns(1ULL, new_mult, new_shift); |
223 | ||
a08ca5d1 SB |
224 | pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", |
225 | bits, r, r_unit, res, wrap); | |
112f38a4 | 226 | |
32fea568 | 227 | /* Enable IRQ time accounting if we have a fast enough sched_clock() */ |
a42c3629 RK |
228 | if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) |
229 | enable_sched_clock_irqtime(); | |
230 | ||
2f0778af MZ |
231 | pr_debug("Registered %pF as sched_clock source\n", read); |
232 | } | |
233 | ||
211baa70 RK |
234 | void __init sched_clock_postinit(void) |
235 | { | |
2f0778af | 236 | /* |
32fea568 | 237 | * If no sched_clock() function has been provided at that point, |
2f0778af MZ |
238 | * make it the final one one. |
239 | */ | |
13dbeb38 | 240 | if (cd.actual_read_sched_clock == jiffy_sched_clock_read) |
e7e3ff1b | 241 | sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); |
2f0778af | 242 | |
a08ca5d1 SB |
243 | update_sched_clock(); |
244 | ||
245 | /* | |
246 | * Start the timer to keep sched_clock() properly updated and | |
247 | * sets the initial epoch. | |
248 | */ | |
249 | hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
250 | sched_clock_timer.function = sched_clock_poll; | |
251 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); | |
211baa70 | 252 | } |
f153d017 | 253 | |
13dbeb38 DT |
254 | /* |
255 | * Clock read function for use when the clock is suspended. | |
256 | * | |
257 | * This function makes it appear to sched_clock() as if the clock | |
258 | * stopped counting at its last update. | |
1809bfa4 DT |
259 | * |
260 | * This function must only be called from the critical | |
261 | * section in sched_clock(). It relies on the read_seqcount_retry() | |
262 | * at the end of the critical section to be sure we observe the | |
32fea568 | 263 | * correct copy of 'epoch_cyc'. |
13dbeb38 DT |
264 | */ |
265 | static u64 notrace suspended_sched_clock_read(void) | |
266 | { | |
1809bfa4 DT |
267 | unsigned long seq = raw_read_seqcount(&cd.seq); |
268 | ||
269 | return cd.read_data[seq & 1].epoch_cyc; | |
13dbeb38 DT |
270 | } |
271 | ||
f153d017 RK |
272 | static int sched_clock_suspend(void) |
273 | { | |
1809bfa4 | 274 | struct clock_read_data *rd = &cd.read_data[0]; |
cf7c9c17 | 275 | |
f723aa18 SB |
276 | update_sched_clock(); |
277 | hrtimer_cancel(&sched_clock_timer); | |
13dbeb38 | 278 | rd->read_sched_clock = suspended_sched_clock_read; |
32fea568 | 279 | |
f153d017 RK |
280 | return 0; |
281 | } | |
282 | ||
237ec6f2 CC |
283 | static void sched_clock_resume(void) |
284 | { | |
1809bfa4 | 285 | struct clock_read_data *rd = &cd.read_data[0]; |
cf7c9c17 | 286 | |
13dbeb38 | 287 | rd->epoch_cyc = cd.actual_read_sched_clock(); |
f723aa18 | 288 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); |
13dbeb38 | 289 | rd->read_sched_clock = cd.actual_read_sched_clock; |
237ec6f2 CC |
290 | } |
291 | ||
f153d017 | 292 | static struct syscore_ops sched_clock_ops = { |
32fea568 IM |
293 | .suspend = sched_clock_suspend, |
294 | .resume = sched_clock_resume, | |
f153d017 RK |
295 | }; |
296 | ||
297 | static int __init sched_clock_syscore_init(void) | |
298 | { | |
299 | register_syscore_ops(&sched_clock_ops); | |
32fea568 | 300 | |
f153d017 RK |
301 | return 0; |
302 | } | |
303 | device_initcall(sched_clock_syscore_init); |