Commit | Line | Data |
---|---|---|
112f38a4 RK |
1 | /* |
2 | * sched_clock.c: support for extending counters to full 64-bit ns counter | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | #include <linux/clocksource.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/jiffies.h> | |
a08ca5d1 | 11 | #include <linux/ktime.h> |
112f38a4 | 12 | #include <linux/kernel.h> |
a42c3629 | 13 | #include <linux/moduleparam.h> |
112f38a4 | 14 | #include <linux/sched.h> |
f153d017 | 15 | #include <linux/syscore_ops.h> |
a08ca5d1 | 16 | #include <linux/hrtimer.h> |
38ff87f7 | 17 | #include <linux/sched_clock.h> |
85c3d2dd | 18 | #include <linux/seqlock.h> |
e7e3ff1b | 19 | #include <linux/bitops.h> |
112f38a4 | 20 | |
cf7c9c17 DT |
21 | /** |
22 | * struct clock_read_data - data required to read from sched_clock | |
23 | * | |
24 | * @epoch_ns: sched_clock value at last update | |
25 | * @epoch_cyc: Clock cycle value at last update | |
26 | * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit | |
27 | * clocks | |
28 | * @read_sched_clock: Current clock source (or dummy source when suspended) | |
29 | * @mult: Multipler for scaled math conversion | |
30 | * @shift: Shift value for scaled math conversion | |
31 | * @suspended: Flag to indicate if the clock is suspended (stopped) | |
32 | * | |
33 | * Care must be taken when updating this structure; it is read by | |
34 | * some very hot code paths. It occupies <=48 bytes and, when combined | |
35 | * with the seqcount used to synchronize access, comfortably fits into | |
36 | * a 64 byte cache line. | |
37 | */ | |
38 | struct clock_read_data { | |
2f0778af | 39 | u64 epoch_ns; |
e7e3ff1b | 40 | u64 epoch_cyc; |
cf7c9c17 DT |
41 | u64 sched_clock_mask; |
42 | u64 (*read_sched_clock)(void); | |
2f0778af MZ |
43 | u32 mult; |
44 | u32 shift; | |
237ec6f2 | 45 | bool suspended; |
2f0778af MZ |
46 | }; |
47 | ||
cf7c9c17 DT |
48 | /** |
49 | * struct clock_data - all data needed for sched_clock (including | |
50 | * registration of a new clock source) | |
51 | * | |
52 | * @seq: Sequence counter for protecting updates. | |
53 | * @read_data: Data required to read from sched_clock. | |
54 | * @wrap_kt: Duration for which clock can run before wrapping | |
55 | * @rate: Tick rate of the registered clock | |
56 | * @actual_read_sched_clock: Registered clock read function | |
57 | * | |
58 | * The ordering of this structure has been chosen to optimize cache | |
59 | * performance. In particular seq and read_data (combined) should fit | |
60 | * into a single 64 byte cache line. | |
61 | */ | |
62 | struct clock_data { | |
63 | seqcount_t seq; | |
64 | struct clock_read_data read_data; | |
65 | ktime_t wrap_kt; | |
66 | unsigned long rate; | |
67 | }; | |
68 | ||
a08ca5d1 | 69 | static struct hrtimer sched_clock_timer; |
a42c3629 RK |
70 | static int irqtime = -1; |
71 | ||
72 | core_param(irqtime, irqtime, int, 0400); | |
2f0778af | 73 | |
e7e3ff1b | 74 | static u64 notrace jiffy_sched_clock_read(void) |
2f0778af | 75 | { |
e7e3ff1b SB |
76 | /* |
77 | * We don't need to use get_jiffies_64 on 32-bit arches here | |
78 | * because we register with BITS_PER_LONG | |
79 | */ | |
80 | return (u64)(jiffies - INITIAL_JIFFIES); | |
81 | } | |
82 | ||
cf7c9c17 DT |
83 | static struct clock_data cd ____cacheline_aligned = { |
84 | .read_data = { .mult = NSEC_PER_SEC / HZ, | |
85 | .read_sched_clock = jiffy_sched_clock_read, }, | |
86 | }; | |
2f0778af | 87 | |
cea15092 | 88 | static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) |
2f0778af MZ |
89 | { |
90 | return (cyc * mult) >> shift; | |
91 | } | |
92 | ||
b4042cea | 93 | unsigned long long notrace sched_clock(void) |
2f0778af | 94 | { |
8710e914 | 95 | u64 cyc, res; |
85c3d2dd | 96 | unsigned long seq; |
cf7c9c17 | 97 | struct clock_read_data *rd = &cd.read_data; |
336ae118 | 98 | |
2f0778af | 99 | do { |
7a06c41c | 100 | seq = raw_read_seqcount_begin(&cd.seq); |
8710e914 | 101 | |
cf7c9c17 DT |
102 | res = rd->epoch_ns; |
103 | if (!rd->suspended) { | |
104 | cyc = rd->read_sched_clock(); | |
105 | cyc = (cyc - rd->epoch_cyc) & rd->sched_clock_mask; | |
106 | res += cyc_to_ns(cyc, rd->mult, rd->shift); | |
8710e914 | 107 | } |
85c3d2dd | 108 | } while (read_seqcount_retry(&cd.seq, seq)); |
2f0778af | 109 | |
8710e914 | 110 | return res; |
2f0778af MZ |
111 | } |
112 | ||
113 | /* | |
114 | * Atomically update the sched_clock epoch. | |
115 | */ | |
116 | static void notrace update_sched_clock(void) | |
117 | { | |
118 | unsigned long flags; | |
e7e3ff1b | 119 | u64 cyc; |
2f0778af | 120 | u64 ns; |
cf7c9c17 | 121 | struct clock_read_data *rd = &cd.read_data; |
2f0778af | 122 | |
cf7c9c17 DT |
123 | cyc = rd->read_sched_clock(); |
124 | ns = rd->epoch_ns + | |
125 | cyc_to_ns((cyc - rd->epoch_cyc) & rd->sched_clock_mask, | |
126 | rd->mult, rd->shift); | |
85c3d2dd | 127 | |
2f0778af | 128 | raw_local_irq_save(flags); |
7a06c41c | 129 | raw_write_seqcount_begin(&cd.seq); |
cf7c9c17 DT |
130 | rd->epoch_ns = ns; |
131 | rd->epoch_cyc = cyc; | |
7a06c41c | 132 | raw_write_seqcount_end(&cd.seq); |
2f0778af MZ |
133 | raw_local_irq_restore(flags); |
134 | } | |
112f38a4 | 135 | |
a08ca5d1 | 136 | static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) |
112f38a4 | 137 | { |
2f0778af | 138 | update_sched_clock(); |
a08ca5d1 SB |
139 | hrtimer_forward_now(hrt, cd.wrap_kt); |
140 | return HRTIMER_RESTART; | |
112f38a4 RK |
141 | } |
142 | ||
e7e3ff1b SB |
143 | void __init sched_clock_register(u64 (*read)(void), int bits, |
144 | unsigned long rate) | |
112f38a4 | 145 | { |
5ae8aabe SB |
146 | u64 res, wrap, new_mask, new_epoch, cyc, ns; |
147 | u32 new_mult, new_shift; | |
a08ca5d1 | 148 | unsigned long r; |
112f38a4 | 149 | char r_unit; |
cf7c9c17 | 150 | struct clock_read_data *rd = &cd.read_data; |
112f38a4 | 151 | |
c115739d RH |
152 | if (cd.rate > rate) |
153 | return; | |
154 | ||
2f0778af | 155 | WARN_ON(!irqs_disabled()); |
112f38a4 RK |
156 | |
157 | /* calculate the mult/shift to convert counter ticks to ns. */ | |
5ae8aabe SB |
158 | clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); |
159 | ||
160 | new_mask = CLOCKSOURCE_MASK(bits); | |
8710e914 | 161 | cd.rate = rate; |
5ae8aabe | 162 | |
362fde04 | 163 | /* calculate how many nanosecs until we risk wrapping */ |
fb82fe2f | 164 | wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL); |
8710e914 | 165 | cd.wrap_kt = ns_to_ktime(wrap); |
5ae8aabe SB |
166 | |
167 | /* update epoch for new counter and update epoch_ns from old counter*/ | |
168 | new_epoch = read(); | |
cf7c9c17 DT |
169 | cyc = rd->read_sched_clock(); |
170 | ns = rd->epoch_ns + | |
171 | cyc_to_ns((cyc - rd->epoch_cyc) & rd->sched_clock_mask, | |
172 | rd->mult, rd->shift); | |
5ae8aabe SB |
173 | |
174 | raw_write_seqcount_begin(&cd.seq); | |
cf7c9c17 DT |
175 | rd->read_sched_clock = read; |
176 | rd->sched_clock_mask = new_mask; | |
177 | rd->mult = new_mult; | |
178 | rd->shift = new_shift; | |
179 | rd->epoch_cyc = new_epoch; | |
180 | rd->epoch_ns = ns; | |
5ae8aabe | 181 | raw_write_seqcount_end(&cd.seq); |
112f38a4 RK |
182 | |
183 | r = rate; | |
184 | if (r >= 4000000) { | |
185 | r /= 1000000; | |
186 | r_unit = 'M'; | |
2f0778af | 187 | } else if (r >= 1000) { |
112f38a4 RK |
188 | r /= 1000; |
189 | r_unit = 'k'; | |
2f0778af MZ |
190 | } else |
191 | r_unit = ' '; | |
112f38a4 | 192 | |
112f38a4 | 193 | /* calculate the ns resolution of this counter */ |
5ae8aabe SB |
194 | res = cyc_to_ns(1ULL, new_mult, new_shift); |
195 | ||
a08ca5d1 SB |
196 | pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", |
197 | bits, r, r_unit, res, wrap); | |
112f38a4 | 198 | |
a42c3629 RK |
199 | /* Enable IRQ time accounting if we have a fast enough sched_clock */ |
200 | if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) | |
201 | enable_sched_clock_irqtime(); | |
202 | ||
2f0778af MZ |
203 | pr_debug("Registered %pF as sched_clock source\n", read); |
204 | } | |
205 | ||
211baa70 RK |
206 | void __init sched_clock_postinit(void) |
207 | { | |
2f0778af MZ |
208 | /* |
209 | * If no sched_clock function has been provided at that point, | |
210 | * make it the final one one. | |
211 | */ | |
cf7c9c17 | 212 | if (cd.read_data.read_sched_clock == jiffy_sched_clock_read) |
e7e3ff1b | 213 | sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); |
2f0778af | 214 | |
a08ca5d1 SB |
215 | update_sched_clock(); |
216 | ||
217 | /* | |
218 | * Start the timer to keep sched_clock() properly updated and | |
219 | * sets the initial epoch. | |
220 | */ | |
221 | hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
222 | sched_clock_timer.function = sched_clock_poll; | |
223 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); | |
211baa70 | 224 | } |
f153d017 RK |
225 | |
226 | static int sched_clock_suspend(void) | |
227 | { | |
cf7c9c17 DT |
228 | struct clock_read_data *rd = &cd.read_data; |
229 | ||
f723aa18 SB |
230 | update_sched_clock(); |
231 | hrtimer_cancel(&sched_clock_timer); | |
cf7c9c17 | 232 | rd->suspended = true; |
f153d017 RK |
233 | return 0; |
234 | } | |
235 | ||
237ec6f2 CC |
236 | static void sched_clock_resume(void) |
237 | { | |
cf7c9c17 DT |
238 | struct clock_read_data *rd = &cd.read_data; |
239 | ||
240 | rd->epoch_cyc = rd->read_sched_clock(); | |
f723aa18 | 241 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); |
cf7c9c17 | 242 | rd->suspended = false; |
237ec6f2 CC |
243 | } |
244 | ||
f153d017 RK |
245 | static struct syscore_ops sched_clock_ops = { |
246 | .suspend = sched_clock_suspend, | |
237ec6f2 | 247 | .resume = sched_clock_resume, |
f153d017 RK |
248 | }; |
249 | ||
250 | static int __init sched_clock_syscore_init(void) | |
251 | { | |
252 | register_syscore_ops(&sched_clock_ops); | |
253 | return 0; | |
254 | } | |
255 | device_initcall(sched_clock_syscore_init); |