mm/page_alloc: prevent merging between isolated and other pageblocks
[linux-2.6-block.git] / kernel / time / sched_clock.c
CommitLineData
112f38a4 1/*
32fea568
IM
2 * sched_clock.c: Generic sched_clock() support, to extend low level
3 * hardware time counters to full 64-bit ns values.
112f38a4
RK
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 */
9#include <linux/clocksource.h>
10#include <linux/init.h>
11#include <linux/jiffies.h>
a08ca5d1 12#include <linux/ktime.h>
112f38a4 13#include <linux/kernel.h>
a42c3629 14#include <linux/moduleparam.h>
112f38a4 15#include <linux/sched.h>
f153d017 16#include <linux/syscore_ops.h>
a08ca5d1 17#include <linux/hrtimer.h>
38ff87f7 18#include <linux/sched_clock.h>
85c3d2dd 19#include <linux/seqlock.h>
e7e3ff1b 20#include <linux/bitops.h>
112f38a4 21
cf7c9c17 22/**
32fea568 23 * struct clock_read_data - data required to read from sched_clock()
cf7c9c17 24 *
32fea568
IM
25 * @epoch_ns: sched_clock() value at last update
26 * @epoch_cyc: Clock cycle value at last update.
cf7c9c17 27 * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit
32fea568
IM
28 * clocks.
29 * @read_sched_clock: Current clock source (or dummy source when suspended).
30 * @mult: Multipler for scaled math conversion.
31 * @shift: Shift value for scaled math conversion.
cf7c9c17
DT
32 *
33 * Care must be taken when updating this structure; it is read by
13dbeb38 34 * some very hot code paths. It occupies <=40 bytes and, when combined
cf7c9c17
DT
35 * with the seqcount used to synchronize access, comfortably fits into
36 * a 64 byte cache line.
37 */
38struct clock_read_data {
2f0778af 39 u64 epoch_ns;
e7e3ff1b 40 u64 epoch_cyc;
cf7c9c17
DT
41 u64 sched_clock_mask;
42 u64 (*read_sched_clock)(void);
2f0778af
MZ
43 u32 mult;
44 u32 shift;
45};
46
cf7c9c17 47/**
32fea568 48 * struct clock_data - all data needed for sched_clock() (including
cf7c9c17
DT
49 * registration of a new clock source)
50 *
1809bfa4
DT
51 * @seq: Sequence counter for protecting updates. The lowest
52 * bit is the index for @read_data.
cf7c9c17 53 * @read_data: Data required to read from sched_clock.
32fea568
IM
54 * @wrap_kt: Duration for which clock can run before wrapping.
55 * @rate: Tick rate of the registered clock.
56 * @actual_read_sched_clock: Registered hardware level clock read function.
cf7c9c17
DT
57 *
58 * The ordering of this structure has been chosen to optimize cache
32fea568
IM
59 * performance. In particular 'seq' and 'read_data[0]' (combined) should fit
60 * into a single 64-byte cache line.
cf7c9c17
DT
61 */
62struct clock_data {
32fea568
IM
63 seqcount_t seq;
64 struct clock_read_data read_data[2];
65 ktime_t wrap_kt;
66 unsigned long rate;
67
13dbeb38 68 u64 (*actual_read_sched_clock)(void);
cf7c9c17
DT
69};
70
a08ca5d1 71static struct hrtimer sched_clock_timer;
a42c3629
RK
72static int irqtime = -1;
73
74core_param(irqtime, irqtime, int, 0400);
2f0778af 75
e7e3ff1b 76static u64 notrace jiffy_sched_clock_read(void)
2f0778af 77{
e7e3ff1b
SB
78 /*
79 * We don't need to use get_jiffies_64 on 32-bit arches here
80 * because we register with BITS_PER_LONG
81 */
82 return (u64)(jiffies - INITIAL_JIFFIES);
83}
84
cf7c9c17 85static struct clock_data cd ____cacheline_aligned = {
1809bfa4
DT
86 .read_data[0] = { .mult = NSEC_PER_SEC / HZ,
87 .read_sched_clock = jiffy_sched_clock_read, },
13dbeb38 88 .actual_read_sched_clock = jiffy_sched_clock_read,
cf7c9c17 89};
2f0778af 90
cea15092 91static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift)
2f0778af
MZ
92{
93 return (cyc * mult) >> shift;
94}
95
b4042cea 96unsigned long long notrace sched_clock(void)
2f0778af 97{
8710e914 98 u64 cyc, res;
85c3d2dd 99 unsigned long seq;
1809bfa4 100 struct clock_read_data *rd;
336ae118 101
2f0778af 102 do {
1809bfa4
DT
103 seq = raw_read_seqcount(&cd.seq);
104 rd = cd.read_data + (seq & 1);
8710e914 105
13dbeb38
DT
106 cyc = (rd->read_sched_clock() - rd->epoch_cyc) &
107 rd->sched_clock_mask;
108 res = rd->epoch_ns + cyc_to_ns(cyc, rd->mult, rd->shift);
85c3d2dd 109 } while (read_seqcount_retry(&cd.seq, seq));
2f0778af 110
8710e914 111 return res;
2f0778af
MZ
112}
113
1809bfa4
DT
114/*
115 * Updating the data required to read the clock.
116 *
32fea568 117 * sched_clock() will never observe mis-matched data even if called from
1809bfa4 118 * an NMI. We do this by maintaining an odd/even copy of the data and
32fea568
IM
119 * steering sched_clock() to one or the other using a sequence counter.
120 * In order to preserve the data cache profile of sched_clock() as much
1809bfa4
DT
121 * as possible the system reverts back to the even copy when the update
122 * completes; the odd copy is used *only* during an update.
123 */
124static void update_clock_read_data(struct clock_read_data *rd)
125{
126 /* update the backup (odd) copy with the new data */
127 cd.read_data[1] = *rd;
128
129 /* steer readers towards the odd copy */
130 raw_write_seqcount_latch(&cd.seq);
131
132 /* now its safe for us to update the normal (even) copy */
133 cd.read_data[0] = *rd;
134
135 /* switch readers back to the even copy */
136 raw_write_seqcount_latch(&cd.seq);
137}
138
2f0778af 139/*
32fea568 140 * Atomically update the sched_clock() epoch.
2f0778af 141 */
9fee69a8 142static void update_sched_clock(void)
2f0778af 143{
e7e3ff1b 144 u64 cyc;
2f0778af 145 u64 ns;
1809bfa4
DT
146 struct clock_read_data rd;
147
148 rd = cd.read_data[0];
2f0778af 149
13dbeb38 150 cyc = cd.actual_read_sched_clock();
32fea568 151 ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift);
1809bfa4
DT
152
153 rd.epoch_ns = ns;
154 rd.epoch_cyc = cyc;
155
156 update_clock_read_data(&rd);
2f0778af 157}
112f38a4 158
a08ca5d1 159static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
112f38a4 160{
2f0778af 161 update_sched_clock();
a08ca5d1 162 hrtimer_forward_now(hrt, cd.wrap_kt);
32fea568 163
a08ca5d1 164 return HRTIMER_RESTART;
112f38a4
RK
165}
166
32fea568
IM
167void __init
168sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
112f38a4 169{
5ae8aabe
SB
170 u64 res, wrap, new_mask, new_epoch, cyc, ns;
171 u32 new_mult, new_shift;
a08ca5d1 172 unsigned long r;
112f38a4 173 char r_unit;
1809bfa4 174 struct clock_read_data rd;
112f38a4 175
c115739d
RH
176 if (cd.rate > rate)
177 return;
178
2f0778af 179 WARN_ON(!irqs_disabled());
112f38a4 180
32fea568 181 /* Calculate the mult/shift to convert counter ticks to ns. */
5ae8aabe
SB
182 clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600);
183
184 new_mask = CLOCKSOURCE_MASK(bits);
8710e914 185 cd.rate = rate;
5ae8aabe 186
32fea568 187 /* Calculate how many nanosecs until we risk wrapping */
fb82fe2f 188 wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL);
8710e914 189 cd.wrap_kt = ns_to_ktime(wrap);
5ae8aabe 190
1809bfa4
DT
191 rd = cd.read_data[0];
192
32fea568 193 /* Update epoch for new counter and update 'epoch_ns' from old counter*/
5ae8aabe 194 new_epoch = read();
13dbeb38 195 cyc = cd.actual_read_sched_clock();
32fea568 196 ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift);
13dbeb38 197 cd.actual_read_sched_clock = read;
5ae8aabe 198
32fea568
IM
199 rd.read_sched_clock = read;
200 rd.sched_clock_mask = new_mask;
201 rd.mult = new_mult;
202 rd.shift = new_shift;
203 rd.epoch_cyc = new_epoch;
204 rd.epoch_ns = ns;
205
1809bfa4 206 update_clock_read_data(&rd);
112f38a4
RK
207
208 r = rate;
209 if (r >= 4000000) {
210 r /= 1000000;
211 r_unit = 'M';
32fea568
IM
212 } else {
213 if (r >= 1000) {
214 r /= 1000;
215 r_unit = 'k';
216 } else {
217 r_unit = ' ';
218 }
219 }
220
221 /* Calculate the ns resolution of this counter */
5ae8aabe
SB
222 res = cyc_to_ns(1ULL, new_mult, new_shift);
223
a08ca5d1
SB
224 pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
225 bits, r, r_unit, res, wrap);
112f38a4 226
32fea568 227 /* Enable IRQ time accounting if we have a fast enough sched_clock() */
a42c3629
RK
228 if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
229 enable_sched_clock_irqtime();
230
2f0778af
MZ
231 pr_debug("Registered %pF as sched_clock source\n", read);
232}
233
211baa70
RK
234void __init sched_clock_postinit(void)
235{
2f0778af 236 /*
32fea568 237 * If no sched_clock() function has been provided at that point,
2f0778af
MZ
238 * make it the final one one.
239 */
13dbeb38 240 if (cd.actual_read_sched_clock == jiffy_sched_clock_read)
e7e3ff1b 241 sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ);
2f0778af 242
a08ca5d1
SB
243 update_sched_clock();
244
245 /*
246 * Start the timer to keep sched_clock() properly updated and
247 * sets the initial epoch.
248 */
249 hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
250 sched_clock_timer.function = sched_clock_poll;
251 hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
211baa70 252}
f153d017 253
13dbeb38
DT
254/*
255 * Clock read function for use when the clock is suspended.
256 *
257 * This function makes it appear to sched_clock() as if the clock
258 * stopped counting at its last update.
1809bfa4
DT
259 *
260 * This function must only be called from the critical
261 * section in sched_clock(). It relies on the read_seqcount_retry()
262 * at the end of the critical section to be sure we observe the
32fea568 263 * correct copy of 'epoch_cyc'.
13dbeb38
DT
264 */
265static u64 notrace suspended_sched_clock_read(void)
266{
1809bfa4
DT
267 unsigned long seq = raw_read_seqcount(&cd.seq);
268
269 return cd.read_data[seq & 1].epoch_cyc;
13dbeb38
DT
270}
271
f153d017
RK
272static int sched_clock_suspend(void)
273{
1809bfa4 274 struct clock_read_data *rd = &cd.read_data[0];
cf7c9c17 275
f723aa18
SB
276 update_sched_clock();
277 hrtimer_cancel(&sched_clock_timer);
13dbeb38 278 rd->read_sched_clock = suspended_sched_clock_read;
32fea568 279
f153d017
RK
280 return 0;
281}
282
237ec6f2
CC
283static void sched_clock_resume(void)
284{
1809bfa4 285 struct clock_read_data *rd = &cd.read_data[0];
cf7c9c17 286
13dbeb38 287 rd->epoch_cyc = cd.actual_read_sched_clock();
f723aa18 288 hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL);
13dbeb38 289 rd->read_sched_clock = cd.actual_read_sched_clock;
237ec6f2
CC
290}
291
f153d017 292static struct syscore_ops sched_clock_ops = {
32fea568
IM
293 .suspend = sched_clock_suspend,
294 .resume = sched_clock_resume,
f153d017
RK
295};
296
297static int __init sched_clock_syscore_init(void)
298{
299 register_syscore_ops(&sched_clock_ops);
32fea568 300
f153d017
RK
301 return 0;
302}
303device_initcall(sched_clock_syscore_init);