Commit | Line | Data |
---|---|---|
42f77542 RB |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (C) 2007 MIPS Technologies, Inc. | |
7 | * Copyright (C) 2007 Ralf Baechle <ralf@linux-mips.org> | |
8 | */ | |
9 | #include <linux/clockchips.h> | |
10 | #include <linux/interrupt.h> | |
21e1a03e | 11 | #include <linux/cpufreq.h> |
42f77542 | 12 | #include <linux/percpu.h> |
631330f5 | 13 | #include <linux/smp.h> |
ca4d3e67 | 14 | #include <linux/irq.h> |
42f77542 RB |
15 | |
16 | #include <asm/time.h> | |
8531a35e KK |
17 | #include <asm/cevt-r4k.h> |
18 | ||
42f77542 | 19 | static int mips_next_event(unsigned long delta, |
70342287 | 20 | struct clock_event_device *evt) |
42f77542 RB |
21 | { |
22 | unsigned int cnt; | |
23 | int res; | |
24 | ||
42f77542 RB |
25 | cnt = read_c0_count(); |
26 | cnt += delta; | |
27 | write_c0_compare(cnt); | |
5878fc93 | 28 | res = ((int)(read_c0_count() - cnt) >= 0) ? -ETIME : 0; |
42f77542 RB |
29 | return res; |
30 | } | |
31 | ||
1fa40555 JH |
32 | /** |
33 | * calculate_min_delta() - Calculate a good minimum delta for mips_next_event(). | |
34 | * | |
35 | * Running under virtualisation can introduce overhead into mips_next_event() in | |
36 | * the form of hypervisor emulation of CP0_Count/CP0_Compare registers, | |
37 | * potentially with an unnatural frequency, which makes a fixed min_delta_ns | |
38 | * value inappropriate as it may be too small. | |
39 | * | |
40 | * It can also introduce occasional latency from the guest being descheduled. | |
41 | * | |
42 | * This function calculates a good minimum delta based roughly on the 75th | |
43 | * percentile of the time taken to do the mips_next_event() sequence, in order | |
44 | * to handle potentially higher overhead while also eliminating outliers due to | |
45 | * unpredictable hypervisor latency (which can be handled by retries). | |
46 | * | |
47 | * Return: An appropriate minimum delta for the clock event device. | |
48 | */ | |
49 | static unsigned int calculate_min_delta(void) | |
50 | { | |
51 | unsigned int cnt, i, j, k, l; | |
52 | unsigned int buf1[4], buf2[3]; | |
53 | unsigned int min_delta; | |
54 | ||
55 | /* | |
56 | * Calculate the median of 5 75th percentiles of 5 samples of how long | |
57 | * it takes to set CP0_Compare = CP0_Count + delta. | |
58 | */ | |
59 | for (i = 0; i < 5; ++i) { | |
60 | for (j = 0; j < 5; ++j) { | |
61 | /* | |
62 | * This is like the code in mips_next_event(), and | |
63 | * directly measures the borderline "safe" delta. | |
64 | */ | |
65 | cnt = read_c0_count(); | |
66 | write_c0_compare(cnt); | |
67 | cnt = read_c0_count() - cnt; | |
68 | ||
69 | /* Sorted insert into buf1 */ | |
70 | for (k = 0; k < j; ++k) { | |
71 | if (cnt < buf1[k]) { | |
72 | l = min_t(unsigned int, | |
73 | j, ARRAY_SIZE(buf1) - 1); | |
74 | for (; l > k; --l) | |
75 | buf1[l] = buf1[l - 1]; | |
76 | break; | |
77 | } | |
78 | } | |
79 | if (k < ARRAY_SIZE(buf1)) | |
80 | buf1[k] = cnt; | |
81 | } | |
82 | ||
83 | /* Sorted insert of 75th percentile into buf2 */ | |
9d7f29cd | 84 | for (k = 0; k < i && k < ARRAY_SIZE(buf2); ++k) { |
1fa40555 JH |
85 | if (buf1[ARRAY_SIZE(buf1) - 1] < buf2[k]) { |
86 | l = min_t(unsigned int, | |
87 | i, ARRAY_SIZE(buf2) - 1); | |
88 | for (; l > k; --l) | |
89 | buf2[l] = buf2[l - 1]; | |
90 | break; | |
91 | } | |
92 | } | |
93 | if (k < ARRAY_SIZE(buf2)) | |
94 | buf2[k] = buf1[ARRAY_SIZE(buf1) - 1]; | |
95 | } | |
96 | ||
97 | /* Use 2 * median of 75th percentiles */ | |
98 | min_delta = buf2[ARRAY_SIZE(buf2) - 1] * 2; | |
99 | ||
100 | /* Don't go too low */ | |
101 | if (min_delta < 0x300) | |
102 | min_delta = 0x300; | |
103 | ||
104 | pr_debug("%s: median 75th percentile=%#x, min_delta=%#x\n", | |
105 | __func__, buf2[ARRAY_SIZE(buf2) - 1], min_delta); | |
106 | return min_delta; | |
107 | } | |
108 | ||
8531a35e KK |
109 | DEFINE_PER_CPU(struct clock_event_device, mips_clockevent_device); |
110 | int cp0_timer_irq_installed; | |
42f77542 | 111 | |
19971c0b JH |
112 | /* |
113 | * Possibly handle a performance counter interrupt. | |
114 | * Return true if the timer interrupt should not be checked | |
115 | */ | |
116 | static inline int handle_perf_irq(int r2) | |
117 | { | |
118 | /* | |
119 | * The performance counter overflow interrupt may be shared with the | |
120 | * timer interrupt (cp0_perfcount_irq < 0). If it is and a | |
121 | * performance counter has overflowed (perf_irq() == IRQ_HANDLED) | |
122 | * and we can't reliably determine if a counter interrupt has also | |
123 | * happened (!r2) then don't check for a timer interrupt. | |
124 | */ | |
125 | return (cp0_perfcount_irq < 0) && | |
126 | perf_irq() == IRQ_HANDLED && | |
127 | !r2; | |
128 | } | |
129 | ||
8531a35e | 130 | irqreturn_t c0_compare_interrupt(int irq, void *dev_id) |
42f77542 | 131 | { |
54dac950 | 132 | const int r2 = cpu_has_mips_r2_r6; |
42f77542 RB |
133 | struct clock_event_device *cd; |
134 | int cpu = smp_processor_id(); | |
135 | ||
136 | /* | |
137 | * Suckage alert: | |
138 | * Before R2 of the architecture there was no way to see if a | |
139 | * performance counter interrupt was pending, so we have to run | |
140 | * the performance counter interrupt handler anyway. | |
141 | */ | |
142 | if (handle_perf_irq(r2)) | |
f0c5b894 | 143 | return IRQ_HANDLED; |
42f77542 RB |
144 | |
145 | /* | |
70342287 | 146 | * The same applies to performance counter interrupts. But with the |
42f77542 RB |
147 | * above we now know that the reason we got here must be a timer |
148 | * interrupt. Being the paranoiacs we are we check anyway. | |
149 | */ | |
3ba5040a | 150 | if (!r2 || (read_c0_cause() & CAUSEF_TI)) { |
8531a35e KK |
151 | /* Clear Count/Compare Interrupt */ |
152 | write_c0_compare(read_c0_compare()); | |
42f77542 RB |
153 | cd = &per_cpu(mips_clockevent_device, cpu); |
154 | cd->event_handler(cd); | |
f0c5b894 RB |
155 | |
156 | return IRQ_HANDLED; | |
42f77542 RB |
157 | } |
158 | ||
f0c5b894 | 159 | return IRQ_NONE; |
42f77542 RB |
160 | } |
161 | ||
8531a35e | 162 | struct irqaction c0_compare_irqaction = { |
42f77542 | 163 | .handler = c0_compare_interrupt, |
7dfe8198 JH |
164 | /* |
165 | * IRQF_SHARED: The timer interrupt may be shared with other interrupts | |
166 | * such as perf counter and FDC interrupts. | |
167 | */ | |
168 | .flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED, | |
42f77542 RB |
169 | .name = "timer", |
170 | }; | |
171 | ||
42f77542 | 172 | |
8531a35e | 173 | void mips_event_handler(struct clock_event_device *dev) |
42f77542 RB |
174 | { |
175 | } | |
176 | ||
177 | /* | |
178 | * FIXME: This doesn't hold for the relocated E9000 compare interrupt. | |
179 | */ | |
180 | static int c0_compare_int_pending(void) | |
181 | { | |
ae58d882 | 182 | /* When cpu_has_mips_r2, this checks Cause.TI instead of Cause.IP7 */ |
010c108d | 183 | return (read_c0_cause() >> cp0_compare_irq_shift) & (1ul << CAUSEB_IP); |
42f77542 RB |
184 | } |
185 | ||
8531a35e KK |
186 | /* |
187 | * Compare interrupt can be routed and latched outside the core, | |
4f1a1eb5 AC |
188 | * so wait up to worst case number of cycle counter ticks for timer interrupt |
189 | * changes to propagate to the cause register. | |
8531a35e | 190 | */ |
4f1a1eb5 | 191 | #define COMPARE_INT_SEEN_TICKS 50 |
8531a35e KK |
192 | |
193 | int c0_compare_int_usable(void) | |
42f77542 | 194 | { |
3a6c43a7 | 195 | unsigned int delta; |
42f77542 RB |
196 | unsigned int cnt; |
197 | ||
9843b030 SL |
198 | #ifdef CONFIG_KVM_GUEST |
199 | return 1; | |
200 | #endif | |
201 | ||
42f77542 | 202 | /* |
70342287 | 203 | * IP7 already pending? Try to clear it by acking the timer. |
42f77542 RB |
204 | */ |
205 | if (c0_compare_int_pending()) { | |
4f1a1eb5 AC |
206 | cnt = read_c0_count(); |
207 | write_c0_compare(cnt); | |
208 | back_to_back_c0_hazard(); | |
209 | while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) | |
210 | if (!c0_compare_int_pending()) | |
211 | break; | |
42f77542 RB |
212 | if (c0_compare_int_pending()) |
213 | return 0; | |
214 | } | |
215 | ||
3a6c43a7 AN |
216 | for (delta = 0x10; delta <= 0x400000; delta <<= 1) { |
217 | cnt = read_c0_count(); | |
218 | cnt += delta; | |
219 | write_c0_compare(cnt); | |
4f1a1eb5 | 220 | back_to_back_c0_hazard(); |
3a6c43a7 AN |
221 | if ((int)(read_c0_count() - cnt) < 0) |
222 | break; | |
223 | /* increase delta if the timer was already expired */ | |
224 | } | |
42f77542 | 225 | |
c637fecb | 226 | while ((int)(read_c0_count() - cnt) <= 0) |
42f77542 RB |
227 | ; /* Wait for expiry */ |
228 | ||
4f1a1eb5 AC |
229 | while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) |
230 | if (c0_compare_int_pending()) | |
231 | break; | |
42f77542 RB |
232 | if (!c0_compare_int_pending()) |
233 | return 0; | |
4f1a1eb5 AC |
234 | cnt = read_c0_count(); |
235 | write_c0_compare(cnt); | |
236 | back_to_back_c0_hazard(); | |
237 | while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) | |
238 | if (!c0_compare_int_pending()) | |
239 | break; | |
42f77542 RB |
240 | if (c0_compare_int_pending()) |
241 | return 0; | |
242 | ||
243 | /* | |
244 | * Feels like a real count / compare timer. | |
245 | */ | |
246 | return 1; | |
247 | } | |
248 | ||
ec0b9d35 BH |
249 | unsigned int __weak get_c0_compare_int(void) |
250 | { | |
251 | return MIPS_CPU_IRQ_BASE + cp0_compare_irq; | |
252 | } | |
253 | ||
21e1a03e SS |
254 | #ifdef CONFIG_CPU_FREQ |
255 | ||
256 | static unsigned long mips_ref_freq; | |
257 | ||
258 | static int r4k_cpufreq_callback(struct notifier_block *nb, | |
259 | unsigned long val, void *data) | |
260 | { | |
261 | struct cpufreq_freqs *freq = data; | |
262 | struct clock_event_device *cd; | |
263 | unsigned long rate; | |
264 | int cpu; | |
265 | ||
266 | if (!mips_ref_freq) | |
267 | mips_ref_freq = freq->old; | |
268 | ||
269 | if (val == CPUFREQ_POSTCHANGE) { | |
270 | rate = cpufreq_scale(mips_hpt_frequency, mips_ref_freq, | |
271 | freq->new); | |
272 | ||
273 | for_each_cpu(cpu, freq->policy->cpus) { | |
274 | cd = &per_cpu(mips_clockevent_device, cpu); | |
275 | ||
276 | clockevents_update_freq(cd, rate); | |
277 | } | |
278 | } | |
279 | ||
280 | return 0; | |
281 | } | |
282 | ||
283 | static struct notifier_block r4k_cpufreq_notifier = { | |
284 | .notifier_call = r4k_cpufreq_callback, | |
285 | }; | |
286 | ||
287 | static int __init r4k_register_cpufreq_notifier(void) | |
288 | { | |
289 | return cpufreq_register_notifier(&r4k_cpufreq_notifier, | |
290 | CPUFREQ_TRANSITION_NOTIFIER); | |
291 | ||
292 | } | |
293 | core_initcall(r4k_register_cpufreq_notifier); | |
294 | ||
295 | #endif /* !CONFIG_CPU_FREQ */ | |
296 | ||
078a55fc | 297 | int r4k_clockevent_init(void) |
42f77542 | 298 | { |
ac8fd122 | 299 | unsigned long flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED; |
42f77542 RB |
300 | unsigned int cpu = smp_processor_id(); |
301 | struct clock_event_device *cd; | |
1fa40555 | 302 | unsigned int irq, min_delta; |
42f77542 | 303 | |
22df3f53 | 304 | if (!cpu_has_counter || !mips_hpt_frequency) |
5aa85c9f | 305 | return -ENXIO; |
42f77542 | 306 | |
42f77542 | 307 | if (!c0_compare_int_usable()) |
5aa85c9f | 308 | return -ENXIO; |
42f77542 | 309 | |
38760d40 RB |
310 | /* |
311 | * With vectored interrupts things are getting platform specific. | |
312 | * get_c0_compare_int is a hook to allow a platform to return the | |
ec0b9d35 | 313 | * interrupt number of its liking. |
38760d40 | 314 | */ |
ec0b9d35 | 315 | irq = get_c0_compare_int(); |
38760d40 | 316 | |
42f77542 RB |
317 | cd = &per_cpu(mips_clockevent_device, cpu); |
318 | ||
319 | cd->name = "MIPS"; | |
5977d682 | 320 | cd->features = CLOCK_EVT_FEAT_ONESHOT | |
d8107efd PB |
321 | CLOCK_EVT_FEAT_C3STOP | |
322 | CLOCK_EVT_FEAT_PERCPU; | |
42f77542 | 323 | |
1fa40555 | 324 | min_delta = calculate_min_delta(); |
42f77542 RB |
325 | |
326 | cd->rating = 300; | |
327 | cd->irq = irq; | |
320ab2b0 | 328 | cd->cpumask = cpumask_of(cpu); |
42f77542 | 329 | cd->set_next_event = mips_next_event; |
42f77542 RB |
330 | cd->event_handler = mips_event_handler; |
331 | ||
6dabf2b7 | 332 | clockevents_config_and_register(cd, mips_hpt_frequency, min_delta, 0x7fffffff); |
42f77542 | 333 | |
aea68639 | 334 | if (cp0_timer_irq_installed) |
5aa85c9f | 335 | return 0; |
38760d40 RB |
336 | |
337 | cp0_timer_irq_installed = 1; | |
338 | ||
49e6e07e | 339 | if (request_irq(irq, c0_compare_interrupt, flags, "timer", |
340 | c0_compare_interrupt)) | |
ac8fd122 | 341 | pr_err("Failed to request irq %d (timer)\n", irq); |
5aa85c9f RB |
342 | |
343 | return 0; | |
42f77542 | 344 | } |
8531a35e | 345 |