Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * Precise Delay Loops for i386 | |
4 | * | |
5 | * Copyright (C) 1993 Linus Torvalds | |
6 | * Copyright (C) 1997 Martin Mares <mj@atrey.karlin.mff.cuni.cz> | |
e01b70ef | 7 | * Copyright (C) 2008 Jiri Hladky <hladky _dot_ jiri _at_ gmail _dot_ com> |
1da177e4 LT |
8 | * |
9 | * The __delay function must _NOT_ be inlined as its execution time | |
10 | * depends wildly on alignment on many x86 processors. The additional | |
11 | * jump magic is needed to get the timing stable on all the CPU's | |
12 | * we have to worry about. | |
13 | */ | |
14 | ||
e683014c | 15 | #include <linux/export.h> |
1da177e4 | 16 | #include <linux/sched.h> |
941e492b | 17 | #include <linux/timex.h> |
35d5d08a | 18 | #include <linux/preempt.h> |
1da177e4 | 19 | #include <linux/delay.h> |
6f84fa2f | 20 | |
1da177e4 LT |
21 | #include <asm/processor.h> |
22 | #include <asm/delay.h> | |
23 | #include <asm/timer.h> | |
b466bdb6 | 24 | #include <asm/mwait.h> |
1da177e4 LT |
25 | |
26 | #ifdef CONFIG_SMP | |
6f84fa2f | 27 | # include <asm/smp.h> |
1da177e4 LT |
28 | #endif |
29 | ||
e8824890 TG |
30 | static void delay_loop(u64 __loops); |
31 | ||
32 | /* | |
33 | * Calibration and selection of the delay mechanism happens only once | |
34 | * during boot. | |
35 | */ | |
36 | static void (*delay_fn)(u64) __ro_after_init = delay_loop; | |
46f90c7a | 37 | static void (*delay_halt_fn)(u64 start, u64 cycles) __ro_after_init; |
e8824890 | 38 | |
6f84fa2f | 39 | /* simple loop based delay: */ |
e8824890 | 40 | static void delay_loop(u64 __loops) |
6f84fa2f | 41 | { |
e8824890 TG |
42 | unsigned long loops = (unsigned long)__loops; |
43 | ||
f0fbf0ab | 44 | asm volatile( |
e01b70ef JH |
45 | " test %0,%0 \n" |
46 | " jz 3f \n" | |
47 | " jmp 1f \n" | |
48 | ||
49 | ".align 16 \n" | |
50 | "1: jmp 2f \n" | |
51 | ||
52 | ".align 16 \n" | |
ff1b15b6 | 53 | "2: dec %0 \n" |
e01b70ef | 54 | " jnz 2b \n" |
ff1b15b6 | 55 | "3: dec %0 \n" |
e01b70ef | 56 | |
b86eb740 AF |
57 | : "+a" (loops) |
58 | : | |
e01b70ef | 59 | ); |
6f84fa2f | 60 | } |
61 | ||
62 | /* TSC based delay: */ | |
e8824890 | 63 | static void delay_tsc(u64 cycles) |
6f84fa2f | 64 | { |
e8824890 | 65 | u64 bclock, now; |
5c1ea082 | 66 | int cpu; |
6f84fa2f | 67 | |
5c1ea082 SR |
68 | preempt_disable(); |
69 | cpu = smp_processor_id(); | |
03b9730b | 70 | bclock = rdtsc_ordered(); |
5c1ea082 | 71 | for (;;) { |
03b9730b | 72 | now = rdtsc_ordered(); |
e8824890 | 73 | if ((now - bclock) >= cycles) |
5c1ea082 SR |
74 | break; |
75 | ||
76 | /* Allow RT tasks to run */ | |
77 | preempt_enable(); | |
78 | rep_nop(); | |
79 | preempt_disable(); | |
80 | ||
81 | /* | |
82 | * It is possible that we moved to another CPU, and | |
83 | * since TSC's are per-cpu we need to calculate | |
84 | * that. The delay must guarantee that we wait "at | |
85 | * least" the amount of time. Being moved to another | |
86 | * CPU could make the wait longer but we just need to | |
87 | * make sure we waited long enough. Rebalance the | |
88 | * counter for this CPU. | |
89 | */ | |
90 | if (unlikely(cpu != smp_processor_id())) { | |
e8824890 | 91 | cycles -= (now - bclock); |
5c1ea082 | 92 | cpu = smp_processor_id(); |
03b9730b | 93 | bclock = rdtsc_ordered(); |
5c1ea082 SR |
94 | } |
95 | } | |
35d5d08a | 96 | preempt_enable(); |
6f84fa2f | 97 | } |
98 | ||
cec5f268 KMP |
99 | /* |
100 | * On Intel the TPAUSE instruction waits until any of: | |
101 | * 1) the TSC counter exceeds the value provided in EDX:EAX | |
102 | * 2) global timeout in IA32_UMWAIT_CONTROL is exceeded | |
103 | * 3) an external interrupt occurs | |
104 | */ | |
105 | static void delay_halt_tpause(u64 start, u64 cycles) | |
106 | { | |
107 | u64 until = start + cycles; | |
108 | u32 eax, edx; | |
109 | ||
110 | eax = lower_32_bits(until); | |
111 | edx = upper_32_bits(until); | |
112 | ||
113 | /* | |
114 | * Hard code the deeper (C0.2) sleep state because exit latency is | |
115 | * small compared to the "microseconds" that usleep() will delay. | |
116 | */ | |
117 | __tpause(TPAUSE_C02_STATE, edx, eax); | |
118 | } | |
119 | ||
b466bdb6 HR |
120 | /* |
121 | * On some AMD platforms, MWAITX has a configurable 32-bit timer, that | |
e8824890 TG |
122 | * counts with TSC frequency. The input value is the number of TSC cycles |
123 | * to wait. MWAITX will also exit when the timer expires. | |
b466bdb6 | 124 | */ |
46f90c7a | 125 | static void delay_halt_mwaitx(u64 unused, u64 cycles) |
b466bdb6 | 126 | { |
46f90c7a KMP |
127 | u64 delay; |
128 | ||
129 | delay = min_t(u64, MWAITX_MAX_WAIT_CYCLES, cycles); | |
130 | /* | |
131 | * Use cpu_tss_rw as a cacheline-aligned, seldomly accessed per-cpu | |
132 | * variable as the monitor target. | |
133 | */ | |
134 | __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0); | |
135 | ||
136 | /* | |
137 | * AMD, like Intel, supports the EAX hint and EAX=0xf means, do not | |
138 | * enter any deep C-state and we use it here in delay() to minimize | |
139 | * wakeup latency. | |
140 | */ | |
141 | __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE); | |
142 | } | |
143 | ||
144 | /* | |
145 | * Call a vendor specific function to delay for a given amount of time. Because | |
146 | * these functions may return earlier than requested, check for actual elapsed | |
147 | * time and call again until done. | |
148 | */ | |
149 | static void delay_halt(u64 __cycles) | |
150 | { | |
151 | u64 start, end, cycles = __cycles; | |
b466bdb6 | 152 | |
88d879d2 JN |
153 | /* |
154 | * Timer value of 0 causes MWAITX to wait indefinitely, unless there | |
155 | * is a store on the memory monitored by MONITORX. | |
156 | */ | |
e8824890 | 157 | if (!cycles) |
88d879d2 JN |
158 | return; |
159 | ||
b466bdb6 HR |
160 | start = rdtsc_ordered(); |
161 | ||
162 | for (;;) { | |
46f90c7a | 163 | delay_halt_fn(start, cycles); |
b466bdb6 HR |
164 | end = rdtsc_ordered(); |
165 | ||
e8824890 | 166 | if (cycles <= end - start) |
b466bdb6 HR |
167 | break; |
168 | ||
e8824890 | 169 | cycles -= end - start; |
b466bdb6 HR |
170 | start = end; |
171 | } | |
172 | } | |
173 | ||
e8824890 | 174 | void __init use_tsc_delay(void) |
6f84fa2f | 175 | { |
b466bdb6 HR |
176 | if (delay_fn == delay_loop) |
177 | delay_fn = delay_tsc; | |
178 | } | |
179 | ||
cec5f268 KMP |
180 | void __init use_tpause_delay(void) |
181 | { | |
182 | delay_halt_fn = delay_halt_tpause; | |
183 | delay_fn = delay_halt; | |
184 | } | |
185 | ||
b466bdb6 HR |
186 | void use_mwaitx_delay(void) |
187 | { | |
46f90c7a KMP |
188 | delay_halt_fn = delay_halt_mwaitx; |
189 | delay_fn = delay_halt; | |
6f84fa2f | 190 | } |
191 | ||
a18e3690 | 192 | int read_current_timer(unsigned long *timer_val) |
6f84fa2f | 193 | { |
194 | if (delay_fn == delay_tsc) { | |
4ea1636b | 195 | *timer_val = rdtsc(); |
6f84fa2f | 196 | return 0; |
197 | } | |
198 | return -1; | |
199 | } | |
1da177e4 LT |
200 | |
201 | void __delay(unsigned long loops) | |
202 | { | |
6f84fa2f | 203 | delay_fn(loops); |
1da177e4 | 204 | } |
f0fbf0ab | 205 | EXPORT_SYMBOL(__delay); |
1da177e4 | 206 | |
81423c37 | 207 | noinline void __const_udelay(unsigned long xloops) |
1da177e4 | 208 | { |
4c45c516 | 209 | unsigned long lpj = this_cpu_read(cpu_info.loops_per_jiffy) ? : loops_per_jiffy; |
1da177e4 | 210 | int d0; |
6f84fa2f | 211 | |
1da177e4 | 212 | xloops *= 4; |
f0fbf0ab | 213 | asm("mull %%edx" |
1da177e4 | 214 | :"=d" (xloops), "=&a" (d0) |
4c45c516 | 215 | :"1" (xloops), "0" (lpj * (HZ / 4))); |
6f84fa2f | 216 | |
217 | __delay(++xloops); | |
1da177e4 | 218 | } |
f0fbf0ab | 219 | EXPORT_SYMBOL(__const_udelay); |
1da177e4 LT |
220 | |
221 | void __udelay(unsigned long usecs) | |
222 | { | |
6f84fa2f | 223 | __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ |
1da177e4 | 224 | } |
f0fbf0ab | 225 | EXPORT_SYMBOL(__udelay); |
1da177e4 LT |
226 | |
227 | void __ndelay(unsigned long nsecs) | |
228 | { | |
6f84fa2f | 229 | __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ |
1da177e4 | 230 | } |
129f6946 | 231 | EXPORT_SYMBOL(__ndelay); |