Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
82023503 GC |
2 | #include <linux/cpumask.h> |
3 | #include <linux/interrupt.h> | |
82023503 GC |
4 | |
5 | #include <linux/mm.h> | |
6 | #include <linux/delay.h> | |
7 | #include <linux/spinlock.h> | |
8 | #include <linux/kernel_stat.h> | |
9 | #include <linux/mc146818rtc.h> | |
10 | #include <linux/cache.h> | |
82023503 | 11 | #include <linux/cpu.h> |
82023503 GC |
12 | |
13 | #include <asm/smp.h> | |
14 | #include <asm/mtrr.h> | |
15 | #include <asm/tlbflush.h> | |
16 | #include <asm/mmu_context.h> | |
7b6aa335 | 17 | #include <asm/apic.h> |
82023503 | 18 | #include <asm/proto.h> |
43f39890 | 19 | #include <asm/ipi.h> |
82023503 | 20 | |
1a8aa8ac DV |
21 | void __default_send_IPI_shortcut(unsigned int shortcut, int vector, unsigned int dest) |
22 | { | |
23 | /* | |
24 | * Subtle. In the case of the 'never do double writes' workaround | |
25 | * we have to lock out interrupts to be safe. As we don't care | |
26 | * of the value read we use an atomic rmw access to avoid costly | |
27 | * cli/sti. Otherwise we use an even cheaper single atomic write | |
28 | * to the APIC. | |
29 | */ | |
30 | unsigned int cfg; | |
31 | ||
32 | /* | |
33 | * Wait for idle. | |
34 | */ | |
35 | __xapic_wait_icr_idle(); | |
36 | ||
37 | /* | |
38 | * No need to touch the target chip field | |
39 | */ | |
40 | cfg = __prepare_ICR(shortcut, vector, dest); | |
41 | ||
42 | /* | |
43 | * Send the IPI. The write to APIC_ICR fires this off. | |
44 | */ | |
45 | native_apic_mem_write(APIC_ICR, cfg); | |
46 | } | |
47 | ||
48 | /* | |
49 | * This is used to send an IPI with no shorthand notation (the destination is | |
50 | * specified in bits 56 to 63 of the ICR). | |
51 | */ | |
52 | void __default_send_IPI_dest_field(unsigned int mask, int vector, unsigned int dest) | |
53 | { | |
54 | unsigned long cfg; | |
55 | ||
56 | /* | |
57 | * Wait for idle. | |
58 | */ | |
59 | if (unlikely(vector == NMI_VECTOR)) | |
60 | safe_apic_wait_icr_idle(); | |
61 | else | |
62 | __xapic_wait_icr_idle(); | |
63 | ||
64 | /* | |
65 | * prepare target chip field | |
66 | */ | |
67 | cfg = __prepare_ICR2(mask); | |
68 | native_apic_mem_write(APIC_ICR2, cfg); | |
69 | ||
70 | /* | |
71 | * program the ICR | |
72 | */ | |
73 | cfg = __prepare_ICR(0, vector, dest); | |
74 | ||
75 | /* | |
76 | * Send the IPI. The write to APIC_ICR fires this off. | |
77 | */ | |
78 | native_apic_mem_write(APIC_ICR, cfg); | |
79 | } | |
80 | ||
53be0fac TG |
81 | void default_send_IPI_single_phys(int cpu, int vector) |
82 | { | |
83 | unsigned long flags; | |
84 | ||
85 | local_irq_save(flags); | |
86 | __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, cpu), | |
87 | vector, APIC_DEST_PHYSICAL); | |
88 | local_irq_restore(flags); | |
89 | } | |
90 | ||
c5e95482 YL |
91 | void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector) |
92 | { | |
93 | unsigned long query_cpu; | |
94 | unsigned long flags; | |
95 | ||
96 | /* | |
97 | * Hack. The clustered APIC addressing mode doesn't allow us to send | |
98 | * to an arbitrary mask, so I do a unicast to each CPU instead. | |
99 | * - mbligh | |
100 | */ | |
101 | local_irq_save(flags); | |
102 | for_each_cpu(query_cpu, mask) { | |
103 | __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, | |
104 | query_cpu), vector, APIC_DEST_PHYSICAL); | |
105 | } | |
106 | local_irq_restore(flags); | |
107 | } | |
108 | ||
109 | void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask, | |
110 | int vector) | |
111 | { | |
112 | unsigned int this_cpu = smp_processor_id(); | |
113 | unsigned int query_cpu; | |
114 | unsigned long flags; | |
115 | ||
116 | /* See Hack comment above */ | |
117 | ||
118 | local_irq_save(flags); | |
119 | for_each_cpu(query_cpu, mask) { | |
120 | if (query_cpu == this_cpu) | |
121 | continue; | |
122 | __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, | |
123 | query_cpu), vector, APIC_DEST_PHYSICAL); | |
124 | } | |
125 | local_irq_restore(flags); | |
126 | } | |
127 | ||
7e29393b TG |
128 | /* |
129 | * Helper function for APICs which insist on cpumasks | |
130 | */ | |
131 | void default_send_IPI_single(int cpu, int vector) | |
132 | { | |
133 | apic->send_IPI_mask(cpumask_of(cpu), vector); | |
134 | } | |
135 | ||
1245e166 TH |
136 | #ifdef CONFIG_X86_32 |
137 | ||
c5e95482 YL |
138 | void default_send_IPI_mask_sequence_logical(const struct cpumask *mask, |
139 | int vector) | |
140 | { | |
141 | unsigned long flags; | |
142 | unsigned int query_cpu; | |
143 | ||
144 | /* | |
145 | * Hack. The clustered APIC addressing mode doesn't allow us to send | |
146 | * to an arbitrary mask, so I do a unicasts to each CPU instead. This | |
147 | * should be modified to do 1 message per cluster ID - mbligh | |
148 | */ | |
149 | ||
150 | local_irq_save(flags); | |
151 | for_each_cpu(query_cpu, mask) | |
152 | __default_send_IPI_dest_field( | |
6f802c4b TH |
153 | early_per_cpu(x86_cpu_to_logical_apicid, query_cpu), |
154 | vector, apic->dest_logical); | |
c5e95482 YL |
155 | local_irq_restore(flags); |
156 | } | |
157 | ||
158 | void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask, | |
159 | int vector) | |
160 | { | |
161 | unsigned long flags; | |
162 | unsigned int query_cpu; | |
163 | unsigned int this_cpu = smp_processor_id(); | |
164 | ||
165 | /* See Hack comment above */ | |
166 | ||
167 | local_irq_save(flags); | |
168 | for_each_cpu(query_cpu, mask) { | |
169 | if (query_cpu == this_cpu) | |
170 | continue; | |
171 | __default_send_IPI_dest_field( | |
6f802c4b TH |
172 | early_per_cpu(x86_cpu_to_logical_apicid, query_cpu), |
173 | vector, apic->dest_logical); | |
c5e95482 YL |
174 | } |
175 | local_irq_restore(flags); | |
176 | } | |
177 | ||
c5e95482 YL |
178 | /* |
179 | * This is only used on smaller machines. | |
180 | */ | |
181 | void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector) | |
182 | { | |
183 | unsigned long mask = cpumask_bits(cpumask)[0]; | |
184 | unsigned long flags; | |
185 | ||
e3f0f36d | 186 | if (!mask) |
83d349f3 LT |
187 | return; |
188 | ||
c5e95482 YL |
189 | local_irq_save(flags); |
190 | WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]); | |
191 | __default_send_IPI_dest_field(mask, vector, apic->dest_logical); | |
192 | local_irq_restore(flags); | |
193 | } | |
194 | ||
195 | void default_send_IPI_allbutself(int vector) | |
196 | { | |
197 | /* | |
198 | * if there are no other CPUs in the system then we get an APIC send | |
199 | * error if we try to broadcast, thus avoid sending IPIs in this case. | |
200 | */ | |
201 | if (!(num_online_cpus() > 1)) | |
202 | return; | |
203 | ||
204 | __default_local_send_IPI_allbutself(vector); | |
205 | } | |
206 | ||
207 | void default_send_IPI_all(int vector) | |
208 | { | |
209 | __default_local_send_IPI_all(vector); | |
210 | } | |
211 | ||
dac5f412 | 212 | void default_send_IPI_self(int vector) |
82023503 | 213 | { |
43f39890 | 214 | __default_send_IPI_shortcut(APIC_DEST_SELF, vector, apic->dest_logical); |
82023503 GC |
215 | } |
216 | ||
217 | /* must come after the send_IPI functions above for inlining */ | |
82023503 GC |
218 | static int convert_apicid_to_cpu(int apic_id) |
219 | { | |
220 | int i; | |
221 | ||
222 | for_each_possible_cpu(i) { | |
223 | if (per_cpu(x86_cpu_to_apicid, i) == apic_id) | |
224 | return i; | |
225 | } | |
226 | return -1; | |
227 | } | |
228 | ||
229 | int safe_smp_processor_id(void) | |
230 | { | |
231 | int apicid, cpuid; | |
232 | ||
93984fbd | 233 | if (!boot_cpu_has(X86_FEATURE_APIC)) |
82023503 GC |
234 | return 0; |
235 | ||
236 | apicid = hard_smp_processor_id(); | |
237 | if (apicid == BAD_APICID) | |
238 | return 0; | |
239 | ||
240 | cpuid = convert_apicid_to_cpu(apicid); | |
241 | ||
242 | return cpuid >= 0 ? cpuid : 0; | |
243 | } | |
244 | #endif |