Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * SN2 Platform specific SMP Support | |
3 | * | |
4 | * This file is subject to the terms and conditions of the GNU General Public | |
5 | * License. See the file "COPYING" in the main directory of this archive | |
6 | * for more details. | |
7 | * | |
470ceb05 | 8 | * Copyright (C) 2000-2005 Silicon Graphics, Inc. All rights reserved. |
1da177e4 LT |
9 | */ |
10 | ||
11 | #include <linux/init.h> | |
12 | #include <linux/kernel.h> | |
13 | #include <linux/spinlock.h> | |
14 | #include <linux/threads.h> | |
15 | #include <linux/sched.h> | |
16 | #include <linux/smp.h> | |
17 | #include <linux/interrupt.h> | |
18 | #include <linux/irq.h> | |
19 | #include <linux/mmzone.h> | |
20 | #include <linux/module.h> | |
21 | #include <linux/bitops.h> | |
22 | #include <linux/nodemask.h> | |
470ceb05 JS |
23 | #include <linux/proc_fs.h> |
24 | #include <linux/seq_file.h> | |
1da177e4 LT |
25 | |
26 | #include <asm/processor.h> | |
27 | #include <asm/irq.h> | |
28 | #include <asm/sal.h> | |
29 | #include <asm/system.h> | |
30 | #include <asm/delay.h> | |
31 | #include <asm/io.h> | |
32 | #include <asm/smp.h> | |
33 | #include <asm/tlb.h> | |
34 | #include <asm/numa.h> | |
35 | #include <asm/hw_irq.h> | |
36 | #include <asm/current.h> | |
37 | #include <asm/sn/sn_cpuid.h> | |
38 | #include <asm/sn/sn_sal.h> | |
39 | #include <asm/sn/addrs.h> | |
40 | #include <asm/sn/shub_mmr.h> | |
41 | #include <asm/sn/nodepda.h> | |
42 | #include <asm/sn/rw_mmr.h> | |
43 | ||
470ceb05 JS |
44 | DEFINE_PER_CPU(struct ptc_stats, ptcstats); |
45 | DECLARE_PER_CPU(struct ptc_stats, ptcstats); | |
1da177e4 LT |
46 | |
47 | static __cacheline_aligned DEFINE_SPINLOCK(sn2_global_ptc_lock); | |
48 | ||
470ceb05 JS |
49 | void sn2_ptc_deadlock_recovery(short *, short, int, volatile unsigned long *, unsigned long data0, |
50 | volatile unsigned long *, unsigned long data1); | |
51 | ||
52 | #ifdef DEBUG_PTC | |
53 | /* | |
54 | * ptctest: | |
55 | * | |
56 | * xyz - 3 digit hex number: | |
57 | * x - Force PTC purges to use shub: | |
58 | * 0 - no force | |
59 | * 1 - force | |
60 | * y - interupt enable | |
61 | * 0 - disable interrupts | |
62 | * 1 - leave interuupts enabled | |
63 | * z - type of lock: | |
64 | * 0 - global lock | |
65 | * 1 - node local lock | |
66 | * 2 - no lock | |
67 | * | |
68 | * Note: on shub1, only ptctest == 0 is supported. Don't try other values! | |
69 | */ | |
70 | ||
71 | static unsigned int sn2_ptctest = 0; | |
72 | ||
73 | static int __init ptc_test(char *str) | |
74 | { | |
75 | get_option(&str, &sn2_ptctest); | |
76 | return 1; | |
77 | } | |
78 | __setup("ptctest=", ptc_test); | |
79 | ||
80 | static inline int ptc_lock(unsigned long *flagp) | |
81 | { | |
82 | unsigned long opt = sn2_ptctest & 255; | |
83 | ||
84 | switch (opt) { | |
85 | case 0x00: | |
86 | spin_lock_irqsave(&sn2_global_ptc_lock, *flagp); | |
87 | break; | |
88 | case 0x01: | |
89 | spin_lock_irqsave(&sn_nodepda->ptc_lock, *flagp); | |
90 | break; | |
91 | case 0x02: | |
92 | local_irq_save(*flagp); | |
93 | break; | |
94 | case 0x10: | |
95 | spin_lock(&sn2_global_ptc_lock); | |
96 | break; | |
97 | case 0x11: | |
98 | spin_lock(&sn_nodepda->ptc_lock); | |
99 | break; | |
100 | case 0x12: | |
101 | break; | |
102 | default: | |
103 | BUG(); | |
104 | } | |
105 | return opt; | |
106 | } | |
107 | ||
108 | static inline void ptc_unlock(unsigned long flags, int opt) | |
109 | { | |
110 | switch (opt) { | |
111 | case 0x00: | |
112 | spin_unlock_irqrestore(&sn2_global_ptc_lock, flags); | |
113 | break; | |
114 | case 0x01: | |
115 | spin_unlock_irqrestore(&sn_nodepda->ptc_lock, flags); | |
116 | break; | |
117 | case 0x02: | |
118 | local_irq_restore(flags); | |
119 | break; | |
120 | case 0x10: | |
121 | spin_unlock(&sn2_global_ptc_lock); | |
122 | break; | |
123 | case 0x11: | |
124 | spin_unlock(&sn_nodepda->ptc_lock); | |
125 | break; | |
126 | case 0x12: | |
127 | break; | |
128 | default: | |
129 | BUG(); | |
130 | } | |
131 | } | |
132 | #else | |
133 | ||
134 | #define sn2_ptctest 0 | |
135 | ||
136 | static inline int ptc_lock(unsigned long *flagp) | |
137 | { | |
138 | spin_lock_irqsave(&sn2_global_ptc_lock, *flagp); | |
139 | return 0; | |
140 | } | |
141 | ||
142 | static inline void ptc_unlock(unsigned long flags, int opt) | |
143 | { | |
144 | spin_unlock_irqrestore(&sn2_global_ptc_lock, flags); | |
145 | } | |
146 | #endif | |
147 | ||
148 | struct ptc_stats { | |
149 | unsigned long ptc_l; | |
150 | unsigned long change_rid; | |
151 | unsigned long shub_ptc_flushes; | |
152 | unsigned long nodes_flushed; | |
153 | unsigned long deadlocks; | |
154 | unsigned long lock_itc_clocks; | |
155 | unsigned long shub_itc_clocks; | |
156 | unsigned long shub_itc_clocks_max; | |
157 | }; | |
1da177e4 LT |
158 | |
159 | static inline unsigned long wait_piowc(void) | |
160 | { | |
161 | volatile unsigned long *piows, zeroval; | |
162 | unsigned long ws; | |
163 | ||
164 | piows = pda->pio_write_status_addr; | |
165 | zeroval = pda->pio_write_status_val; | |
166 | do { | |
167 | cpu_relax(); | |
168 | } while (((ws = *piows) & SH_PIO_WRITE_STATUS_PENDING_WRITE_COUNT_MASK) != zeroval); | |
169 | return ws; | |
170 | } | |
171 | ||
172 | void sn_tlb_migrate_finish(struct mm_struct *mm) | |
173 | { | |
174 | if (mm == current->mm) | |
175 | flush_tlb_mm(mm); | |
176 | } | |
177 | ||
178 | /** | |
179 | * sn2_global_tlb_purge - globally purge translation cache of virtual address range | |
180 | * @start: start of virtual address range | |
181 | * @end: end of virtual address range | |
182 | * @nbits: specifies number of bytes to purge per instruction (num = 1<<(nbits & 0xfc)) | |
183 | * | |
184 | * Purges the translation caches of all processors of the given virtual address | |
185 | * range. | |
186 | * | |
187 | * Note: | |
188 | * - cpu_vm_mask is a bit mask that indicates which cpus have loaded the context. | |
189 | * - cpu_vm_mask is converted into a nodemask of the nodes containing the | |
190 | * cpus in cpu_vm_mask. | |
191 | * - if only one bit is set in cpu_vm_mask & it is the current cpu, | |
192 | * then only the local TLB needs to be flushed. This flushing can be done | |
193 | * using ptc.l. This is the common case & avoids the global spinlock. | |
194 | * - if multiple cpus have loaded the context, then flushing has to be | |
195 | * done with ptc.g/MMRs under protection of the global ptc_lock. | |
196 | */ | |
197 | ||
198 | void | |
199 | sn2_global_tlb_purge(unsigned long start, unsigned long end, | |
200 | unsigned long nbits) | |
201 | { | |
470ceb05 | 202 | int i, opt, shub1, cnode, mynasid, cpu, lcpu = 0, nasid, flushed = 0; |
1da177e4 | 203 | volatile unsigned long *ptc0, *ptc1; |
470ceb05 | 204 | unsigned long itc, itc2, flags, data0 = 0, data1 = 0; |
1da177e4 LT |
205 | struct mm_struct *mm = current->active_mm; |
206 | short nasids[MAX_NUMNODES], nix; | |
207 | nodemask_t nodes_flushed; | |
208 | ||
209 | nodes_clear(nodes_flushed); | |
210 | i = 0; | |
211 | ||
212 | for_each_cpu_mask(cpu, mm->cpu_vm_mask) { | |
213 | cnode = cpu_to_node(cpu); | |
214 | node_set(cnode, nodes_flushed); | |
215 | lcpu = cpu; | |
216 | i++; | |
217 | } | |
218 | ||
219 | preempt_disable(); | |
220 | ||
221 | if (likely(i == 1 && lcpu == smp_processor_id())) { | |
222 | do { | |
223 | ia64_ptcl(start, nbits << 2); | |
224 | start += (1UL << nbits); | |
225 | } while (start < end); | |
226 | ia64_srlz_i(); | |
470ceb05 | 227 | __get_cpu_var(ptcstats).ptc_l++; |
1da177e4 LT |
228 | preempt_enable(); |
229 | return; | |
230 | } | |
231 | ||
232 | if (atomic_read(&mm->mm_users) == 1) { | |
233 | flush_tlb_mm(mm); | |
470ceb05 | 234 | __get_cpu_var(ptcstats).change_rid++; |
1da177e4 LT |
235 | preempt_enable(); |
236 | return; | |
237 | } | |
238 | ||
470ceb05 | 239 | itc = ia64_get_itc(); |
1da177e4 LT |
240 | nix = 0; |
241 | for_each_node_mask(cnode, nodes_flushed) | |
242 | nasids[nix++] = cnodeid_to_nasid(cnode); | |
243 | ||
244 | shub1 = is_shub1(); | |
245 | if (shub1) { | |
246 | data0 = (1UL << SH1_PTC_0_A_SHFT) | | |
247 | (nbits << SH1_PTC_0_PS_SHFT) | | |
248 | ((ia64_get_rr(start) >> 8) << SH1_PTC_0_RID_SHFT) | | |
249 | (1UL << SH1_PTC_0_START_SHFT); | |
250 | ptc0 = (long *)GLOBAL_MMR_PHYS_ADDR(0, SH1_PTC_0); | |
251 | ptc1 = (long *)GLOBAL_MMR_PHYS_ADDR(0, SH1_PTC_1); | |
252 | } else { | |
253 | data0 = (1UL << SH2_PTC_A_SHFT) | | |
254 | (nbits << SH2_PTC_PS_SHFT) | | |
255 | (1UL << SH2_PTC_START_SHFT); | |
256 | ptc0 = (long *)GLOBAL_MMR_PHYS_ADDR(0, SH2_PTC + | |
257 | ((ia64_get_rr(start) >> 8) << SH2_PTC_RID_SHFT) ); | |
258 | ptc1 = NULL; | |
259 | } | |
260 | ||
261 | ||
262 | mynasid = get_nasid(); | |
263 | ||
470ceb05 JS |
264 | itc = ia64_get_itc(); |
265 | opt = ptc_lock(&flags); | |
266 | itc2 = ia64_get_itc(); | |
267 | __get_cpu_var(ptcstats).lock_itc_clocks += itc2 - itc; | |
268 | __get_cpu_var(ptcstats).shub_ptc_flushes++; | |
269 | __get_cpu_var(ptcstats).nodes_flushed += nix; | |
1da177e4 LT |
270 | |
271 | do { | |
272 | if (shub1) | |
273 | data1 = start | (1UL << SH1_PTC_1_START_SHFT); | |
274 | else | |
275 | data0 = (data0 & ~SH2_PTC_ADDR_MASK) | (start & SH2_PTC_ADDR_MASK); | |
276 | for (i = 0; i < nix; i++) { | |
277 | nasid = nasids[i]; | |
470ceb05 | 278 | if ((!(sn2_ptctest & 3)) && unlikely(nasid == mynasid)) { |
1da177e4 LT |
279 | ia64_ptcga(start, nbits << 2); |
280 | ia64_srlz_i(); | |
281 | } else { | |
282 | ptc0 = CHANGE_NASID(nasid, ptc0); | |
283 | if (ptc1) | |
284 | ptc1 = CHANGE_NASID(nasid, ptc1); | |
285 | pio_atomic_phys_write_mmrs(ptc0, data0, ptc1, | |
286 | data1); | |
287 | flushed = 1; | |
288 | } | |
289 | } | |
1da177e4 LT |
290 | if (flushed |
291 | && (wait_piowc() & | |
470ceb05 JS |
292 | (SH_PIO_WRITE_STATUS_WRITE_DEADLOCK_MASK))) { |
293 | sn2_ptc_deadlock_recovery(nasids, nix, mynasid, ptc0, data0, ptc1, data1); | |
1da177e4 LT |
294 | } |
295 | ||
296 | start += (1UL << nbits); | |
297 | ||
298 | } while (start < end); | |
299 | ||
470ceb05 JS |
300 | itc2 = ia64_get_itc() - itc2; |
301 | __get_cpu_var(ptcstats).shub_itc_clocks += itc2; | |
302 | if (itc2 > __get_cpu_var(ptcstats).shub_itc_clocks_max) | |
303 | __get_cpu_var(ptcstats).shub_itc_clocks_max = itc2; | |
304 | ||
305 | ptc_unlock(flags, opt); | |
1da177e4 LT |
306 | |
307 | preempt_enable(); | |
308 | } | |
309 | ||
310 | /* | |
311 | * sn2_ptc_deadlock_recovery | |
312 | * | |
313 | * Recover from PTC deadlocks conditions. Recovery requires stepping thru each | |
314 | * TLB flush transaction. The recovery sequence is somewhat tricky & is | |
315 | * coded in assembly language. | |
316 | */ | |
470ceb05 | 317 | void sn2_ptc_deadlock_recovery(short *nasids, short nix, int mynasid, volatile unsigned long *ptc0, unsigned long data0, |
1da177e4 LT |
318 | volatile unsigned long *ptc1, unsigned long data1) |
319 | { | |
320 | extern void sn2_ptc_deadlock_recovery_core(volatile unsigned long *, unsigned long, | |
321 | volatile unsigned long *, unsigned long, volatile unsigned long *, unsigned long); | |
470ceb05 JS |
322 | short nasid, i; |
323 | unsigned long *piows, zeroval; | |
1da177e4 | 324 | |
470ceb05 | 325 | __get_cpu_var(ptcstats).deadlocks++; |
1da177e4 | 326 | |
470ceb05 | 327 | piows = (unsigned long *) pda->pio_write_status_addr; |
1da177e4 LT |
328 | zeroval = pda->pio_write_status_val; |
329 | ||
470ceb05 JS |
330 | for (i=0; i < nix; i++) { |
331 | nasid = nasids[i]; | |
332 | if (!(sn2_ptctest & 3) && nasid == mynasid) | |
1da177e4 | 333 | continue; |
1da177e4 LT |
334 | ptc0 = CHANGE_NASID(nasid, ptc0); |
335 | if (ptc1) | |
336 | ptc1 = CHANGE_NASID(nasid, ptc1); | |
337 | sn2_ptc_deadlock_recovery_core(ptc0, data0, ptc1, data1, piows, zeroval); | |
338 | } | |
470ceb05 | 339 | |
1da177e4 LT |
340 | } |
341 | ||
342 | /** | |
343 | * sn_send_IPI_phys - send an IPI to a Nasid and slice | |
344 | * @nasid: nasid to receive the interrupt (may be outside partition) | |
345 | * @physid: physical cpuid to receive the interrupt. | |
346 | * @vector: command to send | |
347 | * @delivery_mode: delivery mechanism | |
348 | * | |
349 | * Sends an IPI (interprocessor interrupt) to the processor specified by | |
350 | * @physid | |
351 | * | |
352 | * @delivery_mode can be one of the following | |
353 | * | |
354 | * %IA64_IPI_DM_INT - pend an interrupt | |
355 | * %IA64_IPI_DM_PMI - pend a PMI | |
356 | * %IA64_IPI_DM_NMI - pend an NMI | |
357 | * %IA64_IPI_DM_INIT - pend an INIT interrupt | |
358 | */ | |
359 | void sn_send_IPI_phys(int nasid, long physid, int vector, int delivery_mode) | |
360 | { | |
361 | long val; | |
362 | unsigned long flags = 0; | |
363 | volatile long *p; | |
364 | ||
365 | p = (long *)GLOBAL_MMR_PHYS_ADDR(nasid, SH_IPI_INT); | |
366 | val = (1UL << SH_IPI_INT_SEND_SHFT) | | |
367 | (physid << SH_IPI_INT_PID_SHFT) | | |
368 | ((long)delivery_mode << SH_IPI_INT_TYPE_SHFT) | | |
369 | ((long)vector << SH_IPI_INT_IDX_SHFT) | | |
370 | (0x000feeUL << SH_IPI_INT_BASE_SHFT); | |
371 | ||
372 | mb(); | |
373 | if (enable_shub_wars_1_1()) { | |
374 | spin_lock_irqsave(&sn2_global_ptc_lock, flags); | |
375 | } | |
376 | pio_phys_write_mmr(p, val); | |
377 | if (enable_shub_wars_1_1()) { | |
378 | wait_piowc(); | |
379 | spin_unlock_irqrestore(&sn2_global_ptc_lock, flags); | |
380 | } | |
381 | ||
382 | } | |
383 | ||
384 | EXPORT_SYMBOL(sn_send_IPI_phys); | |
385 | ||
386 | /** | |
387 | * sn2_send_IPI - send an IPI to a processor | |
388 | * @cpuid: target of the IPI | |
389 | * @vector: command to send | |
390 | * @delivery_mode: delivery mechanism | |
391 | * @redirect: redirect the IPI? | |
392 | * | |
393 | * Sends an IPI (InterProcessor Interrupt) to the processor specified by | |
394 | * @cpuid. @vector specifies the command to send, while @delivery_mode can | |
395 | * be one of the following | |
396 | * | |
397 | * %IA64_IPI_DM_INT - pend an interrupt | |
398 | * %IA64_IPI_DM_PMI - pend a PMI | |
399 | * %IA64_IPI_DM_NMI - pend an NMI | |
400 | * %IA64_IPI_DM_INIT - pend an INIT interrupt | |
401 | */ | |
402 | void sn2_send_IPI(int cpuid, int vector, int delivery_mode, int redirect) | |
403 | { | |
404 | long physid; | |
405 | int nasid; | |
406 | ||
407 | physid = cpu_physical_id(cpuid); | |
408 | nasid = cpuid_to_nasid(cpuid); | |
409 | ||
410 | /* the following is used only when starting cpus at boot time */ | |
411 | if (unlikely(nasid == -1)) | |
412 | ia64_sn_get_sapic_info(physid, &nasid, NULL, NULL); | |
413 | ||
414 | sn_send_IPI_phys(nasid, physid, vector, delivery_mode); | |
415 | } | |
470ceb05 JS |
416 | |
417 | #ifdef CONFIG_PROC_FS | |
418 | ||
419 | #define PTC_BASENAME "sgi_sn/ptc_statistics" | |
420 | ||
421 | static void *sn2_ptc_seq_start(struct seq_file *file, loff_t * offset) | |
422 | { | |
423 | if (*offset < NR_CPUS) | |
424 | return offset; | |
425 | return NULL; | |
426 | } | |
427 | ||
428 | static void *sn2_ptc_seq_next(struct seq_file *file, void *data, loff_t * offset) | |
429 | { | |
430 | (*offset)++; | |
431 | if (*offset < NR_CPUS) | |
432 | return offset; | |
433 | return NULL; | |
434 | } | |
435 | ||
436 | static void sn2_ptc_seq_stop(struct seq_file *file, void *data) | |
437 | { | |
438 | } | |
439 | ||
440 | static int sn2_ptc_seq_show(struct seq_file *file, void *data) | |
441 | { | |
442 | struct ptc_stats *stat; | |
443 | int cpu; | |
444 | ||
445 | cpu = *(loff_t *) data; | |
446 | ||
447 | if (!cpu) { | |
448 | seq_printf(file, "# ptc_l change_rid shub_ptc_flushes shub_nodes_flushed deadlocks lock_nsec shub_nsec shub_nsec_max\n"); | |
449 | seq_printf(file, "# ptctest %d\n", sn2_ptctest); | |
450 | } | |
451 | ||
452 | if (cpu < NR_CPUS && cpu_online(cpu)) { | |
453 | stat = &per_cpu(ptcstats, cpu); | |
454 | seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld %ld\n", cpu, stat->ptc_l, | |
455 | stat->change_rid, stat->shub_ptc_flushes, stat->nodes_flushed, | |
456 | stat->deadlocks, | |
457 | 1000 * stat->lock_itc_clocks / per_cpu(cpu_info, cpu).cyc_per_usec, | |
458 | 1000 * stat->shub_itc_clocks / per_cpu(cpu_info, cpu).cyc_per_usec, | |
459 | 1000 * stat->shub_itc_clocks_max / per_cpu(cpu_info, cpu).cyc_per_usec); | |
460 | } | |
461 | ||
462 | return 0; | |
463 | } | |
464 | ||
465 | static struct seq_operations sn2_ptc_seq_ops = { | |
466 | .start = sn2_ptc_seq_start, | |
467 | .next = sn2_ptc_seq_next, | |
468 | .stop = sn2_ptc_seq_stop, | |
469 | .show = sn2_ptc_seq_show | |
470 | }; | |
471 | ||
472 | int sn2_ptc_proc_open(struct inode *inode, struct file *file) | |
473 | { | |
474 | return seq_open(file, &sn2_ptc_seq_ops); | |
475 | } | |
476 | ||
477 | static struct file_operations proc_sn2_ptc_operations = { | |
478 | .open = sn2_ptc_proc_open, | |
479 | .read = seq_read, | |
480 | .llseek = seq_lseek, | |
481 | .release = seq_release, | |
482 | }; | |
483 | ||
484 | static struct proc_dir_entry *proc_sn2_ptc; | |
485 | ||
486 | static int __init sn2_ptc_init(void) | |
487 | { | |
488 | if (!(proc_sn2_ptc = create_proc_entry(PTC_BASENAME, 0444, NULL))) { | |
489 | printk(KERN_ERR "unable to create %s proc entry", PTC_BASENAME); | |
490 | return -EINVAL; | |
491 | } | |
492 | proc_sn2_ptc->proc_fops = &proc_sn2_ptc_operations; | |
493 | spin_lock_init(&sn2_global_ptc_lock); | |
494 | return 0; | |
495 | } | |
496 | ||
497 | static void __exit sn2_ptc_exit(void) | |
498 | { | |
499 | remove_proc_entry(PTC_BASENAME, NULL); | |
500 | } | |
501 | ||
502 | module_init(sn2_ptc_init); | |
503 | module_exit(sn2_ptc_exit); | |
504 | #endif /* CONFIG_PROC_FS */ | |
505 |