Commit | Line | Data |
---|---|---|
f615136c MF |
1 | /* |
2 | * Xtensa SMP support functions. | |
3 | * | |
4 | * This file is subject to the terms and conditions of the GNU General Public | |
5 | * License. See the file "COPYING" in the main directory of this archive | |
6 | * for more details. | |
7 | * | |
8 | * Copyright (C) 2008 - 2013 Tensilica Inc. | |
9 | * | |
10 | * Chris Zankel <chris@zankel.net> | |
11 | * Joe Taylor <joe@tensilica.com> | |
12 | * Pete Delaney <piet@tensilica.com | |
13 | */ | |
14 | ||
15 | #include <linux/cpu.h> | |
16 | #include <linux/cpumask.h> | |
17 | #include <linux/delay.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <linux/irqdomain.h> | |
21 | #include <linux/irq.h> | |
22 | #include <linux/kdebug.h> | |
23 | #include <linux/module.h> | |
68e21be2 | 24 | #include <linux/sched/mm.h> |
ef8bd77f | 25 | #include <linux/sched/hotplug.h> |
68db0cf1 | 26 | #include <linux/sched/task_stack.h> |
f615136c MF |
27 | #include <linux/reboot.h> |
28 | #include <linux/seq_file.h> | |
29 | #include <linux/smp.h> | |
30 | #include <linux/thread_info.h> | |
31 | ||
32 | #include <asm/cacheflush.h> | |
33 | #include <asm/kdebug.h> | |
34 | #include <asm/mmu_context.h> | |
35 | #include <asm/mxregs.h> | |
36 | #include <asm/platform.h> | |
37 | #include <asm/tlbflush.h> | |
38 | #include <asm/traps.h> | |
39 | ||
40 | #ifdef CONFIG_SMP | |
41 | # if XCHAL_HAVE_S32C1I == 0 | |
42 | # error "The S32C1I option is required for SMP." | |
43 | # endif | |
44 | #endif | |
45 | ||
49b424fe MF |
46 | static void system_invalidate_dcache_range(unsigned long start, |
47 | unsigned long size); | |
48 | static void system_flush_invalidate_dcache_range(unsigned long start, | |
49 | unsigned long size); | |
50 | ||
f615136c MF |
51 | /* IPI (Inter Process Interrupt) */ |
52 | ||
53 | #define IPI_IRQ 0 | |
54 | ||
55 | static irqreturn_t ipi_interrupt(int irq, void *dev_id); | |
56 | static struct irqaction ipi_irqaction = { | |
57 | .handler = ipi_interrupt, | |
58 | .flags = IRQF_PERCPU, | |
59 | .name = "ipi", | |
60 | }; | |
61 | ||
62 | void ipi_init(void) | |
63 | { | |
64 | unsigned irq = irq_create_mapping(NULL, IPI_IRQ); | |
65 | setup_irq(irq, &ipi_irqaction); | |
66 | } | |
67 | ||
68 | static inline unsigned int get_core_count(void) | |
69 | { | |
70 | /* Bits 18..21 of SYSCFGID contain the core count minus 1. */ | |
71 | unsigned int syscfgid = get_er(SYSCFGID); | |
72 | return ((syscfgid >> 18) & 0xf) + 1; | |
73 | } | |
74 | ||
75 | static inline int get_core_id(void) | |
76 | { | |
77 | /* Bits 0...18 of SYSCFGID contain the core id */ | |
78 | unsigned int core_id = get_er(SYSCFGID); | |
79 | return core_id & 0x3fff; | |
80 | } | |
81 | ||
82 | void __init smp_prepare_cpus(unsigned int max_cpus) | |
83 | { | |
84 | unsigned i; | |
85 | ||
86 | for (i = 0; i < max_cpus; ++i) | |
87 | set_cpu_present(i, true); | |
88 | } | |
89 | ||
90 | void __init smp_init_cpus(void) | |
91 | { | |
92 | unsigned i; | |
93 | unsigned int ncpus = get_core_count(); | |
94 | unsigned int core_id = get_core_id(); | |
95 | ||
96 | pr_info("%s: Core Count = %d\n", __func__, ncpus); | |
97 | pr_info("%s: Core Id = %d\n", __func__, core_id); | |
98 | ||
99 | for (i = 0; i < ncpus; ++i) | |
100 | set_cpu_possible(i, true); | |
101 | } | |
102 | ||
103 | void __init smp_prepare_boot_cpu(void) | |
104 | { | |
105 | unsigned int cpu = smp_processor_id(); | |
106 | BUG_ON(cpu != 0); | |
107 | cpu_asid_cache(cpu) = ASID_USER_FIRST; | |
108 | } | |
109 | ||
110 | void __init smp_cpus_done(unsigned int max_cpus) | |
111 | { | |
112 | } | |
113 | ||
114 | static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */ | |
115 | static DECLARE_COMPLETION(cpu_running); | |
116 | ||
49b424fe | 117 | void secondary_start_kernel(void) |
f615136c MF |
118 | { |
119 | struct mm_struct *mm = &init_mm; | |
120 | unsigned int cpu = smp_processor_id(); | |
121 | ||
122 | init_mmu(); | |
123 | ||
124 | #ifdef CONFIG_DEBUG_KERNEL | |
125 | if (boot_secondary_processors == 0) { | |
126 | pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n", | |
127 | __func__, boot_secondary_processors, cpu); | |
128 | for (;;) | |
129 | __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL)); | |
130 | } | |
131 | ||
132 | pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n", | |
133 | __func__, boot_secondary_processors, cpu); | |
134 | #endif | |
135 | /* Init EXCSAVE1 */ | |
136 | ||
137 | secondary_trap_init(); | |
138 | ||
139 | /* All kernel threads share the same mm context. */ | |
140 | ||
3fce371b | 141 | mmget(mm); |
f1f10076 | 142 | mmgrab(mm); |
f615136c MF |
143 | current->active_mm = mm; |
144 | cpumask_set_cpu(cpu, mm_cpumask(mm)); | |
145 | enter_lazy_tlb(mm, current); | |
146 | ||
147 | preempt_disable(); | |
148 | trace_hardirqs_off(); | |
149 | ||
150 | calibrate_delay(); | |
151 | ||
152 | notify_cpu_starting(cpu); | |
153 | ||
154 | secondary_init_irq(); | |
155 | local_timer_setup(cpu); | |
156 | ||
abf0ea65 KT |
157 | set_cpu_online(cpu, true); |
158 | ||
f615136c MF |
159 | local_irq_enable(); |
160 | ||
f615136c MF |
161 | complete(&cpu_running); |
162 | ||
fc6d73d6 | 163 | cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); |
f615136c MF |
164 | } |
165 | ||
166 | static void mx_cpu_start(void *p) | |
167 | { | |
168 | unsigned cpu = (unsigned)p; | |
169 | unsigned long run_stall_mask = get_er(MPSCORE); | |
170 | ||
171 | set_er(run_stall_mask & ~(1u << cpu), MPSCORE); | |
172 | pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", | |
173 | __func__, cpu, run_stall_mask, get_er(MPSCORE)); | |
174 | } | |
175 | ||
176 | static void mx_cpu_stop(void *p) | |
177 | { | |
178 | unsigned cpu = (unsigned)p; | |
179 | unsigned long run_stall_mask = get_er(MPSCORE); | |
180 | ||
181 | set_er(run_stall_mask | (1u << cpu), MPSCORE); | |
182 | pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", | |
183 | __func__, cpu, run_stall_mask, get_er(MPSCORE)); | |
184 | } | |
185 | ||
49b424fe MF |
186 | #ifdef CONFIG_HOTPLUG_CPU |
187 | unsigned long cpu_start_id __cacheline_aligned; | |
188 | #endif | |
f615136c MF |
189 | unsigned long cpu_start_ccount; |
190 | ||
191 | static int boot_secondary(unsigned int cpu, struct task_struct *ts) | |
192 | { | |
193 | unsigned long timeout = jiffies + msecs_to_jiffies(1000); | |
194 | unsigned long ccount; | |
195 | int i; | |
196 | ||
49b424fe MF |
197 | #ifdef CONFIG_HOTPLUG_CPU |
198 | cpu_start_id = cpu; | |
199 | system_flush_invalidate_dcache_range( | |
200 | (unsigned long)&cpu_start_id, sizeof(cpu_start_id)); | |
201 | #endif | |
f615136c MF |
202 | smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1); |
203 | ||
204 | for (i = 0; i < 2; ++i) { | |
205 | do | |
206 | ccount = get_ccount(); | |
207 | while (!ccount); | |
208 | ||
209 | cpu_start_ccount = ccount; | |
210 | ||
211 | while (time_before(jiffies, timeout)) { | |
212 | mb(); | |
213 | if (!cpu_start_ccount) | |
214 | break; | |
215 | } | |
216 | ||
217 | if (cpu_start_ccount) { | |
218 | smp_call_function_single(0, mx_cpu_stop, | |
219 | (void *)cpu, 1); | |
220 | cpu_start_ccount = 0; | |
221 | return -EIO; | |
222 | } | |
223 | } | |
224 | return 0; | |
225 | } | |
226 | ||
227 | int __cpu_up(unsigned int cpu, struct task_struct *idle) | |
228 | { | |
229 | int ret = 0; | |
230 | ||
231 | if (cpu_asid_cache(cpu) == 0) | |
232 | cpu_asid_cache(cpu) = ASID_USER_FIRST; | |
233 | ||
234 | start_info.stack = (unsigned long)task_pt_regs(idle); | |
235 | wmb(); | |
236 | ||
237 | pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n", | |
238 | __func__, cpu, idle, start_info.stack); | |
239 | ||
240 | ret = boot_secondary(cpu, idle); | |
241 | if (ret == 0) { | |
242 | wait_for_completion_timeout(&cpu_running, | |
243 | msecs_to_jiffies(1000)); | |
244 | if (!cpu_online(cpu)) | |
245 | ret = -EIO; | |
246 | } | |
247 | ||
248 | if (ret) | |
249 | pr_err("CPU %u failed to boot\n", cpu); | |
250 | ||
251 | return ret; | |
252 | } | |
253 | ||
49b424fe MF |
254 | #ifdef CONFIG_HOTPLUG_CPU |
255 | ||
256 | /* | |
257 | * __cpu_disable runs on the processor to be shutdown. | |
258 | */ | |
259 | int __cpu_disable(void) | |
260 | { | |
261 | unsigned int cpu = smp_processor_id(); | |
262 | ||
263 | /* | |
264 | * Take this CPU offline. Once we clear this, we can't return, | |
265 | * and we must not schedule until we're ready to give up the cpu. | |
266 | */ | |
267 | set_cpu_online(cpu, false); | |
268 | ||
269 | /* | |
270 | * OK - migrate IRQs away from this CPU | |
271 | */ | |
272 | migrate_irqs(); | |
273 | ||
274 | /* | |
275 | * Flush user cache and TLB mappings, and then remove this CPU | |
276 | * from the vm mask set of all processes. | |
277 | */ | |
278 | local_flush_cache_all(); | |
279 | local_flush_tlb_all(); | |
280 | invalidate_page_directory(); | |
281 | ||
282 | clear_tasks_mm_cpumask(cpu); | |
283 | ||
284 | return 0; | |
285 | } | |
286 | ||
287 | static void platform_cpu_kill(unsigned int cpu) | |
288 | { | |
289 | smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true); | |
290 | } | |
291 | ||
292 | /* | |
293 | * called on the thread which is asking for a CPU to be shutdown - | |
294 | * waits until shutdown has completed, or it is timed out. | |
295 | */ | |
296 | void __cpu_die(unsigned int cpu) | |
297 | { | |
298 | unsigned long timeout = jiffies + msecs_to_jiffies(1000); | |
299 | while (time_before(jiffies, timeout)) { | |
300 | system_invalidate_dcache_range((unsigned long)&cpu_start_id, | |
301 | sizeof(cpu_start_id)); | |
302 | if (cpu_start_id == -cpu) { | |
303 | platform_cpu_kill(cpu); | |
304 | return; | |
305 | } | |
306 | } | |
307 | pr_err("CPU%u: unable to kill\n", cpu); | |
308 | } | |
309 | ||
310 | void arch_cpu_idle_dead(void) | |
311 | { | |
312 | cpu_die(); | |
313 | } | |
314 | /* | |
315 | * Called from the idle thread for the CPU which has been shutdown. | |
316 | * | |
317 | * Note that we disable IRQs here, but do not re-enable them | |
318 | * before returning to the caller. This is also the behaviour | |
319 | * of the other hotplug-cpu capable cores, so presumably coming | |
320 | * out of idle fixes this. | |
321 | */ | |
322 | void __ref cpu_die(void) | |
323 | { | |
324 | idle_task_exit(); | |
325 | local_irq_disable(); | |
326 | __asm__ __volatile__( | |
327 | " movi a2, cpu_restart\n" | |
328 | " jx a2\n"); | |
329 | } | |
330 | ||
331 | #endif /* CONFIG_HOTPLUG_CPU */ | |
332 | ||
f615136c MF |
333 | enum ipi_msg_type { |
334 | IPI_RESCHEDULE = 0, | |
335 | IPI_CALL_FUNC, | |
336 | IPI_CPU_STOP, | |
337 | IPI_MAX | |
338 | }; | |
339 | ||
340 | static const struct { | |
341 | const char *short_text; | |
342 | const char *long_text; | |
343 | } ipi_text[] = { | |
344 | { .short_text = "RES", .long_text = "Rescheduling interrupts" }, | |
345 | { .short_text = "CAL", .long_text = "Function call interrupts" }, | |
346 | { .short_text = "DIE", .long_text = "CPU shutdown interrupts" }, | |
347 | }; | |
348 | ||
349 | struct ipi_data { | |
350 | unsigned long ipi_count[IPI_MAX]; | |
351 | }; | |
352 | ||
353 | static DEFINE_PER_CPU(struct ipi_data, ipi_data); | |
354 | ||
355 | static void send_ipi_message(const struct cpumask *callmask, | |
356 | enum ipi_msg_type msg_id) | |
357 | { | |
358 | int index; | |
359 | unsigned long mask = 0; | |
360 | ||
361 | for_each_cpu(index, callmask) | |
362 | if (index != smp_processor_id()) | |
363 | mask |= 1 << index; | |
364 | ||
365 | set_er(mask, MIPISET(msg_id)); | |
366 | } | |
367 | ||
368 | void arch_send_call_function_ipi_mask(const struct cpumask *mask) | |
369 | { | |
370 | send_ipi_message(mask, IPI_CALL_FUNC); | |
371 | } | |
372 | ||
373 | void arch_send_call_function_single_ipi(int cpu) | |
374 | { | |
375 | send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); | |
376 | } | |
377 | ||
378 | void smp_send_reschedule(int cpu) | |
379 | { | |
380 | send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); | |
381 | } | |
382 | ||
383 | void smp_send_stop(void) | |
384 | { | |
385 | struct cpumask targets; | |
386 | ||
387 | cpumask_copy(&targets, cpu_online_mask); | |
388 | cpumask_clear_cpu(smp_processor_id(), &targets); | |
389 | send_ipi_message(&targets, IPI_CPU_STOP); | |
390 | } | |
391 | ||
392 | static void ipi_cpu_stop(unsigned int cpu) | |
393 | { | |
394 | set_cpu_online(cpu, false); | |
395 | machine_halt(); | |
396 | } | |
397 | ||
398 | irqreturn_t ipi_interrupt(int irq, void *dev_id) | |
399 | { | |
400 | unsigned int cpu = smp_processor_id(); | |
401 | struct ipi_data *ipi = &per_cpu(ipi_data, cpu); | |
402 | unsigned int msg; | |
403 | unsigned i; | |
404 | ||
405 | msg = get_er(MIPICAUSE(cpu)); | |
406 | for (i = 0; i < IPI_MAX; i++) | |
407 | if (msg & (1 << i)) { | |
408 | set_er(1 << i, MIPICAUSE(cpu)); | |
409 | ++ipi->ipi_count[i]; | |
410 | } | |
411 | ||
412 | if (msg & (1 << IPI_RESCHEDULE)) | |
413 | scheduler_ipi(); | |
414 | if (msg & (1 << IPI_CALL_FUNC)) | |
415 | generic_smp_call_function_interrupt(); | |
416 | if (msg & (1 << IPI_CPU_STOP)) | |
417 | ipi_cpu_stop(cpu); | |
418 | ||
419 | return IRQ_HANDLED; | |
420 | } | |
421 | ||
422 | void show_ipi_list(struct seq_file *p, int prec) | |
423 | { | |
424 | unsigned int cpu; | |
425 | unsigned i; | |
426 | ||
427 | for (i = 0; i < IPI_MAX; ++i) { | |
428 | seq_printf(p, "%*s:", prec, ipi_text[i].short_text); | |
429 | for_each_online_cpu(cpu) | |
430 | seq_printf(p, " %10lu", | |
431 | per_cpu(ipi_data, cpu).ipi_count[i]); | |
432 | seq_printf(p, " %s\n", ipi_text[i].long_text); | |
433 | } | |
434 | } | |
435 | ||
436 | int setup_profiling_timer(unsigned int multiplier) | |
437 | { | |
438 | pr_debug("setup_profiling_timer %d\n", multiplier); | |
439 | return 0; | |
440 | } | |
441 | ||
442 | /* TLB flush functions */ | |
443 | ||
444 | struct flush_data { | |
445 | struct vm_area_struct *vma; | |
446 | unsigned long addr1; | |
447 | unsigned long addr2; | |
448 | }; | |
449 | ||
450 | static void ipi_flush_tlb_all(void *arg) | |
451 | { | |
452 | local_flush_tlb_all(); | |
453 | } | |
454 | ||
455 | void flush_tlb_all(void) | |
456 | { | |
457 | on_each_cpu(ipi_flush_tlb_all, NULL, 1); | |
458 | } | |
459 | ||
460 | static void ipi_flush_tlb_mm(void *arg) | |
461 | { | |
462 | local_flush_tlb_mm(arg); | |
463 | } | |
464 | ||
465 | void flush_tlb_mm(struct mm_struct *mm) | |
466 | { | |
467 | on_each_cpu(ipi_flush_tlb_mm, mm, 1); | |
468 | } | |
469 | ||
470 | static void ipi_flush_tlb_page(void *arg) | |
471 | { | |
472 | struct flush_data *fd = arg; | |
473 | local_flush_tlb_page(fd->vma, fd->addr1); | |
474 | } | |
475 | ||
476 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) | |
477 | { | |
478 | struct flush_data fd = { | |
479 | .vma = vma, | |
480 | .addr1 = addr, | |
481 | }; | |
482 | on_each_cpu(ipi_flush_tlb_page, &fd, 1); | |
483 | } | |
484 | ||
485 | static void ipi_flush_tlb_range(void *arg) | |
486 | { | |
487 | struct flush_data *fd = arg; | |
488 | local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); | |
489 | } | |
490 | ||
491 | void flush_tlb_range(struct vm_area_struct *vma, | |
492 | unsigned long start, unsigned long end) | |
493 | { | |
494 | struct flush_data fd = { | |
495 | .vma = vma, | |
496 | .addr1 = start, | |
497 | .addr2 = end, | |
498 | }; | |
499 | on_each_cpu(ipi_flush_tlb_range, &fd, 1); | |
500 | } | |
501 | ||
04c6b3e2 MF |
502 | static void ipi_flush_tlb_kernel_range(void *arg) |
503 | { | |
504 | struct flush_data *fd = arg; | |
505 | local_flush_tlb_kernel_range(fd->addr1, fd->addr2); | |
506 | } | |
507 | ||
508 | void flush_tlb_kernel_range(unsigned long start, unsigned long end) | |
509 | { | |
510 | struct flush_data fd = { | |
511 | .addr1 = start, | |
512 | .addr2 = end, | |
513 | }; | |
514 | on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1); | |
515 | } | |
516 | ||
f615136c MF |
517 | /* Cache flush functions */ |
518 | ||
519 | static void ipi_flush_cache_all(void *arg) | |
520 | { | |
521 | local_flush_cache_all(); | |
522 | } | |
523 | ||
524 | void flush_cache_all(void) | |
525 | { | |
526 | on_each_cpu(ipi_flush_cache_all, NULL, 1); | |
527 | } | |
528 | ||
529 | static void ipi_flush_cache_page(void *arg) | |
530 | { | |
531 | struct flush_data *fd = arg; | |
532 | local_flush_cache_page(fd->vma, fd->addr1, fd->addr2); | |
533 | } | |
534 | ||
535 | void flush_cache_page(struct vm_area_struct *vma, | |
536 | unsigned long address, unsigned long pfn) | |
537 | { | |
538 | struct flush_data fd = { | |
539 | .vma = vma, | |
540 | .addr1 = address, | |
541 | .addr2 = pfn, | |
542 | }; | |
543 | on_each_cpu(ipi_flush_cache_page, &fd, 1); | |
544 | } | |
545 | ||
546 | static void ipi_flush_cache_range(void *arg) | |
547 | { | |
548 | struct flush_data *fd = arg; | |
549 | local_flush_cache_range(fd->vma, fd->addr1, fd->addr2); | |
550 | } | |
551 | ||
552 | void flush_cache_range(struct vm_area_struct *vma, | |
553 | unsigned long start, unsigned long end) | |
554 | { | |
555 | struct flush_data fd = { | |
556 | .vma = vma, | |
557 | .addr1 = start, | |
558 | .addr2 = end, | |
559 | }; | |
560 | on_each_cpu(ipi_flush_cache_range, &fd, 1); | |
561 | } | |
562 | ||
563 | static void ipi_flush_icache_range(void *arg) | |
564 | { | |
565 | struct flush_data *fd = arg; | |
566 | local_flush_icache_range(fd->addr1, fd->addr2); | |
567 | } | |
568 | ||
569 | void flush_icache_range(unsigned long start, unsigned long end) | |
570 | { | |
571 | struct flush_data fd = { | |
572 | .addr1 = start, | |
573 | .addr2 = end, | |
574 | }; | |
575 | on_each_cpu(ipi_flush_icache_range, &fd, 1); | |
576 | } | |
e3560305 | 577 | EXPORT_SYMBOL(flush_icache_range); |
49b424fe MF |
578 | |
579 | /* ------------------------------------------------------------------------- */ | |
580 | ||
581 | static void ipi_invalidate_dcache_range(void *arg) | |
582 | { | |
583 | struct flush_data *fd = arg; | |
584 | __invalidate_dcache_range(fd->addr1, fd->addr2); | |
585 | } | |
586 | ||
587 | static void system_invalidate_dcache_range(unsigned long start, | |
588 | unsigned long size) | |
589 | { | |
590 | struct flush_data fd = { | |
591 | .addr1 = start, | |
592 | .addr2 = size, | |
593 | }; | |
594 | on_each_cpu(ipi_invalidate_dcache_range, &fd, 1); | |
595 | } | |
596 | ||
597 | static void ipi_flush_invalidate_dcache_range(void *arg) | |
598 | { | |
599 | struct flush_data *fd = arg; | |
600 | __flush_invalidate_dcache_range(fd->addr1, fd->addr2); | |
601 | } | |
602 | ||
603 | static void system_flush_invalidate_dcache_range(unsigned long start, | |
604 | unsigned long size) | |
605 | { | |
606 | struct flush_data fd = { | |
607 | .addr1 = start, | |
608 | .addr2 = size, | |
609 | }; | |
610 | on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1); | |
611 | } |