Commit | Line | Data |
---|---|---|
1c33be57 NP |
1 | /* |
2 | * arch/arm/common/bL_switcher.c -- big.LITTLE cluster switcher core driver | |
3 | * | |
4 | * Created by: Nicolas Pitre, March 2012 | |
5 | * Copyright: (C) 2012-2013 Linaro Limited | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | */ | |
11 | ||
0577fee2 | 12 | #include <linux/atomic.h> |
1c33be57 NP |
13 | #include <linux/init.h> |
14 | #include <linux/kernel.h> | |
15 | #include <linux/module.h> | |
16 | #include <linux/sched.h> | |
17 | #include <linux/interrupt.h> | |
18 | #include <linux/cpu_pm.h> | |
71ce1dee | 19 | #include <linux/cpu.h> |
3f09d479 | 20 | #include <linux/cpumask.h> |
71ce1dee NP |
21 | #include <linux/kthread.h> |
22 | #include <linux/wait.h> | |
3f09d479 LP |
23 | #include <linux/clockchips.h> |
24 | #include <linux/hrtimer.h> | |
25 | #include <linux/tick.h> | |
491990e2 | 26 | #include <linux/notifier.h> |
1c33be57 | 27 | #include <linux/mm.h> |
c0f43751 | 28 | #include <linux/mutex.h> |
0577fee2 | 29 | #include <linux/spinlock.h> |
1c33be57 | 30 | #include <linux/string.h> |
6b7437ae | 31 | #include <linux/sysfs.h> |
1c33be57 | 32 | #include <linux/irqchip/arm-gic.h> |
c4821c05 | 33 | #include <linux/moduleparam.h> |
1c33be57 NP |
34 | |
35 | #include <asm/smp_plat.h> | |
36 | #include <asm/suspend.h> | |
37 | #include <asm/mcpm.h> | |
38 | #include <asm/bL_switcher.h> | |
39 | ||
40 | ||
41 | /* | |
42 | * Use our own MPIDR accessors as the generic ones in asm/cputype.h have | |
43 | * __attribute_const__ and we don't want the compiler to assume any | |
44 | * constness here as the value _does_ change along some code paths. | |
45 | */ | |
46 | ||
47 | static int read_mpidr(void) | |
48 | { | |
49 | unsigned int id; | |
50 | asm volatile ("mrc p15, 0, %0, c0, c0, 5" : "=r" (id)); | |
51 | return id & MPIDR_HWID_BITMASK; | |
52 | } | |
53 | ||
54 | /* | |
55 | * bL switcher core code. | |
56 | */ | |
57 | ||
108a9640 | 58 | static void bL_do_switch(void *_arg) |
1c33be57 | 59 | { |
38c35d4f | 60 | unsigned ib_mpidr, ib_cpu, ib_cluster; |
108a9640 | 61 | long volatile handshake, **handshake_ptr = _arg; |
1c33be57 | 62 | |
1c33be57 NP |
63 | pr_debug("%s\n", __func__); |
64 | ||
38c35d4f NP |
65 | ib_mpidr = cpu_logical_map(smp_processor_id()); |
66 | ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0); | |
67 | ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1); | |
1c33be57 | 68 | |
108a9640 NP |
69 | /* Advertise our handshake location */ |
70 | if (handshake_ptr) { | |
71 | handshake = 0; | |
72 | *handshake_ptr = &handshake; | |
73 | } else | |
74 | handshake = -1; | |
75 | ||
1c33be57 NP |
76 | /* |
77 | * Our state has been saved at this point. Let's release our | |
78 | * inbound CPU. | |
79 | */ | |
38c35d4f | 80 | mcpm_set_entry_vector(ib_cpu, ib_cluster, cpu_resume); |
1c33be57 NP |
81 | sev(); |
82 | ||
83 | /* | |
84 | * From this point, we must assume that our counterpart CPU might | |
85 | * have taken over in its parallel world already, as if execution | |
86 | * just returned from cpu_suspend(). It is therefore important to | |
87 | * be very careful not to make any change the other guy is not | |
88 | * expecting. This is why we need stack isolation. | |
89 | * | |
90 | * Fancy under cover tasks could be performed here. For now | |
91 | * we have none. | |
92 | */ | |
93 | ||
108a9640 NP |
94 | /* |
95 | * Let's wait until our inbound is alive. | |
96 | */ | |
97 | while (!handshake) { | |
98 | wfe(); | |
99 | smp_mb(); | |
100 | } | |
101 | ||
1c33be57 NP |
102 | /* Let's put ourself down. */ |
103 | mcpm_cpu_power_down(); | |
104 | ||
105 | /* should never get here */ | |
106 | BUG(); | |
107 | } | |
108 | ||
109 | /* | |
c052de26 NP |
110 | * Stack isolation. To ensure 'current' remains valid, we just use another |
111 | * piece of our thread's stack space which should be fairly lightly used. | |
112 | * The selected area starts just above the thread_info structure located | |
113 | * at the very bottom of the stack, aligned to a cache line, and indexed | |
114 | * with the cluster number. | |
1c33be57 | 115 | */ |
c052de26 | 116 | #define STACK_SIZE 512 |
1c33be57 NP |
117 | extern void call_with_stack(void (*fn)(void *), void *arg, void *sp); |
118 | static int bL_switchpoint(unsigned long _arg) | |
119 | { | |
120 | unsigned int mpidr = read_mpidr(); | |
1c33be57 | 121 | unsigned int clusterid = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
c052de26 | 122 | void *stack = current_thread_info() + 1; |
1c33be57 | 123 | stack = PTR_ALIGN(stack, L1_CACHE_BYTES); |
c052de26 | 124 | stack += clusterid * STACK_SIZE + STACK_SIZE; |
1c33be57 NP |
125 | call_with_stack(bL_do_switch, (void *)_arg, stack); |
126 | BUG(); | |
127 | } | |
128 | ||
129 | /* | |
130 | * Generic switcher interface | |
131 | */ | |
132 | ||
ed96762e | 133 | static unsigned int bL_gic_id[MAX_CPUS_PER_CLUSTER][MAX_NR_CLUSTERS]; |
38c35d4f | 134 | static int bL_switcher_cpu_pairing[NR_CPUS]; |
ed96762e | 135 | |
1c33be57 NP |
136 | /* |
137 | * bL_switch_to - Switch to a specific cluster for the current CPU | |
138 | * @new_cluster_id: the ID of the cluster to switch to. | |
139 | * | |
140 | * This function must be called on the CPU to be switched. | |
141 | * Returns 0 on success, else a negative status code. | |
142 | */ | |
143 | static int bL_switch_to(unsigned int new_cluster_id) | |
144 | { | |
38c35d4f NP |
145 | unsigned int mpidr, this_cpu, that_cpu; |
146 | unsigned int ob_mpidr, ob_cpu, ob_cluster, ib_mpidr, ib_cpu, ib_cluster; | |
3f09d479 LP |
147 | struct tick_device *tdev; |
148 | enum clock_event_mode tdev_mode; | |
108a9640 | 149 | long volatile *handshake_ptr; |
1c33be57 NP |
150 | int ret; |
151 | ||
38c35d4f NP |
152 | this_cpu = smp_processor_id(); |
153 | ob_mpidr = read_mpidr(); | |
154 | ob_cpu = MPIDR_AFFINITY_LEVEL(ob_mpidr, 0); | |
155 | ob_cluster = MPIDR_AFFINITY_LEVEL(ob_mpidr, 1); | |
156 | BUG_ON(cpu_logical_map(this_cpu) != ob_mpidr); | |
1c33be57 | 157 | |
38c35d4f | 158 | if (new_cluster_id == ob_cluster) |
1c33be57 NP |
159 | return 0; |
160 | ||
38c35d4f NP |
161 | that_cpu = bL_switcher_cpu_pairing[this_cpu]; |
162 | ib_mpidr = cpu_logical_map(that_cpu); | |
163 | ib_cpu = MPIDR_AFFINITY_LEVEL(ib_mpidr, 0); | |
164 | ib_cluster = MPIDR_AFFINITY_LEVEL(ib_mpidr, 1); | |
165 | ||
166 | pr_debug("before switch: CPU %d MPIDR %#x -> %#x\n", | |
167 | this_cpu, ob_mpidr, ib_mpidr); | |
1c33be57 NP |
168 | |
169 | /* Close the gate for our entry vectors */ | |
38c35d4f NP |
170 | mcpm_set_entry_vector(ob_cpu, ob_cluster, NULL); |
171 | mcpm_set_entry_vector(ib_cpu, ib_cluster, NULL); | |
1c33be57 NP |
172 | |
173 | /* | |
174 | * Let's wake up the inbound CPU now in case it requires some delay | |
175 | * to come online, but leave it gated in our entry vector code. | |
176 | */ | |
38c35d4f | 177 | ret = mcpm_cpu_power_up(ib_cpu, ib_cluster); |
1c33be57 NP |
178 | if (ret) { |
179 | pr_err("%s: mcpm_cpu_power_up() returned %d\n", __func__, ret); | |
180 | return ret; | |
181 | } | |
182 | ||
183 | /* | |
184 | * From this point we are entering the switch critical zone | |
185 | * and can't take any interrupts anymore. | |
186 | */ | |
187 | local_irq_disable(); | |
188 | local_fiq_disable(); | |
189 | ||
1c33be57 | 190 | /* redirect GIC's SGIs to our counterpart */ |
38c35d4f | 191 | gic_migrate_target(bL_gic_id[ib_cpu][ib_cluster]); |
1c33be57 NP |
192 | |
193 | /* | |
194 | * Raise a SGI on the inbound CPU to make sure it doesn't stall | |
195 | * in a possible WFI, such as in mcpm_power_down(). | |
196 | */ | |
197 | arch_send_wakeup_ipi_mask(cpumask_of(this_cpu)); | |
198 | ||
3f09d479 LP |
199 | tdev = tick_get_device(this_cpu); |
200 | if (tdev && !cpumask_equal(tdev->evtdev->cpumask, cpumask_of(this_cpu))) | |
201 | tdev = NULL; | |
202 | if (tdev) { | |
203 | tdev_mode = tdev->evtdev->mode; | |
204 | clockevents_set_mode(tdev->evtdev, CLOCK_EVT_MODE_SHUTDOWN); | |
205 | } | |
206 | ||
1c33be57 NP |
207 | ret = cpu_pm_enter(); |
208 | ||
209 | /* we can not tolerate errors at this point */ | |
210 | if (ret) | |
211 | panic("%s: cpu_pm_enter() returned %d\n", __func__, ret); | |
212 | ||
38c35d4f NP |
213 | /* Swap the physical CPUs in the logical map for this logical CPU. */ |
214 | cpu_logical_map(this_cpu) = ib_mpidr; | |
215 | cpu_logical_map(that_cpu) = ob_mpidr; | |
1c33be57 NP |
216 | |
217 | /* Let's do the actual CPU switch. */ | |
108a9640 | 218 | ret = cpu_suspend((unsigned long)&handshake_ptr, bL_switchpoint); |
1c33be57 NP |
219 | if (ret > 0) |
220 | panic("%s: cpu_suspend() returned %d\n", __func__, ret); | |
221 | ||
222 | /* We are executing on the inbound CPU at this point */ | |
223 | mpidr = read_mpidr(); | |
38c35d4f NP |
224 | pr_debug("after switch: CPU %d MPIDR %#x\n", this_cpu, mpidr); |
225 | BUG_ON(mpidr != ib_mpidr); | |
1c33be57 NP |
226 | |
227 | mcpm_cpu_powered_up(); | |
228 | ||
229 | ret = cpu_pm_exit(); | |
230 | ||
3f09d479 LP |
231 | if (tdev) { |
232 | clockevents_set_mode(tdev->evtdev, tdev_mode); | |
233 | clockevents_program_event(tdev->evtdev, | |
234 | tdev->evtdev->next_event, 1); | |
235 | } | |
236 | ||
1c33be57 NP |
237 | local_fiq_enable(); |
238 | local_irq_enable(); | |
239 | ||
108a9640 NP |
240 | *handshake_ptr = 1; |
241 | dsb_sev(); | |
242 | ||
1c33be57 NP |
243 | if (ret) |
244 | pr_err("%s exiting with error %d\n", __func__, ret); | |
245 | return ret; | |
246 | } | |
247 | ||
71ce1dee | 248 | struct bL_thread { |
0577fee2 | 249 | spinlock_t lock; |
71ce1dee NP |
250 | struct task_struct *task; |
251 | wait_queue_head_t wq; | |
252 | int wanted_cluster; | |
6b7437ae | 253 | struct completion started; |
0577fee2 DM |
254 | bL_switch_completion_handler completer; |
255 | void *completer_cookie; | |
1c33be57 NP |
256 | }; |
257 | ||
71ce1dee NP |
258 | static struct bL_thread bL_threads[NR_CPUS]; |
259 | ||
260 | static int bL_switcher_thread(void *arg) | |
261 | { | |
262 | struct bL_thread *t = arg; | |
263 | struct sched_param param = { .sched_priority = 1 }; | |
264 | int cluster; | |
0577fee2 DM |
265 | bL_switch_completion_handler completer; |
266 | void *completer_cookie; | |
71ce1dee NP |
267 | |
268 | sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); | |
6b7437ae | 269 | complete(&t->started); |
71ce1dee NP |
270 | |
271 | do { | |
272 | if (signal_pending(current)) | |
273 | flush_signals(current); | |
274 | wait_event_interruptible(t->wq, | |
275 | t->wanted_cluster != -1 || | |
276 | kthread_should_stop()); | |
0577fee2 DM |
277 | |
278 | spin_lock(&t->lock); | |
279 | cluster = t->wanted_cluster; | |
280 | completer = t->completer; | |
281 | completer_cookie = t->completer_cookie; | |
282 | t->wanted_cluster = -1; | |
283 | t->completer = NULL; | |
284 | spin_unlock(&t->lock); | |
285 | ||
286 | if (cluster != -1) { | |
71ce1dee | 287 | bL_switch_to(cluster); |
0577fee2 DM |
288 | |
289 | if (completer) | |
290 | completer(completer_cookie); | |
291 | } | |
71ce1dee NP |
292 | } while (!kthread_should_stop()); |
293 | ||
294 | return 0; | |
295 | } | |
296 | ||
6b7437ae | 297 | static struct task_struct *bL_switcher_thread_create(int cpu, void *arg) |
1c33be57 | 298 | { |
71ce1dee NP |
299 | struct task_struct *task; |
300 | ||
301 | task = kthread_create_on_node(bL_switcher_thread, arg, | |
302 | cpu_to_node(cpu), "kswitcher_%d", cpu); | |
303 | if (!IS_ERR(task)) { | |
304 | kthread_bind(task, cpu); | |
305 | wake_up_process(task); | |
306 | } else | |
307 | pr_err("%s failed for CPU %d\n", __func__, cpu); | |
308 | return task; | |
1c33be57 NP |
309 | } |
310 | ||
311 | /* | |
0577fee2 DM |
312 | * bL_switch_request_cb - Switch to a specific cluster for the given CPU, |
313 | * with completion notification via a callback | |
1c33be57 NP |
314 | * |
315 | * @cpu: the CPU to switch | |
316 | * @new_cluster_id: the ID of the cluster to switch to. | |
0577fee2 DM |
317 | * @completer: switch completion callback. if non-NULL, |
318 | * @completer(@completer_cookie) will be called on completion of | |
319 | * the switch, in non-atomic context. | |
320 | * @completer_cookie: opaque context argument for @completer. | |
1c33be57 | 321 | * |
71ce1dee NP |
322 | * This function causes a cluster switch on the given CPU by waking up |
323 | * the appropriate switcher thread. This function may or may not return | |
324 | * before the switch has occurred. | |
0577fee2 DM |
325 | * |
326 | * If a @completer callback function is supplied, it will be called when | |
327 | * the switch is complete. This can be used to determine asynchronously | |
328 | * when the switch is complete, regardless of when bL_switch_request() | |
329 | * returns. When @completer is supplied, no new switch request is permitted | |
330 | * for the affected CPU until after the switch is complete, and @completer | |
331 | * has returned. | |
1c33be57 | 332 | */ |
0577fee2 DM |
333 | int bL_switch_request_cb(unsigned int cpu, unsigned int new_cluster_id, |
334 | bL_switch_completion_handler completer, | |
335 | void *completer_cookie) | |
1c33be57 | 336 | { |
71ce1dee | 337 | struct bL_thread *t; |
1c33be57 | 338 | |
71ce1dee NP |
339 | if (cpu >= ARRAY_SIZE(bL_threads)) { |
340 | pr_err("%s: cpu %d out of bounds\n", __func__, cpu); | |
341 | return -EINVAL; | |
1c33be57 | 342 | } |
1c33be57 | 343 | |
71ce1dee | 344 | t = &bL_threads[cpu]; |
0577fee2 | 345 | |
71ce1dee NP |
346 | if (IS_ERR(t->task)) |
347 | return PTR_ERR(t->task); | |
348 | if (!t->task) | |
349 | return -ESRCH; | |
350 | ||
0577fee2 DM |
351 | spin_lock(&t->lock); |
352 | if (t->completer) { | |
353 | spin_unlock(&t->lock); | |
354 | return -EBUSY; | |
355 | } | |
356 | t->completer = completer; | |
357 | t->completer_cookie = completer_cookie; | |
71ce1dee | 358 | t->wanted_cluster = new_cluster_id; |
0577fee2 | 359 | spin_unlock(&t->lock); |
71ce1dee NP |
360 | wake_up(&t->wq); |
361 | return 0; | |
1c33be57 | 362 | } |
0577fee2 | 363 | EXPORT_SYMBOL_GPL(bL_switch_request_cb); |
71ce1dee | 364 | |
9797a0e9 NP |
365 | /* |
366 | * Activation and configuration code. | |
367 | */ | |
368 | ||
c0f43751 | 369 | static DEFINE_MUTEX(bL_switcher_activation_lock); |
491990e2 | 370 | static BLOCKING_NOTIFIER_HEAD(bL_activation_notifier); |
6b7437ae | 371 | static unsigned int bL_switcher_active; |
38c35d4f | 372 | static unsigned int bL_switcher_cpu_original_cluster[NR_CPUS]; |
9797a0e9 NP |
373 | static cpumask_t bL_switcher_removed_logical_cpus; |
374 | ||
491990e2 DM |
375 | int bL_switcher_register_notifier(struct notifier_block *nb) |
376 | { | |
377 | return blocking_notifier_chain_register(&bL_activation_notifier, nb); | |
378 | } | |
379 | EXPORT_SYMBOL_GPL(bL_switcher_register_notifier); | |
380 | ||
381 | int bL_switcher_unregister_notifier(struct notifier_block *nb) | |
382 | { | |
383 | return blocking_notifier_chain_unregister(&bL_activation_notifier, nb); | |
384 | } | |
385 | EXPORT_SYMBOL_GPL(bL_switcher_unregister_notifier); | |
386 | ||
387 | static int bL_activation_notify(unsigned long val) | |
388 | { | |
389 | int ret; | |
390 | ||
391 | ret = blocking_notifier_call_chain(&bL_activation_notifier, val, NULL); | |
392 | if (ret & NOTIFY_STOP_MASK) | |
393 | pr_err("%s: notifier chain failed with status 0x%x\n", | |
394 | __func__, ret); | |
395 | return notifier_to_errno(ret); | |
396 | } | |
397 | ||
6b7437ae | 398 | static void bL_switcher_restore_cpus(void) |
9797a0e9 NP |
399 | { |
400 | int i; | |
401 | ||
402 | for_each_cpu(i, &bL_switcher_removed_logical_cpus) | |
403 | cpu_up(i); | |
404 | } | |
405 | ||
6b7437ae | 406 | static int bL_switcher_halve_cpus(void) |
9797a0e9 | 407 | { |
38c35d4f NP |
408 | int i, j, cluster_0, gic_id, ret; |
409 | unsigned int cpu, cluster, mask; | |
410 | cpumask_t available_cpus; | |
9797a0e9 | 411 | |
38c35d4f NP |
412 | /* First pass to validate what we have */ |
413 | mask = 0; | |
9797a0e9 | 414 | for_each_online_cpu(i) { |
38c35d4f NP |
415 | cpu = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0); |
416 | cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1); | |
9797a0e9 NP |
417 | if (cluster >= 2) { |
418 | pr_err("%s: only dual cluster systems are supported\n", __func__); | |
419 | return -EINVAL; | |
420 | } | |
38c35d4f NP |
421 | if (WARN_ON(cpu >= MAX_CPUS_PER_CLUSTER)) |
422 | return -EINVAL; | |
423 | mask |= (1 << cluster); | |
9797a0e9 | 424 | } |
38c35d4f NP |
425 | if (mask != 3) { |
426 | pr_err("%s: no CPU pairing possible\n", __func__); | |
9797a0e9 NP |
427 | return -EINVAL; |
428 | } | |
429 | ||
38c35d4f NP |
430 | /* |
431 | * Now let's do the pairing. We match each CPU with another CPU | |
432 | * from a different cluster. To get a uniform scheduling behavior | |
433 | * without fiddling with CPU topology and compute capacity data, | |
434 | * we'll use logical CPUs initially belonging to the same cluster. | |
435 | */ | |
436 | memset(bL_switcher_cpu_pairing, -1, sizeof(bL_switcher_cpu_pairing)); | |
437 | cpumask_copy(&available_cpus, cpu_online_mask); | |
438 | cluster_0 = -1; | |
439 | for_each_cpu(i, &available_cpus) { | |
440 | int match = -1; | |
441 | cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1); | |
442 | if (cluster_0 == -1) | |
443 | cluster_0 = cluster; | |
444 | if (cluster != cluster_0) | |
445 | continue; | |
446 | cpumask_clear_cpu(i, &available_cpus); | |
447 | for_each_cpu(j, &available_cpus) { | |
448 | cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(j), 1); | |
9797a0e9 | 449 | /* |
38c35d4f NP |
450 | * Let's remember the last match to create "odd" |
451 | * pairings on purpose in order for other code not | |
452 | * to assume any relation between physical and | |
453 | * logical CPU numbers. | |
9797a0e9 | 454 | */ |
38c35d4f NP |
455 | if (cluster != cluster_0) |
456 | match = j; | |
457 | } | |
458 | if (match != -1) { | |
459 | bL_switcher_cpu_pairing[i] = match; | |
460 | cpumask_clear_cpu(match, &available_cpus); | |
461 | pr_info("CPU%d paired with CPU%d\n", i, match); | |
462 | } | |
463 | } | |
464 | ||
465 | /* | |
466 | * Now we disable the unwanted CPUs i.e. everything that has no | |
467 | * pairing information (that includes the pairing counterparts). | |
468 | */ | |
469 | cpumask_clear(&bL_switcher_removed_logical_cpus); | |
470 | for_each_online_cpu(i) { | |
471 | cpu = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0); | |
472 | cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 1); | |
473 | ||
474 | /* Let's take note of the GIC ID for this CPU */ | |
475 | gic_id = gic_get_cpu_id(i); | |
476 | if (gic_id < 0) { | |
477 | pr_err("%s: bad GIC ID for CPU %d\n", __func__, i); | |
478 | bL_switcher_restore_cpus(); | |
479 | return -EINVAL; | |
480 | } | |
481 | bL_gic_id[cpu][cluster] = gic_id; | |
482 | pr_info("GIC ID for CPU %u cluster %u is %u\n", | |
483 | cpu, cluster, gic_id); | |
484 | ||
485 | if (bL_switcher_cpu_pairing[i] != -1) { | |
486 | bL_switcher_cpu_original_cluster[i] = cluster; | |
487 | continue; | |
9797a0e9 NP |
488 | } |
489 | ||
490 | ret = cpu_down(i); | |
491 | if (ret) { | |
492 | bL_switcher_restore_cpus(); | |
493 | return ret; | |
494 | } | |
495 | cpumask_set_cpu(i, &bL_switcher_removed_logical_cpus); | |
496 | } | |
497 | ||
498 | return 0; | |
499 | } | |
500 | ||
6b7437ae | 501 | static int bL_switcher_enable(void) |
71ce1dee | 502 | { |
9797a0e9 | 503 | int cpu, ret; |
71ce1dee | 504 | |
c0f43751 | 505 | mutex_lock(&bL_switcher_activation_lock); |
6b7437ae NP |
506 | cpu_hotplug_driver_lock(); |
507 | if (bL_switcher_active) { | |
508 | cpu_hotplug_driver_unlock(); | |
c0f43751 | 509 | mutex_unlock(&bL_switcher_activation_lock); |
6b7437ae | 510 | return 0; |
9797a0e9 NP |
511 | } |
512 | ||
6b7437ae NP |
513 | pr_info("big.LITTLE switcher initializing\n"); |
514 | ||
491990e2 DM |
515 | ret = bL_activation_notify(BL_NOTIFY_PRE_ENABLE); |
516 | if (ret) | |
517 | goto error; | |
518 | ||
9797a0e9 | 519 | ret = bL_switcher_halve_cpus(); |
491990e2 DM |
520 | if (ret) |
521 | goto error; | |
9797a0e9 | 522 | |
71ce1dee NP |
523 | for_each_online_cpu(cpu) { |
524 | struct bL_thread *t = &bL_threads[cpu]; | |
0577fee2 | 525 | spin_lock_init(&t->lock); |
71ce1dee | 526 | init_waitqueue_head(&t->wq); |
6b7437ae | 527 | init_completion(&t->started); |
71ce1dee NP |
528 | t->wanted_cluster = -1; |
529 | t->task = bL_switcher_thread_create(cpu, t); | |
530 | } | |
6b7437ae NP |
531 | |
532 | bL_switcher_active = 1; | |
491990e2 | 533 | bL_activation_notify(BL_NOTIFY_POST_ENABLE); |
71ce1dee | 534 | pr_info("big.LITTLE switcher initialized\n"); |
491990e2 DM |
535 | goto out; |
536 | ||
537 | error: | |
538 | pr_warn("big.LITTLE switcher initialization failed\n"); | |
539 | bL_activation_notify(BL_NOTIFY_POST_DISABLE); | |
c0f43751 | 540 | |
491990e2 | 541 | out: |
c0f43751 DM |
542 | cpu_hotplug_driver_unlock(); |
543 | mutex_unlock(&bL_switcher_activation_lock); | |
491990e2 | 544 | return ret; |
71ce1dee NP |
545 | } |
546 | ||
6b7437ae NP |
547 | #ifdef CONFIG_SYSFS |
548 | ||
549 | static void bL_switcher_disable(void) | |
550 | { | |
38c35d4f | 551 | unsigned int cpu, cluster; |
6b7437ae NP |
552 | struct bL_thread *t; |
553 | struct task_struct *task; | |
554 | ||
c0f43751 | 555 | mutex_lock(&bL_switcher_activation_lock); |
6b7437ae | 556 | cpu_hotplug_driver_lock(); |
491990e2 DM |
557 | |
558 | if (!bL_switcher_active) | |
559 | goto out; | |
560 | ||
561 | if (bL_activation_notify(BL_NOTIFY_PRE_DISABLE) != 0) { | |
562 | bL_activation_notify(BL_NOTIFY_POST_ENABLE); | |
563 | goto out; | |
6b7437ae | 564 | } |
491990e2 | 565 | |
6b7437ae NP |
566 | bL_switcher_active = 0; |
567 | ||
568 | /* | |
569 | * To deactivate the switcher, we must shut down the switcher | |
570 | * threads to prevent any other requests from being accepted. | |
571 | * Then, if the final cluster for given logical CPU is not the | |
572 | * same as the original one, we'll recreate a switcher thread | |
573 | * just for the purpose of switching the CPU back without any | |
574 | * possibility for interference from external requests. | |
575 | */ | |
576 | for_each_online_cpu(cpu) { | |
6b7437ae NP |
577 | t = &bL_threads[cpu]; |
578 | task = t->task; | |
579 | t->task = NULL; | |
580 | if (!task || IS_ERR(task)) | |
581 | continue; | |
582 | kthread_stop(task); | |
583 | /* no more switch may happen on this CPU at this point */ | |
584 | cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 1); | |
585 | if (cluster == bL_switcher_cpu_original_cluster[cpu]) | |
586 | continue; | |
587 | init_completion(&t->started); | |
588 | t->wanted_cluster = bL_switcher_cpu_original_cluster[cpu]; | |
589 | task = bL_switcher_thread_create(cpu, t); | |
590 | if (!IS_ERR(task)) { | |
591 | wait_for_completion(&t->started); | |
592 | kthread_stop(task); | |
593 | cluster = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 1); | |
594 | if (cluster == bL_switcher_cpu_original_cluster[cpu]) | |
595 | continue; | |
596 | } | |
597 | /* If execution gets here, we're in trouble. */ | |
598 | pr_crit("%s: unable to restore original cluster for CPU %d\n", | |
599 | __func__, cpu); | |
38c35d4f NP |
600 | pr_crit("%s: CPU %d can't be restored\n", |
601 | __func__, bL_switcher_cpu_pairing[cpu]); | |
602 | cpumask_clear_cpu(bL_switcher_cpu_pairing[cpu], | |
603 | &bL_switcher_removed_logical_cpus); | |
6b7437ae NP |
604 | } |
605 | ||
606 | bL_switcher_restore_cpus(); | |
491990e2 DM |
607 | bL_activation_notify(BL_NOTIFY_POST_DISABLE); |
608 | ||
609 | out: | |
6b7437ae | 610 | cpu_hotplug_driver_unlock(); |
c0f43751 | 611 | mutex_unlock(&bL_switcher_activation_lock); |
6b7437ae NP |
612 | } |
613 | ||
614 | static ssize_t bL_switcher_active_show(struct kobject *kobj, | |
615 | struct kobj_attribute *attr, char *buf) | |
616 | { | |
617 | return sprintf(buf, "%u\n", bL_switcher_active); | |
618 | } | |
619 | ||
620 | static ssize_t bL_switcher_active_store(struct kobject *kobj, | |
621 | struct kobj_attribute *attr, const char *buf, size_t count) | |
622 | { | |
623 | int ret; | |
624 | ||
625 | switch (buf[0]) { | |
626 | case '0': | |
627 | bL_switcher_disable(); | |
628 | ret = 0; | |
629 | break; | |
630 | case '1': | |
631 | ret = bL_switcher_enable(); | |
632 | break; | |
633 | default: | |
634 | ret = -EINVAL; | |
635 | } | |
636 | ||
637 | return (ret >= 0) ? count : ret; | |
638 | } | |
639 | ||
640 | static struct kobj_attribute bL_switcher_active_attr = | |
641 | __ATTR(active, 0644, bL_switcher_active_show, bL_switcher_active_store); | |
642 | ||
643 | static struct attribute *bL_switcher_attrs[] = { | |
644 | &bL_switcher_active_attr.attr, | |
645 | NULL, | |
646 | }; | |
647 | ||
648 | static struct attribute_group bL_switcher_attr_group = { | |
649 | .attrs = bL_switcher_attrs, | |
650 | }; | |
651 | ||
652 | static struct kobject *bL_switcher_kobj; | |
653 | ||
654 | static int __init bL_switcher_sysfs_init(void) | |
655 | { | |
656 | int ret; | |
657 | ||
658 | bL_switcher_kobj = kobject_create_and_add("bL_switcher", kernel_kobj); | |
659 | if (!bL_switcher_kobj) | |
660 | return -ENOMEM; | |
661 | ret = sysfs_create_group(bL_switcher_kobj, &bL_switcher_attr_group); | |
662 | if (ret) | |
663 | kobject_put(bL_switcher_kobj); | |
664 | return ret; | |
665 | } | |
666 | ||
667 | #endif /* CONFIG_SYSFS */ | |
668 | ||
c0f43751 DM |
669 | bool bL_switcher_get_enabled(void) |
670 | { | |
671 | mutex_lock(&bL_switcher_activation_lock); | |
672 | ||
673 | return bL_switcher_active; | |
674 | } | |
675 | EXPORT_SYMBOL_GPL(bL_switcher_get_enabled); | |
676 | ||
677 | void bL_switcher_put_enabled(void) | |
678 | { | |
679 | mutex_unlock(&bL_switcher_activation_lock); | |
680 | } | |
681 | EXPORT_SYMBOL_GPL(bL_switcher_put_enabled); | |
682 | ||
27261435 NP |
683 | /* |
684 | * Veto any CPU hotplug operation on those CPUs we've removed | |
685 | * while the switcher is active. | |
686 | * We're just not ready to deal with that given the trickery involved. | |
687 | */ | |
688 | static int bL_switcher_hotplug_callback(struct notifier_block *nfb, | |
689 | unsigned long action, void *hcpu) | |
690 | { | |
691 | if (bL_switcher_active) { | |
692 | int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu]; | |
693 | switch (action & 0xf) { | |
694 | case CPU_UP_PREPARE: | |
695 | case CPU_DOWN_PREPARE: | |
696 | if (pairing == -1) | |
697 | return NOTIFY_BAD; | |
698 | } | |
699 | } | |
700 | return NOTIFY_DONE; | |
701 | } | |
702 | ||
c4821c05 NP |
703 | static bool no_bL_switcher; |
704 | core_param(no_bL_switcher, no_bL_switcher, bool, 0644); | |
705 | ||
6b7437ae NP |
706 | static int __init bL_switcher_init(void) |
707 | { | |
708 | int ret; | |
709 | ||
710 | if (MAX_NR_CLUSTERS != 2) { | |
711 | pr_err("%s: only dual cluster systems are supported\n", __func__); | |
712 | return -EINVAL; | |
713 | } | |
714 | ||
27261435 NP |
715 | cpu_notifier(bL_switcher_hotplug_callback, 0); |
716 | ||
c4821c05 NP |
717 | if (!no_bL_switcher) { |
718 | ret = bL_switcher_enable(); | |
719 | if (ret) | |
720 | return ret; | |
721 | } | |
6b7437ae NP |
722 | |
723 | #ifdef CONFIG_SYSFS | |
724 | ret = bL_switcher_sysfs_init(); | |
725 | if (ret) | |
726 | pr_err("%s: unable to create sysfs entry\n", __func__); | |
727 | #endif | |
728 | ||
729 | return 0; | |
730 | } | |
731 | ||
71ce1dee | 732 | late_initcall(bL_switcher_init); |