Commit | Line | Data |
---|---|---|
ffdb5976 | 1 | /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
e5582ca2 RR |
2 | * GPL v2 and any later version. |
3 | */ | |
1da177e4 LT |
4 | #include <linux/cpu.h> |
5 | #include <linux/err.h> | |
ee527cd3 PB |
6 | #include <linux/kthread.h> |
7 | #include <linux/module.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/stop_machine.h> | |
1da177e4 | 10 | #include <linux/syscalls.h> |
a12bb444 BH |
11 | #include <linux/interrupt.h> |
12 | ||
1da177e4 | 13 | #include <asm/atomic.h> |
1da177e4 LT |
14 | #include <asm/uaccess.h> |
15 | ||
ffdb5976 | 16 | /* This controls the threads on each CPU. */ |
1da177e4 | 17 | enum stopmachine_state { |
ffdb5976 RR |
18 | /* Dummy starting state for thread. */ |
19 | STOPMACHINE_NONE, | |
20 | /* Awaiting everyone to be scheduled. */ | |
1da177e4 | 21 | STOPMACHINE_PREPARE, |
ffdb5976 | 22 | /* Disable interrupts. */ |
1da177e4 | 23 | STOPMACHINE_DISABLE_IRQ, |
ffdb5976 | 24 | /* Run the function */ |
5c2aed62 | 25 | STOPMACHINE_RUN, |
ffdb5976 | 26 | /* Exit */ |
1da177e4 LT |
27 | STOPMACHINE_EXIT, |
28 | }; | |
ffdb5976 | 29 | static enum stopmachine_state state; |
1da177e4 | 30 | |
5c2aed62 JB |
31 | struct stop_machine_data { |
32 | int (*fn)(void *); | |
33 | void *data; | |
ffdb5976 RR |
34 | int fnret; |
35 | }; | |
5c2aed62 | 36 | |
ffdb5976 RR |
37 | /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ |
38 | static unsigned int num_threads; | |
39 | static atomic_t thread_ack; | |
40 | static struct completion finished; | |
41 | static DEFINE_MUTEX(lock); | |
1da177e4 | 42 | |
ffdb5976 | 43 | static void set_state(enum stopmachine_state newstate) |
1da177e4 | 44 | { |
ffdb5976 RR |
45 | /* Reset ack counter. */ |
46 | atomic_set(&thread_ack, num_threads); | |
47 | smp_wmb(); | |
48 | state = newstate; | |
1da177e4 LT |
49 | } |
50 | ||
ffdb5976 RR |
51 | /* Last one to ack a state moves to the next state. */ |
52 | static void ack_state(void) | |
1da177e4 | 53 | { |
ffdb5976 RR |
54 | if (atomic_dec_and_test(&thread_ack)) { |
55 | /* If we're the last one to ack the EXIT, we're finished. */ | |
56 | if (state == STOPMACHINE_EXIT) | |
57 | complete(&finished); | |
58 | else | |
59 | set_state(state + 1); | |
60 | } | |
1da177e4 LT |
61 | } |
62 | ||
ffdb5976 RR |
63 | /* This is the actual thread which stops the CPU. It exits by itself rather |
64 | * than waiting for kthread_stop(), because it's easier for hotplug CPU. */ | |
65 | static int stop_cpu(struct stop_machine_data *smdata) | |
1da177e4 | 66 | { |
ffdb5976 RR |
67 | enum stopmachine_state curstate = STOPMACHINE_NONE; |
68 | int uninitialized_var(ret); | |
1da177e4 | 69 | |
ffdb5976 RR |
70 | /* Simple state machine */ |
71 | do { | |
72 | /* Chill out and ensure we re-read stopmachine_state. */ | |
3401a61e | 73 | cpu_relax(); |
ffdb5976 RR |
74 | if (state != curstate) { |
75 | curstate = state; | |
76 | switch (curstate) { | |
77 | case STOPMACHINE_DISABLE_IRQ: | |
78 | local_irq_disable(); | |
79 | hard_irq_disable(); | |
80 | break; | |
81 | case STOPMACHINE_RUN: | |
82 | /* |= allows error detection if functions on | |
83 | * multiple CPUs. */ | |
84 | smdata->fnret |= smdata->fn(smdata->data); | |
85 | break; | |
86 | default: | |
87 | break; | |
88 | } | |
89 | ack_state(); | |
90 | } | |
91 | } while (curstate != STOPMACHINE_EXIT); | |
1da177e4 | 92 | |
1da177e4 | 93 | local_irq_enable(); |
ffdb5976 | 94 | do_exit(0); |
1da177e4 LT |
95 | } |
96 | ||
ffdb5976 RR |
97 | /* Callback for CPUs which aren't supposed to do anything. */ |
98 | static int chill(void *unused) | |
5c2aed62 | 99 | { |
ffdb5976 | 100 | return 0; |
5c2aed62 | 101 | } |
1da177e4 | 102 | |
ffdb5976 | 103 | int __stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) |
1da177e4 | 104 | { |
ffdb5976 RR |
105 | int i, err; |
106 | struct stop_machine_data active, idle; | |
107 | struct task_struct **threads; | |
108 | ||
109 | active.fn = fn; | |
110 | active.data = data; | |
111 | active.fnret = 0; | |
112 | idle.fn = chill; | |
113 | idle.data = NULL; | |
114 | ||
115 | /* If they don't care which cpu fn runs on, just pick one. */ | |
116 | if (cpu == NR_CPUS) | |
117 | cpu = any_online_cpu(cpu_online_map); | |
118 | ||
119 | /* This could be too big for stack on large machines. */ | |
120 | threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL); | |
121 | if (!threads) | |
122 | return -ENOMEM; | |
123 | ||
124 | /* Set up initial state. */ | |
125 | mutex_lock(&lock); | |
126 | init_completion(&finished); | |
127 | num_threads = num_online_cpus(); | |
128 | set_state(STOPMACHINE_PREPARE); | |
1da177e4 | 129 | |
ffdb5976 RR |
130 | for_each_online_cpu(i) { |
131 | struct stop_machine_data *smdata; | |
132 | struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; | |
1da177e4 | 133 | |
ffdb5976 RR |
134 | if (cpu == ALL_CPUS || i == cpu) |
135 | smdata = &active; | |
136 | else | |
137 | smdata = &idle; | |
138 | ||
139 | threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u", | |
140 | i); | |
141 | if (IS_ERR(threads[i])) { | |
142 | err = PTR_ERR(threads[i]); | |
143 | threads[i] = NULL; | |
144 | goto kill_threads; | |
145 | } | |
1da177e4 | 146 | |
ffdb5976 RR |
147 | /* Place it onto correct cpu. */ |
148 | kthread_bind(threads[i], i); | |
1da177e4 | 149 | |
ffdb5976 RR |
150 | /* Make it highest prio. */ |
151 | if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, ¶m)) | |
152 | BUG(); | |
153 | } | |
1da177e4 | 154 | |
ffdb5976 RR |
155 | /* We've created all the threads. Wake them all: hold this CPU so one |
156 | * doesn't hit this CPU until we're ready. */ | |
157 | cpu = get_cpu(); | |
158 | for_each_online_cpu(i) | |
159 | wake_up_process(threads[i]); | |
5c2aed62 | 160 | |
ffdb5976 RR |
161 | /* This will release the thread on our CPU. */ |
162 | put_cpu(); | |
163 | wait_for_completion(&finished); | |
164 | mutex_unlock(&lock); | |
1da177e4 | 165 | |
ffdb5976 | 166 | kfree(threads); |
1da177e4 | 167 | |
ffdb5976 | 168 | return active.fnret; |
1da177e4 | 169 | |
ffdb5976 RR |
170 | kill_threads: |
171 | for_each_online_cpu(i) | |
172 | if (threads[i]) | |
173 | kthread_stop(threads[i]); | |
174 | mutex_unlock(&lock); | |
85653af7 | 175 | |
ffdb5976 RR |
176 | kfree(threads); |
177 | return err; | |
1da177e4 LT |
178 | } |
179 | ||
180 | int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) | |
181 | { | |
1da177e4 LT |
182 | int ret; |
183 | ||
184 | /* No CPUs can come up or down during this. */ | |
86ef5c9a | 185 | get_online_cpus(); |
ffdb5976 | 186 | ret = __stop_machine_run(fn, data, cpu); |
86ef5c9a | 187 | put_online_cpus(); |
1da177e4 LT |
188 | |
189 | return ret; | |
190 | } | |
ee527cd3 | 191 | EXPORT_SYMBOL_GPL(stop_machine_run); |