Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #include <linux/stop_machine.h> |
2 | #include <linux/kthread.h> | |
3 | #include <linux/sched.h> | |
4 | #include <linux/cpu.h> | |
5 | #include <linux/err.h> | |
6 | #include <linux/syscalls.h> | |
7 | #include <asm/atomic.h> | |
8 | #include <asm/semaphore.h> | |
9 | #include <asm/uaccess.h> | |
10 | ||
11 | /* Since we effect priority and affinity (both of which are visible | |
12 | * to, and settable by outside processes) we do indirection via a | |
13 | * kthread. */ | |
14 | ||
15 | /* Thread to stop each CPU in user context. */ | |
16 | enum stopmachine_state { | |
17 | STOPMACHINE_WAIT, | |
18 | STOPMACHINE_PREPARE, | |
19 | STOPMACHINE_DISABLE_IRQ, | |
20 | STOPMACHINE_EXIT, | |
21 | }; | |
22 | ||
23 | static enum stopmachine_state stopmachine_state; | |
24 | static unsigned int stopmachine_num_threads; | |
25 | static atomic_t stopmachine_thread_ack; | |
26 | static DECLARE_MUTEX(stopmachine_mutex); | |
27 | ||
d8cb7c1d | 28 | static int stopmachine(void *cpu) |
1da177e4 LT |
29 | { |
30 | int irqs_disabled = 0; | |
31 | int prepared = 0; | |
32 | ||
d8cb7c1d AM |
33 | set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu)); |
34 | ||
1da177e4 | 35 | /* Ack: we are alive */ |
d59dd462 | 36 | smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ |
1da177e4 LT |
37 | atomic_inc(&stopmachine_thread_ack); |
38 | ||
39 | /* Simple state machine */ | |
40 | while (stopmachine_state != STOPMACHINE_EXIT) { | |
41 | if (stopmachine_state == STOPMACHINE_DISABLE_IRQ | |
42 | && !irqs_disabled) { | |
43 | local_irq_disable(); | |
44 | irqs_disabled = 1; | |
45 | /* Ack: irqs disabled. */ | |
d59dd462 | 46 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
47 | atomic_inc(&stopmachine_thread_ack); |
48 | } else if (stopmachine_state == STOPMACHINE_PREPARE | |
49 | && !prepared) { | |
50 | /* Everyone is in place, hold CPU. */ | |
51 | preempt_disable(); | |
52 | prepared = 1; | |
d59dd462 | 53 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
54 | atomic_inc(&stopmachine_thread_ack); |
55 | } | |
56 | /* Yield in first stage: migration threads need to | |
57 | * help our sisters onto their CPUs. */ | |
58 | if (!prepared && !irqs_disabled) | |
59 | yield(); | |
60 | else | |
61 | cpu_relax(); | |
62 | } | |
63 | ||
64 | /* Ack: we are exiting. */ | |
d59dd462 | 65 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
66 | atomic_inc(&stopmachine_thread_ack); |
67 | ||
68 | if (irqs_disabled) | |
69 | local_irq_enable(); | |
70 | if (prepared) | |
71 | preempt_enable(); | |
72 | ||
73 | return 0; | |
74 | } | |
75 | ||
76 | /* Change the thread state */ | |
77 | static void stopmachine_set_state(enum stopmachine_state state) | |
78 | { | |
79 | atomic_set(&stopmachine_thread_ack, 0); | |
d59dd462 | 80 | smp_wmb(); |
1da177e4 LT |
81 | stopmachine_state = state; |
82 | while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) | |
83 | cpu_relax(); | |
84 | } | |
85 | ||
86 | static int stop_machine(void) | |
87 | { | |
d8cb7c1d | 88 | int i, ret = 0; |
1da177e4 | 89 | struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; |
1da177e4 LT |
90 | |
91 | /* One high-prio thread per cpu. We'll do this one. */ | |
ed653a64 | 92 | sched_setscheduler(current, SCHED_FIFO, ¶m); |
1da177e4 LT |
93 | |
94 | atomic_set(&stopmachine_thread_ack, 0); | |
95 | stopmachine_num_threads = 0; | |
96 | stopmachine_state = STOPMACHINE_WAIT; | |
97 | ||
98 | for_each_online_cpu(i) { | |
39c715b7 | 99 | if (i == raw_smp_processor_id()) |
1da177e4 | 100 | continue; |
d8cb7c1d AM |
101 | ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL); |
102 | if (ret < 0) | |
1da177e4 LT |
103 | break; |
104 | stopmachine_num_threads++; | |
105 | } | |
106 | ||
107 | /* Wait for them all to come to life. */ | |
108 | while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) | |
109 | yield(); | |
110 | ||
111 | /* If some failed, kill them all. */ | |
112 | if (ret < 0) { | |
113 | stopmachine_set_state(STOPMACHINE_EXIT); | |
1da177e4 LT |
114 | return ret; |
115 | } | |
116 | ||
1da177e4 | 117 | /* Now they are all started, make them hold the CPUs, ready. */ |
4557398f | 118 | preempt_disable(); |
1da177e4 LT |
119 | stopmachine_set_state(STOPMACHINE_PREPARE); |
120 | ||
121 | /* Make them disable irqs. */ | |
4557398f | 122 | local_irq_disable(); |
1da177e4 LT |
123 | stopmachine_set_state(STOPMACHINE_DISABLE_IRQ); |
124 | ||
125 | return 0; | |
126 | } | |
127 | ||
128 | static void restart_machine(void) | |
129 | { | |
130 | stopmachine_set_state(STOPMACHINE_EXIT); | |
131 | local_irq_enable(); | |
4557398f | 132 | preempt_enable_no_resched(); |
1da177e4 LT |
133 | } |
134 | ||
135 | struct stop_machine_data | |
136 | { | |
137 | int (*fn)(void *); | |
138 | void *data; | |
139 | struct completion done; | |
140 | }; | |
141 | ||
142 | static int do_stop(void *_smdata) | |
143 | { | |
144 | struct stop_machine_data *smdata = _smdata; | |
145 | int ret; | |
146 | ||
147 | ret = stop_machine(); | |
148 | if (ret == 0) { | |
149 | ret = smdata->fn(smdata->data); | |
150 | restart_machine(); | |
151 | } | |
152 | ||
153 | /* We're done: you can kthread_stop us now */ | |
154 | complete(&smdata->done); | |
155 | ||
156 | /* Wait for kthread_stop */ | |
157 | set_current_state(TASK_INTERRUPTIBLE); | |
158 | while (!kthread_should_stop()) { | |
159 | schedule(); | |
160 | set_current_state(TASK_INTERRUPTIBLE); | |
161 | } | |
162 | __set_current_state(TASK_RUNNING); | |
163 | return ret; | |
164 | } | |
165 | ||
166 | struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, | |
167 | unsigned int cpu) | |
168 | { | |
169 | struct stop_machine_data smdata; | |
170 | struct task_struct *p; | |
171 | ||
172 | smdata.fn = fn; | |
173 | smdata.data = data; | |
174 | init_completion(&smdata.done); | |
175 | ||
176 | down(&stopmachine_mutex); | |
177 | ||
178 | /* If they don't care which CPU fn runs on, bind to any online one. */ | |
179 | if (cpu == NR_CPUS) | |
39c715b7 | 180 | cpu = raw_smp_processor_id(); |
1da177e4 LT |
181 | |
182 | p = kthread_create(do_stop, &smdata, "kstopmachine"); | |
183 | if (!IS_ERR(p)) { | |
184 | kthread_bind(p, cpu); | |
185 | wake_up_process(p); | |
186 | wait_for_completion(&smdata.done); | |
187 | } | |
188 | up(&stopmachine_mutex); | |
189 | return p; | |
190 | } | |
191 | ||
192 | int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) | |
193 | { | |
194 | struct task_struct *p; | |
195 | int ret; | |
196 | ||
197 | /* No CPUs can come up or down during this. */ | |
198 | lock_cpu_hotplug(); | |
199 | p = __stop_machine_run(fn, data, cpu); | |
200 | if (!IS_ERR(p)) | |
201 | ret = kthread_stop(p); | |
202 | else | |
203 | ret = PTR_ERR(p); | |
204 | unlock_cpu_hotplug(); | |
205 | ||
206 | return ret; | |
207 | } |