Commit | Line | Data |
---|---|---|
98211489 ZB |
1 | /* -*- mode: c; c-basic-offset: 8; -*- |
2 | * | |
3 | * vim: noexpandtab sw=8 ts=8 sts=0: | |
4 | * | |
5 | * Copyright (C) 2005 Oracle. All rights reserved. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public | |
9 | * License as published by the Free Software Foundation; either | |
10 | * version 2 of the License, or (at your option) any later version. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 | * General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public | |
18 | * License along with this program; if not, write to the | |
19 | * Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
20 | * Boston, MA 021110-1307, USA. | |
21 | */ | |
22 | ||
23 | /* This quorum hack is only here until we transition to some more rational | |
24 | * approach that is driven from userspace. Honest. No foolin'. | |
25 | * | |
26 | * Imagine two nodes lose network connectivity to each other but they're still | |
27 | * up and operating in every other way. Presumably a network timeout indicates | |
28 | * that a node is broken and should be recovered. They can't both recover each | |
29 | * other and both carry on without serialising their access to the file system. | |
30 | * They need to decide who is authoritative. Now extend that problem to | |
31 | * arbitrary groups of nodes losing connectivity between each other. | |
32 | * | |
33 | * So we declare that a node which has given up on connecting to a majority | |
34 | * of nodes who are still heartbeating will fence itself. | |
35 | * | |
36 | * There are huge opportunities for races here. After we give up on a node's | |
37 | * connection we need to wait long enough to give heartbeat an opportunity | |
38 | * to declare the node as truly dead. We also need to be careful with the | |
39 | * race between when we see a node start heartbeating and when we connect | |
40 | * to it. | |
41 | * | |
42 | * So nodes that are in this transtion put a hold on the quorum decision | |
43 | * with a counter. As they fall out of this transition they drop the count | |
44 | * and if they're the last, they fire off the decision. | |
45 | */ | |
46 | #include <linux/kernel.h> | |
47 | #include <linux/slab.h> | |
48 | #include <linux/workqueue.h> | |
bebe6f12 | 49 | #include <linux/reboot.h> |
98211489 ZB |
50 | |
51 | #include "heartbeat.h" | |
52 | #include "nodemanager.h" | |
53 | #define MLOG_MASK_PREFIX ML_QUORUM | |
54 | #include "masklog.h" | |
55 | #include "quorum.h" | |
56 | ||
57 | static struct o2quo_state { | |
58 | spinlock_t qs_lock; | |
59 | struct work_struct qs_work; | |
60 | int qs_pending; | |
61 | int qs_heartbeating; | |
62 | unsigned long qs_hb_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; | |
63 | int qs_connected; | |
64 | unsigned long qs_conn_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; | |
65 | int qs_holds; | |
66 | unsigned long qs_hold_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; | |
67 | } o2quo_state; | |
68 | ||
69 | /* this is horribly heavy-handed. It should instead flip the file | |
70 | * system RO and call some userspace script. */ | |
71 | static void o2quo_fence_self(void) | |
72 | { | |
73 | /* panic spins with interrupts enabled. with preempt | |
74 | * threads can still schedule, etc, etc */ | |
75 | o2hb_stop_all_regions(); | |
bebe6f12 | 76 | |
f6656d26 SM |
77 | switch (o2nm_single_cluster->cl_fence_method) { |
78 | case O2NM_FENCE_PANIC: | |
79 | panic("*** ocfs2 is very sorry to be fencing this system by " | |
80 | "panicing ***\n"); | |
81 | break; | |
82 | default: | |
83 | WARN_ON(o2nm_single_cluster->cl_fence_method >= | |
84 | O2NM_FENCE_METHODS); | |
85 | case O2NM_FENCE_RESET: | |
86 | printk(KERN_ERR "*** ocfs2 is very sorry to be fencing this " | |
87 | "system by restarting ***\n"); | |
88 | emergency_restart(); | |
89 | break; | |
90 | }; | |
98211489 ZB |
91 | } |
92 | ||
93 | /* Indicate that a timeout occured on a hearbeat region write. The | |
94 | * other nodes in the cluster may consider us dead at that time so we | |
95 | * want to "fence" ourselves so that we don't scribble on the disk | |
96 | * after they think they've recovered us. This can't solve all | |
97 | * problems related to writeout after recovery but this hack can at | |
98 | * least close some of those gaps. When we have real fencing, this can | |
99 | * go away as our node would be fenced externally before other nodes | |
100 | * begin recovery. */ | |
101 | void o2quo_disk_timeout(void) | |
102 | { | |
103 | o2quo_fence_self(); | |
104 | } | |
105 | ||
c4028958 | 106 | static void o2quo_make_decision(struct work_struct *work) |
98211489 ZB |
107 | { |
108 | int quorum; | |
109 | int lowest_hb, lowest_reachable = 0, fence = 0; | |
110 | struct o2quo_state *qs = &o2quo_state; | |
111 | ||
112 | spin_lock(&qs->qs_lock); | |
113 | ||
114 | lowest_hb = find_first_bit(qs->qs_hb_bm, O2NM_MAX_NODES); | |
115 | if (lowest_hb != O2NM_MAX_NODES) | |
116 | lowest_reachable = test_bit(lowest_hb, qs->qs_conn_bm); | |
117 | ||
118 | mlog(0, "heartbeating: %d, connected: %d, " | |
119 | "lowest: %d (%sreachable)\n", qs->qs_heartbeating, | |
120 | qs->qs_connected, lowest_hb, lowest_reachable ? "" : "un"); | |
121 | ||
122 | if (!test_bit(o2nm_this_node(), qs->qs_hb_bm) || | |
123 | qs->qs_heartbeating == 1) | |
124 | goto out; | |
125 | ||
126 | if (qs->qs_heartbeating & 1) { | |
127 | /* the odd numbered cluster case is straight forward -- | |
128 | * if we can't talk to the majority we're hosed */ | |
129 | quorum = (qs->qs_heartbeating + 1)/2; | |
130 | if (qs->qs_connected < quorum) { | |
131 | mlog(ML_ERROR, "fencing this node because it is " | |
132 | "only connected to %u nodes and %u is needed " | |
133 | "to make a quorum out of %u heartbeating nodes\n", | |
134 | qs->qs_connected, quorum, | |
135 | qs->qs_heartbeating); | |
136 | fence = 1; | |
137 | } | |
138 | } else { | |
139 | /* the even numbered cluster adds the possibility of each half | |
140 | * of the cluster being able to talk amongst themselves.. in | |
141 | * that case we're hosed if we can't talk to the group that has | |
142 | * the lowest numbered node */ | |
143 | quorum = qs->qs_heartbeating / 2; | |
144 | if (qs->qs_connected < quorum) { | |
145 | mlog(ML_ERROR, "fencing this node because it is " | |
146 | "only connected to %u nodes and %u is needed " | |
147 | "to make a quorum out of %u heartbeating nodes\n", | |
148 | qs->qs_connected, quorum, | |
149 | qs->qs_heartbeating); | |
150 | fence = 1; | |
151 | } | |
152 | else if ((qs->qs_connected == quorum) && | |
153 | !lowest_reachable) { | |
154 | mlog(ML_ERROR, "fencing this node because it is " | |
155 | "connected to a half-quorum of %u out of %u " | |
156 | "nodes which doesn't include the lowest active " | |
157 | "node %u\n", quorum, qs->qs_heartbeating, | |
158 | lowest_hb); | |
159 | fence = 1; | |
160 | } | |
161 | } | |
162 | ||
163 | out: | |
164 | spin_unlock(&qs->qs_lock); | |
165 | if (fence) | |
166 | o2quo_fence_self(); | |
167 | } | |
168 | ||
169 | static void o2quo_set_hold(struct o2quo_state *qs, u8 node) | |
170 | { | |
171 | assert_spin_locked(&qs->qs_lock); | |
172 | ||
173 | if (!test_and_set_bit(node, qs->qs_hold_bm)) { | |
174 | qs->qs_holds++; | |
175 | mlog_bug_on_msg(qs->qs_holds == O2NM_MAX_NODES, | |
176 | "node %u\n", node); | |
177 | mlog(0, "node %u, %d total\n", node, qs->qs_holds); | |
178 | } | |
179 | } | |
180 | ||
181 | static void o2quo_clear_hold(struct o2quo_state *qs, u8 node) | |
182 | { | |
183 | assert_spin_locked(&qs->qs_lock); | |
184 | ||
185 | if (test_and_clear_bit(node, qs->qs_hold_bm)) { | |
186 | mlog(0, "node %u, %d total\n", node, qs->qs_holds - 1); | |
187 | if (--qs->qs_holds == 0) { | |
188 | if (qs->qs_pending) { | |
189 | qs->qs_pending = 0; | |
190 | schedule_work(&qs->qs_work); | |
191 | } | |
192 | } | |
193 | mlog_bug_on_msg(qs->qs_holds < 0, "node %u, holds %d\n", | |
194 | node, qs->qs_holds); | |
195 | } | |
196 | } | |
197 | ||
198 | /* as a node comes up we delay the quorum decision until we know the fate of | |
199 | * the connection. the hold will be droped in conn_up or hb_down. it might be | |
200 | * perpetuated by con_err until hb_down. if we already have a conn, we might | |
201 | * be dropping a hold that conn_up got. */ | |
202 | void o2quo_hb_up(u8 node) | |
203 | { | |
204 | struct o2quo_state *qs = &o2quo_state; | |
205 | ||
206 | spin_lock(&qs->qs_lock); | |
207 | ||
208 | qs->qs_heartbeating++; | |
209 | mlog_bug_on_msg(qs->qs_heartbeating == O2NM_MAX_NODES, | |
210 | "node %u\n", node); | |
211 | mlog_bug_on_msg(test_bit(node, qs->qs_hb_bm), "node %u\n", node); | |
212 | set_bit(node, qs->qs_hb_bm); | |
213 | ||
214 | mlog(0, "node %u, %d total\n", node, qs->qs_heartbeating); | |
215 | ||
216 | if (!test_bit(node, qs->qs_conn_bm)) | |
217 | o2quo_set_hold(qs, node); | |
218 | else | |
219 | o2quo_clear_hold(qs, node); | |
220 | ||
221 | spin_unlock(&qs->qs_lock); | |
222 | } | |
223 | ||
224 | /* hb going down releases any holds we might have had due to this node from | |
225 | * conn_up, conn_err, or hb_up */ | |
226 | void o2quo_hb_down(u8 node) | |
227 | { | |
228 | struct o2quo_state *qs = &o2quo_state; | |
229 | ||
230 | spin_lock(&qs->qs_lock); | |
231 | ||
232 | qs->qs_heartbeating--; | |
233 | mlog_bug_on_msg(qs->qs_heartbeating < 0, | |
234 | "node %u, %d heartbeating\n", | |
235 | node, qs->qs_heartbeating); | |
236 | mlog_bug_on_msg(!test_bit(node, qs->qs_hb_bm), "node %u\n", node); | |
237 | clear_bit(node, qs->qs_hb_bm); | |
238 | ||
239 | mlog(0, "node %u, %d total\n", node, qs->qs_heartbeating); | |
240 | ||
241 | o2quo_clear_hold(qs, node); | |
242 | ||
243 | spin_unlock(&qs->qs_lock); | |
244 | } | |
245 | ||
246 | /* this tells us that we've decided that the node is still heartbeating | |
247 | * even though we've lost it's conn. it must only be called after conn_err | |
248 | * and indicates that we must now make a quorum decision in the future, | |
249 | * though we might be doing so after waiting for holds to drain. Here | |
250 | * we'll be dropping the hold from conn_err. */ | |
251 | void o2quo_hb_still_up(u8 node) | |
252 | { | |
253 | struct o2quo_state *qs = &o2quo_state; | |
254 | ||
255 | spin_lock(&qs->qs_lock); | |
256 | ||
257 | mlog(0, "node %u\n", node); | |
258 | ||
259 | qs->qs_pending = 1; | |
260 | o2quo_clear_hold(qs, node); | |
261 | ||
262 | spin_unlock(&qs->qs_lock); | |
263 | } | |
264 | ||
265 | /* This is analagous to hb_up. as a node's connection comes up we delay the | |
266 | * quorum decision until we see it heartbeating. the hold will be droped in | |
267 | * hb_up or hb_down. it might be perpetuated by con_err until hb_down. if | |
268 | * it's already heartbeating we we might be dropping a hold that conn_up got. | |
269 | * */ | |
270 | void o2quo_conn_up(u8 node) | |
271 | { | |
272 | struct o2quo_state *qs = &o2quo_state; | |
273 | ||
274 | spin_lock(&qs->qs_lock); | |
275 | ||
276 | qs->qs_connected++; | |
277 | mlog_bug_on_msg(qs->qs_connected == O2NM_MAX_NODES, | |
278 | "node %u\n", node); | |
279 | mlog_bug_on_msg(test_bit(node, qs->qs_conn_bm), "node %u\n", node); | |
280 | set_bit(node, qs->qs_conn_bm); | |
281 | ||
282 | mlog(0, "node %u, %d total\n", node, qs->qs_connected); | |
283 | ||
284 | if (!test_bit(node, qs->qs_hb_bm)) | |
285 | o2quo_set_hold(qs, node); | |
286 | else | |
287 | o2quo_clear_hold(qs, node); | |
288 | ||
289 | spin_unlock(&qs->qs_lock); | |
290 | } | |
291 | ||
292 | /* we've decided that we won't ever be connecting to the node again. if it's | |
293 | * still heartbeating we grab a hold that will delay decisions until either the | |
294 | * node stops heartbeating from hb_down or the caller decides that the node is | |
295 | * still up and calls still_up */ | |
296 | void o2quo_conn_err(u8 node) | |
297 | { | |
298 | struct o2quo_state *qs = &o2quo_state; | |
299 | ||
300 | spin_lock(&qs->qs_lock); | |
301 | ||
302 | if (test_bit(node, qs->qs_conn_bm)) { | |
303 | qs->qs_connected--; | |
304 | mlog_bug_on_msg(qs->qs_connected < 0, | |
305 | "node %u, connected %d\n", | |
306 | node, qs->qs_connected); | |
307 | ||
308 | clear_bit(node, qs->qs_conn_bm); | |
309 | } | |
310 | ||
311 | mlog(0, "node %u, %d total\n", node, qs->qs_connected); | |
312 | ||
313 | if (test_bit(node, qs->qs_hb_bm)) | |
314 | o2quo_set_hold(qs, node); | |
315 | ||
316 | spin_unlock(&qs->qs_lock); | |
317 | } | |
318 | ||
319 | void o2quo_init(void) | |
320 | { | |
321 | struct o2quo_state *qs = &o2quo_state; | |
322 | ||
323 | spin_lock_init(&qs->qs_lock); | |
c4028958 | 324 | INIT_WORK(&qs->qs_work, o2quo_make_decision); |
98211489 ZB |
325 | } |
326 | ||
327 | void o2quo_exit(void) | |
328 | { | |
329 | flush_scheduled_work(); | |
330 | } |