Commit | Line | Data |
---|---|---|
0b61f8a4 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 | 2 | /* |
7b718769 | 3 | * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. |
c7e8f268 | 4 | * Copyright (c) 2008 Dave Chinner |
7b718769 | 5 | * All Rights Reserved. |
1da177e4 | 6 | */ |
1da177e4 | 7 | #include "xfs.h" |
a844f451 | 8 | #include "xfs_fs.h" |
5467b34b | 9 | #include "xfs_shared.h" |
4fb6e8ad | 10 | #include "xfs_format.h" |
239880ef DC |
11 | #include "xfs_log_format.h" |
12 | #include "xfs_trans_resv.h" | |
1da177e4 | 13 | #include "xfs_mount.h" |
239880ef | 14 | #include "xfs_trans.h" |
1da177e4 | 15 | #include "xfs_trans_priv.h" |
9e4c109a | 16 | #include "xfs_trace.h" |
e9e899a2 | 17 | #include "xfs_errortag.h" |
1da177e4 | 18 | #include "xfs_error.h" |
239880ef | 19 | #include "xfs_log.h" |
0020a190 | 20 | #include "xfs_log_priv.h" |
1da177e4 | 21 | |
1da177e4 | 22 | #ifdef DEBUG |
cd4a3c50 DC |
23 | /* |
24 | * Check that the list is sorted as it should be. | |
d686d12d DC |
25 | * |
26 | * Called with the ail lock held, but we don't want to assert fail with it | |
27 | * held otherwise we'll lock everything up and won't be able to debug the | |
28 | * cause. Hence we sample and check the state under the AIL lock and return if | |
29 | * everything is fine, otherwise we drop the lock and run the ASSERT checks. | |
30 | * Asserts may not be fatal, so pick the lock back up and continue onwards. | |
cd4a3c50 DC |
31 | */ |
32 | STATIC void | |
33 | xfs_ail_check( | |
d686d12d DC |
34 | struct xfs_ail *ailp, |
35 | struct xfs_log_item *lip) | |
daebba1b | 36 | __must_hold(&ailp->ail_lock) |
cd4a3c50 | 37 | { |
d686d12d DC |
38 | struct xfs_log_item *prev_lip; |
39 | struct xfs_log_item *next_lip; | |
40 | xfs_lsn_t prev_lsn = NULLCOMMITLSN; | |
41 | xfs_lsn_t next_lsn = NULLCOMMITLSN; | |
42 | xfs_lsn_t lsn; | |
43 | bool in_ail; | |
44 | ||
cd4a3c50 | 45 | |
57e80956 | 46 | if (list_empty(&ailp->ail_head)) |
cd4a3c50 DC |
47 | return; |
48 | ||
49 | /* | |
d686d12d | 50 | * Sample then check the next and previous entries are valid. |
cd4a3c50 | 51 | */ |
d686d12d DC |
52 | in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags); |
53 | prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail); | |
57e80956 | 54 | if (&prev_lip->li_ail != &ailp->ail_head) |
d686d12d DC |
55 | prev_lsn = prev_lip->li_lsn; |
56 | next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail); | |
57 | if (&next_lip->li_ail != &ailp->ail_head) | |
58 | next_lsn = next_lip->li_lsn; | |
59 | lsn = lip->li_lsn; | |
cd4a3c50 | 60 | |
d686d12d DC |
61 | if (in_ail && |
62 | (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) && | |
63 | (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0)) | |
64 | return; | |
cd4a3c50 | 65 | |
d686d12d DC |
66 | spin_unlock(&ailp->ail_lock); |
67 | ASSERT(in_ail); | |
68 | ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0); | |
69 | ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0); | |
70 | spin_lock(&ailp->ail_lock); | |
cd4a3c50 DC |
71 | } |
72 | #else /* !DEBUG */ | |
de08dbc1 | 73 | #define xfs_ail_check(a,l) |
1da177e4 LT |
74 | #endif /* DEBUG */ |
75 | ||
cd4a3c50 | 76 | /* |
fd074841 DC |
77 | * Return a pointer to the last item in the AIL. If the AIL is empty, then |
78 | * return NULL. | |
79 | */ | |
efe2330f | 80 | static struct xfs_log_item * |
fd074841 DC |
81 | xfs_ail_max( |
82 | struct xfs_ail *ailp) | |
83 | { | |
57e80956 | 84 | if (list_empty(&ailp->ail_head)) |
fd074841 DC |
85 | return NULL; |
86 | ||
efe2330f | 87 | return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail); |
fd074841 DC |
88 | } |
89 | ||
cd4a3c50 DC |
90 | /* |
91 | * Return a pointer to the item which follows the given item in the AIL. If | |
92 | * the given item is the last item in the list, then return NULL. | |
93 | */ | |
efe2330f | 94 | static struct xfs_log_item * |
cd4a3c50 | 95 | xfs_ail_next( |
efe2330f CH |
96 | struct xfs_ail *ailp, |
97 | struct xfs_log_item *lip) | |
cd4a3c50 | 98 | { |
57e80956 | 99 | if (lip->li_ail.next == &ailp->ail_head) |
cd4a3c50 DC |
100 | return NULL; |
101 | ||
efe2330f | 102 | return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail); |
cd4a3c50 | 103 | } |
1da177e4 LT |
104 | |
105 | /* | |
cd4a3c50 DC |
106 | * This is called by the log manager code to determine the LSN of the tail of |
107 | * the log. This is exactly the LSN of the first item in the AIL. If the AIL | |
108 | * is empty, then this function returns 0. | |
1da177e4 | 109 | * |
cd4a3c50 DC |
110 | * We need the AIL lock in order to get a coherent read of the lsn of the last |
111 | * item in the AIL. | |
1da177e4 | 112 | */ |
8eb807bd DC |
113 | static xfs_lsn_t |
114 | __xfs_ail_min_lsn( | |
115 | struct xfs_ail *ailp) | |
116 | { | |
117 | struct xfs_log_item *lip = xfs_ail_min(ailp); | |
118 | ||
119 | if (lip) | |
120 | return lip->li_lsn; | |
121 | return 0; | |
122 | } | |
123 | ||
1da177e4 | 124 | xfs_lsn_t |
fd074841 | 125 | xfs_ail_min_lsn( |
efe2330f | 126 | struct xfs_ail *ailp) |
1da177e4 | 127 | { |
8eb807bd | 128 | xfs_lsn_t lsn; |
1da177e4 | 129 | |
57e80956 | 130 | spin_lock(&ailp->ail_lock); |
8eb807bd | 131 | lsn = __xfs_ail_min_lsn(ailp); |
57e80956 | 132 | spin_unlock(&ailp->ail_lock); |
1da177e4 LT |
133 | |
134 | return lsn; | |
135 | } | |
136 | ||
fd074841 DC |
137 | /* |
138 | * Return the maximum lsn held in the AIL, or zero if the AIL is empty. | |
139 | */ | |
140 | static xfs_lsn_t | |
141 | xfs_ail_max_lsn( | |
efe2330f | 142 | struct xfs_ail *ailp) |
fd074841 | 143 | { |
efe2330f CH |
144 | xfs_lsn_t lsn = 0; |
145 | struct xfs_log_item *lip; | |
fd074841 | 146 | |
57e80956 | 147 | spin_lock(&ailp->ail_lock); |
fd074841 DC |
148 | lip = xfs_ail_max(ailp); |
149 | if (lip) | |
150 | lsn = lip->li_lsn; | |
57e80956 | 151 | spin_unlock(&ailp->ail_lock); |
fd074841 DC |
152 | |
153 | return lsn; | |
154 | } | |
155 | ||
27d8d5fe | 156 | /* |
af3e4022 DC |
157 | * The cursor keeps track of where our current traversal is up to by tracking |
158 | * the next item in the list for us. However, for this to be safe, removing an | |
159 | * object from the AIL needs to invalidate any cursor that points to it. hence | |
160 | * the traversal cursor needs to be linked to the struct xfs_ail so that | |
161 | * deletion can search all the active cursors for invalidation. | |
27d8d5fe | 162 | */ |
5b00f14f | 163 | STATIC void |
27d8d5fe DC |
164 | xfs_trans_ail_cursor_init( |
165 | struct xfs_ail *ailp, | |
166 | struct xfs_ail_cursor *cur) | |
167 | { | |
168 | cur->item = NULL; | |
57e80956 | 169 | list_add_tail(&cur->list, &ailp->ail_cursors); |
27d8d5fe DC |
170 | } |
171 | ||
27d8d5fe | 172 | /* |
af3e4022 DC |
173 | * Get the next item in the traversal and advance the cursor. If the cursor |
174 | * was invalidated (indicated by a lip of 1), restart the traversal. | |
27d8d5fe | 175 | */ |
5b00f14f | 176 | struct xfs_log_item * |
27d8d5fe DC |
177 | xfs_trans_ail_cursor_next( |
178 | struct xfs_ail *ailp, | |
179 | struct xfs_ail_cursor *cur) | |
180 | { | |
181 | struct xfs_log_item *lip = cur->item; | |
182 | ||
db9d67d6 | 183 | if ((uintptr_t)lip & 1) |
27d8d5fe | 184 | lip = xfs_ail_min(ailp); |
16b59029 DC |
185 | if (lip) |
186 | cur->item = xfs_ail_next(ailp, lip); | |
27d8d5fe DC |
187 | return lip; |
188 | } | |
189 | ||
27d8d5fe | 190 | /* |
af3e4022 DC |
191 | * When the traversal is complete, we need to remove the cursor from the list |
192 | * of traversing cursors. | |
27d8d5fe DC |
193 | */ |
194 | void | |
195 | xfs_trans_ail_cursor_done( | |
af3e4022 | 196 | struct xfs_ail_cursor *cur) |
27d8d5fe | 197 | { |
af3e4022 DC |
198 | cur->item = NULL; |
199 | list_del_init(&cur->list); | |
27d8d5fe DC |
200 | } |
201 | ||
5b00f14f | 202 | /* |
af3e4022 DC |
203 | * Invalidate any cursor that is pointing to this item. This is called when an |
204 | * item is removed from the AIL. Any cursor pointing to this object is now | |
205 | * invalid and the traversal needs to be terminated so it doesn't reference a | |
206 | * freed object. We set the low bit of the cursor item pointer so we can | |
207 | * distinguish between an invalidation and the end of the list when getting the | |
208 | * next item from the cursor. | |
5b00f14f DC |
209 | */ |
210 | STATIC void | |
211 | xfs_trans_ail_cursor_clear( | |
212 | struct xfs_ail *ailp, | |
213 | struct xfs_log_item *lip) | |
214 | { | |
215 | struct xfs_ail_cursor *cur; | |
216 | ||
57e80956 | 217 | list_for_each_entry(cur, &ailp->ail_cursors, list) { |
5b00f14f DC |
218 | if (cur->item == lip) |
219 | cur->item = (struct xfs_log_item *) | |
db9d67d6 | 220 | ((uintptr_t)cur->item | 1); |
5b00f14f DC |
221 | } |
222 | } | |
223 | ||
249a8c11 | 224 | /* |
16b59029 DC |
225 | * Find the first item in the AIL with the given @lsn by searching in ascending |
226 | * LSN order and initialise the cursor to point to the next item for a | |
227 | * ascending traversal. Pass a @lsn of zero to initialise the cursor to the | |
228 | * first item in the AIL. Returns NULL if the list is empty. | |
249a8c11 | 229 | */ |
efe2330f | 230 | struct xfs_log_item * |
5b00f14f | 231 | xfs_trans_ail_cursor_first( |
27d8d5fe DC |
232 | struct xfs_ail *ailp, |
233 | struct xfs_ail_cursor *cur, | |
234 | xfs_lsn_t lsn) | |
249a8c11 | 235 | { |
efe2330f | 236 | struct xfs_log_item *lip; |
249a8c11 | 237 | |
5b00f14f | 238 | xfs_trans_ail_cursor_init(ailp, cur); |
16b59029 DC |
239 | |
240 | if (lsn == 0) { | |
241 | lip = xfs_ail_min(ailp); | |
5b00f14f | 242 | goto out; |
16b59029 | 243 | } |
249a8c11 | 244 | |
57e80956 | 245 | list_for_each_entry(lip, &ailp->ail_head, li_ail) { |
5b00f14f | 246 | if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0) |
7ee49acf | 247 | goto out; |
535f6b37 | 248 | } |
16b59029 DC |
249 | return NULL; |
250 | ||
5b00f14f | 251 | out: |
16b59029 DC |
252 | if (lip) |
253 | cur->item = xfs_ail_next(ailp, lip); | |
5b00f14f | 254 | return lip; |
249a8c11 DC |
255 | } |
256 | ||
1d8c95a3 DC |
257 | static struct xfs_log_item * |
258 | __xfs_trans_ail_cursor_last( | |
259 | struct xfs_ail *ailp, | |
260 | xfs_lsn_t lsn) | |
261 | { | |
efe2330f | 262 | struct xfs_log_item *lip; |
1d8c95a3 | 263 | |
57e80956 | 264 | list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) { |
1d8c95a3 DC |
265 | if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0) |
266 | return lip; | |
267 | } | |
268 | return NULL; | |
269 | } | |
270 | ||
271 | /* | |
16b59029 DC |
272 | * Find the last item in the AIL with the given @lsn by searching in descending |
273 | * LSN order and initialise the cursor to point to that item. If there is no | |
274 | * item with the value of @lsn, then it sets the cursor to the last item with an | |
275 | * LSN lower than @lsn. Returns NULL if the list is empty. | |
1d8c95a3 DC |
276 | */ |
277 | struct xfs_log_item * | |
278 | xfs_trans_ail_cursor_last( | |
279 | struct xfs_ail *ailp, | |
280 | struct xfs_ail_cursor *cur, | |
281 | xfs_lsn_t lsn) | |
282 | { | |
283 | xfs_trans_ail_cursor_init(ailp, cur); | |
284 | cur->item = __xfs_trans_ail_cursor_last(ailp, lsn); | |
285 | return cur->item; | |
286 | } | |
287 | ||
288 | /* | |
16b59029 | 289 | * Splice the log item list into the AIL at the given LSN. We splice to the |
1d8c95a3 DC |
290 | * tail of the given LSN to maintain insert order for push traversals. The |
291 | * cursor is optional, allowing repeated updates to the same LSN to avoid | |
e44f4112 | 292 | * repeated traversals. This should not be called with an empty list. |
cd4a3c50 DC |
293 | */ |
294 | static void | |
295 | xfs_ail_splice( | |
1d8c95a3 DC |
296 | struct xfs_ail *ailp, |
297 | struct xfs_ail_cursor *cur, | |
298 | struct list_head *list, | |
299 | xfs_lsn_t lsn) | |
cd4a3c50 | 300 | { |
e44f4112 AE |
301 | struct xfs_log_item *lip; |
302 | ||
303 | ASSERT(!list_empty(list)); | |
cd4a3c50 | 304 | |
1d8c95a3 | 305 | /* |
e44f4112 AE |
306 | * Use the cursor to determine the insertion point if one is |
307 | * provided. If not, or if the one we got is not valid, | |
308 | * find the place in the AIL where the items belong. | |
1d8c95a3 | 309 | */ |
e44f4112 | 310 | lip = cur ? cur->item : NULL; |
db9d67d6 | 311 | if (!lip || (uintptr_t)lip & 1) |
1d8c95a3 DC |
312 | lip = __xfs_trans_ail_cursor_last(ailp, lsn); |
313 | ||
e44f4112 AE |
314 | /* |
315 | * If a cursor is provided, we know we're processing the AIL | |
316 | * in lsn order, and future items to be spliced in will | |
317 | * follow the last one being inserted now. Update the | |
318 | * cursor to point to that last item, now while we have a | |
319 | * reliable pointer to it. | |
320 | */ | |
321 | if (cur) | |
322 | cur->item = list_entry(list->prev, struct xfs_log_item, li_ail); | |
cd4a3c50 | 323 | |
1d8c95a3 | 324 | /* |
e44f4112 AE |
325 | * Finally perform the splice. Unless the AIL was empty, |
326 | * lip points to the item in the AIL _after_ which the new | |
327 | * items should go. If lip is null the AIL was empty, so | |
328 | * the new items go at the head of the AIL. | |
1d8c95a3 | 329 | */ |
e44f4112 AE |
330 | if (lip) |
331 | list_splice(list, &lip->li_ail); | |
332 | else | |
57e80956 | 333 | list_splice(list, &ailp->ail_head); |
cd4a3c50 DC |
334 | } |
335 | ||
336 | /* | |
337 | * Delete the given item from the AIL. Return a pointer to the item. | |
338 | */ | |
339 | static void | |
340 | xfs_ail_delete( | |
efe2330f CH |
341 | struct xfs_ail *ailp, |
342 | struct xfs_log_item *lip) | |
cd4a3c50 DC |
343 | { |
344 | xfs_ail_check(ailp, lip); | |
345 | list_del(&lip->li_ail); | |
346 | xfs_trans_ail_cursor_clear(ailp, lip); | |
347 | } | |
348 | ||
cb6ad099 BF |
349 | /* |
350 | * Requeue a failed buffer for writeback. | |
351 | * | |
352 | * We clear the log item failed state here as well, but we have to be careful | |
353 | * about reference counts because the only active reference counts on the buffer | |
354 | * may be the failed log items. Hence if we clear the log item failed state | |
355 | * before queuing the buffer for IO we can release all active references to | |
356 | * the buffer and free it, leading to use after free problems in | |
357 | * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which | |
358 | * order we process them in - the buffer is locked, and we own the buffer list | |
359 | * so nothing on them is going to change while we are performing this action. | |
360 | * | |
361 | * Hence we can safely queue the buffer for IO before we clear the failed log | |
362 | * item state, therefore always having an active reference to the buffer and | |
363 | * avoiding the transient zero-reference state that leads to use-after-free. | |
364 | */ | |
365 | static inline int | |
366 | xfsaild_resubmit_item( | |
367 | struct xfs_log_item *lip, | |
368 | struct list_head *buffer_list) | |
369 | { | |
370 | struct xfs_buf *bp = lip->li_buf; | |
371 | ||
372 | if (!xfs_buf_trylock(bp)) | |
373 | return XFS_ITEM_LOCKED; | |
374 | ||
375 | if (!xfs_buf_delwri_queue(bp, buffer_list)) { | |
376 | xfs_buf_unlock(bp); | |
377 | return XFS_ITEM_FLUSHING; | |
378 | } | |
379 | ||
380 | /* protected by ail_lock */ | |
298f7bec DC |
381 | list_for_each_entry(lip, &bp->b_li_list, li_bio_list) { |
382 | if (bp->b_flags & _XBF_INODES) | |
383 | clear_bit(XFS_LI_FAILED, &lip->li_flags); | |
384 | else | |
385 | xfs_clear_li_failed(lip); | |
386 | } | |
cb6ad099 BF |
387 | |
388 | xfs_buf_unlock(bp); | |
389 | return XFS_ITEM_SUCCESS; | |
390 | } | |
391 | ||
7f4d01f3 BF |
392 | static inline uint |
393 | xfsaild_push_item( | |
394 | struct xfs_ail *ailp, | |
395 | struct xfs_log_item *lip) | |
396 | { | |
397 | /* | |
398 | * If log item pinning is enabled, skip the push and track the item as | |
399 | * pinned. This can help induce head-behind-tail conditions. | |
400 | */ | |
8eda8721 | 401 | if (XFS_TEST_ERROR(false, ailp->ail_log->l_mp, XFS_ERRTAG_LOG_ITEM_PIN)) |
7f4d01f3 BF |
402 | return XFS_ITEM_PINNED; |
403 | ||
e8b78db7 CH |
404 | /* |
405 | * Consider the item pinned if a push callback is not defined so the | |
406 | * caller will force the log. This should only happen for intent items | |
407 | * as they are unpinned once the associated done item is committed to | |
408 | * the on-disk log. | |
409 | */ | |
410 | if (!lip->li_ops->iop_push) | |
411 | return XFS_ITEM_PINNED; | |
cb6ad099 BF |
412 | if (test_bit(XFS_LI_FAILED, &lip->li_flags)) |
413 | return xfsaild_resubmit_item(lip, &ailp->ail_buf_list); | |
57e80956 | 414 | return lip->li_ops->iop_push(lip, &ailp->ail_buf_list); |
7f4d01f3 BF |
415 | } |
416 | ||
0030807c CH |
417 | static long |
418 | xfsaild_push( | |
419 | struct xfs_ail *ailp) | |
249a8c11 | 420 | { |
8eda8721 | 421 | struct xfs_mount *mp = ailp->ail_log->l_mp; |
af3e4022 | 422 | struct xfs_ail_cursor cur; |
efe2330f | 423 | struct xfs_log_item *lip; |
9e7004e7 | 424 | xfs_lsn_t lsn; |
e5827a00 | 425 | xfs_lsn_t target = NULLCOMMITLSN; |
43ff2122 | 426 | long tout; |
9e7004e7 | 427 | int stuck = 0; |
43ff2122 | 428 | int flushing = 0; |
9e7004e7 | 429 | int count = 0; |
1da177e4 | 430 | |
670ce93f | 431 | /* |
43ff2122 | 432 | * If we encountered pinned items or did not finish writing out all |
0020a190 DC |
433 | * buffers the last time we ran, force a background CIL push to get the |
434 | * items unpinned in the near future. We do not wait on the CIL push as | |
435 | * that could stall us for seconds if there is enough background IO | |
436 | * load. Stalling for that long when the tail of the log is pinned and | |
437 | * needs flushing will hard stop the transaction subsystem when log | |
438 | * space runs out. | |
670ce93f | 439 | */ |
57e80956 MW |
440 | if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 && |
441 | (!list_empty_careful(&ailp->ail_buf_list) || | |
43ff2122 | 442 | xfs_ail_min_lsn(ailp))) { |
57e80956 | 443 | ailp->ail_log_flush = 0; |
43ff2122 | 444 | |
ff6d6af2 | 445 | XFS_STATS_INC(mp, xs_push_ail_flush); |
8eda8721 | 446 | xlog_cil_flush(ailp->ail_log); |
670ce93f DC |
447 | } |
448 | ||
57e80956 | 449 | spin_lock(&ailp->ail_lock); |
8375f922 | 450 | |
941fbdfd DC |
451 | /* |
452 | * If we have a sync push waiter, we always have to push till the AIL is | |
453 | * empty. Update the target to point to the end of the AIL so that | |
454 | * capture updates that occur after the sync push waiter has gone to | |
455 | * sleep. | |
456 | */ | |
457 | if (waitqueue_active(&ailp->ail_empty)) { | |
458 | lip = xfs_ail_max(ailp); | |
459 | if (lip) | |
460 | target = lip->li_lsn; | |
461 | } else { | |
462 | /* barrier matches the ail_target update in xfs_ail_push() */ | |
463 | smp_rmb(); | |
464 | target = ailp->ail_target; | |
465 | ailp->ail_target_prev = target; | |
466 | } | |
8375f922 | 467 | |
f376b45e | 468 | /* we're done if the AIL is empty or our push has reached the end */ |
57e80956 | 469 | lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn); |
f376b45e | 470 | if (!lip) |
9e7004e7 | 471 | goto out_done; |
1da177e4 | 472 | |
ff6d6af2 | 473 | XFS_STATS_INC(mp, xs_push_ail); |
1da177e4 | 474 | |
e5827a00 DW |
475 | ASSERT(target != NULLCOMMITLSN); |
476 | ||
249a8c11 | 477 | lsn = lip->li_lsn; |
50e86686 | 478 | while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) { |
249a8c11 | 479 | int lock_result; |
43ff2122 | 480 | |
1da177e4 | 481 | /* |
904c17e6 | 482 | * Note that iop_push may unlock and reacquire the AIL lock. We |
43ff2122 CH |
483 | * rely on the AIL cursor implementation to be able to deal with |
484 | * the dropped lock. | |
1da177e4 | 485 | */ |
7f4d01f3 | 486 | lock_result = xfsaild_push_item(ailp, lip); |
1da177e4 | 487 | switch (lock_result) { |
249a8c11 | 488 | case XFS_ITEM_SUCCESS: |
ff6d6af2 | 489 | XFS_STATS_INC(mp, xs_push_ail_success); |
9e4c109a CH |
490 | trace_xfs_ail_push(lip); |
491 | ||
57e80956 | 492 | ailp->ail_last_pushed_lsn = lsn; |
1da177e4 LT |
493 | break; |
494 | ||
43ff2122 CH |
495 | case XFS_ITEM_FLUSHING: |
496 | /* | |
cf085a1b | 497 | * The item or its backing buffer is already being |
43ff2122 CH |
498 | * flushed. The typical reason for that is that an |
499 | * inode buffer is locked because we already pushed the | |
500 | * updates to it as part of inode clustering. | |
501 | * | |
b63da6c8 | 502 | * We do not want to stop flushing just because lots |
cf085a1b | 503 | * of items are already being flushed, but we need to |
43ff2122 | 504 | * re-try the flushing relatively soon if most of the |
cf085a1b | 505 | * AIL is being flushed. |
43ff2122 | 506 | */ |
ff6d6af2 | 507 | XFS_STATS_INC(mp, xs_push_ail_flushing); |
43ff2122 CH |
508 | trace_xfs_ail_flushing(lip); |
509 | ||
510 | flushing++; | |
57e80956 | 511 | ailp->ail_last_pushed_lsn = lsn; |
1da177e4 LT |
512 | break; |
513 | ||
249a8c11 | 514 | case XFS_ITEM_PINNED: |
ff6d6af2 | 515 | XFS_STATS_INC(mp, xs_push_ail_pinned); |
9e4c109a CH |
516 | trace_xfs_ail_pinned(lip); |
517 | ||
249a8c11 | 518 | stuck++; |
57e80956 | 519 | ailp->ail_log_flush++; |
1da177e4 | 520 | break; |
249a8c11 | 521 | case XFS_ITEM_LOCKED: |
ff6d6af2 | 522 | XFS_STATS_INC(mp, xs_push_ail_locked); |
9e4c109a | 523 | trace_xfs_ail_locked(lip); |
43ff2122 | 524 | |
249a8c11 | 525 | stuck++; |
1da177e4 | 526 | break; |
249a8c11 | 527 | default: |
1da177e4 LT |
528 | ASSERT(0); |
529 | break; | |
530 | } | |
531 | ||
249a8c11 | 532 | count++; |
1da177e4 | 533 | |
249a8c11 DC |
534 | /* |
535 | * Are there too many items we can't do anything with? | |
43ff2122 | 536 | * |
b63da6c8 | 537 | * If we are skipping too many items because we can't flush |
249a8c11 DC |
538 | * them or they are already being flushed, we back off and |
539 | * given them time to complete whatever operation is being | |
540 | * done. i.e. remove pressure from the AIL while we can't make | |
541 | * progress so traversals don't slow down further inserts and | |
542 | * removals to/from the AIL. | |
543 | * | |
544 | * The value of 100 is an arbitrary magic number based on | |
545 | * observation. | |
546 | */ | |
547 | if (stuck > 100) | |
548 | break; | |
549 | ||
af3e4022 | 550 | lip = xfs_trans_ail_cursor_next(ailp, &cur); |
249a8c11 DC |
551 | if (lip == NULL) |
552 | break; | |
249a8c11 | 553 | lsn = lip->li_lsn; |
1da177e4 | 554 | } |
f376b45e BF |
555 | |
556 | out_done: | |
e4a1e29c | 557 | xfs_trans_ail_cursor_done(&cur); |
57e80956 | 558 | spin_unlock(&ailp->ail_lock); |
1da177e4 | 559 | |
57e80956 MW |
560 | if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list)) |
561 | ailp->ail_log_flush++; | |
d808f617 | 562 | |
43ff2122 | 563 | if (!count || XFS_LSN_CMP(lsn, target) >= 0) { |
92d9cd10 | 564 | /* |
43ff2122 CH |
565 | * We reached the target or the AIL is empty, so wait a bit |
566 | * longer for I/O to complete and remove pushed items from the | |
567 | * AIL before we start the next scan from the start of the AIL. | |
92d9cd10 | 568 | */ |
453eac8a | 569 | tout = 50; |
57e80956 | 570 | ailp->ail_last_pushed_lsn = 0; |
43ff2122 | 571 | } else if (((stuck + flushing) * 100) / count > 90) { |
249a8c11 | 572 | /* |
43ff2122 CH |
573 | * Either there is a lot of contention on the AIL or we are |
574 | * stuck due to operations in progress. "Stuck" in this case | |
575 | * is defined as >90% of the items we tried to push were stuck. | |
249a8c11 DC |
576 | * |
577 | * Backoff a bit more to allow some I/O to complete before | |
43ff2122 CH |
578 | * restarting from the start of the AIL. This prevents us from |
579 | * spinning on the same items, and if they are pinned will all | |
580 | * the restart to issue a log force to unpin the stuck items. | |
249a8c11 | 581 | */ |
453eac8a | 582 | tout = 20; |
57e80956 | 583 | ailp->ail_last_pushed_lsn = 0; |
43ff2122 CH |
584 | } else { |
585 | /* | |
586 | * Assume we have more work to do in a short while. | |
587 | */ | |
588 | tout = 10; | |
1da177e4 | 589 | } |
0bf6a5bd | 590 | |
0030807c CH |
591 | return tout; |
592 | } | |
593 | ||
594 | static int | |
595 | xfsaild( | |
596 | void *data) | |
597 | { | |
598 | struct xfs_ail *ailp = data; | |
599 | long tout = 0; /* milliseconds */ | |
10a98cb1 | 600 | unsigned int noreclaim_flag; |
0030807c | 601 | |
10a98cb1 | 602 | noreclaim_flag = memalloc_noreclaim_save(); |
18f1df4e | 603 | set_freezable(); |
43ff2122 | 604 | |
0bd89676 | 605 | while (1) { |
0030807c | 606 | if (tout && tout <= 20) |
f5d39b02 | 607 | set_current_state(TASK_KILLABLE|TASK_FREEZABLE); |
0030807c | 608 | else |
f5d39b02 | 609 | set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); |
0bd89676 HT |
610 | |
611 | /* | |
efc3289c BF |
612 | * Check kthread_should_stop() after we set the task state to |
613 | * guarantee that we either see the stop bit and exit or the | |
614 | * task state is reset to runnable such that it's not scheduled | |
615 | * out indefinitely and detects the stop bit at next iteration. | |
0bd89676 HT |
616 | * A memory barrier is included in above task state set to |
617 | * serialize again kthread_stop(). | |
618 | */ | |
619 | if (kthread_should_stop()) { | |
620 | __set_current_state(TASK_RUNNING); | |
efc3289c BF |
621 | |
622 | /* | |
623 | * The caller forces out the AIL before stopping the | |
624 | * thread in the common case, which means the delwri | |
625 | * queue is drained. In the shutdown case, the queue may | |
626 | * still hold relogged buffers that haven't been | |
627 | * submitted because they were pinned since added to the | |
628 | * queue. | |
629 | * | |
630 | * Log I/O error processing stales the underlying buffer | |
631 | * and clears the delwri state, expecting the buf to be | |
632 | * removed on the next submission attempt. That won't | |
633 | * happen if we're shutting down, so this is the last | |
634 | * opportunity to release such buffers from the queue. | |
635 | */ | |
636 | ASSERT(list_empty(&ailp->ail_buf_list) || | |
8eda8721 | 637 | xlog_is_shutdown(ailp->ail_log)); |
efc3289c | 638 | xfs_buf_delwri_cancel(&ailp->ail_buf_list); |
0bd89676 HT |
639 | break; |
640 | } | |
8375f922 | 641 | |
57e80956 | 642 | spin_lock(&ailp->ail_lock); |
8375f922 BF |
643 | |
644 | /* | |
645 | * Idle if the AIL is empty and we are not racing with a target | |
646 | * update. We check the AIL after we set the task to a sleep | |
57e80956 | 647 | * state to guarantee that we either catch an ail_target update |
8375f922 BF |
648 | * or that a wake_up resets the state to TASK_RUNNING. |
649 | * Otherwise, we run the risk of sleeping indefinitely. | |
650 | * | |
57e80956 | 651 | * The barrier matches the ail_target update in xfs_ail_push(). |
8375f922 BF |
652 | */ |
653 | smp_rmb(); | |
654 | if (!xfs_ail_min(ailp) && | |
f376b45e BF |
655 | ailp->ail_target == ailp->ail_target_prev && |
656 | list_empty(&ailp->ail_buf_list)) { | |
57e80956 | 657 | spin_unlock(&ailp->ail_lock); |
f5d39b02 | 658 | schedule(); |
8375f922 BF |
659 | tout = 0; |
660 | continue; | |
661 | } | |
57e80956 | 662 | spin_unlock(&ailp->ail_lock); |
8375f922 BF |
663 | |
664 | if (tout) | |
f5d39b02 | 665 | schedule_timeout(msecs_to_jiffies(tout)); |
8375f922 BF |
666 | |
667 | __set_current_state(TASK_RUNNING); | |
0030807c CH |
668 | |
669 | try_to_freeze(); | |
670 | ||
671 | tout = xfsaild_push(ailp); | |
672 | } | |
673 | ||
10a98cb1 | 674 | memalloc_noreclaim_restore(noreclaim_flag); |
0030807c | 675 | return 0; |
453eac8a | 676 | } |
1da177e4 | 677 | |
0bf6a5bd DC |
678 | /* |
679 | * This routine is called to move the tail of the AIL forward. It does this by | |
680 | * trying to flush items in the AIL whose lsns are below the given | |
681 | * threshold_lsn. | |
682 | * | |
683 | * The push is run asynchronously in a workqueue, which means the caller needs | |
684 | * to handle waiting on the async flush for space to become available. | |
685 | * We don't want to interrupt any push that is in progress, hence we only queue | |
cf085a1b | 686 | * work if we set the pushing bit appropriately. |
0bf6a5bd DC |
687 | * |
688 | * We do this unlocked - we only need to know whether there is anything in the | |
689 | * AIL at the time we are called. We don't need to access the contents of | |
690 | * any of the objects, so the lock is not needed. | |
691 | */ | |
692 | void | |
fd074841 | 693 | xfs_ail_push( |
efe2330f CH |
694 | struct xfs_ail *ailp, |
695 | xfs_lsn_t threshold_lsn) | |
0bf6a5bd | 696 | { |
efe2330f | 697 | struct xfs_log_item *lip; |
0bf6a5bd DC |
698 | |
699 | lip = xfs_ail_min(ailp); | |
8eda8721 | 700 | if (!lip || xlog_is_shutdown(ailp->ail_log) || |
57e80956 | 701 | XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0) |
0bf6a5bd DC |
702 | return; |
703 | ||
704 | /* | |
705 | * Ensure that the new target is noticed in push code before it clears | |
706 | * the XFS_AIL_PUSHING_BIT. | |
707 | */ | |
708 | smp_wmb(); | |
57e80956 | 709 | xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn); |
0030807c CH |
710 | smp_wmb(); |
711 | ||
57e80956 | 712 | wake_up_process(ailp->ail_task); |
0bf6a5bd | 713 | } |
1da177e4 | 714 | |
fd074841 DC |
715 | /* |
716 | * Push out all items in the AIL immediately | |
717 | */ | |
718 | void | |
719 | xfs_ail_push_all( | |
720 | struct xfs_ail *ailp) | |
721 | { | |
722 | xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp); | |
723 | ||
724 | if (threshold_lsn) | |
725 | xfs_ail_push(ailp, threshold_lsn); | |
726 | } | |
727 | ||
211e4d43 CH |
728 | /* |
729 | * Push out all items in the AIL immediately and wait until the AIL is empty. | |
730 | */ | |
731 | void | |
732 | xfs_ail_push_all_sync( | |
733 | struct xfs_ail *ailp) | |
734 | { | |
211e4d43 CH |
735 | DEFINE_WAIT(wait); |
736 | ||
57e80956 | 737 | spin_lock(&ailp->ail_lock); |
fc93812c | 738 | while (xfs_ail_max(ailp) != NULL) { |
57e80956 | 739 | prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE); |
57e80956 MW |
740 | wake_up_process(ailp->ail_task); |
741 | spin_unlock(&ailp->ail_lock); | |
211e4d43 | 742 | schedule(); |
57e80956 | 743 | spin_lock(&ailp->ail_lock); |
211e4d43 | 744 | } |
57e80956 | 745 | spin_unlock(&ailp->ail_lock); |
211e4d43 | 746 | |
57e80956 | 747 | finish_wait(&ailp->ail_empty, &wait); |
211e4d43 CH |
748 | } |
749 | ||
4165994a DC |
750 | void |
751 | xfs_ail_update_finish( | |
752 | struct xfs_ail *ailp, | |
8eb807bd | 753 | xfs_lsn_t old_lsn) __releases(ailp->ail_lock) |
4165994a | 754 | { |
8eda8721 | 755 | struct xlog *log = ailp->ail_log; |
4165994a | 756 | |
8eb807bd DC |
757 | /* if the tail lsn hasn't changed, don't do updates or wakeups. */ |
758 | if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) { | |
4165994a DC |
759 | spin_unlock(&ailp->ail_lock); |
760 | return; | |
761 | } | |
762 | ||
8eda8721 DC |
763 | if (!xlog_is_shutdown(log)) |
764 | xlog_assign_tail_lsn_locked(log->l_mp); | |
4165994a DC |
765 | |
766 | if (list_empty(&ailp->ail_head)) | |
767 | wake_up_all(&ailp->ail_empty); | |
768 | spin_unlock(&ailp->ail_lock); | |
8eda8721 | 769 | xfs_log_space_wake(log->l_mp); |
4165994a DC |
770 | } |
771 | ||
0e57f6a3 DC |
772 | /* |
773 | * xfs_trans_ail_update - bulk AIL insertion operation. | |
774 | * | |
775 | * @xfs_trans_ail_update takes an array of log items that all need to be | |
776 | * positioned at the same LSN in the AIL. If an item is not in the AIL, it will | |
777 | * be added. Otherwise, it will be repositioned by removing it and re-adding | |
778 | * it to the AIL. If we move the first item in the AIL, update the log tail to | |
779 | * match the new minimum LSN in the AIL. | |
780 | * | |
781 | * This function takes the AIL lock once to execute the update operations on | |
782 | * all the items in the array, and as such should not be called with the AIL | |
783 | * lock held. As a result, once we have the AIL lock, we need to check each log | |
784 | * item LSN to confirm it needs to be moved forward in the AIL. | |
785 | * | |
786 | * To optimise the insert operation, we delete all the items from the AIL in | |
787 | * the first pass, moving them into a temporary list, then splice the temporary | |
788 | * list into the correct position in the AIL. This avoids needing to do an | |
789 | * insert operation on every item. | |
790 | * | |
791 | * This function must be called with the AIL lock held. The lock is dropped | |
792 | * before returning. | |
793 | */ | |
794 | void | |
795 | xfs_trans_ail_update_bulk( | |
796 | struct xfs_ail *ailp, | |
1d8c95a3 | 797 | struct xfs_ail_cursor *cur, |
0e57f6a3 DC |
798 | struct xfs_log_item **log_items, |
799 | int nr_items, | |
57e80956 | 800 | xfs_lsn_t lsn) __releases(ailp->ail_lock) |
0e57f6a3 | 801 | { |
efe2330f | 802 | struct xfs_log_item *mlip; |
8eb807bd | 803 | xfs_lsn_t tail_lsn = 0; |
0e57f6a3 DC |
804 | int i; |
805 | LIST_HEAD(tmp); | |
806 | ||
e44f4112 | 807 | ASSERT(nr_items > 0); /* Not required, but true. */ |
0e57f6a3 DC |
808 | mlip = xfs_ail_min(ailp); |
809 | ||
810 | for (i = 0; i < nr_items; i++) { | |
811 | struct xfs_log_item *lip = log_items[i]; | |
22525c17 | 812 | if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) { |
0e57f6a3 DC |
813 | /* check if we really need to move the item */ |
814 | if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0) | |
815 | continue; | |
816 | ||
750b9c90 | 817 | trace_xfs_ail_move(lip, lip->li_lsn, lsn); |
8eb807bd DC |
818 | if (mlip == lip && !tail_lsn) |
819 | tail_lsn = lip->li_lsn; | |
820 | ||
0e57f6a3 | 821 | xfs_ail_delete(ailp, lip); |
0e57f6a3 | 822 | } else { |
750b9c90 | 823 | trace_xfs_ail_insert(lip, 0, lsn); |
0e57f6a3 DC |
824 | } |
825 | lip->li_lsn = lsn; | |
826 | list_add(&lip->li_ail, &tmp); | |
827 | } | |
828 | ||
e44f4112 AE |
829 | if (!list_empty(&tmp)) |
830 | xfs_ail_splice(ailp, cur, &tmp, lsn); | |
0e57f6a3 | 831 | |
8eb807bd | 832 | xfs_ail_update_finish(ailp, tail_lsn); |
86a37174 DW |
833 | } |
834 | ||
835 | /* Insert a log item into the AIL. */ | |
836 | void | |
837 | xfs_trans_ail_insert( | |
838 | struct xfs_ail *ailp, | |
839 | struct xfs_log_item *lip, | |
840 | xfs_lsn_t lsn) | |
841 | { | |
842 | spin_lock(&ailp->ail_lock); | |
843 | xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn); | |
0e57f6a3 DC |
844 | } |
845 | ||
8eb807bd DC |
846 | /* |
847 | * Delete one log item from the AIL. | |
848 | * | |
849 | * If this item was at the tail of the AIL, return the LSN of the log item so | |
850 | * that we can use it to check if the LSN of the tail of the log has moved | |
851 | * when finishing up the AIL delete process in xfs_ail_update_finish(). | |
852 | */ | |
853 | xfs_lsn_t | |
27af1bbf CH |
854 | xfs_ail_delete_one( |
855 | struct xfs_ail *ailp, | |
d3a304b6 | 856 | struct xfs_log_item *lip) |
27af1bbf CH |
857 | { |
858 | struct xfs_log_item *mlip = xfs_ail_min(ailp); | |
8eb807bd | 859 | xfs_lsn_t lsn = lip->li_lsn; |
27af1bbf CH |
860 | |
861 | trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn); | |
862 | xfs_ail_delete(ailp, lip); | |
22525c17 | 863 | clear_bit(XFS_LI_IN_AIL, &lip->li_flags); |
27af1bbf CH |
864 | lip->li_lsn = 0; |
865 | ||
8eb807bd DC |
866 | if (mlip == lip) |
867 | return lsn; | |
868 | return 0; | |
27af1bbf CH |
869 | } |
870 | ||
30136832 | 871 | void |
27af1bbf | 872 | xfs_trans_ail_delete( |
27af1bbf | 873 | struct xfs_log_item *lip, |
4165994a | 874 | int shutdown_type) |
30136832 | 875 | { |
849274c1 | 876 | struct xfs_ail *ailp = lip->li_ailp; |
b5f17bec | 877 | struct xlog *log = ailp->ail_log; |
8eb807bd | 878 | xfs_lsn_t tail_lsn; |
30136832 | 879 | |
849274c1 | 880 | spin_lock(&ailp->ail_lock); |
22525c17 | 881 | if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { |
57e80956 | 882 | spin_unlock(&ailp->ail_lock); |
b5f17bec DC |
883 | if (shutdown_type && !xlog_is_shutdown(log)) { |
884 | xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE, | |
27af1bbf CH |
885 | "%s: attempting to delete a log item that is not in the AIL", |
886 | __func__); | |
b5f17bec | 887 | xlog_force_shutdown(log, shutdown_type); |
30136832 | 888 | } |
27af1bbf | 889 | return; |
30136832 DC |
890 | } |
891 | ||
2b3cf093 | 892 | /* xfs_ail_update_finish() drops the AIL lock */ |
e98084b8 | 893 | xfs_clear_li_failed(lip); |
8eb807bd DC |
894 | tail_lsn = xfs_ail_delete_one(ailp, lip); |
895 | xfs_ail_update_finish(ailp, tail_lsn); | |
30136832 | 896 | } |
1da177e4 | 897 | |
249a8c11 | 898 | int |
1da177e4 LT |
899 | xfs_trans_ail_init( |
900 | xfs_mount_t *mp) | |
901 | { | |
82fa9012 DC |
902 | struct xfs_ail *ailp; |
903 | ||
904 | ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL); | |
905 | if (!ailp) | |
2451337d | 906 | return -ENOMEM; |
82fa9012 | 907 | |
8eda8721 | 908 | ailp->ail_log = mp->m_log; |
57e80956 MW |
909 | INIT_LIST_HEAD(&ailp->ail_head); |
910 | INIT_LIST_HEAD(&ailp->ail_cursors); | |
911 | spin_lock_init(&ailp->ail_lock); | |
912 | INIT_LIST_HEAD(&ailp->ail_buf_list); | |
913 | init_waitqueue_head(&ailp->ail_empty); | |
0030807c | 914 | |
57e80956 | 915 | ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s", |
8eda8721 | 916 | mp->m_super->s_id); |
57e80956 | 917 | if (IS_ERR(ailp->ail_task)) |
0030807c CH |
918 | goto out_free_ailp; |
919 | ||
27d8d5fe DC |
920 | mp->m_ail = ailp; |
921 | return 0; | |
0030807c CH |
922 | |
923 | out_free_ailp: | |
924 | kmem_free(ailp); | |
2451337d | 925 | return -ENOMEM; |
249a8c11 DC |
926 | } |
927 | ||
928 | void | |
929 | xfs_trans_ail_destroy( | |
930 | xfs_mount_t *mp) | |
931 | { | |
82fa9012 DC |
932 | struct xfs_ail *ailp = mp->m_ail; |
933 | ||
57e80956 | 934 | kthread_stop(ailp->ail_task); |
82fa9012 | 935 | kmem_free(ailp); |
1da177e4 | 936 | } |