Commit | Line | Data |
---|---|---|
7f3283ab DH |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* netfs cookie management | |
3 | * | |
4 | * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. | |
5 | * Written by David Howells (dhowells@redhat.com) | |
6 | * | |
7 | * See Documentation/filesystems/caching/netfs-api.rst for more information on | |
8 | * the netfs API. | |
9 | */ | |
10 | ||
11 | #define FSCACHE_DEBUG_LEVEL COOKIE | |
12 | #include <linux/module.h> | |
13 | #include <linux/slab.h> | |
14 | #include "internal.h" | |
15 | ||
16 | struct kmem_cache *fscache_cookie_jar; | |
17 | ||
12bb21a2 DH |
18 | static void fscache_cookie_lru_timed_out(struct timer_list *timer); |
19 | static void fscache_cookie_lru_worker(struct work_struct *work); | |
5d00e426 DH |
20 | static void fscache_cookie_worker(struct work_struct *work); |
21 | static void fscache_unhash_cookie(struct fscache_cookie *cookie); | |
d24af13e | 22 | static void fscache_perform_invalidation(struct fscache_cookie *cookie); |
7f3283ab DH |
23 | |
24 | #define fscache_cookie_hash_shift 15 | |
25 | static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; | |
26 | static LIST_HEAD(fscache_cookies); | |
27 | static DEFINE_RWLOCK(fscache_cookies_lock); | |
12bb21a2 DH |
28 | static LIST_HEAD(fscache_cookie_lru); |
29 | static DEFINE_SPINLOCK(fscache_cookie_lru_lock); | |
30 | DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); | |
31 | static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); | |
d24af13e | 32 | static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD"; |
19517e53 | 33 | static unsigned int fscache_lru_cookie_timeout = 10 * HZ; |
7f3283ab DH |
34 | |
35 | void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) | |
36 | { | |
37 | const u8 *k; | |
38 | ||
39 | pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", | |
40 | prefix, | |
41 | cookie->debug_id, | |
42 | cookie->flags, | |
43 | atomic_read(&cookie->n_active), | |
44 | atomic_read(&cookie->n_accesses), | |
45 | fscache_cookie_states[cookie->state]); | |
46 | pr_err("%c-cookie V=%08x [%s]\n", | |
47 | prefix, | |
48 | cookie->volume->debug_id, | |
49 | cookie->volume->key); | |
50 | ||
51 | k = (cookie->key_len <= sizeof(cookie->inline_key)) ? | |
52 | cookie->inline_key : cookie->key; | |
53 | pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); | |
54 | } | |
55 | ||
56 | static void fscache_free_cookie(struct fscache_cookie *cookie) | |
57 | { | |
12bb21a2 DH |
58 | if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { |
59 | spin_lock(&fscache_cookie_lru_lock); | |
60 | list_del_init(&cookie->commit_link); | |
61 | spin_unlock(&fscache_cookie_lru_lock); | |
62 | fscache_stat_d(&fscache_n_cookies_lru); | |
63 | fscache_stat(&fscache_n_cookies_lru_removed); | |
64 | } | |
65 | ||
7f3283ab DH |
66 | if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { |
67 | fscache_print_cookie(cookie, 'F'); | |
68 | return; | |
69 | } | |
70 | ||
71 | write_lock(&fscache_cookies_lock); | |
72 | list_del(&cookie->proc_link); | |
73 | write_unlock(&fscache_cookies_lock); | |
74 | if (cookie->aux_len > sizeof(cookie->inline_aux)) | |
75 | kfree(cookie->aux); | |
76 | if (cookie->key_len > sizeof(cookie->inline_key)) | |
77 | kfree(cookie->key); | |
78 | fscache_stat_d(&fscache_n_cookies); | |
79 | kmem_cache_free(fscache_cookie_jar, cookie); | |
80 | } | |
81 | ||
5d00e426 DH |
82 | static void __fscache_queue_cookie(struct fscache_cookie *cookie) |
83 | { | |
84 | if (!queue_work(fscache_wq, &cookie->work)) | |
85 | fscache_put_cookie(cookie, fscache_cookie_put_over_queued); | |
86 | } | |
87 | ||
88 | static void fscache_queue_cookie(struct fscache_cookie *cookie, | |
89 | enum fscache_cookie_trace where) | |
90 | { | |
91 | fscache_get_cookie(cookie, where); | |
92 | __fscache_queue_cookie(cookie); | |
93 | } | |
94 | ||
a7733fb6 DH |
95 | /* |
96 | * Initialise the access gate on a cookie by setting a flag to prevent the | |
97 | * state machine from being queued when the access counter transitions to 0. | |
98 | * We're only interested in this when we withdraw caching services from the | |
99 | * cookie. | |
100 | */ | |
101 | static void fscache_init_access_gate(struct fscache_cookie *cookie) | |
102 | { | |
103 | int n_accesses; | |
104 | ||
105 | n_accesses = atomic_read(&cookie->n_accesses); | |
106 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
107 | n_accesses, fscache_access_cache_pin); | |
108 | set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); | |
109 | } | |
110 | ||
111 | /** | |
112 | * fscache_end_cookie_access - Unpin a cache at the end of an access. | |
113 | * @cookie: A data file cookie | |
114 | * @why: An indication of the circumstances of the access for tracing | |
115 | * | |
116 | * Unpin a cache cookie after we've accessed it and bring a deferred | |
117 | * relinquishment or withdrawal state into effect. | |
118 | * | |
119 | * The @why indicator is provided for tracing purposes. | |
120 | */ | |
121 | void fscache_end_cookie_access(struct fscache_cookie *cookie, | |
122 | enum fscache_access_trace why) | |
123 | { | |
124 | int n_accesses; | |
125 | ||
126 | smp_mb__before_atomic(); | |
127 | n_accesses = atomic_dec_return(&cookie->n_accesses); | |
128 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
129 | n_accesses, why); | |
130 | if (n_accesses == 0 && | |
5d00e426 DH |
131 | !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) |
132 | fscache_queue_cookie(cookie, fscache_cookie_get_end_access); | |
a7733fb6 DH |
133 | } |
134 | EXPORT_SYMBOL(fscache_end_cookie_access); | |
135 | ||
136 | /* | |
137 | * Pin the cache behind a cookie so that we can access it. | |
138 | */ | |
139 | static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, | |
140 | enum fscache_access_trace why) | |
141 | { | |
142 | int n_accesses; | |
143 | ||
144 | n_accesses = atomic_inc_return(&cookie->n_accesses); | |
145 | smp_mb__after_atomic(); /* (Future) read state after is-caching. | |
146 | * Reread n_accesses after is-caching | |
147 | */ | |
148 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
149 | n_accesses, why); | |
150 | } | |
151 | ||
152 | /** | |
153 | * fscache_begin_cookie_access - Pin a cache so data can be accessed | |
154 | * @cookie: A data file cookie | |
155 | * @why: An indication of the circumstances of the access for tracing | |
156 | * | |
157 | * Attempt to pin the cache to prevent it from going away whilst we're | |
158 | * accessing data and returns true if successful. This works as follows: | |
159 | * | |
160 | * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not | |
161 | * set), we return false to indicate access was not permitted. | |
162 | * | |
163 | * (2) If the cookie is being cached, we increment its n_accesses count and | |
164 | * then recheck the IS_CACHING flag, ending the access if it got cleared. | |
165 | * | |
166 | * (3) When we end the access, we decrement the cookie's n_accesses and wake | |
167 | * up the any waiters if it reaches 0. | |
168 | * | |
169 | * (4) Whilst the cookie is actively being cached, its n_accesses is kept | |
170 | * artificially incremented to prevent wakeups from happening. | |
171 | * | |
172 | * (5) When the cache is taken offline or if the cookie is culled, the flag is | |
173 | * cleared to prevent new accesses, the cookie's n_accesses is decremented | |
174 | * and we wait for it to become 0. | |
175 | * | |
176 | * The @why indicator are merely provided for tracing purposes. | |
177 | */ | |
178 | bool fscache_begin_cookie_access(struct fscache_cookie *cookie, | |
179 | enum fscache_access_trace why) | |
180 | { | |
181 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) | |
182 | return false; | |
183 | __fscache_begin_cookie_access(cookie, why); | |
184 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || | |
185 | !fscache_cache_is_live(cookie->volume->cache)) { | |
186 | fscache_end_cookie_access(cookie, fscache_access_unlive); | |
187 | return false; | |
188 | } | |
189 | return true; | |
190 | } | |
191 | ||
7f3283ab DH |
192 | static inline void wake_up_cookie_state(struct fscache_cookie *cookie) |
193 | { | |
194 | /* Use a barrier to ensure that waiters see the state variable | |
195 | * change, as spin_unlock doesn't guarantee a barrier. | |
196 | * | |
197 | * See comments over wake_up_bit() and waitqueue_active(). | |
198 | */ | |
199 | smp_mb(); | |
200 | wake_up_var(&cookie->state); | |
201 | } | |
202 | ||
5d00e426 DH |
203 | /* |
204 | * Change the state a cookie is at and wake up anyone waiting for that. Impose | |
205 | * an ordering between the stuff stored in the cookie and the state member. | |
206 | * Paired with fscache_cookie_state(). | |
207 | */ | |
7f3283ab DH |
208 | static void __fscache_set_cookie_state(struct fscache_cookie *cookie, |
209 | enum fscache_cookie_state state) | |
210 | { | |
5d00e426 | 211 | smp_store_release(&cookie->state, state); |
7f3283ab DH |
212 | } |
213 | ||
5d00e426 DH |
214 | static void fscache_set_cookie_state(struct fscache_cookie *cookie, |
215 | enum fscache_cookie_state state) | |
7f3283ab | 216 | { |
7f3283ab | 217 | spin_lock(&cookie->lock); |
5d00e426 | 218 | __fscache_set_cookie_state(cookie, state); |
7f3283ab | 219 | spin_unlock(&cookie->lock); |
5d00e426 DH |
220 | wake_up_cookie_state(cookie); |
221 | } | |
222 | ||
223 | /** | |
224 | * fscache_cookie_lookup_negative - Note negative lookup | |
225 | * @cookie: The cookie that was being looked up | |
226 | * | |
227 | * Note that some part of the metadata path in the cache doesn't exist and so | |
228 | * we can release any waiting readers in the certain knowledge that there's | |
229 | * nothing for them to actually read. | |
230 | * | |
231 | * This function uses no locking and must only be called from the state machine. | |
232 | */ | |
233 | void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) | |
234 | { | |
235 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
236 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); | |
237 | } | |
238 | EXPORT_SYMBOL(fscache_cookie_lookup_negative); | |
239 | ||
d24af13e DH |
240 | /** |
241 | * fscache_resume_after_invalidation - Allow I/O to resume after invalidation | |
242 | * @cookie: The cookie that was invalidated | |
243 | * | |
244 | * Tell fscache that invalidation is sufficiently complete that I/O can be | |
245 | * allowed again. | |
246 | */ | |
247 | void fscache_resume_after_invalidation(struct fscache_cookie *cookie) | |
248 | { | |
249 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); | |
250 | } | |
251 | EXPORT_SYMBOL(fscache_resume_after_invalidation); | |
252 | ||
5d00e426 DH |
253 | /** |
254 | * fscache_caching_failed - Report that a failure stopped caching on a cookie | |
255 | * @cookie: The cookie that was affected | |
256 | * | |
257 | * Tell fscache that caching on a cookie needs to be stopped due to some sort | |
258 | * of failure. | |
259 | * | |
260 | * This function uses no locking and must only be called from the state machine. | |
261 | */ | |
262 | void fscache_caching_failed(struct fscache_cookie *cookie) | |
263 | { | |
264 | clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); | |
265 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); | |
7f3283ab | 266 | } |
5d00e426 | 267 | EXPORT_SYMBOL(fscache_caching_failed); |
7f3283ab DH |
268 | |
269 | /* | |
270 | * Set the index key in a cookie. The cookie struct has space for a 16-byte | |
271 | * key plus length and hash, but if that's not big enough, it's instead a | |
272 | * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then | |
273 | * the key data. | |
274 | */ | |
275 | static int fscache_set_key(struct fscache_cookie *cookie, | |
276 | const void *index_key, size_t index_key_len) | |
277 | { | |
278 | void *buf; | |
279 | size_t buf_size; | |
280 | ||
281 | buf_size = round_up(index_key_len, sizeof(__le32)); | |
282 | ||
283 | if (index_key_len > sizeof(cookie->inline_key)) { | |
284 | buf = kzalloc(buf_size, GFP_KERNEL); | |
285 | if (!buf) | |
286 | return -ENOMEM; | |
287 | cookie->key = buf; | |
288 | } else { | |
289 | buf = cookie->inline_key; | |
290 | } | |
291 | ||
292 | memcpy(buf, index_key, index_key_len); | |
293 | cookie->key_hash = fscache_hash(cookie->volume->key_hash, | |
294 | buf, buf_size); | |
295 | return 0; | |
296 | } | |
297 | ||
298 | static bool fscache_cookie_same(const struct fscache_cookie *a, | |
299 | const struct fscache_cookie *b) | |
300 | { | |
301 | const void *ka, *kb; | |
302 | ||
303 | if (a->key_hash != b->key_hash || | |
304 | a->volume != b->volume || | |
305 | a->key_len != b->key_len) | |
306 | return false; | |
307 | ||
308 | if (a->key_len <= sizeof(a->inline_key)) { | |
309 | ka = &a->inline_key; | |
310 | kb = &b->inline_key; | |
311 | } else { | |
312 | ka = a->key; | |
313 | kb = b->key; | |
314 | } | |
315 | return memcmp(ka, kb, a->key_len) == 0; | |
316 | } | |
317 | ||
318 | static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); | |
319 | ||
320 | /* | |
321 | * Allocate a cookie. | |
322 | */ | |
323 | static struct fscache_cookie *fscache_alloc_cookie( | |
324 | struct fscache_volume *volume, | |
325 | u8 advice, | |
326 | const void *index_key, size_t index_key_len, | |
327 | const void *aux_data, size_t aux_data_len, | |
328 | loff_t object_size) | |
329 | { | |
330 | struct fscache_cookie *cookie; | |
331 | ||
332 | /* allocate and initialise a cookie */ | |
333 | cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); | |
334 | if (!cookie) | |
335 | return NULL; | |
336 | fscache_stat(&fscache_n_cookies); | |
337 | ||
338 | cookie->volume = volume; | |
339 | cookie->advice = advice; | |
340 | cookie->key_len = index_key_len; | |
341 | cookie->aux_len = aux_data_len; | |
342 | cookie->object_size = object_size; | |
343 | if (object_size == 0) | |
344 | __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
345 | ||
346 | if (fscache_set_key(cookie, index_key, index_key_len) < 0) | |
347 | goto nomem; | |
348 | ||
349 | if (cookie->aux_len <= sizeof(cookie->inline_aux)) { | |
350 | memcpy(cookie->inline_aux, aux_data, cookie->aux_len); | |
351 | } else { | |
352 | cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); | |
353 | if (!cookie->aux) | |
354 | goto nomem; | |
355 | } | |
356 | ||
357 | refcount_set(&cookie->ref, 1); | |
358 | cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); | |
7f3283ab DH |
359 | spin_lock_init(&cookie->lock); |
360 | INIT_LIST_HEAD(&cookie->commit_link); | |
5d00e426 DH |
361 | INIT_WORK(&cookie->work, fscache_cookie_worker); |
362 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
7f3283ab DH |
363 | |
364 | write_lock(&fscache_cookies_lock); | |
365 | list_add_tail(&cookie->proc_link, &fscache_cookies); | |
366 | write_unlock(&fscache_cookies_lock); | |
367 | fscache_see_cookie(cookie, fscache_cookie_new_acquire); | |
368 | return cookie; | |
369 | ||
370 | nomem: | |
371 | fscache_free_cookie(cookie); | |
372 | return NULL; | |
373 | } | |
374 | ||
5c4588ae YH |
375 | static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie) |
376 | { | |
377 | return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED; | |
378 | } | |
379 | ||
7f3283ab DH |
380 | static void fscache_wait_on_collision(struct fscache_cookie *candidate, |
381 | struct fscache_cookie *wait_for) | |
382 | { | |
383 | enum fscache_cookie_state *statep = &wait_for->state; | |
384 | ||
5c4588ae | 385 | wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for), |
7f3283ab | 386 | 20 * HZ); |
5c4588ae | 387 | if (!fscache_cookie_is_dropped(wait_for)) { |
7f3283ab DH |
388 | pr_notice("Potential collision c=%08x old: c=%08x", |
389 | candidate->debug_id, wait_for->debug_id); | |
5c4588ae | 390 | wait_var_event(statep, fscache_cookie_is_dropped(wait_for)); |
7f3283ab DH |
391 | } |
392 | } | |
393 | ||
394 | /* | |
395 | * Attempt to insert the new cookie into the hash. If there's a collision, we | |
396 | * wait for the old cookie to complete if it's being relinquished and an error | |
397 | * otherwise. | |
398 | */ | |
399 | static bool fscache_hash_cookie(struct fscache_cookie *candidate) | |
400 | { | |
401 | struct fscache_cookie *cursor, *wait_for = NULL; | |
402 | struct hlist_bl_head *h; | |
403 | struct hlist_bl_node *p; | |
404 | unsigned int bucket; | |
405 | ||
406 | bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); | |
407 | h = &fscache_cookie_hash[bucket]; | |
408 | ||
409 | hlist_bl_lock(h); | |
410 | hlist_bl_for_each_entry(cursor, p, h, hash_link) { | |
411 | if (fscache_cookie_same(candidate, cursor)) { | |
412 | if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) | |
413 | goto collision; | |
414 | wait_for = fscache_get_cookie(cursor, | |
415 | fscache_cookie_get_hash_collision); | |
416 | break; | |
417 | } | |
418 | } | |
419 | ||
420 | fscache_get_volume(candidate->volume, fscache_volume_get_cookie); | |
421 | atomic_inc(&candidate->volume->n_cookies); | |
422 | hlist_bl_add_head(&candidate->hash_link, h); | |
423 | set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); | |
424 | hlist_bl_unlock(h); | |
425 | ||
426 | if (wait_for) { | |
427 | fscache_wait_on_collision(candidate, wait_for); | |
428 | fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); | |
429 | } | |
430 | return true; | |
431 | ||
432 | collision: | |
433 | trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), | |
434 | fscache_cookie_collision); | |
435 | pr_err("Duplicate cookie detected\n"); | |
436 | fscache_print_cookie(cursor, 'O'); | |
437 | fscache_print_cookie(candidate, 'N'); | |
438 | hlist_bl_unlock(h); | |
439 | return false; | |
440 | } | |
441 | ||
442 | /* | |
443 | * Request a cookie to represent a data storage object within a volume. | |
444 | * | |
445 | * We never let on to the netfs about errors. We may set a negative cookie | |
446 | * pointer, but that's okay | |
447 | */ | |
448 | struct fscache_cookie *__fscache_acquire_cookie( | |
449 | struct fscache_volume *volume, | |
450 | u8 advice, | |
451 | const void *index_key, size_t index_key_len, | |
452 | const void *aux_data, size_t aux_data_len, | |
453 | loff_t object_size) | |
454 | { | |
455 | struct fscache_cookie *cookie; | |
456 | ||
457 | _enter("V=%x", volume->debug_id); | |
458 | ||
459 | if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) | |
460 | return NULL; | |
461 | if (!aux_data || !aux_data_len) { | |
462 | aux_data = NULL; | |
463 | aux_data_len = 0; | |
464 | } | |
465 | ||
466 | fscache_stat(&fscache_n_acquires); | |
467 | ||
468 | cookie = fscache_alloc_cookie(volume, advice, | |
469 | index_key, index_key_len, | |
470 | aux_data, aux_data_len, | |
471 | object_size); | |
472 | if (!cookie) { | |
473 | fscache_stat(&fscache_n_acquires_oom); | |
474 | return NULL; | |
475 | } | |
476 | ||
477 | if (!fscache_hash_cookie(cookie)) { | |
478 | fscache_see_cookie(cookie, fscache_cookie_discard); | |
479 | fscache_free_cookie(cookie); | |
480 | return NULL; | |
481 | } | |
482 | ||
483 | trace_fscache_acquire(cookie); | |
484 | fscache_stat(&fscache_n_acquires_ok); | |
485 | _leave(" = c=%08x", cookie->debug_id); | |
486 | return cookie; | |
487 | } | |
488 | EXPORT_SYMBOL(__fscache_acquire_cookie); | |
489 | ||
5d00e426 DH |
490 | /* |
491 | * Prepare a cache object to be written to. | |
492 | */ | |
493 | static void fscache_prepare_to_write(struct fscache_cookie *cookie) | |
494 | { | |
495 | cookie->volume->cache->ops->prepare_to_write(cookie); | |
496 | } | |
497 | ||
498 | /* | |
499 | * Look up a cookie in the cache. | |
500 | */ | |
501 | static void fscache_perform_lookup(struct fscache_cookie *cookie) | |
502 | { | |
503 | enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; | |
504 | bool need_withdraw = false; | |
505 | ||
506 | _enter(""); | |
507 | ||
508 | if (!cookie->volume->cache_priv) { | |
509 | fscache_create_volume(cookie->volume, true); | |
510 | if (!cookie->volume->cache_priv) { | |
511 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
512 | goto out; | |
513 | } | |
514 | } | |
515 | ||
516 | if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { | |
517 | if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) | |
518 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
519 | need_withdraw = true; | |
520 | _leave(" [fail]"); | |
521 | goto out; | |
522 | } | |
523 | ||
524 | fscache_see_cookie(cookie, fscache_cookie_see_active); | |
85e4ea10 DH |
525 | spin_lock(&cookie->lock); |
526 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) | |
527 | __fscache_set_cookie_state(cookie, | |
528 | FSCACHE_COOKIE_STATE_INVALIDATING); | |
529 | else | |
530 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); | |
531 | spin_unlock(&cookie->lock); | |
532 | wake_up_cookie_state(cookie); | |
5d00e426 DH |
533 | trace = fscache_access_lookup_cookie_end; |
534 | ||
535 | out: | |
536 | fscache_end_cookie_access(cookie, trace); | |
537 | if (need_withdraw) | |
538 | fscache_withdraw_cookie(cookie); | |
539 | fscache_end_volume_access(cookie->volume, cookie, trace); | |
540 | } | |
541 | ||
12bb21a2 DH |
542 | /* |
543 | * Begin the process of looking up a cookie. We offload the actual process to | |
544 | * a worker thread. | |
545 | */ | |
546 | static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) | |
547 | { | |
548 | if (will_modify) { | |
549 | set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); | |
550 | set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); | |
551 | } | |
552 | if (!fscache_begin_volume_access(cookie->volume, cookie, | |
553 | fscache_access_lookup_cookie)) | |
554 | return false; | |
555 | ||
556 | __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); | |
557 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); | |
558 | set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); | |
559 | set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); | |
560 | return true; | |
561 | } | |
562 | ||
563 | /* | |
564 | * Start using the cookie for I/O. This prevents the backing object from being | |
565 | * reaped by VM pressure. | |
566 | */ | |
567 | void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) | |
568 | { | |
569 | enum fscache_cookie_state state; | |
570 | bool queue = false; | |
e6435f1e | 571 | int n_active; |
12bb21a2 DH |
572 | |
573 | _enter("c=%08x", cookie->debug_id); | |
574 | ||
575 | if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), | |
576 | "Trying to use relinquished cookie\n")) | |
577 | return; | |
578 | ||
579 | spin_lock(&cookie->lock); | |
580 | ||
e6435f1e DH |
581 | n_active = atomic_inc_return(&cookie->n_active); |
582 | trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref), | |
583 | n_active, atomic_read(&cookie->n_accesses), | |
584 | will_modify ? | |
585 | fscache_active_use_modify : fscache_active_use); | |
12bb21a2 DH |
586 | |
587 | again: | |
588 | state = fscache_cookie_state(cookie); | |
589 | switch (state) { | |
590 | case FSCACHE_COOKIE_STATE_QUIESCENT: | |
591 | queue = fscache_begin_lookup(cookie, will_modify); | |
592 | break; | |
593 | ||
594 | case FSCACHE_COOKIE_STATE_LOOKING_UP: | |
595 | case FSCACHE_COOKIE_STATE_CREATING: | |
596 | if (will_modify) | |
597 | set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); | |
598 | break; | |
599 | case FSCACHE_COOKIE_STATE_ACTIVE: | |
d24af13e | 600 | case FSCACHE_COOKIE_STATE_INVALIDATING: |
12bb21a2 DH |
601 | if (will_modify && |
602 | !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { | |
603 | set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); | |
604 | queue = true; | |
605 | } | |
606 | break; | |
607 | ||
608 | case FSCACHE_COOKIE_STATE_FAILED: | |
609 | case FSCACHE_COOKIE_STATE_WITHDRAWING: | |
610 | break; | |
611 | ||
612 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: | |
613 | spin_unlock(&cookie->lock); | |
614 | wait_var_event(&cookie->state, | |
615 | fscache_cookie_state(cookie) != | |
616 | FSCACHE_COOKIE_STATE_LRU_DISCARDING); | |
617 | spin_lock(&cookie->lock); | |
618 | goto again; | |
619 | ||
620 | case FSCACHE_COOKIE_STATE_DROPPED: | |
621 | case FSCACHE_COOKIE_STATE_RELINQUISHING: | |
622 | WARN(1, "Can't use cookie in state %u\n", state); | |
623 | break; | |
624 | } | |
625 | ||
626 | spin_unlock(&cookie->lock); | |
627 | if (queue) | |
628 | fscache_queue_cookie(cookie, fscache_cookie_get_use_work); | |
629 | _leave(""); | |
630 | } | |
631 | EXPORT_SYMBOL(__fscache_use_cookie); | |
632 | ||
633 | static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) | |
634 | { | |
635 | clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); | |
636 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) | |
637 | return; | |
638 | ||
639 | cookie->unused_at = jiffies; | |
640 | spin_lock(&fscache_cookie_lru_lock); | |
641 | if (list_empty(&cookie->commit_link)) { | |
642 | fscache_get_cookie(cookie, fscache_cookie_get_lru); | |
643 | fscache_stat(&fscache_n_cookies_lru); | |
644 | } | |
645 | list_move_tail(&cookie->commit_link, &fscache_cookie_lru); | |
646 | ||
647 | spin_unlock(&fscache_cookie_lru_lock); | |
648 | timer_reduce(&fscache_cookie_lru_timer, | |
649 | jiffies + fscache_lru_cookie_timeout); | |
650 | } | |
651 | ||
652 | /* | |
653 | * Stop using the cookie for I/O. | |
654 | */ | |
655 | void __fscache_unuse_cookie(struct fscache_cookie *cookie, | |
656 | const void *aux_data, const loff_t *object_size) | |
657 | { | |
e6435f1e DH |
658 | unsigned int debug_id = cookie->debug_id; |
659 | unsigned int r = refcount_read(&cookie->ref); | |
660 | unsigned int a = atomic_read(&cookie->n_accesses); | |
661 | unsigned int c; | |
662 | ||
12bb21a2 DH |
663 | if (aux_data || object_size) |
664 | __fscache_update_cookie(cookie, aux_data, object_size); | |
665 | ||
e6435f1e DH |
666 | /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ |
667 | c = atomic_fetch_add_unless(&cookie->n_active, -1, 1); | |
668 | if (c != 1) { | |
669 | trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse); | |
670 | return; | |
12bb21a2 | 671 | } |
e6435f1e DH |
672 | |
673 | spin_lock(&cookie->lock); | |
674 | r = refcount_read(&cookie->ref); | |
675 | a = atomic_read(&cookie->n_accesses); | |
676 | c = atomic_dec_return(&cookie->n_active); | |
677 | trace_fscache_active(debug_id, r, c, a, fscache_active_unuse); | |
678 | if (c == 0) | |
679 | fscache_unuse_cookie_locked(cookie); | |
680 | spin_unlock(&cookie->lock); | |
12bb21a2 DH |
681 | } |
682 | EXPORT_SYMBOL(__fscache_unuse_cookie); | |
683 | ||
5d00e426 DH |
684 | /* |
685 | * Perform work upon the cookie, such as committing its cache state, | |
686 | * relinquishing it or withdrawing the backing cache. We're protected from the | |
687 | * cache going away under us as object withdrawal must come through this | |
688 | * non-reentrant work item. | |
689 | */ | |
690 | static void fscache_cookie_state_machine(struct fscache_cookie *cookie) | |
691 | { | |
692 | enum fscache_cookie_state state; | |
693 | bool wake = false; | |
694 | ||
695 | _enter("c=%x", cookie->debug_id); | |
696 | ||
697 | again: | |
698 | spin_lock(&cookie->lock); | |
699 | again_locked: | |
700 | state = cookie->state; | |
701 | switch (state) { | |
702 | case FSCACHE_COOKIE_STATE_QUIESCENT: | |
703 | /* The QUIESCENT state is jumped to the LOOKING_UP state by | |
704 | * fscache_use_cookie(). | |
705 | */ | |
706 | ||
707 | if (atomic_read(&cookie->n_accesses) == 0 && | |
708 | test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { | |
709 | __fscache_set_cookie_state(cookie, | |
710 | FSCACHE_COOKIE_STATE_RELINQUISHING); | |
711 | wake = true; | |
712 | goto again_locked; | |
713 | } | |
714 | break; | |
715 | ||
716 | case FSCACHE_COOKIE_STATE_LOOKING_UP: | |
717 | spin_unlock(&cookie->lock); | |
718 | fscache_init_access_gate(cookie); | |
719 | fscache_perform_lookup(cookie); | |
720 | goto again; | |
721 | ||
d24af13e DH |
722 | case FSCACHE_COOKIE_STATE_INVALIDATING: |
723 | spin_unlock(&cookie->lock); | |
724 | fscache_perform_invalidation(cookie); | |
725 | goto again; | |
726 | ||
5d00e426 DH |
727 | case FSCACHE_COOKIE_STATE_ACTIVE: |
728 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { | |
729 | spin_unlock(&cookie->lock); | |
730 | fscache_prepare_to_write(cookie); | |
731 | spin_lock(&cookie->lock); | |
732 | } | |
12bb21a2 DH |
733 | if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { |
734 | __fscache_set_cookie_state(cookie, | |
735 | FSCACHE_COOKIE_STATE_LRU_DISCARDING); | |
736 | wake = true; | |
737 | goto again_locked; | |
738 | } | |
5d00e426 DH |
739 | fallthrough; |
740 | ||
741 | case FSCACHE_COOKIE_STATE_FAILED: | |
fb24771f JL |
742 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) |
743 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); | |
744 | ||
5d00e426 DH |
745 | if (atomic_read(&cookie->n_accesses) != 0) |
746 | break; | |
747 | if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { | |
748 | __fscache_set_cookie_state(cookie, | |
749 | FSCACHE_COOKIE_STATE_RELINQUISHING); | |
750 | wake = true; | |
751 | goto again_locked; | |
752 | } | |
753 | if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { | |
754 | __fscache_set_cookie_state(cookie, | |
755 | FSCACHE_COOKIE_STATE_WITHDRAWING); | |
756 | wake = true; | |
757 | goto again_locked; | |
758 | } | |
759 | break; | |
760 | ||
12bb21a2 | 761 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
5d00e426 DH |
762 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
763 | case FSCACHE_COOKIE_STATE_WITHDRAWING: | |
764 | if (cookie->cache_priv) { | |
765 | spin_unlock(&cookie->lock); | |
766 | cookie->volume->cache->ops->withdraw_cookie(cookie); | |
767 | spin_lock(&cookie->lock); | |
768 | } | |
769 | ||
85e4ea10 DH |
770 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) |
771 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); | |
772 | ||
5d00e426 DH |
773 | switch (state) { |
774 | case FSCACHE_COOKIE_STATE_RELINQUISHING: | |
775 | fscache_see_cookie(cookie, fscache_cookie_see_relinquish); | |
776 | fscache_unhash_cookie(cookie); | |
777 | __fscache_set_cookie_state(cookie, | |
778 | FSCACHE_COOKIE_STATE_DROPPED); | |
779 | wake = true; | |
780 | goto out; | |
12bb21a2 DH |
781 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
782 | fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); | |
783 | break; | |
5d00e426 DH |
784 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
785 | fscache_see_cookie(cookie, fscache_cookie_see_withdraw); | |
786 | break; | |
787 | default: | |
788 | BUG(); | |
789 | } | |
790 | ||
791 | clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); | |
792 | clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); | |
793 | clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); | |
794 | clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); | |
795 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
796 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
797 | wake = true; | |
798 | goto again_locked; | |
799 | ||
800 | case FSCACHE_COOKIE_STATE_DROPPED: | |
801 | break; | |
802 | ||
803 | default: | |
804 | WARN_ONCE(1, "Cookie %x in unexpected state %u\n", | |
805 | cookie->debug_id, state); | |
806 | break; | |
807 | } | |
808 | ||
809 | out: | |
810 | spin_unlock(&cookie->lock); | |
811 | if (wake) | |
812 | wake_up_cookie_state(cookie); | |
813 | _leave(""); | |
814 | } | |
815 | ||
816 | static void fscache_cookie_worker(struct work_struct *work) | |
817 | { | |
818 | struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); | |
819 | ||
820 | fscache_see_cookie(cookie, fscache_cookie_see_work); | |
821 | fscache_cookie_state_machine(cookie); | |
822 | fscache_put_cookie(cookie, fscache_cookie_put_work); | |
823 | } | |
824 | ||
825 | /* | |
826 | * Wait for the object to become inactive. The cookie's work item will be | |
827 | * scheduled when someone transitions n_accesses to 0 - but if someone's | |
828 | * already done that, schedule it anyway. | |
829 | */ | |
830 | static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) | |
831 | { | |
832 | int n_accesses; | |
833 | bool unpinned; | |
834 | ||
835 | unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); | |
836 | ||
837 | /* Need to read the access count after unpinning */ | |
838 | n_accesses = atomic_read(&cookie->n_accesses); | |
839 | if (unpinned) | |
840 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
841 | n_accesses, fscache_access_cache_unpin); | |
842 | if (n_accesses == 0) | |
843 | fscache_queue_cookie(cookie, fscache_cookie_get_end_access); | |
844 | } | |
845 | ||
12bb21a2 DH |
846 | static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) |
847 | { | |
848 | fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); | |
849 | ||
850 | spin_lock(&cookie->lock); | |
851 | if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || | |
852 | time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || | |
853 | atomic_read(&cookie->n_active) > 0) { | |
854 | spin_unlock(&cookie->lock); | |
855 | fscache_stat(&fscache_n_cookies_lru_removed); | |
856 | } else { | |
857 | set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); | |
858 | spin_unlock(&cookie->lock); | |
859 | fscache_stat(&fscache_n_cookies_lru_expired); | |
860 | _debug("lru c=%x", cookie->debug_id); | |
861 | __fscache_withdraw_cookie(cookie); | |
862 | } | |
863 | ||
864 | fscache_put_cookie(cookie, fscache_cookie_put_lru); | |
865 | } | |
866 | ||
867 | static void fscache_cookie_lru_worker(struct work_struct *work) | |
868 | { | |
869 | struct fscache_cookie *cookie; | |
870 | unsigned long unused_at; | |
871 | ||
872 | spin_lock(&fscache_cookie_lru_lock); | |
873 | ||
874 | while (!list_empty(&fscache_cookie_lru)) { | |
875 | cookie = list_first_entry(&fscache_cookie_lru, | |
876 | struct fscache_cookie, commit_link); | |
877 | unused_at = cookie->unused_at + fscache_lru_cookie_timeout; | |
878 | if (time_before(jiffies, unused_at)) { | |
879 | timer_reduce(&fscache_cookie_lru_timer, unused_at); | |
880 | break; | |
881 | } | |
882 | ||
883 | list_del_init(&cookie->commit_link); | |
884 | fscache_stat_d(&fscache_n_cookies_lru); | |
885 | spin_unlock(&fscache_cookie_lru_lock); | |
886 | fscache_cookie_lru_do_one(cookie); | |
887 | spin_lock(&fscache_cookie_lru_lock); | |
888 | } | |
889 | ||
890 | spin_unlock(&fscache_cookie_lru_lock); | |
891 | } | |
892 | ||
893 | static void fscache_cookie_lru_timed_out(struct timer_list *timer) | |
894 | { | |
895 | queue_work(fscache_wq, &fscache_cookie_lru_work); | |
896 | } | |
897 | ||
898 | static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) | |
899 | { | |
900 | bool need_put = false; | |
901 | ||
902 | if (!list_empty(&cookie->commit_link)) { | |
903 | spin_lock(&fscache_cookie_lru_lock); | |
904 | if (!list_empty(&cookie->commit_link)) { | |
905 | list_del_init(&cookie->commit_link); | |
906 | fscache_stat_d(&fscache_n_cookies_lru); | |
907 | fscache_stat(&fscache_n_cookies_lru_dropped); | |
908 | need_put = true; | |
909 | } | |
910 | spin_unlock(&fscache_cookie_lru_lock); | |
911 | if (need_put) | |
912 | fscache_put_cookie(cookie, fscache_cookie_put_lru); | |
913 | } | |
914 | } | |
915 | ||
7f3283ab DH |
916 | /* |
917 | * Remove a cookie from the hash table. | |
918 | */ | |
919 | static void fscache_unhash_cookie(struct fscache_cookie *cookie) | |
920 | { | |
921 | struct hlist_bl_head *h; | |
922 | unsigned int bucket; | |
923 | ||
924 | bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); | |
925 | h = &fscache_cookie_hash[bucket]; | |
926 | ||
927 | hlist_bl_lock(h); | |
928 | hlist_bl_del(&cookie->hash_link); | |
929 | clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); | |
930 | hlist_bl_unlock(h); | |
5d00e426 | 931 | fscache_stat(&fscache_n_relinquishes_dropped); |
7f3283ab DH |
932 | } |
933 | ||
5d00e426 | 934 | static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) |
7f3283ab | 935 | { |
12bb21a2 | 936 | fscache_cookie_drop_from_lru(cookie); |
5d00e426 DH |
937 | __fscache_withdraw_cookie(cookie); |
938 | } | |
7f3283ab | 939 | |
5d00e426 DH |
940 | /** |
941 | * fscache_withdraw_cookie - Mark a cookie for withdrawal | |
942 | * @cookie: The cookie to be withdrawn. | |
943 | * | |
944 | * Allow the cache backend to withdraw the backing for a cookie for its own | |
945 | * reasons, even if that cookie is in active use. | |
946 | */ | |
947 | void fscache_withdraw_cookie(struct fscache_cookie *cookie) | |
948 | { | |
949 | set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); | |
950 | fscache_drop_withdraw_cookie(cookie); | |
7f3283ab | 951 | } |
5d00e426 | 952 | EXPORT_SYMBOL(fscache_withdraw_cookie); |
7f3283ab DH |
953 | |
954 | /* | |
955 | * Allow the netfs to release a cookie back to the cache. | |
956 | * - the object will be marked as recyclable on disk if retire is true | |
957 | */ | |
958 | void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) | |
959 | { | |
960 | fscache_stat(&fscache_n_relinquishes); | |
961 | if (retire) | |
962 | fscache_stat(&fscache_n_relinquishes_retire); | |
963 | ||
964 | _enter("c=%08x{%d},%d", | |
965 | cookie->debug_id, atomic_read(&cookie->n_active), retire); | |
966 | ||
967 | if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), | |
968 | "Cookie c=%x already relinquished\n", cookie->debug_id)) | |
969 | return; | |
970 | ||
971 | if (retire) | |
972 | set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); | |
973 | trace_fscache_relinquish(cookie, retire); | |
974 | ||
975 | ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); | |
976 | ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); | |
977 | atomic_dec(&cookie->volume->n_cookies); | |
978 | ||
5d00e426 DH |
979 | if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { |
980 | set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); | |
981 | fscache_drop_withdraw_cookie(cookie); | |
982 | } else { | |
983 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); | |
984 | fscache_unhash_cookie(cookie); | |
985 | } | |
7f3283ab DH |
986 | fscache_put_cookie(cookie, fscache_cookie_put_relinquish); |
987 | } | |
988 | EXPORT_SYMBOL(__fscache_relinquish_cookie); | |
989 | ||
990 | /* | |
991 | * Drop a reference to a cookie. | |
992 | */ | |
993 | void fscache_put_cookie(struct fscache_cookie *cookie, | |
994 | enum fscache_cookie_trace where) | |
995 | { | |
996 | struct fscache_volume *volume = cookie->volume; | |
997 | unsigned int cookie_debug_id = cookie->debug_id; | |
998 | bool zero; | |
999 | int ref; | |
1000 | ||
1001 | zero = __refcount_dec_and_test(&cookie->ref, &ref); | |
1002 | trace_fscache_cookie(cookie_debug_id, ref - 1, where); | |
1003 | if (zero) { | |
1004 | fscache_free_cookie(cookie); | |
1005 | fscache_put_volume(volume, fscache_volume_put_cookie); | |
1006 | } | |
1007 | } | |
1008 | EXPORT_SYMBOL(fscache_put_cookie); | |
1009 | ||
1010 | /* | |
1011 | * Get a reference to a cookie. | |
1012 | */ | |
1013 | struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, | |
1014 | enum fscache_cookie_trace where) | |
1015 | { | |
1016 | int ref; | |
1017 | ||
1018 | __refcount_inc(&cookie->ref, &ref); | |
1019 | trace_fscache_cookie(cookie->debug_id, ref + 1, where); | |
1020 | return cookie; | |
1021 | } | |
1022 | EXPORT_SYMBOL(fscache_get_cookie); | |
1023 | ||
d24af13e DH |
1024 | /* |
1025 | * Ask the cache to effect invalidation of a cookie. | |
1026 | */ | |
1027 | static void fscache_perform_invalidation(struct fscache_cookie *cookie) | |
1028 | { | |
1029 | if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) | |
1030 | fscache_caching_failed(cookie); | |
1031 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); | |
1032 | } | |
1033 | ||
1034 | /* | |
1035 | * Invalidate an object. | |
1036 | */ | |
1037 | void __fscache_invalidate(struct fscache_cookie *cookie, | |
1038 | const void *aux_data, loff_t new_size, | |
1039 | unsigned int flags) | |
1040 | { | |
1041 | bool is_caching; | |
1042 | ||
1043 | _enter("c=%x", cookie->debug_id); | |
1044 | ||
1045 | fscache_stat(&fscache_n_invalidates); | |
1046 | ||
1047 | if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), | |
1048 | "Trying to invalidate relinquished cookie\n")) | |
1049 | return; | |
1050 | ||
1051 | if ((flags & FSCACHE_INVAL_DIO_WRITE) && | |
1052 | test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) | |
1053 | return; | |
1054 | ||
1055 | spin_lock(&cookie->lock); | |
1056 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
1057 | fscache_update_aux(cookie, aux_data, &new_size); | |
1058 | cookie->inval_counter++; | |
1059 | trace_fscache_invalidate(cookie, new_size); | |
1060 | ||
1061 | switch (cookie->state) { | |
1062 | case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ | |
1063 | default: | |
1064 | spin_unlock(&cookie->lock); | |
1065 | _leave(" [no %u]", cookie->state); | |
1066 | return; | |
1067 | ||
1068 | case FSCACHE_COOKIE_STATE_LOOKING_UP: | |
fb24771f JL |
1069 | if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) |
1070 | __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie); | |
85e4ea10 | 1071 | fallthrough; |
d24af13e DH |
1072 | case FSCACHE_COOKIE_STATE_CREATING: |
1073 | spin_unlock(&cookie->lock); | |
1074 | _leave(" [look %x]", cookie->inval_counter); | |
1075 | return; | |
1076 | ||
1077 | case FSCACHE_COOKIE_STATE_ACTIVE: | |
1078 | is_caching = fscache_begin_cookie_access( | |
1079 | cookie, fscache_access_invalidate_cookie); | |
1080 | if (is_caching) | |
1081 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); | |
1082 | spin_unlock(&cookie->lock); | |
1083 | wake_up_cookie_state(cookie); | |
1084 | ||
1085 | if (is_caching) | |
1086 | fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); | |
1087 | _leave(" [inv]"); | |
1088 | return; | |
1089 | } | |
1090 | } | |
1091 | EXPORT_SYMBOL(__fscache_invalidate); | |
1092 | ||
19517e53 | 1093 | #ifdef CONFIG_PROC_FS |
7f3283ab DH |
1094 | /* |
1095 | * Generate a list of extant cookies in /proc/fs/fscache/cookies | |
1096 | */ | |
1097 | static int fscache_cookies_seq_show(struct seq_file *m, void *v) | |
1098 | { | |
1099 | struct fscache_cookie *cookie; | |
1100 | unsigned int keylen = 0, auxlen = 0; | |
1101 | u8 *p; | |
1102 | ||
1103 | if (v == &fscache_cookies) { | |
1104 | seq_puts(m, | |
1105 | "COOKIE VOLUME REF ACT ACC S FL DEF \n" | |
1106 | "======== ======== === === === = == ================\n" | |
1107 | ); | |
1108 | return 0; | |
1109 | } | |
1110 | ||
1111 | cookie = list_entry(v, struct fscache_cookie, proc_link); | |
1112 | ||
1113 | seq_printf(m, | |
1114 | "%08x %08x %3d %3d %3d %c %02lx", | |
1115 | cookie->debug_id, | |
1116 | cookie->volume->debug_id, | |
1117 | refcount_read(&cookie->ref), | |
1118 | atomic_read(&cookie->n_active), | |
1119 | atomic_read(&cookie->n_accesses), | |
1120 | fscache_cookie_states[cookie->state], | |
1121 | cookie->flags); | |
1122 | ||
1123 | keylen = cookie->key_len; | |
1124 | auxlen = cookie->aux_len; | |
1125 | ||
1126 | if (keylen > 0 || auxlen > 0) { | |
1127 | seq_puts(m, " "); | |
1128 | p = keylen <= sizeof(cookie->inline_key) ? | |
1129 | cookie->inline_key : cookie->key; | |
1130 | for (; keylen > 0; keylen--) | |
1131 | seq_printf(m, "%02x", *p++); | |
1132 | if (auxlen > 0) { | |
1133 | seq_puts(m, ", "); | |
1134 | p = auxlen <= sizeof(cookie->inline_aux) ? | |
1135 | cookie->inline_aux : cookie->aux; | |
1136 | for (; auxlen > 0; auxlen--) | |
1137 | seq_printf(m, "%02x", *p++); | |
1138 | } | |
1139 | } | |
1140 | ||
1141 | seq_puts(m, "\n"); | |
1142 | return 0; | |
1143 | } | |
1144 | ||
1145 | static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) | |
1146 | __acquires(fscache_cookies_lock) | |
1147 | { | |
1148 | read_lock(&fscache_cookies_lock); | |
1149 | return seq_list_start_head(&fscache_cookies, *_pos); | |
1150 | } | |
1151 | ||
1152 | static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) | |
1153 | { | |
1154 | return seq_list_next(v, &fscache_cookies, _pos); | |
1155 | } | |
1156 | ||
1157 | static void fscache_cookies_seq_stop(struct seq_file *m, void *v) | |
1158 | __releases(rcu) | |
1159 | { | |
1160 | read_unlock(&fscache_cookies_lock); | |
1161 | } | |
1162 | ||
1163 | ||
1164 | const struct seq_operations fscache_cookies_seq_ops = { | |
1165 | .start = fscache_cookies_seq_start, | |
1166 | .next = fscache_cookies_seq_next, | |
1167 | .stop = fscache_cookies_seq_stop, | |
1168 | .show = fscache_cookies_seq_show, | |
1169 | }; | |
19517e53 | 1170 | #endif |