1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* netfs cookie management
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
7 * See Documentation/filesystems/caching/netfs-api.rst for more information on
11 #define FSCACHE_DEBUG_LEVEL COOKIE
12 #include <linux/module.h>
13 #include <linux/slab.h>
16 struct kmem_cache *fscache_cookie_jar;
18 static void fscache_cookie_lru_timed_out(struct timer_list *timer);
19 static void fscache_cookie_lru_worker(struct work_struct *work);
20 static void fscache_cookie_worker(struct work_struct *work);
21 static void fscache_unhash_cookie(struct fscache_cookie *cookie);
22 static void fscache_perform_invalidation(struct fscache_cookie *cookie);
24 #define fscache_cookie_hash_shift 15
25 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
26 static LIST_HEAD(fscache_cookies);
27 static DEFINE_RWLOCK(fscache_cookies_lock);
28 static LIST_HEAD(fscache_cookie_lru);
29 static DEFINE_SPINLOCK(fscache_cookie_lru_lock);
30 DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out);
31 static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker);
32 static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD";
33 unsigned int fscache_lru_cookie_timeout = 10 * HZ;
35 void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
39 pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n",
43 atomic_read(&cookie->n_active),
44 atomic_read(&cookie->n_accesses),
45 fscache_cookie_states[cookie->state]);
46 pr_err("%c-cookie V=%08x [%s]\n",
48 cookie->volume->debug_id,
51 k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
52 cookie->inline_key : cookie->key;
53 pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k);
56 static void fscache_free_cookie(struct fscache_cookie *cookie)
58 if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) {
59 spin_lock(&fscache_cookie_lru_lock);
60 list_del_init(&cookie->commit_link);
61 spin_unlock(&fscache_cookie_lru_lock);
62 fscache_stat_d(&fscache_n_cookies_lru);
63 fscache_stat(&fscache_n_cookies_lru_removed);
66 if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) {
67 fscache_print_cookie(cookie, 'F');
71 write_lock(&fscache_cookies_lock);
72 list_del(&cookie->proc_link);
73 write_unlock(&fscache_cookies_lock);
74 if (cookie->aux_len > sizeof(cookie->inline_aux))
76 if (cookie->key_len > sizeof(cookie->inline_key))
78 fscache_stat_d(&fscache_n_cookies);
79 kmem_cache_free(fscache_cookie_jar, cookie);
82 static void __fscache_queue_cookie(struct fscache_cookie *cookie)
84 if (!queue_work(fscache_wq, &cookie->work))
85 fscache_put_cookie(cookie, fscache_cookie_put_over_queued);
88 static void fscache_queue_cookie(struct fscache_cookie *cookie,
89 enum fscache_cookie_trace where)
91 fscache_get_cookie(cookie, where);
92 __fscache_queue_cookie(cookie);
96 * Initialise the access gate on a cookie by setting a flag to prevent the
97 * state machine from being queued when the access counter transitions to 0.
98 * We're only interested in this when we withdraw caching services from the
101 static void fscache_init_access_gate(struct fscache_cookie *cookie)
105 n_accesses = atomic_read(&cookie->n_accesses);
106 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
107 n_accesses, fscache_access_cache_pin);
108 set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
112 * fscache_end_cookie_access - Unpin a cache at the end of an access.
113 * @cookie: A data file cookie
114 * @why: An indication of the circumstances of the access for tracing
116 * Unpin a cache cookie after we've accessed it and bring a deferred
117 * relinquishment or withdrawal state into effect.
119 * The @why indicator is provided for tracing purposes.
121 void fscache_end_cookie_access(struct fscache_cookie *cookie,
122 enum fscache_access_trace why)
126 smp_mb__before_atomic();
127 n_accesses = atomic_dec_return(&cookie->n_accesses);
128 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
130 if (n_accesses == 0 &&
131 !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags))
132 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
134 EXPORT_SYMBOL(fscache_end_cookie_access);
137 * Pin the cache behind a cookie so that we can access it.
139 static void __fscache_begin_cookie_access(struct fscache_cookie *cookie,
140 enum fscache_access_trace why)
144 n_accesses = atomic_inc_return(&cookie->n_accesses);
145 smp_mb__after_atomic(); /* (Future) read state after is-caching.
146 * Reread n_accesses after is-caching
148 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
153 * fscache_begin_cookie_access - Pin a cache so data can be accessed
154 * @cookie: A data file cookie
155 * @why: An indication of the circumstances of the access for tracing
157 * Attempt to pin the cache to prevent it from going away whilst we're
158 * accessing data and returns true if successful. This works as follows:
160 * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not
161 * set), we return false to indicate access was not permitted.
163 * (2) If the cookie is being cached, we increment its n_accesses count and
164 * then recheck the IS_CACHING flag, ending the access if it got cleared.
166 * (3) When we end the access, we decrement the cookie's n_accesses and wake
167 * up the any waiters if it reaches 0.
169 * (4) Whilst the cookie is actively being cached, its n_accesses is kept
170 * artificially incremented to prevent wakeups from happening.
172 * (5) When the cache is taken offline or if the cookie is culled, the flag is
173 * cleared to prevent new accesses, the cookie's n_accesses is decremented
174 * and we wait for it to become 0.
176 * The @why indicator are merely provided for tracing purposes.
178 bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
179 enum fscache_access_trace why)
181 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
183 __fscache_begin_cookie_access(cookie, why);
184 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) ||
185 !fscache_cache_is_live(cookie->volume->cache)) {
186 fscache_end_cookie_access(cookie, fscache_access_unlive);
192 static inline void wake_up_cookie_state(struct fscache_cookie *cookie)
194 /* Use a barrier to ensure that waiters see the state variable
195 * change, as spin_unlock doesn't guarantee a barrier.
197 * See comments over wake_up_bit() and waitqueue_active().
200 wake_up_var(&cookie->state);
204 * Change the state a cookie is at and wake up anyone waiting for that. Impose
205 * an ordering between the stuff stored in the cookie and the state member.
206 * Paired with fscache_cookie_state().
208 static void __fscache_set_cookie_state(struct fscache_cookie *cookie,
209 enum fscache_cookie_state state)
211 smp_store_release(&cookie->state, state);
214 static void fscache_set_cookie_state(struct fscache_cookie *cookie,
215 enum fscache_cookie_state state)
217 spin_lock(&cookie->lock);
218 __fscache_set_cookie_state(cookie, state);
219 spin_unlock(&cookie->lock);
220 wake_up_cookie_state(cookie);
224 * fscache_cookie_lookup_negative - Note negative lookup
225 * @cookie: The cookie that was being looked up
227 * Note that some part of the metadata path in the cache doesn't exist and so
228 * we can release any waiting readers in the certain knowledge that there's
229 * nothing for them to actually read.
231 * This function uses no locking and must only be called from the state machine.
233 void fscache_cookie_lookup_negative(struct fscache_cookie *cookie)
235 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
236 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING);
238 EXPORT_SYMBOL(fscache_cookie_lookup_negative);
241 * fscache_resume_after_invalidation - Allow I/O to resume after invalidation
242 * @cookie: The cookie that was invalidated
244 * Tell fscache that invalidation is sufficiently complete that I/O can be
247 void fscache_resume_after_invalidation(struct fscache_cookie *cookie)
249 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
251 EXPORT_SYMBOL(fscache_resume_after_invalidation);
254 * fscache_caching_failed - Report that a failure stopped caching on a cookie
255 * @cookie: The cookie that was affected
257 * Tell fscache that caching on a cookie needs to be stopped due to some sort
260 * This function uses no locking and must only be called from the state machine.
262 void fscache_caching_failed(struct fscache_cookie *cookie)
264 clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
265 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED);
267 EXPORT_SYMBOL(fscache_caching_failed);
270 * Set the index key in a cookie. The cookie struct has space for a 16-byte
271 * key plus length and hash, but if that's not big enough, it's instead a
272 * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
275 static int fscache_set_key(struct fscache_cookie *cookie,
276 const void *index_key, size_t index_key_len)
281 buf_size = round_up(index_key_len, sizeof(__le32));
283 if (index_key_len > sizeof(cookie->inline_key)) {
284 buf = kzalloc(buf_size, GFP_KERNEL);
289 buf = cookie->inline_key;
292 memcpy(buf, index_key, index_key_len);
293 cookie->key_hash = fscache_hash(cookie->volume->key_hash,
298 static bool fscache_cookie_same(const struct fscache_cookie *a,
299 const struct fscache_cookie *b)
303 if (a->key_hash != b->key_hash ||
304 a->volume != b->volume ||
305 a->key_len != b->key_len)
308 if (a->key_len <= sizeof(a->inline_key)) {
315 return memcmp(ka, kb, a->key_len) == 0;
318 static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
323 static struct fscache_cookie *fscache_alloc_cookie(
324 struct fscache_volume *volume,
326 const void *index_key, size_t index_key_len,
327 const void *aux_data, size_t aux_data_len,
330 struct fscache_cookie *cookie;
332 /* allocate and initialise a cookie */
333 cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
336 fscache_stat(&fscache_n_cookies);
338 cookie->volume = volume;
339 cookie->advice = advice;
340 cookie->key_len = index_key_len;
341 cookie->aux_len = aux_data_len;
342 cookie->object_size = object_size;
343 if (object_size == 0)
344 __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
346 if (fscache_set_key(cookie, index_key, index_key_len) < 0)
349 if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
350 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
352 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
357 refcount_set(&cookie->ref, 1);
358 cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
359 spin_lock_init(&cookie->lock);
360 INIT_LIST_HEAD(&cookie->commit_link);
361 INIT_WORK(&cookie->work, fscache_cookie_worker);
362 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
364 write_lock(&fscache_cookies_lock);
365 list_add_tail(&cookie->proc_link, &fscache_cookies);
366 write_unlock(&fscache_cookies_lock);
367 fscache_see_cookie(cookie, fscache_cookie_new_acquire);
371 fscache_free_cookie(cookie);
375 static void fscache_wait_on_collision(struct fscache_cookie *candidate,
376 struct fscache_cookie *wait_for)
378 enum fscache_cookie_state *statep = &wait_for->state;
380 wait_var_event_timeout(statep, READ_ONCE(*statep) == FSCACHE_COOKIE_STATE_DROPPED,
382 if (READ_ONCE(*statep) != FSCACHE_COOKIE_STATE_DROPPED) {
383 pr_notice("Potential collision c=%08x old: c=%08x",
384 candidate->debug_id, wait_for->debug_id);
385 wait_var_event(statep, READ_ONCE(*statep) == FSCACHE_COOKIE_STATE_DROPPED);
390 * Attempt to insert the new cookie into the hash. If there's a collision, we
391 * wait for the old cookie to complete if it's being relinquished and an error
394 static bool fscache_hash_cookie(struct fscache_cookie *candidate)
396 struct fscache_cookie *cursor, *wait_for = NULL;
397 struct hlist_bl_head *h;
398 struct hlist_bl_node *p;
401 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
402 h = &fscache_cookie_hash[bucket];
405 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
406 if (fscache_cookie_same(candidate, cursor)) {
407 if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags))
409 wait_for = fscache_get_cookie(cursor,
410 fscache_cookie_get_hash_collision);
415 fscache_get_volume(candidate->volume, fscache_volume_get_cookie);
416 atomic_inc(&candidate->volume->n_cookies);
417 hlist_bl_add_head(&candidate->hash_link, h);
418 set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags);
422 fscache_wait_on_collision(candidate, wait_for);
423 fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision);
428 trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref),
429 fscache_cookie_collision);
430 pr_err("Duplicate cookie detected\n");
431 fscache_print_cookie(cursor, 'O');
432 fscache_print_cookie(candidate, 'N');
438 * Request a cookie to represent a data storage object within a volume.
440 * We never let on to the netfs about errors. We may set a negative cookie
441 * pointer, but that's okay
443 struct fscache_cookie *__fscache_acquire_cookie(
444 struct fscache_volume *volume,
446 const void *index_key, size_t index_key_len,
447 const void *aux_data, size_t aux_data_len,
450 struct fscache_cookie *cookie;
452 _enter("V=%x", volume->debug_id);
454 if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
456 if (!aux_data || !aux_data_len) {
461 fscache_stat(&fscache_n_acquires);
463 cookie = fscache_alloc_cookie(volume, advice,
464 index_key, index_key_len,
465 aux_data, aux_data_len,
468 fscache_stat(&fscache_n_acquires_oom);
472 if (!fscache_hash_cookie(cookie)) {
473 fscache_see_cookie(cookie, fscache_cookie_discard);
474 fscache_free_cookie(cookie);
478 trace_fscache_acquire(cookie);
479 fscache_stat(&fscache_n_acquires_ok);
480 _leave(" = c=%08x", cookie->debug_id);
483 EXPORT_SYMBOL(__fscache_acquire_cookie);
486 * Prepare a cache object to be written to.
488 static void fscache_prepare_to_write(struct fscache_cookie *cookie)
490 cookie->volume->cache->ops->prepare_to_write(cookie);
494 * Look up a cookie in the cache.
496 static void fscache_perform_lookup(struct fscache_cookie *cookie)
498 enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed;
499 bool need_withdraw = false;
503 if (!cookie->volume->cache_priv) {
504 fscache_create_volume(cookie->volume, true);
505 if (!cookie->volume->cache_priv) {
506 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
511 if (!cookie->volume->cache->ops->lookup_cookie(cookie)) {
512 if (cookie->state != FSCACHE_COOKIE_STATE_FAILED)
513 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
514 need_withdraw = true;
519 fscache_see_cookie(cookie, fscache_cookie_see_active);
520 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
521 trace = fscache_access_lookup_cookie_end;
524 fscache_end_cookie_access(cookie, trace);
526 fscache_withdraw_cookie(cookie);
527 fscache_end_volume_access(cookie->volume, cookie, trace);
531 * Begin the process of looking up a cookie. We offload the actual process to
534 static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify)
537 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
538 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
540 if (!fscache_begin_volume_access(cookie->volume, cookie,
541 fscache_access_lookup_cookie))
544 __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie);
545 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP);
546 set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
547 set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags);
552 * Start using the cookie for I/O. This prevents the backing object from being
553 * reaped by VM pressure.
555 void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify)
557 enum fscache_cookie_state state;
561 _enter("c=%08x", cookie->debug_id);
563 if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
564 "Trying to use relinquished cookie\n"))
567 spin_lock(&cookie->lock);
569 n_active = atomic_inc_return(&cookie->n_active);
570 trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref),
571 n_active, atomic_read(&cookie->n_accesses),
573 fscache_active_use_modify : fscache_active_use);
576 state = fscache_cookie_state(cookie);
578 case FSCACHE_COOKIE_STATE_QUIESCENT:
579 queue = fscache_begin_lookup(cookie, will_modify);
582 case FSCACHE_COOKIE_STATE_LOOKING_UP:
583 case FSCACHE_COOKIE_STATE_CREATING:
585 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
587 case FSCACHE_COOKIE_STATE_ACTIVE:
588 case FSCACHE_COOKIE_STATE_INVALIDATING:
590 !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) {
591 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
596 case FSCACHE_COOKIE_STATE_FAILED:
597 case FSCACHE_COOKIE_STATE_WITHDRAWING:
600 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
601 spin_unlock(&cookie->lock);
602 wait_var_event(&cookie->state,
603 fscache_cookie_state(cookie) !=
604 FSCACHE_COOKIE_STATE_LRU_DISCARDING);
605 spin_lock(&cookie->lock);
608 case FSCACHE_COOKIE_STATE_DROPPED:
609 case FSCACHE_COOKIE_STATE_RELINQUISHING:
610 WARN(1, "Can't use cookie in state %u\n", state);
614 spin_unlock(&cookie->lock);
616 fscache_queue_cookie(cookie, fscache_cookie_get_use_work);
619 EXPORT_SYMBOL(__fscache_use_cookie);
621 static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
623 clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags);
624 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
627 cookie->unused_at = jiffies;
628 spin_lock(&fscache_cookie_lru_lock);
629 if (list_empty(&cookie->commit_link)) {
630 fscache_get_cookie(cookie, fscache_cookie_get_lru);
631 fscache_stat(&fscache_n_cookies_lru);
633 list_move_tail(&cookie->commit_link, &fscache_cookie_lru);
635 spin_unlock(&fscache_cookie_lru_lock);
636 timer_reduce(&fscache_cookie_lru_timer,
637 jiffies + fscache_lru_cookie_timeout);
641 * Stop using the cookie for I/O.
643 void __fscache_unuse_cookie(struct fscache_cookie *cookie,
644 const void *aux_data, const loff_t *object_size)
646 unsigned int debug_id = cookie->debug_id;
647 unsigned int r = refcount_read(&cookie->ref);
648 unsigned int a = atomic_read(&cookie->n_accesses);
651 if (aux_data || object_size)
652 __fscache_update_cookie(cookie, aux_data, object_size);
654 /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
655 c = atomic_fetch_add_unless(&cookie->n_active, -1, 1);
657 trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse);
661 spin_lock(&cookie->lock);
662 r = refcount_read(&cookie->ref);
663 a = atomic_read(&cookie->n_accesses);
664 c = atomic_dec_return(&cookie->n_active);
665 trace_fscache_active(debug_id, r, c, a, fscache_active_unuse);
667 fscache_unuse_cookie_locked(cookie);
668 spin_unlock(&cookie->lock);
670 EXPORT_SYMBOL(__fscache_unuse_cookie);
673 * Perform work upon the cookie, such as committing its cache state,
674 * relinquishing it or withdrawing the backing cache. We're protected from the
675 * cache going away under us as object withdrawal must come through this
676 * non-reentrant work item.
678 static void fscache_cookie_state_machine(struct fscache_cookie *cookie)
680 enum fscache_cookie_state state;
683 _enter("c=%x", cookie->debug_id);
686 spin_lock(&cookie->lock);
688 state = cookie->state;
690 case FSCACHE_COOKIE_STATE_QUIESCENT:
691 /* The QUIESCENT state is jumped to the LOOKING_UP state by
692 * fscache_use_cookie().
695 if (atomic_read(&cookie->n_accesses) == 0 &&
696 test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
697 __fscache_set_cookie_state(cookie,
698 FSCACHE_COOKIE_STATE_RELINQUISHING);
704 case FSCACHE_COOKIE_STATE_LOOKING_UP:
705 spin_unlock(&cookie->lock);
706 fscache_init_access_gate(cookie);
707 fscache_perform_lookup(cookie);
710 case FSCACHE_COOKIE_STATE_INVALIDATING:
711 spin_unlock(&cookie->lock);
712 fscache_perform_invalidation(cookie);
715 case FSCACHE_COOKIE_STATE_ACTIVE:
716 if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) {
717 spin_unlock(&cookie->lock);
718 fscache_prepare_to_write(cookie);
719 spin_lock(&cookie->lock);
721 if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) {
722 __fscache_set_cookie_state(cookie,
723 FSCACHE_COOKIE_STATE_LRU_DISCARDING);
729 case FSCACHE_COOKIE_STATE_FAILED:
730 if (atomic_read(&cookie->n_accesses) != 0)
732 if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
733 __fscache_set_cookie_state(cookie,
734 FSCACHE_COOKIE_STATE_RELINQUISHING);
738 if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) {
739 __fscache_set_cookie_state(cookie,
740 FSCACHE_COOKIE_STATE_WITHDRAWING);
746 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
747 case FSCACHE_COOKIE_STATE_RELINQUISHING:
748 case FSCACHE_COOKIE_STATE_WITHDRAWING:
749 if (cookie->cache_priv) {
750 spin_unlock(&cookie->lock);
751 cookie->volume->cache->ops->withdraw_cookie(cookie);
752 spin_lock(&cookie->lock);
756 case FSCACHE_COOKIE_STATE_RELINQUISHING:
757 fscache_see_cookie(cookie, fscache_cookie_see_relinquish);
758 fscache_unhash_cookie(cookie);
759 __fscache_set_cookie_state(cookie,
760 FSCACHE_COOKIE_STATE_DROPPED);
763 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
764 fscache_see_cookie(cookie, fscache_cookie_see_lru_discard);
766 case FSCACHE_COOKIE_STATE_WITHDRAWING:
767 fscache_see_cookie(cookie, fscache_cookie_see_withdraw);
773 clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags);
774 clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
775 clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
776 clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
777 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
778 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
782 case FSCACHE_COOKIE_STATE_DROPPED:
786 WARN_ONCE(1, "Cookie %x in unexpected state %u\n",
787 cookie->debug_id, state);
792 spin_unlock(&cookie->lock);
794 wake_up_cookie_state(cookie);
798 static void fscache_cookie_worker(struct work_struct *work)
800 struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work);
802 fscache_see_cookie(cookie, fscache_cookie_see_work);
803 fscache_cookie_state_machine(cookie);
804 fscache_put_cookie(cookie, fscache_cookie_put_work);
808 * Wait for the object to become inactive. The cookie's work item will be
809 * scheduled when someone transitions n_accesses to 0 - but if someone's
810 * already done that, schedule it anyway.
812 static void __fscache_withdraw_cookie(struct fscache_cookie *cookie)
817 unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
819 /* Need to read the access count after unpinning */
820 n_accesses = atomic_read(&cookie->n_accesses);
822 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
823 n_accesses, fscache_access_cache_unpin);
825 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
828 static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie)
830 fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one);
832 spin_lock(&cookie->lock);
833 if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE ||
834 time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) ||
835 atomic_read(&cookie->n_active) > 0) {
836 spin_unlock(&cookie->lock);
837 fscache_stat(&fscache_n_cookies_lru_removed);
839 set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
840 spin_unlock(&cookie->lock);
841 fscache_stat(&fscache_n_cookies_lru_expired);
842 _debug("lru c=%x", cookie->debug_id);
843 __fscache_withdraw_cookie(cookie);
846 fscache_put_cookie(cookie, fscache_cookie_put_lru);
849 static void fscache_cookie_lru_worker(struct work_struct *work)
851 struct fscache_cookie *cookie;
852 unsigned long unused_at;
854 spin_lock(&fscache_cookie_lru_lock);
856 while (!list_empty(&fscache_cookie_lru)) {
857 cookie = list_first_entry(&fscache_cookie_lru,
858 struct fscache_cookie, commit_link);
859 unused_at = cookie->unused_at + fscache_lru_cookie_timeout;
860 if (time_before(jiffies, unused_at)) {
861 timer_reduce(&fscache_cookie_lru_timer, unused_at);
865 list_del_init(&cookie->commit_link);
866 fscache_stat_d(&fscache_n_cookies_lru);
867 spin_unlock(&fscache_cookie_lru_lock);
868 fscache_cookie_lru_do_one(cookie);
869 spin_lock(&fscache_cookie_lru_lock);
872 spin_unlock(&fscache_cookie_lru_lock);
875 static void fscache_cookie_lru_timed_out(struct timer_list *timer)
877 queue_work(fscache_wq, &fscache_cookie_lru_work);
880 static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie)
882 bool need_put = false;
884 if (!list_empty(&cookie->commit_link)) {
885 spin_lock(&fscache_cookie_lru_lock);
886 if (!list_empty(&cookie->commit_link)) {
887 list_del_init(&cookie->commit_link);
888 fscache_stat_d(&fscache_n_cookies_lru);
889 fscache_stat(&fscache_n_cookies_lru_dropped);
892 spin_unlock(&fscache_cookie_lru_lock);
894 fscache_put_cookie(cookie, fscache_cookie_put_lru);
899 * Remove a cookie from the hash table.
901 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
903 struct hlist_bl_head *h;
906 bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
907 h = &fscache_cookie_hash[bucket];
910 hlist_bl_del(&cookie->hash_link);
911 clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags);
913 fscache_stat(&fscache_n_relinquishes_dropped);
916 static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie)
918 fscache_cookie_drop_from_lru(cookie);
919 __fscache_withdraw_cookie(cookie);
923 * fscache_withdraw_cookie - Mark a cookie for withdrawal
924 * @cookie: The cookie to be withdrawn.
926 * Allow the cache backend to withdraw the backing for a cookie for its own
927 * reasons, even if that cookie is in active use.
929 void fscache_withdraw_cookie(struct fscache_cookie *cookie)
931 set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
932 fscache_drop_withdraw_cookie(cookie);
934 EXPORT_SYMBOL(fscache_withdraw_cookie);
937 * Allow the netfs to release a cookie back to the cache.
938 * - the object will be marked as recyclable on disk if retire is true
940 void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
942 fscache_stat(&fscache_n_relinquishes);
944 fscache_stat(&fscache_n_relinquishes_retire);
946 _enter("c=%08x{%d},%d",
947 cookie->debug_id, atomic_read(&cookie->n_active), retire);
949 if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
950 "Cookie c=%x already relinquished\n", cookie->debug_id))
954 set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags);
955 trace_fscache_relinquish(cookie, retire);
957 ASSERTCMP(atomic_read(&cookie->n_active), ==, 0);
958 ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0);
959 atomic_dec(&cookie->volume->n_cookies);
961 if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) {
962 set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags);
963 fscache_drop_withdraw_cookie(cookie);
965 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED);
966 fscache_unhash_cookie(cookie);
968 fscache_put_cookie(cookie, fscache_cookie_put_relinquish);
970 EXPORT_SYMBOL(__fscache_relinquish_cookie);
973 * Drop a reference to a cookie.
975 void fscache_put_cookie(struct fscache_cookie *cookie,
976 enum fscache_cookie_trace where)
978 struct fscache_volume *volume = cookie->volume;
979 unsigned int cookie_debug_id = cookie->debug_id;
983 zero = __refcount_dec_and_test(&cookie->ref, &ref);
984 trace_fscache_cookie(cookie_debug_id, ref - 1, where);
986 fscache_free_cookie(cookie);
987 fscache_put_volume(volume, fscache_volume_put_cookie);
990 EXPORT_SYMBOL(fscache_put_cookie);
993 * Get a reference to a cookie.
995 struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie,
996 enum fscache_cookie_trace where)
1000 __refcount_inc(&cookie->ref, &ref);
1001 trace_fscache_cookie(cookie->debug_id, ref + 1, where);
1004 EXPORT_SYMBOL(fscache_get_cookie);
1007 * Ask the cache to effect invalidation of a cookie.
1009 static void fscache_perform_invalidation(struct fscache_cookie *cookie)
1011 if (!cookie->volume->cache->ops->invalidate_cookie(cookie))
1012 fscache_caching_failed(cookie);
1013 fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
1017 * Invalidate an object.
1019 void __fscache_invalidate(struct fscache_cookie *cookie,
1020 const void *aux_data, loff_t new_size,
1025 _enter("c=%x", cookie->debug_id);
1027 fscache_stat(&fscache_n_invalidates);
1029 if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
1030 "Trying to invalidate relinquished cookie\n"))
1033 if ((flags & FSCACHE_INVAL_DIO_WRITE) &&
1034 test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags))
1037 spin_lock(&cookie->lock);
1038 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
1039 fscache_update_aux(cookie, aux_data, &new_size);
1040 cookie->inval_counter++;
1041 trace_fscache_invalidate(cookie, new_size);
1043 switch (cookie->state) {
1044 case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */
1046 spin_unlock(&cookie->lock);
1047 _leave(" [no %u]", cookie->state);
1050 case FSCACHE_COOKIE_STATE_LOOKING_UP:
1051 case FSCACHE_COOKIE_STATE_CREATING:
1052 spin_unlock(&cookie->lock);
1053 _leave(" [look %x]", cookie->inval_counter);
1056 case FSCACHE_COOKIE_STATE_ACTIVE:
1057 is_caching = fscache_begin_cookie_access(
1058 cookie, fscache_access_invalidate_cookie);
1060 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING);
1061 spin_unlock(&cookie->lock);
1062 wake_up_cookie_state(cookie);
1065 fscache_queue_cookie(cookie, fscache_cookie_get_inval_work);
1070 EXPORT_SYMBOL(__fscache_invalidate);
1073 * Generate a list of extant cookies in /proc/fs/fscache/cookies
1075 static int fscache_cookies_seq_show(struct seq_file *m, void *v)
1077 struct fscache_cookie *cookie;
1078 unsigned int keylen = 0, auxlen = 0;
1081 if (v == &fscache_cookies) {
1083 "COOKIE VOLUME REF ACT ACC S FL DEF \n"
1084 "======== ======== === === === = == ================\n"
1089 cookie = list_entry(v, struct fscache_cookie, proc_link);
1092 "%08x %08x %3d %3d %3d %c %02lx",
1094 cookie->volume->debug_id,
1095 refcount_read(&cookie->ref),
1096 atomic_read(&cookie->n_active),
1097 atomic_read(&cookie->n_accesses),
1098 fscache_cookie_states[cookie->state],
1101 keylen = cookie->key_len;
1102 auxlen = cookie->aux_len;
1104 if (keylen > 0 || auxlen > 0) {
1106 p = keylen <= sizeof(cookie->inline_key) ?
1107 cookie->inline_key : cookie->key;
1108 for (; keylen > 0; keylen--)
1109 seq_printf(m, "%02x", *p++);
1112 p = auxlen <= sizeof(cookie->inline_aux) ?
1113 cookie->inline_aux : cookie->aux;
1114 for (; auxlen > 0; auxlen--)
1115 seq_printf(m, "%02x", *p++);
1123 static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
1124 __acquires(fscache_cookies_lock)
1126 read_lock(&fscache_cookies_lock);
1127 return seq_list_start_head(&fscache_cookies, *_pos);
1130 static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
1132 return seq_list_next(v, &fscache_cookies, _pos);
1135 static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
1138 read_unlock(&fscache_cookies_lock);
1142 const struct seq_operations fscache_cookies_seq_ops = {
1143 .start = fscache_cookies_seq_start,
1144 .next = fscache_cookies_seq_next,
1145 .stop = fscache_cookies_seq_stop,
1146 .show = fscache_cookies_seq_show,