projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
fscache: Introduce fscache_cookie_is_dropped()
[linux-2.6-block.git] / fs / fscache / cookie.c
CommitLineData
7f3283ab
DH		 1// SPDX-License-Identifier: GPL-2.0-or-later
2/* netfs cookie management
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 * See Documentation/filesystems/caching/netfs-api.rst for more information on
8 * the netfs API.
9 */
10
11#define FSCACHE_DEBUG_LEVEL COOKIE
12#include <linux/module.h>
13#include <linux/slab.h>
14#include "internal.h"
15
16struct kmem_cache *fscache_cookie_jar;
17
12bb21a2
DH		 18static void fscache_cookie_lru_timed_out(struct timer_list *timer);
19static void fscache_cookie_lru_worker(struct work_struct *work);
5d00e426
DH		 20static void fscache_cookie_worker(struct work_struct *work);
21static void fscache_unhash_cookie(struct fscache_cookie *cookie);
d24af13e 22static void fscache_perform_invalidation(struct fscache_cookie *cookie);
7f3283ab
DH		 23
24#define fscache_cookie_hash_shift 15
25static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
26static LIST_HEAD(fscache_cookies);
27static DEFINE_RWLOCK(fscache_cookies_lock);
12bb21a2
DH		 28static LIST_HEAD(fscache_cookie_lru);
29static DEFINE_SPINLOCK(fscache_cookie_lru_lock);
30DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out);
31static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker);
d24af13e 32static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD";
19517e53 33static unsigned int fscache_lru_cookie_timeout = 10 * HZ;
7f3283ab
DH		 34
35void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
36{
37 const u8 *k;
38
39 pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n",
40 prefix,
41 cookie->debug_id,
42 cookie->flags,
43 atomic_read(&cookie->n_active),
44 atomic_read(&cookie->n_accesses),
45 fscache_cookie_states[cookie->state]);
46 pr_err("%c-cookie V=%08x [%s]\n",
47 prefix,
48 cookie->volume->debug_id,
49 cookie->volume->key);
50
51 k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
52 cookie->inline_key : cookie->key;
53 pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k);
54}
55
56static void fscache_free_cookie(struct fscache_cookie *cookie)
57{
12bb21a2
DH		 58 if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) {
59 spin_lock(&fscache_cookie_lru_lock);
60 list_del_init(&cookie->commit_link);
61 spin_unlock(&fscache_cookie_lru_lock);
62 fscache_stat_d(&fscache_n_cookies_lru);
63 fscache_stat(&fscache_n_cookies_lru_removed);
64 }
65
7f3283ab
DH		 66 if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) {
67 fscache_print_cookie(cookie, 'F');
68 return;
69 }
70
71 write_lock(&fscache_cookies_lock);
72 list_del(&cookie->proc_link);
73 write_unlock(&fscache_cookies_lock);
74 if (cookie->aux_len > sizeof(cookie->inline_aux))
75 kfree(cookie->aux);
76 if (cookie->key_len > sizeof(cookie->inline_key))
77 kfree(cookie->key);
78 fscache_stat_d(&fscache_n_cookies);
79 kmem_cache_free(fscache_cookie_jar, cookie);
80}
81
5d00e426
DH		 82static void __fscache_queue_cookie(struct fscache_cookie *cookie)
83{
84 if (!queue_work(fscache_wq, &cookie->work))
85 fscache_put_cookie(cookie, fscache_cookie_put_over_queued);
86}
87
88static void fscache_queue_cookie(struct fscache_cookie *cookie,
89 enum fscache_cookie_trace where)
90{
91 fscache_get_cookie(cookie, where);
92 __fscache_queue_cookie(cookie);
93}
94
a7733fb6
DH		 95/*
96 * Initialise the access gate on a cookie by setting a flag to prevent the
97 * state machine from being queued when the access counter transitions to 0.
98 * We're only interested in this when we withdraw caching services from the
99 * cookie.
100 */
101static void fscache_init_access_gate(struct fscache_cookie *cookie)
102{
103 int n_accesses;
104
105 n_accesses = atomic_read(&cookie->n_accesses);
106 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
107 n_accesses, fscache_access_cache_pin);
108 set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
109}
110
111/**
112 * fscache_end_cookie_access - Unpin a cache at the end of an access.
113 * @cookie: A data file cookie
114 * @why: An indication of the circumstances of the access for tracing
115 *
116 * Unpin a cache cookie after we've accessed it and bring a deferred
117 * relinquishment or withdrawal state into effect.
118 *
119 * The @why indicator is provided for tracing purposes.
120 */
121void fscache_end_cookie_access(struct fscache_cookie *cookie,
122 enum fscache_access_trace why)
123{
124 int n_accesses;
125
126 smp_mb__before_atomic();
127 n_accesses = atomic_dec_return(&cookie->n_accesses);
128 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
129 n_accesses, why);
130 if (n_accesses == 0 &&
5d00e426
DH		 131 !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags))
132 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
a7733fb6
DH		 133}
134EXPORT_SYMBOL(fscache_end_cookie_access);
135
136/*
137 * Pin the cache behind a cookie so that we can access it.
138 */
139static void __fscache_begin_cookie_access(struct fscache_cookie *cookie,
140 enum fscache_access_trace why)
141{
142 int n_accesses;
143
144 n_accesses = atomic_inc_return(&cookie->n_accesses);
145 smp_mb__after_atomic(); /* (Future) read state after is-caching.
146 * Reread n_accesses after is-caching
147 */
148 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
149 n_accesses, why);
150}
151
152/**
153 * fscache_begin_cookie_access - Pin a cache so data can be accessed
154 * @cookie: A data file cookie
155 * @why: An indication of the circumstances of the access for tracing
156 *
157 * Attempt to pin the cache to prevent it from going away whilst we're
158 * accessing data and returns true if successful. This works as follows:
159 *
160 * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not
161 * set), we return false to indicate access was not permitted.
162 *
163 * (2) If the cookie is being cached, we increment its n_accesses count and
164 * then recheck the IS_CACHING flag, ending the access if it got cleared.
165 *
166 * (3) When we end the access, we decrement the cookie's n_accesses and wake
167 * up the any waiters if it reaches 0.
168 *
169 * (4) Whilst the cookie is actively being cached, its n_accesses is kept
170 * artificially incremented to prevent wakeups from happening.
171 *
172 * (5) When the cache is taken offline or if the cookie is culled, the flag is
173 * cleared to prevent new accesses, the cookie's n_accesses is decremented
174 * and we wait for it to become 0.
175 *
176 * The @why indicator are merely provided for tracing purposes.
177 */
178bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
179 enum fscache_access_trace why)
180{
181 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
182 return false;
183 __fscache_begin_cookie_access(cookie, why);
184 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) ||
185 !fscache_cache_is_live(cookie->volume->cache)) {
186 fscache_end_cookie_access(cookie, fscache_access_unlive);
187 return false;
188 }
189 return true;
190}
191
7f3283ab
DH		 192static inline void wake_up_cookie_state(struct fscache_cookie *cookie)
193{
194 /* Use a barrier to ensure that waiters see the state variable
195 * change, as spin_unlock doesn't guarantee a barrier.
196 *
197 * See comments over wake_up_bit() and waitqueue_active().
198 */
199 smp_mb();
200 wake_up_var(&cookie->state);
201}
202
5d00e426
DH		 203/*
204 * Change the state a cookie is at and wake up anyone waiting for that. Impose
205 * an ordering between the stuff stored in the cookie and the state member.
206 * Paired with fscache_cookie_state().
207 */
7f3283ab
DH		 208static void __fscache_set_cookie_state(struct fscache_cookie *cookie,
209 enum fscache_cookie_state state)
210{
5d00e426 211 smp_store_release(&cookie->state, state);
7f3283ab
DH		 212}
213
5d00e426
DH		 214static void fscache_set_cookie_state(struct fscache_cookie *cookie,
215 enum fscache_cookie_state state)
7f3283ab 216{
7f3283ab 217 spin_lock(&cookie->lock);
5d00e426 218 __fscache_set_cookie_state(cookie, state);
7f3283ab 219 spin_unlock(&cookie->lock);
5d00e426
DH		 220 wake_up_cookie_state(cookie);
221}
222
223/**
224 * fscache_cookie_lookup_negative - Note negative lookup
225 * @cookie: The cookie that was being looked up
226 *
227 * Note that some part of the metadata path in the cache doesn't exist and so
228 * we can release any waiting readers in the certain knowledge that there's
229 * nothing for them to actually read.
230 *
231 * This function uses no locking and must only be called from the state machine.
232 */
233void fscache_cookie_lookup_negative(struct fscache_cookie *cookie)
234{
235 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
236 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING);
237}
238EXPORT_SYMBOL(fscache_cookie_lookup_negative);
239
d24af13e
DH		 240/**
241 * fscache_resume_after_invalidation - Allow I/O to resume after invalidation
242 * @cookie: The cookie that was invalidated
243 *
244 * Tell fscache that invalidation is sufficiently complete that I/O can be
245 * allowed again.
246 */
247void fscache_resume_after_invalidation(struct fscache_cookie *cookie)
248{
249 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
250}
251EXPORT_SYMBOL(fscache_resume_after_invalidation);
252
5d00e426
DH		 253/**
254 * fscache_caching_failed - Report that a failure stopped caching on a cookie
255 * @cookie: The cookie that was affected
256 *
257 * Tell fscache that caching on a cookie needs to be stopped due to some sort
258 * of failure.
259 *
260 * This function uses no locking and must only be called from the state machine.
261 */
262void fscache_caching_failed(struct fscache_cookie *cookie)
263{
264 clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
265 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED);
7f3283ab 266}
5d00e426 267EXPORT_SYMBOL(fscache_caching_failed);
7f3283ab
DH		 268
269/*
270 * Set the index key in a cookie. The cookie struct has space for a 16-byte
271 * key plus length and hash, but if that's not big enough, it's instead a
272 * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
273 * the key data.
274 */
275static int fscache_set_key(struct fscache_cookie *cookie,
276 const void *index_key, size_t index_key_len)
277{
278 void *buf;
279 size_t buf_size;
280
281 buf_size = round_up(index_key_len, sizeof(__le32));
282
283 if (index_key_len > sizeof(cookie->inline_key)) {
284 buf = kzalloc(buf_size, GFP_KERNEL);
285 if (!buf)
286 return -ENOMEM;
287 cookie->key = buf;
288 } else {
289 buf = cookie->inline_key;
290 }
291
292 memcpy(buf, index_key, index_key_len);
293 cookie->key_hash = fscache_hash(cookie->volume->key_hash,
294 buf, buf_size);
295 return 0;
296}
297
298static bool fscache_cookie_same(const struct fscache_cookie *a,
299 const struct fscache_cookie *b)
300{
301 const void *ka, *kb;
302
303 if (a->key_hash != b->key_hash ||
304 a->volume != b->volume ||
305 a->key_len != b->key_len)
306 return false;
307
308 if (a->key_len <= sizeof(a->inline_key)) {
309 ka = &a->inline_key;
310 kb = &b->inline_key;
311 } else {
312 ka = a->key;
313 kb = b->key;
314 }
315 return memcmp(ka, kb, a->key_len) == 0;
316}
317
318static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
319
320/*
321 * Allocate a cookie.
322 */
323static struct fscache_cookie *fscache_alloc_cookie(
324 struct fscache_volume *volume,
325 u8 advice,
326 const void *index_key, size_t index_key_len,
327 const void *aux_data, size_t aux_data_len,
328 loff_t object_size)
329{
330 struct fscache_cookie *cookie;
331
332 /* allocate and initialise a cookie */
333 cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
334 if (!cookie)
335 return NULL;
336 fscache_stat(&fscache_n_cookies);
337
338 cookie->volume = volume;
339 cookie->advice = advice;
340 cookie->key_len = index_key_len;
341 cookie->aux_len = aux_data_len;
342 cookie->object_size = object_size;
343 if (object_size == 0)
344 __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
345
346 if (fscache_set_key(cookie, index_key, index_key_len) < 0)
347 goto nomem;
348
349 if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
350 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
351 } else {
352 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
353 if (!cookie->aux)
354 goto nomem;
355 }
356
357 refcount_set(&cookie->ref, 1);
358 cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
7f3283ab
DH		 359 spin_lock_init(&cookie->lock);
360 INIT_LIST_HEAD(&cookie->commit_link);
5d00e426
DH		 361 INIT_WORK(&cookie->work, fscache_cookie_worker);
362 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
7f3283ab
DH		 363
364 write_lock(&fscache_cookies_lock);
365 list_add_tail(&cookie->proc_link, &fscache_cookies);
366 write_unlock(&fscache_cookies_lock);
367 fscache_see_cookie(cookie, fscache_cookie_new_acquire);
368 return cookie;
369
370nomem:
371 fscache_free_cookie(cookie);
372 return NULL;
373}
374
5c4588ae
YH		 375static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie)
376{
377 return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED;
378}
379
7f3283ab
DH		 380static void fscache_wait_on_collision(struct fscache_cookie *candidate,
381 struct fscache_cookie *wait_for)
382{
383 enum fscache_cookie_state *statep = &wait_for->state;
384
5c4588ae 385 wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for),
7f3283ab 386 20 * HZ);
5c4588ae 387 if (!fscache_cookie_is_dropped(wait_for)) {
7f3283ab
DH		 388 pr_notice("Potential collision c=%08x old: c=%08x",
389 candidate->debug_id, wait_for->debug_id);
5c4588ae 390 wait_var_event(statep, fscache_cookie_is_dropped(wait_for));
7f3283ab
DH		 391 }
392}
393
394/*
395 * Attempt to insert the new cookie into the hash. If there's a collision, we
396 * wait for the old cookie to complete if it's being relinquished and an error
397 * otherwise.
398 */
399static bool fscache_hash_cookie(struct fscache_cookie *candidate)
400{
401 struct fscache_cookie *cursor, *wait_for = NULL;
402 struct hlist_bl_head *h;
403 struct hlist_bl_node *p;
404 unsigned int bucket;
405
406 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
407 h = &fscache_cookie_hash[bucket];
408
409 hlist_bl_lock(h);
410 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
411 if (fscache_cookie_same(candidate, cursor)) {
412 if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags))
413 goto collision;
414 wait_for = fscache_get_cookie(cursor,
415 fscache_cookie_get_hash_collision);
416 break;
417 }
418 }
419
420 fscache_get_volume(candidate->volume, fscache_volume_get_cookie);
421 atomic_inc(&candidate->volume->n_cookies);
422 hlist_bl_add_head(&candidate->hash_link, h);
423 set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags);
424 hlist_bl_unlock(h);
425
426 if (wait_for) {
427 fscache_wait_on_collision(candidate, wait_for);
428 fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision);
429 }
430 return true;
431
432collision:
433 trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref),
434 fscache_cookie_collision);
435 pr_err("Duplicate cookie detected\n");
436 fscache_print_cookie(cursor, 'O');
437 fscache_print_cookie(candidate, 'N');
438 hlist_bl_unlock(h);
439 return false;
440}
441
442/*
443 * Request a cookie to represent a data storage object within a volume.
444 *
445 * We never let on to the netfs about errors. We may set a negative cookie
446 * pointer, but that's okay
447 */
448struct fscache_cookie *__fscache_acquire_cookie(
449 struct fscache_volume *volume,
450 u8 advice,
451 const void *index_key, size_t index_key_len,
452 const void *aux_data, size_t aux_data_len,
453 loff_t object_size)
454{
455 struct fscache_cookie *cookie;
456
457 _enter("V=%x", volume->debug_id);
458
459 if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
460 return NULL;
461 if (!aux_data || !aux_data_len) {
462 aux_data = NULL;
463 aux_data_len = 0;
464 }
465
466 fscache_stat(&fscache_n_acquires);
467
468 cookie = fscache_alloc_cookie(volume, advice,
469 index_key, index_key_len,
470 aux_data, aux_data_len,
471 object_size);
472 if (!cookie) {
473 fscache_stat(&fscache_n_acquires_oom);
474 return NULL;
475 }
476
477 if (!fscache_hash_cookie(cookie)) {
478 fscache_see_cookie(cookie, fscache_cookie_discard);
479 fscache_free_cookie(cookie);
480 return NULL;
481 }
482
483 trace_fscache_acquire(cookie);
484 fscache_stat(&fscache_n_acquires_ok);
485 _leave(" = c=%08x", cookie->debug_id);
486 return cookie;
487}
488EXPORT_SYMBOL(__fscache_acquire_cookie);
489
5d00e426
DH		 490/*
491 * Prepare a cache object to be written to.
492 */
493static void fscache_prepare_to_write(struct fscache_cookie *cookie)
494{
495 cookie->volume->cache->ops->prepare_to_write(cookie);
496}
497
498/*
499 * Look up a cookie in the cache.
500 */
501static void fscache_perform_lookup(struct fscache_cookie *cookie)
502{
503 enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed;
504 bool need_withdraw = false;
505
506 _enter("");
507
508 if (!cookie->volume->cache_priv) {
509 fscache_create_volume(cookie->volume, true);
510 if (!cookie->volume->cache_priv) {
511 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
512 goto out;
513 }
514 }
515
516 if (!cookie->volume->cache->ops->lookup_cookie(cookie)) {
517 if (cookie->state != FSCACHE_COOKIE_STATE_FAILED)
518 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
519 need_withdraw = true;
520 _leave(" [fail]");
521 goto out;
522 }
523
524 fscache_see_cookie(cookie, fscache_cookie_see_active);
525 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
526 trace = fscache_access_lookup_cookie_end;
527
528out:
529 fscache_end_cookie_access(cookie, trace);
530 if (need_withdraw)
531 fscache_withdraw_cookie(cookie);
532 fscache_end_volume_access(cookie->volume, cookie, trace);
533}
534
12bb21a2
DH		 535/*
536 * Begin the process of looking up a cookie. We offload the actual process to
537 * a worker thread.
538 */
539static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify)
540{
541 if (will_modify) {
542 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
543 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
544 }
545 if (!fscache_begin_volume_access(cookie->volume, cookie,
546 fscache_access_lookup_cookie))
547 return false;
548
549 __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie);
550 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP);
551 set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
552 set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags);
553 return true;
554}
555
556/*
557 * Start using the cookie for I/O. This prevents the backing object from being
558 * reaped by VM pressure.
559 */
560void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify)
561{
562 enum fscache_cookie_state state;
563 bool queue = false;
e6435f1e 564 int n_active;
12bb21a2
DH		 565
566 _enter("c=%08x", cookie->debug_id);
567
568 if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
569 "Trying to use relinquished cookie\n"))
570 return;
571
572 spin_lock(&cookie->lock);
573
e6435f1e
DH		 574 n_active = atomic_inc_return(&cookie->n_active);
575 trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref),
576 n_active, atomic_read(&cookie->n_accesses),
577 will_modify ?
578 fscache_active_use_modify : fscache_active_use);
12bb21a2
DH		 579
580again:
581 state = fscache_cookie_state(cookie);
582 switch (state) {
583 case FSCACHE_COOKIE_STATE_QUIESCENT:
584 queue = fscache_begin_lookup(cookie, will_modify);
585 break;
586
587 case FSCACHE_COOKIE_STATE_LOOKING_UP:
588 case FSCACHE_COOKIE_STATE_CREATING:
589 if (will_modify)
590 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
591 break;
592 case FSCACHE_COOKIE_STATE_ACTIVE:
d24af13e 593 case FSCACHE_COOKIE_STATE_INVALIDATING:
12bb21a2
DH		 594 if (will_modify &&
595 !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) {
596 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
597 queue = true;
598 }
599 break;
600
601 case FSCACHE_COOKIE_STATE_FAILED:
602 case FSCACHE_COOKIE_STATE_WITHDRAWING:
603 break;
604
605 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
606 spin_unlock(&cookie->lock);
607 wait_var_event(&cookie->state,
608 fscache_cookie_state(cookie) !=
609 FSCACHE_COOKIE_STATE_LRU_DISCARDING);
610 spin_lock(&cookie->lock);
611 goto again;
612
613 case FSCACHE_COOKIE_STATE_DROPPED:
614 case FSCACHE_COOKIE_STATE_RELINQUISHING:
615 WARN(1, "Can't use cookie in state %u\n", state);
616 break;
617 }
618
619 spin_unlock(&cookie->lock);
620 if (queue)
621 fscache_queue_cookie(cookie, fscache_cookie_get_use_work);
622 _leave("");
623}
624EXPORT_SYMBOL(__fscache_use_cookie);
625
626static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
627{
628 clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags);
629 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
630 return;
631
632 cookie->unused_at = jiffies;
633 spin_lock(&fscache_cookie_lru_lock);
634 if (list_empty(&cookie->commit_link)) {
635 fscache_get_cookie(cookie, fscache_cookie_get_lru);
636 fscache_stat(&fscache_n_cookies_lru);
637 }
638 list_move_tail(&cookie->commit_link, &fscache_cookie_lru);
639
640 spin_unlock(&fscache_cookie_lru_lock);
641 timer_reduce(&fscache_cookie_lru_timer,
642 jiffies + fscache_lru_cookie_timeout);
643}
644
645/*
646 * Stop using the cookie for I/O.
647 */
648void __fscache_unuse_cookie(struct fscache_cookie *cookie,
649 const void *aux_data, const loff_t *object_size)
650{
e6435f1e
DH		 651 unsigned int debug_id = cookie->debug_id;
652 unsigned int r = refcount_read(&cookie->ref);
653 unsigned int a = atomic_read(&cookie->n_accesses);
654 unsigned int c;
655
12bb21a2
DH		 656 if (aux_data || object_size)
657 __fscache_update_cookie(cookie, aux_data, object_size);
658
e6435f1e
DH		 659 /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
660 c = atomic_fetch_add_unless(&cookie->n_active, -1, 1);
661 if (c != 1) {
662 trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse);
663 return;
12bb21a2 664 }
e6435f1e
DH		 665
666 spin_lock(&cookie->lock);
667 r = refcount_read(&cookie->ref);
668 a = atomic_read(&cookie->n_accesses);
669 c = atomic_dec_return(&cookie->n_active);
670 trace_fscache_active(debug_id, r, c, a, fscache_active_unuse);
671 if (c == 0)
672 fscache_unuse_cookie_locked(cookie);
673 spin_unlock(&cookie->lock);
12bb21a2
DH		 674}
675EXPORT_SYMBOL(__fscache_unuse_cookie);
676
5d00e426
DH		 677/*
678 * Perform work upon the cookie, such as committing its cache state,
679 * relinquishing it or withdrawing the backing cache. We're protected from the
680 * cache going away under us as object withdrawal must come through this
681 * non-reentrant work item.
682 */
683static void fscache_cookie_state_machine(struct fscache_cookie *cookie)
684{
685 enum fscache_cookie_state state;
686 bool wake = false;
687
688 _enter("c=%x", cookie->debug_id);
689
690again:
691 spin_lock(&cookie->lock);
692again_locked:
693 state = cookie->state;
694 switch (state) {
695 case FSCACHE_COOKIE_STATE_QUIESCENT:
696 /* The QUIESCENT state is jumped to the LOOKING_UP state by
697 * fscache_use_cookie().
698 */
699
700 if (atomic_read(&cookie->n_accesses) == 0 &&
701 test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
702 __fscache_set_cookie_state(cookie,
703 FSCACHE_COOKIE_STATE_RELINQUISHING);
704 wake = true;
705 goto again_locked;
706 }
707 break;
708
709 case FSCACHE_COOKIE_STATE_LOOKING_UP:
710 spin_unlock(&cookie->lock);
711 fscache_init_access_gate(cookie);
712 fscache_perform_lookup(cookie);
713 goto again;
714
d24af13e
DH		 715 case FSCACHE_COOKIE_STATE_INVALIDATING:
716 spin_unlock(&cookie->lock);
717 fscache_perform_invalidation(cookie);
718 goto again;
719
5d00e426
DH		 720 case FSCACHE_COOKIE_STATE_ACTIVE:
721 if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) {
722 spin_unlock(&cookie->lock);
723 fscache_prepare_to_write(cookie);
724 spin_lock(&cookie->lock);
725 }
12bb21a2
DH		 726 if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) {
727 __fscache_set_cookie_state(cookie,
728 FSCACHE_COOKIE_STATE_LRU_DISCARDING);
729 wake = true;
730 goto again_locked;
731 }
5d00e426
DH		 732 fallthrough;
733
734 case FSCACHE_COOKIE_STATE_FAILED:
735 if (atomic_read(&cookie->n_accesses) != 0)
736 break;
737 if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
738 __fscache_set_cookie_state(cookie,
739 FSCACHE_COOKIE_STATE_RELINQUISHING);
740 wake = true;
741 goto again_locked;
742 }
743 if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) {
744 __fscache_set_cookie_state(cookie,
745 FSCACHE_COOKIE_STATE_WITHDRAWING);
746 wake = true;
747 goto again_locked;
748 }
749 break;
750
12bb21a2 751 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
5d00e426
DH		 752 case FSCACHE_COOKIE_STATE_RELINQUISHING:
753 case FSCACHE_COOKIE_STATE_WITHDRAWING:
754 if (cookie->cache_priv) {
755 spin_unlock(&cookie->lock);
756 cookie->volume->cache->ops->withdraw_cookie(cookie);
757 spin_lock(&cookie->lock);
758 }
759
760 switch (state) {
761 case FSCACHE_COOKIE_STATE_RELINQUISHING:
762 fscache_see_cookie(cookie, fscache_cookie_see_relinquish);
763 fscache_unhash_cookie(cookie);
764 __fscache_set_cookie_state(cookie,
765 FSCACHE_COOKIE_STATE_DROPPED);
766 wake = true;
767 goto out;
12bb21a2
DH		 768 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
769 fscache_see_cookie(cookie, fscache_cookie_see_lru_discard);
770 break;
5d00e426
DH		 771 case FSCACHE_COOKIE_STATE_WITHDRAWING:
772 fscache_see_cookie(cookie, fscache_cookie_see_withdraw);
773 break;
774 default:
775 BUG();
776 }
777
778 clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags);
779 clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
780 clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
781 clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
782 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
783 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
784 wake = true;
785 goto again_locked;
786
787 case FSCACHE_COOKIE_STATE_DROPPED:
788 break;
789
790 default:
791 WARN_ONCE(1, "Cookie %x in unexpected state %u\n",
792 cookie->debug_id, state);
793 break;
794 }
795
796out:
797 spin_unlock(&cookie->lock);
798 if (wake)
799 wake_up_cookie_state(cookie);
800 _leave("");
801}
802
803static void fscache_cookie_worker(struct work_struct *work)
804{
805 struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work);
806
807 fscache_see_cookie(cookie, fscache_cookie_see_work);
808 fscache_cookie_state_machine(cookie);
809 fscache_put_cookie(cookie, fscache_cookie_put_work);
810}
811
812/*
813 * Wait for the object to become inactive. The cookie's work item will be
814 * scheduled when someone transitions n_accesses to 0 - but if someone's
815 * already done that, schedule it anyway.
816 */
817static void __fscache_withdraw_cookie(struct fscache_cookie *cookie)
818{
819 int n_accesses;
820 bool unpinned;
821
822 unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
823
824 /* Need to read the access count after unpinning */
825 n_accesses = atomic_read(&cookie->n_accesses);
826 if (unpinned)
827 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
828 n_accesses, fscache_access_cache_unpin);
829 if (n_accesses == 0)
830 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
831}
832
12bb21a2
DH		 833static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie)
834{
835 fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one);
836
837 spin_lock(&cookie->lock);
838 if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE ||
839 time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) ||
840 atomic_read(&cookie->n_active) > 0) {
841 spin_unlock(&cookie->lock);
842 fscache_stat(&fscache_n_cookies_lru_removed);
843 } else {
844 set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
845 spin_unlock(&cookie->lock);
846 fscache_stat(&fscache_n_cookies_lru_expired);
847 _debug("lru c=%x", cookie->debug_id);
848 __fscache_withdraw_cookie(cookie);
849 }
850
851 fscache_put_cookie(cookie, fscache_cookie_put_lru);
852}
853
854static void fscache_cookie_lru_worker(struct work_struct *work)
855{
856 struct fscache_cookie *cookie;
857 unsigned long unused_at;
858
859 spin_lock(&fscache_cookie_lru_lock);
860
861 while (!list_empty(&fscache_cookie_lru)) {
862 cookie = list_first_entry(&fscache_cookie_lru,
863 struct fscache_cookie, commit_link);
864 unused_at = cookie->unused_at + fscache_lru_cookie_timeout;
865 if (time_before(jiffies, unused_at)) {
866 timer_reduce(&fscache_cookie_lru_timer, unused_at);
867 break;
868 }
869
870 list_del_init(&cookie->commit_link);
871 fscache_stat_d(&fscache_n_cookies_lru);
872 spin_unlock(&fscache_cookie_lru_lock);
873 fscache_cookie_lru_do_one(cookie);
874 spin_lock(&fscache_cookie_lru_lock);
875 }
876
877 spin_unlock(&fscache_cookie_lru_lock);
878}
879
880static void fscache_cookie_lru_timed_out(struct timer_list *timer)
881{
882 queue_work(fscache_wq, &fscache_cookie_lru_work);
883}
884
885static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie)
886{
887 bool need_put = false;
888
889 if (!list_empty(&cookie->commit_link)) {
890 spin_lock(&fscache_cookie_lru_lock);
891 if (!list_empty(&cookie->commit_link)) {
892 list_del_init(&cookie->commit_link);
893 fscache_stat_d(&fscache_n_cookies_lru);
894 fscache_stat(&fscache_n_cookies_lru_dropped);
895 need_put = true;
896 }
897 spin_unlock(&fscache_cookie_lru_lock);
898 if (need_put)
899 fscache_put_cookie(cookie, fscache_cookie_put_lru);
900 }
901}
902
7f3283ab
DH		 903/*
904 * Remove a cookie from the hash table.
905 */
906static void fscache_unhash_cookie(struct fscache_cookie *cookie)
907{
908 struct hlist_bl_head *h;
909 unsigned int bucket;
910
911 bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
912 h = &fscache_cookie_hash[bucket];
913
914 hlist_bl_lock(h);
915 hlist_bl_del(&cookie->hash_link);
916 clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags);
917 hlist_bl_unlock(h);
5d00e426 918 fscache_stat(&fscache_n_relinquishes_dropped);
7f3283ab
DH		 919}
920
5d00e426 921static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie)
7f3283ab 922{
12bb21a2 923 fscache_cookie_drop_from_lru(cookie);
5d00e426
DH		 924 __fscache_withdraw_cookie(cookie);
925}
7f3283ab 926
5d00e426
DH		 927/**
928 * fscache_withdraw_cookie - Mark a cookie for withdrawal
929 * @cookie: The cookie to be withdrawn.
930 *
931 * Allow the cache backend to withdraw the backing for a cookie for its own
932 * reasons, even if that cookie is in active use.
933 */
934void fscache_withdraw_cookie(struct fscache_cookie *cookie)
935{
936 set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
937 fscache_drop_withdraw_cookie(cookie);
7f3283ab 938}
5d00e426 939EXPORT_SYMBOL(fscache_withdraw_cookie);
7f3283ab
DH		 940
941/*
942 * Allow the netfs to release a cookie back to the cache.
943 * - the object will be marked as recyclable on disk if retire is true
944 */
945void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
946{
947 fscache_stat(&fscache_n_relinquishes);
948 if (retire)
949 fscache_stat(&fscache_n_relinquishes_retire);
950
951 _enter("c=%08x{%d},%d",
952 cookie->debug_id, atomic_read(&cookie->n_active), retire);
953
954 if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
955 "Cookie c=%x already relinquished\n", cookie->debug_id))
956 return;
957
958 if (retire)
959 set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags);
960 trace_fscache_relinquish(cookie, retire);
961
962 ASSERTCMP(atomic_read(&cookie->n_active), ==, 0);
963 ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0);
964 atomic_dec(&cookie->volume->n_cookies);
965
5d00e426
DH		 966 if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) {
967 set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags);
968 fscache_drop_withdraw_cookie(cookie);
969 } else {
970 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED);
971 fscache_unhash_cookie(cookie);
972 }
7f3283ab
DH		 973 fscache_put_cookie(cookie, fscache_cookie_put_relinquish);
974}
975EXPORT_SYMBOL(__fscache_relinquish_cookie);
976
977/*
978 * Drop a reference to a cookie.
979 */
980void fscache_put_cookie(struct fscache_cookie *cookie,
981 enum fscache_cookie_trace where)
982{
983 struct fscache_volume *volume = cookie->volume;
984 unsigned int cookie_debug_id = cookie->debug_id;
985 bool zero;
986 int ref;
987
988 zero = __refcount_dec_and_test(&cookie->ref, &ref);
989 trace_fscache_cookie(cookie_debug_id, ref - 1, where);
990 if (zero) {
991 fscache_free_cookie(cookie);
992 fscache_put_volume(volume, fscache_volume_put_cookie);
993 }
994}
995EXPORT_SYMBOL(fscache_put_cookie);
996
997/*
998 * Get a reference to a cookie.
999 */
1000struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie,
1001 enum fscache_cookie_trace where)
1002{
1003 int ref;
1004
1005 __refcount_inc(&cookie->ref, &ref);
1006 trace_fscache_cookie(cookie->debug_id, ref + 1, where);
1007 return cookie;
1008}
1009EXPORT_SYMBOL(fscache_get_cookie);
1010
d24af13e
DH		 1011/*
1012 * Ask the cache to effect invalidation of a cookie.
1013 */
1014static void fscache_perform_invalidation(struct fscache_cookie *cookie)
1015{
1016 if (!cookie->volume->cache->ops->invalidate_cookie(cookie))
1017 fscache_caching_failed(cookie);
1018 fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
1019}
1020
1021/*
1022 * Invalidate an object.
1023 */
1024void __fscache_invalidate(struct fscache_cookie *cookie,
1025 const void *aux_data, loff_t new_size,
1026 unsigned int flags)
1027{
1028 bool is_caching;
1029
1030 _enter("c=%x", cookie->debug_id);
1031
1032 fscache_stat(&fscache_n_invalidates);
1033
1034 if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
1035 "Trying to invalidate relinquished cookie\n"))
1036 return;
1037
1038 if ((flags & FSCACHE_INVAL_DIO_WRITE) &&
1039 test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags))
1040 return;
1041
1042 spin_lock(&cookie->lock);
1043 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
1044 fscache_update_aux(cookie, aux_data, &new_size);
1045 cookie->inval_counter++;
1046 trace_fscache_invalidate(cookie, new_size);
1047
1048 switch (cookie->state) {
1049 case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */
1050 default:
1051 spin_unlock(&cookie->lock);
1052 _leave(" [no %u]", cookie->state);
1053 return;
1054
1055 case FSCACHE_COOKIE_STATE_LOOKING_UP:
1056 case FSCACHE_COOKIE_STATE_CREATING:
1057 spin_unlock(&cookie->lock);
1058 _leave(" [look %x]", cookie->inval_counter);
1059 return;
1060
1061 case FSCACHE_COOKIE_STATE_ACTIVE:
1062 is_caching = fscache_begin_cookie_access(
1063 cookie, fscache_access_invalidate_cookie);
1064 if (is_caching)
1065 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING);
1066 spin_unlock(&cookie->lock);
1067 wake_up_cookie_state(cookie);
1068
1069 if (is_caching)
1070 fscache_queue_cookie(cookie, fscache_cookie_get_inval_work);
1071 _leave(" [inv]");
1072 return;
1073 }
1074}
1075EXPORT_SYMBOL(__fscache_invalidate);
1076
19517e53 1077#ifdef CONFIG_PROC_FS
7f3283ab
DH		 1078/*
1079 * Generate a list of extant cookies in /proc/fs/fscache/cookies
1080 */
1081static int fscache_cookies_seq_show(struct seq_file *m, void *v)
1082{
1083 struct fscache_cookie *cookie;
1084 unsigned int keylen = 0, auxlen = 0;
1085 u8 *p;
1086
1087 if (v == &fscache_cookies) {
1088 seq_puts(m,
1089 "COOKIE VOLUME REF ACT ACC S FL DEF \n"
1090 "======== ======== === === === = == ================\n"
1091 );
1092 return 0;
1093 }
1094
1095 cookie = list_entry(v, struct fscache_cookie, proc_link);
1096
1097 seq_printf(m,
1098 "%08x %08x %3d %3d %3d %c %02lx",
1099 cookie->debug_id,
1100 cookie->volume->debug_id,
1101 refcount_read(&cookie->ref),
1102 atomic_read(&cookie->n_active),
1103 atomic_read(&cookie->n_accesses),
1104 fscache_cookie_states[cookie->state],
1105 cookie->flags);
1106
1107 keylen = cookie->key_len;
1108 auxlen = cookie->aux_len;
1109
1110 if (keylen > 0 || auxlen > 0) {
1111 seq_puts(m, " ");
1112 p = keylen <= sizeof(cookie->inline_key) ?
1113 cookie->inline_key : cookie->key;
1114 for (; keylen > 0; keylen--)
1115 seq_printf(m, "%02x", *p++);
1116 if (auxlen > 0) {
1117 seq_puts(m, ", ");
1118 p = auxlen <= sizeof(cookie->inline_aux) ?
1119 cookie->inline_aux : cookie->aux;
1120 for (; auxlen > 0; auxlen--)
1121 seq_printf(m, "%02x", *p++);
1122 }
1123 }
1124
1125 seq_puts(m, "\n");
1126 return 0;
1127}
1128
1129static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
1130 __acquires(fscache_cookies_lock)
1131{
1132 read_lock(&fscache_cookies_lock);
1133 return seq_list_start_head(&fscache_cookies, *_pos);
1134}
1135
1136static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
1137{
1138 return seq_list_next(v, &fscache_cookies, _pos);
1139}
1140
1141static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
1142 __releases(rcu)
1143{
1144 read_unlock(&fscache_cookies_lock);
1145}
1146
1147
1148const struct seq_operations fscache_cookies_seq_ops = {
1149 .start = fscache_cookies_seq_start,
1150 .next = fscache_cookies_seq_next,
1151 .stop = fscache_cookies_seq_stop,
1152 .show = fscache_cookies_seq_show,
1153};
19517e53 1154#endif
Linux 6.x block layer and io_uring tree(s)