Commit | Line | Data |
---|---|---|
7f3283ab DH |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* netfs cookie management | |
3 | * | |
4 | * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. | |
5 | * Written by David Howells (dhowells@redhat.com) | |
6 | * | |
7 | * See Documentation/filesystems/caching/netfs-api.rst for more information on | |
8 | * the netfs API. | |
9 | */ | |
10 | ||
11 | #define FSCACHE_DEBUG_LEVEL COOKIE | |
12 | #include <linux/module.h> | |
13 | #include <linux/slab.h> | |
14 | #include "internal.h" | |
15 | ||
16 | struct kmem_cache *fscache_cookie_jar; | |
17 | ||
12bb21a2 DH |
18 | static void fscache_cookie_lru_timed_out(struct timer_list *timer); |
19 | static void fscache_cookie_lru_worker(struct work_struct *work); | |
5d00e426 DH |
20 | static void fscache_cookie_worker(struct work_struct *work); |
21 | static void fscache_unhash_cookie(struct fscache_cookie *cookie); | |
d24af13e | 22 | static void fscache_perform_invalidation(struct fscache_cookie *cookie); |
7f3283ab DH |
23 | |
24 | #define fscache_cookie_hash_shift 15 | |
25 | static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; | |
26 | static LIST_HEAD(fscache_cookies); | |
27 | static DEFINE_RWLOCK(fscache_cookies_lock); | |
12bb21a2 DH |
28 | static LIST_HEAD(fscache_cookie_lru); |
29 | static DEFINE_SPINLOCK(fscache_cookie_lru_lock); | |
30 | DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); | |
31 | static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); | |
d24af13e | 32 | static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD"; |
19517e53 | 33 | static unsigned int fscache_lru_cookie_timeout = 10 * HZ; |
7f3283ab DH |
34 | |
35 | void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) | |
36 | { | |
37 | const u8 *k; | |
38 | ||
39 | pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", | |
40 | prefix, | |
41 | cookie->debug_id, | |
42 | cookie->flags, | |
43 | atomic_read(&cookie->n_active), | |
44 | atomic_read(&cookie->n_accesses), | |
45 | fscache_cookie_states[cookie->state]); | |
46 | pr_err("%c-cookie V=%08x [%s]\n", | |
47 | prefix, | |
48 | cookie->volume->debug_id, | |
49 | cookie->volume->key); | |
50 | ||
51 | k = (cookie->key_len <= sizeof(cookie->inline_key)) ? | |
52 | cookie->inline_key : cookie->key; | |
53 | pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); | |
54 | } | |
55 | ||
56 | static void fscache_free_cookie(struct fscache_cookie *cookie) | |
57 | { | |
12bb21a2 DH |
58 | if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { |
59 | spin_lock(&fscache_cookie_lru_lock); | |
60 | list_del_init(&cookie->commit_link); | |
61 | spin_unlock(&fscache_cookie_lru_lock); | |
62 | fscache_stat_d(&fscache_n_cookies_lru); | |
63 | fscache_stat(&fscache_n_cookies_lru_removed); | |
64 | } | |
65 | ||
7f3283ab DH |
66 | if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { |
67 | fscache_print_cookie(cookie, 'F'); | |
68 | return; | |
69 | } | |
70 | ||
71 | write_lock(&fscache_cookies_lock); | |
72 | list_del(&cookie->proc_link); | |
73 | write_unlock(&fscache_cookies_lock); | |
74 | if (cookie->aux_len > sizeof(cookie->inline_aux)) | |
75 | kfree(cookie->aux); | |
76 | if (cookie->key_len > sizeof(cookie->inline_key)) | |
77 | kfree(cookie->key); | |
78 | fscache_stat_d(&fscache_n_cookies); | |
79 | kmem_cache_free(fscache_cookie_jar, cookie); | |
80 | } | |
81 | ||
5d00e426 DH |
82 | static void __fscache_queue_cookie(struct fscache_cookie *cookie) |
83 | { | |
84 | if (!queue_work(fscache_wq, &cookie->work)) | |
85 | fscache_put_cookie(cookie, fscache_cookie_put_over_queued); | |
86 | } | |
87 | ||
88 | static void fscache_queue_cookie(struct fscache_cookie *cookie, | |
89 | enum fscache_cookie_trace where) | |
90 | { | |
91 | fscache_get_cookie(cookie, where); | |
92 | __fscache_queue_cookie(cookie); | |
93 | } | |
94 | ||
a7733fb6 DH |
95 | /* |
96 | * Initialise the access gate on a cookie by setting a flag to prevent the | |
97 | * state machine from being queued when the access counter transitions to 0. | |
98 | * We're only interested in this when we withdraw caching services from the | |
99 | * cookie. | |
100 | */ | |
101 | static void fscache_init_access_gate(struct fscache_cookie *cookie) | |
102 | { | |
103 | int n_accesses; | |
104 | ||
105 | n_accesses = atomic_read(&cookie->n_accesses); | |
106 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
107 | n_accesses, fscache_access_cache_pin); | |
108 | set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); | |
109 | } | |
110 | ||
111 | /** | |
112 | * fscache_end_cookie_access - Unpin a cache at the end of an access. | |
113 | * @cookie: A data file cookie | |
114 | * @why: An indication of the circumstances of the access for tracing | |
115 | * | |
116 | * Unpin a cache cookie after we've accessed it and bring a deferred | |
117 | * relinquishment or withdrawal state into effect. | |
118 | * | |
119 | * The @why indicator is provided for tracing purposes. | |
120 | */ | |
121 | void fscache_end_cookie_access(struct fscache_cookie *cookie, | |
122 | enum fscache_access_trace why) | |
123 | { | |
124 | int n_accesses; | |
125 | ||
126 | smp_mb__before_atomic(); | |
127 | n_accesses = atomic_dec_return(&cookie->n_accesses); | |
128 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
129 | n_accesses, why); | |
130 | if (n_accesses == 0 && | |
5d00e426 DH |
131 | !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) |
132 | fscache_queue_cookie(cookie, fscache_cookie_get_end_access); | |
a7733fb6 DH |
133 | } |
134 | EXPORT_SYMBOL(fscache_end_cookie_access); | |
135 | ||
136 | /* | |
137 | * Pin the cache behind a cookie so that we can access it. | |
138 | */ | |
139 | static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, | |
140 | enum fscache_access_trace why) | |
141 | { | |
142 | int n_accesses; | |
143 | ||
144 | n_accesses = atomic_inc_return(&cookie->n_accesses); | |
145 | smp_mb__after_atomic(); /* (Future) read state after is-caching. | |
146 | * Reread n_accesses after is-caching | |
147 | */ | |
148 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
149 | n_accesses, why); | |
150 | } | |
151 | ||
152 | /** | |
153 | * fscache_begin_cookie_access - Pin a cache so data can be accessed | |
154 | * @cookie: A data file cookie | |
155 | * @why: An indication of the circumstances of the access for tracing | |
156 | * | |
157 | * Attempt to pin the cache to prevent it from going away whilst we're | |
158 | * accessing data and returns true if successful. This works as follows: | |
159 | * | |
160 | * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not | |
161 | * set), we return false to indicate access was not permitted. | |
162 | * | |
163 | * (2) If the cookie is being cached, we increment its n_accesses count and | |
164 | * then recheck the IS_CACHING flag, ending the access if it got cleared. | |
165 | * | |
166 | * (3) When we end the access, we decrement the cookie's n_accesses and wake | |
167 | * up the any waiters if it reaches 0. | |
168 | * | |
169 | * (4) Whilst the cookie is actively being cached, its n_accesses is kept | |
170 | * artificially incremented to prevent wakeups from happening. | |
171 | * | |
172 | * (5) When the cache is taken offline or if the cookie is culled, the flag is | |
173 | * cleared to prevent new accesses, the cookie's n_accesses is decremented | |
174 | * and we wait for it to become 0. | |
175 | * | |
176 | * The @why indicator are merely provided for tracing purposes. | |
177 | */ | |
178 | bool fscache_begin_cookie_access(struct fscache_cookie *cookie, | |
179 | enum fscache_access_trace why) | |
180 | { | |
181 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) | |
182 | return false; | |
183 | __fscache_begin_cookie_access(cookie, why); | |
184 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || | |
185 | !fscache_cache_is_live(cookie->volume->cache)) { | |
186 | fscache_end_cookie_access(cookie, fscache_access_unlive); | |
187 | return false; | |
188 | } | |
189 | return true; | |
190 | } | |
191 | ||
7f3283ab DH |
192 | static inline void wake_up_cookie_state(struct fscache_cookie *cookie) |
193 | { | |
194 | /* Use a barrier to ensure that waiters see the state variable | |
195 | * change, as spin_unlock doesn't guarantee a barrier. | |
196 | * | |
197 | * See comments over wake_up_bit() and waitqueue_active(). | |
198 | */ | |
199 | smp_mb(); | |
200 | wake_up_var(&cookie->state); | |
201 | } | |
202 | ||
5d00e426 DH |
203 | /* |
204 | * Change the state a cookie is at and wake up anyone waiting for that. Impose | |
205 | * an ordering between the stuff stored in the cookie and the state member. | |
206 | * Paired with fscache_cookie_state(). | |
207 | */ | |
7f3283ab DH |
208 | static void __fscache_set_cookie_state(struct fscache_cookie *cookie, |
209 | enum fscache_cookie_state state) | |
210 | { | |
5d00e426 | 211 | smp_store_release(&cookie->state, state); |
7f3283ab DH |
212 | } |
213 | ||
5d00e426 DH |
214 | static void fscache_set_cookie_state(struct fscache_cookie *cookie, |
215 | enum fscache_cookie_state state) | |
7f3283ab | 216 | { |
7f3283ab | 217 | spin_lock(&cookie->lock); |
5d00e426 | 218 | __fscache_set_cookie_state(cookie, state); |
7f3283ab | 219 | spin_unlock(&cookie->lock); |
5d00e426 DH |
220 | wake_up_cookie_state(cookie); |
221 | } | |
222 | ||
223 | /** | |
224 | * fscache_cookie_lookup_negative - Note negative lookup | |
225 | * @cookie: The cookie that was being looked up | |
226 | * | |
227 | * Note that some part of the metadata path in the cache doesn't exist and so | |
228 | * we can release any waiting readers in the certain knowledge that there's | |
229 | * nothing for them to actually read. | |
230 | * | |
231 | * This function uses no locking and must only be called from the state machine. | |
232 | */ | |
233 | void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) | |
234 | { | |
235 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
236 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); | |
237 | } | |
238 | EXPORT_SYMBOL(fscache_cookie_lookup_negative); | |
239 | ||
d24af13e DH |
240 | /** |
241 | * fscache_resume_after_invalidation - Allow I/O to resume after invalidation | |
242 | * @cookie: The cookie that was invalidated | |
243 | * | |
244 | * Tell fscache that invalidation is sufficiently complete that I/O can be | |
245 | * allowed again. | |
246 | */ | |
247 | void fscache_resume_after_invalidation(struct fscache_cookie *cookie) | |
248 | { | |
249 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); | |
250 | } | |
251 | EXPORT_SYMBOL(fscache_resume_after_invalidation); | |
252 | ||
5d00e426 DH |
253 | /** |
254 | * fscache_caching_failed - Report that a failure stopped caching on a cookie | |
255 | * @cookie: The cookie that was affected | |
256 | * | |
257 | * Tell fscache that caching on a cookie needs to be stopped due to some sort | |
258 | * of failure. | |
259 | * | |
260 | * This function uses no locking and must only be called from the state machine. | |
261 | */ | |
262 | void fscache_caching_failed(struct fscache_cookie *cookie) | |
263 | { | |
264 | clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); | |
265 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); | |
1a1e3aca JL |
266 | trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), |
267 | fscache_cookie_failed); | |
7f3283ab | 268 | } |
5d00e426 | 269 | EXPORT_SYMBOL(fscache_caching_failed); |
7f3283ab DH |
270 | |
271 | /* | |
272 | * Set the index key in a cookie. The cookie struct has space for a 16-byte | |
273 | * key plus length and hash, but if that's not big enough, it's instead a | |
274 | * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then | |
275 | * the key data. | |
276 | */ | |
277 | static int fscache_set_key(struct fscache_cookie *cookie, | |
278 | const void *index_key, size_t index_key_len) | |
279 | { | |
280 | void *buf; | |
281 | size_t buf_size; | |
282 | ||
283 | buf_size = round_up(index_key_len, sizeof(__le32)); | |
284 | ||
285 | if (index_key_len > sizeof(cookie->inline_key)) { | |
286 | buf = kzalloc(buf_size, GFP_KERNEL); | |
287 | if (!buf) | |
288 | return -ENOMEM; | |
289 | cookie->key = buf; | |
290 | } else { | |
291 | buf = cookie->inline_key; | |
292 | } | |
293 | ||
294 | memcpy(buf, index_key, index_key_len); | |
295 | cookie->key_hash = fscache_hash(cookie->volume->key_hash, | |
296 | buf, buf_size); | |
297 | return 0; | |
298 | } | |
299 | ||
300 | static bool fscache_cookie_same(const struct fscache_cookie *a, | |
301 | const struct fscache_cookie *b) | |
302 | { | |
303 | const void *ka, *kb; | |
304 | ||
305 | if (a->key_hash != b->key_hash || | |
306 | a->volume != b->volume || | |
307 | a->key_len != b->key_len) | |
308 | return false; | |
309 | ||
310 | if (a->key_len <= sizeof(a->inline_key)) { | |
311 | ka = &a->inline_key; | |
312 | kb = &b->inline_key; | |
313 | } else { | |
314 | ka = a->key; | |
315 | kb = b->key; | |
316 | } | |
317 | return memcmp(ka, kb, a->key_len) == 0; | |
318 | } | |
319 | ||
320 | static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); | |
321 | ||
322 | /* | |
323 | * Allocate a cookie. | |
324 | */ | |
325 | static struct fscache_cookie *fscache_alloc_cookie( | |
326 | struct fscache_volume *volume, | |
327 | u8 advice, | |
328 | const void *index_key, size_t index_key_len, | |
329 | const void *aux_data, size_t aux_data_len, | |
330 | loff_t object_size) | |
331 | { | |
332 | struct fscache_cookie *cookie; | |
333 | ||
334 | /* allocate and initialise a cookie */ | |
335 | cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); | |
336 | if (!cookie) | |
337 | return NULL; | |
338 | fscache_stat(&fscache_n_cookies); | |
339 | ||
340 | cookie->volume = volume; | |
341 | cookie->advice = advice; | |
342 | cookie->key_len = index_key_len; | |
343 | cookie->aux_len = aux_data_len; | |
344 | cookie->object_size = object_size; | |
345 | if (object_size == 0) | |
346 | __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
347 | ||
348 | if (fscache_set_key(cookie, index_key, index_key_len) < 0) | |
349 | goto nomem; | |
350 | ||
351 | if (cookie->aux_len <= sizeof(cookie->inline_aux)) { | |
352 | memcpy(cookie->inline_aux, aux_data, cookie->aux_len); | |
353 | } else { | |
354 | cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); | |
355 | if (!cookie->aux) | |
356 | goto nomem; | |
357 | } | |
358 | ||
359 | refcount_set(&cookie->ref, 1); | |
360 | cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); | |
7f3283ab DH |
361 | spin_lock_init(&cookie->lock); |
362 | INIT_LIST_HEAD(&cookie->commit_link); | |
5d00e426 DH |
363 | INIT_WORK(&cookie->work, fscache_cookie_worker); |
364 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
7f3283ab DH |
365 | |
366 | write_lock(&fscache_cookies_lock); | |
367 | list_add_tail(&cookie->proc_link, &fscache_cookies); | |
368 | write_unlock(&fscache_cookies_lock); | |
369 | fscache_see_cookie(cookie, fscache_cookie_new_acquire); | |
370 | return cookie; | |
371 | ||
372 | nomem: | |
373 | fscache_free_cookie(cookie); | |
374 | return NULL; | |
375 | } | |
376 | ||
5c4588ae YH |
377 | static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie) |
378 | { | |
379 | return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED; | |
380 | } | |
381 | ||
7f3283ab DH |
382 | static void fscache_wait_on_collision(struct fscache_cookie *candidate, |
383 | struct fscache_cookie *wait_for) | |
384 | { | |
385 | enum fscache_cookie_state *statep = &wait_for->state; | |
386 | ||
5c4588ae | 387 | wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for), |
7f3283ab | 388 | 20 * HZ); |
5c4588ae | 389 | if (!fscache_cookie_is_dropped(wait_for)) { |
7f3283ab DH |
390 | pr_notice("Potential collision c=%08x old: c=%08x", |
391 | candidate->debug_id, wait_for->debug_id); | |
5c4588ae | 392 | wait_var_event(statep, fscache_cookie_is_dropped(wait_for)); |
7f3283ab DH |
393 | } |
394 | } | |
395 | ||
396 | /* | |
397 | * Attempt to insert the new cookie into the hash. If there's a collision, we | |
398 | * wait for the old cookie to complete if it's being relinquished and an error | |
399 | * otherwise. | |
400 | */ | |
401 | static bool fscache_hash_cookie(struct fscache_cookie *candidate) | |
402 | { | |
403 | struct fscache_cookie *cursor, *wait_for = NULL; | |
404 | struct hlist_bl_head *h; | |
405 | struct hlist_bl_node *p; | |
406 | unsigned int bucket; | |
407 | ||
408 | bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); | |
409 | h = &fscache_cookie_hash[bucket]; | |
410 | ||
411 | hlist_bl_lock(h); | |
412 | hlist_bl_for_each_entry(cursor, p, h, hash_link) { | |
413 | if (fscache_cookie_same(candidate, cursor)) { | |
414 | if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) | |
415 | goto collision; | |
416 | wait_for = fscache_get_cookie(cursor, | |
417 | fscache_cookie_get_hash_collision); | |
418 | break; | |
419 | } | |
420 | } | |
421 | ||
422 | fscache_get_volume(candidate->volume, fscache_volume_get_cookie); | |
423 | atomic_inc(&candidate->volume->n_cookies); | |
424 | hlist_bl_add_head(&candidate->hash_link, h); | |
425 | set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); | |
426 | hlist_bl_unlock(h); | |
427 | ||
428 | if (wait_for) { | |
429 | fscache_wait_on_collision(candidate, wait_for); | |
430 | fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); | |
431 | } | |
432 | return true; | |
433 | ||
434 | collision: | |
435 | trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), | |
436 | fscache_cookie_collision); | |
437 | pr_err("Duplicate cookie detected\n"); | |
438 | fscache_print_cookie(cursor, 'O'); | |
439 | fscache_print_cookie(candidate, 'N'); | |
440 | hlist_bl_unlock(h); | |
441 | return false; | |
442 | } | |
443 | ||
444 | /* | |
445 | * Request a cookie to represent a data storage object within a volume. | |
446 | * | |
447 | * We never let on to the netfs about errors. We may set a negative cookie | |
448 | * pointer, but that's okay | |
449 | */ | |
450 | struct fscache_cookie *__fscache_acquire_cookie( | |
451 | struct fscache_volume *volume, | |
452 | u8 advice, | |
453 | const void *index_key, size_t index_key_len, | |
454 | const void *aux_data, size_t aux_data_len, | |
455 | loff_t object_size) | |
456 | { | |
457 | struct fscache_cookie *cookie; | |
458 | ||
459 | _enter("V=%x", volume->debug_id); | |
460 | ||
461 | if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) | |
462 | return NULL; | |
463 | if (!aux_data || !aux_data_len) { | |
464 | aux_data = NULL; | |
465 | aux_data_len = 0; | |
466 | } | |
467 | ||
468 | fscache_stat(&fscache_n_acquires); | |
469 | ||
470 | cookie = fscache_alloc_cookie(volume, advice, | |
471 | index_key, index_key_len, | |
472 | aux_data, aux_data_len, | |
473 | object_size); | |
474 | if (!cookie) { | |
475 | fscache_stat(&fscache_n_acquires_oom); | |
476 | return NULL; | |
477 | } | |
478 | ||
479 | if (!fscache_hash_cookie(cookie)) { | |
480 | fscache_see_cookie(cookie, fscache_cookie_discard); | |
481 | fscache_free_cookie(cookie); | |
482 | return NULL; | |
483 | } | |
484 | ||
485 | trace_fscache_acquire(cookie); | |
486 | fscache_stat(&fscache_n_acquires_ok); | |
487 | _leave(" = c=%08x", cookie->debug_id); | |
488 | return cookie; | |
489 | } | |
490 | EXPORT_SYMBOL(__fscache_acquire_cookie); | |
491 | ||
5d00e426 DH |
492 | /* |
493 | * Prepare a cache object to be written to. | |
494 | */ | |
495 | static void fscache_prepare_to_write(struct fscache_cookie *cookie) | |
496 | { | |
497 | cookie->volume->cache->ops->prepare_to_write(cookie); | |
498 | } | |
499 | ||
500 | /* | |
501 | * Look up a cookie in the cache. | |
502 | */ | |
503 | static void fscache_perform_lookup(struct fscache_cookie *cookie) | |
504 | { | |
505 | enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; | |
506 | bool need_withdraw = false; | |
507 | ||
508 | _enter(""); | |
509 | ||
510 | if (!cookie->volume->cache_priv) { | |
511 | fscache_create_volume(cookie->volume, true); | |
512 | if (!cookie->volume->cache_priv) { | |
513 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
514 | goto out; | |
515 | } | |
516 | } | |
517 | ||
518 | if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { | |
519 | if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) | |
520 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
521 | need_withdraw = true; | |
522 | _leave(" [fail]"); | |
523 | goto out; | |
524 | } | |
525 | ||
526 | fscache_see_cookie(cookie, fscache_cookie_see_active); | |
85e4ea10 DH |
527 | spin_lock(&cookie->lock); |
528 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) | |
529 | __fscache_set_cookie_state(cookie, | |
530 | FSCACHE_COOKIE_STATE_INVALIDATING); | |
531 | else | |
532 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); | |
533 | spin_unlock(&cookie->lock); | |
534 | wake_up_cookie_state(cookie); | |
5d00e426 DH |
535 | trace = fscache_access_lookup_cookie_end; |
536 | ||
537 | out: | |
538 | fscache_end_cookie_access(cookie, trace); | |
539 | if (need_withdraw) | |
540 | fscache_withdraw_cookie(cookie); | |
541 | fscache_end_volume_access(cookie->volume, cookie, trace); | |
542 | } | |
543 | ||
12bb21a2 DH |
544 | /* |
545 | * Begin the process of looking up a cookie. We offload the actual process to | |
546 | * a worker thread. | |
547 | */ | |
548 | static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) | |
549 | { | |
550 | if (will_modify) { | |
551 | set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); | |
552 | set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); | |
553 | } | |
554 | if (!fscache_begin_volume_access(cookie->volume, cookie, | |
555 | fscache_access_lookup_cookie)) | |
556 | return false; | |
557 | ||
558 | __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); | |
559 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); | |
560 | set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); | |
561 | set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); | |
562 | return true; | |
563 | } | |
564 | ||
565 | /* | |
566 | * Start using the cookie for I/O. This prevents the backing object from being | |
567 | * reaped by VM pressure. | |
568 | */ | |
569 | void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) | |
570 | { | |
571 | enum fscache_cookie_state state; | |
572 | bool queue = false; | |
e6435f1e | 573 | int n_active; |
12bb21a2 DH |
574 | |
575 | _enter("c=%08x", cookie->debug_id); | |
576 | ||
577 | if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), | |
578 | "Trying to use relinquished cookie\n")) | |
579 | return; | |
580 | ||
581 | spin_lock(&cookie->lock); | |
582 | ||
e6435f1e DH |
583 | n_active = atomic_inc_return(&cookie->n_active); |
584 | trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref), | |
585 | n_active, atomic_read(&cookie->n_accesses), | |
586 | will_modify ? | |
587 | fscache_active_use_modify : fscache_active_use); | |
12bb21a2 DH |
588 | |
589 | again: | |
590 | state = fscache_cookie_state(cookie); | |
591 | switch (state) { | |
592 | case FSCACHE_COOKIE_STATE_QUIESCENT: | |
593 | queue = fscache_begin_lookup(cookie, will_modify); | |
594 | break; | |
595 | ||
596 | case FSCACHE_COOKIE_STATE_LOOKING_UP: | |
597 | case FSCACHE_COOKIE_STATE_CREATING: | |
598 | if (will_modify) | |
599 | set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); | |
600 | break; | |
601 | case FSCACHE_COOKIE_STATE_ACTIVE: | |
d24af13e | 602 | case FSCACHE_COOKIE_STATE_INVALIDATING: |
12bb21a2 DH |
603 | if (will_modify && |
604 | !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { | |
605 | set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); | |
606 | queue = true; | |
607 | } | |
b5b52de3 DW |
608 | /* |
609 | * We could race with cookie_lru which may set LRU_DISCARD bit | |
610 | * but has yet to run the cookie state machine. If this happens | |
611 | * and another thread tries to use the cookie, clear LRU_DISCARD | |
612 | * so we don't end up withdrawing the cookie while in use. | |
613 | */ | |
614 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) | |
615 | fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear); | |
12bb21a2 DH |
616 | break; |
617 | ||
618 | case FSCACHE_COOKIE_STATE_FAILED: | |
619 | case FSCACHE_COOKIE_STATE_WITHDRAWING: | |
620 | break; | |
621 | ||
622 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: | |
623 | spin_unlock(&cookie->lock); | |
624 | wait_var_event(&cookie->state, | |
625 | fscache_cookie_state(cookie) != | |
626 | FSCACHE_COOKIE_STATE_LRU_DISCARDING); | |
627 | spin_lock(&cookie->lock); | |
628 | goto again; | |
629 | ||
630 | case FSCACHE_COOKIE_STATE_DROPPED: | |
631 | case FSCACHE_COOKIE_STATE_RELINQUISHING: | |
632 | WARN(1, "Can't use cookie in state %u\n", state); | |
633 | break; | |
634 | } | |
635 | ||
636 | spin_unlock(&cookie->lock); | |
637 | if (queue) | |
638 | fscache_queue_cookie(cookie, fscache_cookie_get_use_work); | |
639 | _leave(""); | |
640 | } | |
641 | EXPORT_SYMBOL(__fscache_use_cookie); | |
642 | ||
643 | static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) | |
644 | { | |
645 | clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); | |
646 | if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) | |
647 | return; | |
648 | ||
649 | cookie->unused_at = jiffies; | |
650 | spin_lock(&fscache_cookie_lru_lock); | |
651 | if (list_empty(&cookie->commit_link)) { | |
652 | fscache_get_cookie(cookie, fscache_cookie_get_lru); | |
653 | fscache_stat(&fscache_n_cookies_lru); | |
654 | } | |
655 | list_move_tail(&cookie->commit_link, &fscache_cookie_lru); | |
656 | ||
657 | spin_unlock(&fscache_cookie_lru_lock); | |
658 | timer_reduce(&fscache_cookie_lru_timer, | |
659 | jiffies + fscache_lru_cookie_timeout); | |
660 | } | |
661 | ||
662 | /* | |
663 | * Stop using the cookie for I/O. | |
664 | */ | |
665 | void __fscache_unuse_cookie(struct fscache_cookie *cookie, | |
666 | const void *aux_data, const loff_t *object_size) | |
667 | { | |
e6435f1e DH |
668 | unsigned int debug_id = cookie->debug_id; |
669 | unsigned int r = refcount_read(&cookie->ref); | |
670 | unsigned int a = atomic_read(&cookie->n_accesses); | |
671 | unsigned int c; | |
672 | ||
12bb21a2 DH |
673 | if (aux_data || object_size) |
674 | __fscache_update_cookie(cookie, aux_data, object_size); | |
675 | ||
e6435f1e DH |
676 | /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ |
677 | c = atomic_fetch_add_unless(&cookie->n_active, -1, 1); | |
678 | if (c != 1) { | |
679 | trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse); | |
680 | return; | |
12bb21a2 | 681 | } |
e6435f1e DH |
682 | |
683 | spin_lock(&cookie->lock); | |
684 | r = refcount_read(&cookie->ref); | |
685 | a = atomic_read(&cookie->n_accesses); | |
686 | c = atomic_dec_return(&cookie->n_active); | |
687 | trace_fscache_active(debug_id, r, c, a, fscache_active_unuse); | |
688 | if (c == 0) | |
689 | fscache_unuse_cookie_locked(cookie); | |
690 | spin_unlock(&cookie->lock); | |
12bb21a2 DH |
691 | } |
692 | EXPORT_SYMBOL(__fscache_unuse_cookie); | |
693 | ||
5d00e426 DH |
694 | /* |
695 | * Perform work upon the cookie, such as committing its cache state, | |
696 | * relinquishing it or withdrawing the backing cache. We're protected from the | |
697 | * cache going away under us as object withdrawal must come through this | |
698 | * non-reentrant work item. | |
699 | */ | |
700 | static void fscache_cookie_state_machine(struct fscache_cookie *cookie) | |
701 | { | |
702 | enum fscache_cookie_state state; | |
703 | bool wake = false; | |
704 | ||
705 | _enter("c=%x", cookie->debug_id); | |
706 | ||
707 | again: | |
708 | spin_lock(&cookie->lock); | |
709 | again_locked: | |
710 | state = cookie->state; | |
711 | switch (state) { | |
712 | case FSCACHE_COOKIE_STATE_QUIESCENT: | |
713 | /* The QUIESCENT state is jumped to the LOOKING_UP state by | |
714 | * fscache_use_cookie(). | |
715 | */ | |
716 | ||
717 | if (atomic_read(&cookie->n_accesses) == 0 && | |
718 | test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { | |
719 | __fscache_set_cookie_state(cookie, | |
720 | FSCACHE_COOKIE_STATE_RELINQUISHING); | |
721 | wake = true; | |
722 | goto again_locked; | |
723 | } | |
724 | break; | |
725 | ||
726 | case FSCACHE_COOKIE_STATE_LOOKING_UP: | |
727 | spin_unlock(&cookie->lock); | |
728 | fscache_init_access_gate(cookie); | |
729 | fscache_perform_lookup(cookie); | |
730 | goto again; | |
731 | ||
d24af13e DH |
732 | case FSCACHE_COOKIE_STATE_INVALIDATING: |
733 | spin_unlock(&cookie->lock); | |
734 | fscache_perform_invalidation(cookie); | |
735 | goto again; | |
736 | ||
5d00e426 DH |
737 | case FSCACHE_COOKIE_STATE_ACTIVE: |
738 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { | |
739 | spin_unlock(&cookie->lock); | |
740 | fscache_prepare_to_write(cookie); | |
741 | spin_lock(&cookie->lock); | |
742 | } | |
12bb21a2 DH |
743 | if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { |
744 | __fscache_set_cookie_state(cookie, | |
745 | FSCACHE_COOKIE_STATE_LRU_DISCARDING); | |
746 | wake = true; | |
747 | goto again_locked; | |
748 | } | |
5d00e426 DH |
749 | fallthrough; |
750 | ||
751 | case FSCACHE_COOKIE_STATE_FAILED: | |
fb24771f JL |
752 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) |
753 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); | |
754 | ||
5d00e426 DH |
755 | if (atomic_read(&cookie->n_accesses) != 0) |
756 | break; | |
757 | if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { | |
758 | __fscache_set_cookie_state(cookie, | |
759 | FSCACHE_COOKIE_STATE_RELINQUISHING); | |
760 | wake = true; | |
761 | goto again_locked; | |
762 | } | |
763 | if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { | |
764 | __fscache_set_cookie_state(cookie, | |
765 | FSCACHE_COOKIE_STATE_WITHDRAWING); | |
766 | wake = true; | |
767 | goto again_locked; | |
768 | } | |
769 | break; | |
770 | ||
12bb21a2 | 771 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
5d00e426 DH |
772 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
773 | case FSCACHE_COOKIE_STATE_WITHDRAWING: | |
774 | if (cookie->cache_priv) { | |
775 | spin_unlock(&cookie->lock); | |
776 | cookie->volume->cache->ops->withdraw_cookie(cookie); | |
777 | spin_lock(&cookie->lock); | |
778 | } | |
779 | ||
85e4ea10 DH |
780 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) |
781 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); | |
782 | ||
5d00e426 DH |
783 | switch (state) { |
784 | case FSCACHE_COOKIE_STATE_RELINQUISHING: | |
785 | fscache_see_cookie(cookie, fscache_cookie_see_relinquish); | |
786 | fscache_unhash_cookie(cookie); | |
787 | __fscache_set_cookie_state(cookie, | |
788 | FSCACHE_COOKIE_STATE_DROPPED); | |
789 | wake = true; | |
790 | goto out; | |
12bb21a2 DH |
791 | case FSCACHE_COOKIE_STATE_LRU_DISCARDING: |
792 | fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); | |
793 | break; | |
5d00e426 DH |
794 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
795 | fscache_see_cookie(cookie, fscache_cookie_see_withdraw); | |
796 | break; | |
797 | default: | |
798 | BUG(); | |
799 | } | |
800 | ||
801 | clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); | |
802 | clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); | |
803 | clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); | |
804 | clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); | |
805 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
806 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); | |
807 | wake = true; | |
808 | goto again_locked; | |
809 | ||
810 | case FSCACHE_COOKIE_STATE_DROPPED: | |
811 | break; | |
812 | ||
813 | default: | |
814 | WARN_ONCE(1, "Cookie %x in unexpected state %u\n", | |
815 | cookie->debug_id, state); | |
816 | break; | |
817 | } | |
818 | ||
819 | out: | |
820 | spin_unlock(&cookie->lock); | |
821 | if (wake) | |
822 | wake_up_cookie_state(cookie); | |
823 | _leave(""); | |
824 | } | |
825 | ||
826 | static void fscache_cookie_worker(struct work_struct *work) | |
827 | { | |
828 | struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); | |
829 | ||
830 | fscache_see_cookie(cookie, fscache_cookie_see_work); | |
831 | fscache_cookie_state_machine(cookie); | |
832 | fscache_put_cookie(cookie, fscache_cookie_put_work); | |
833 | } | |
834 | ||
835 | /* | |
836 | * Wait for the object to become inactive. The cookie's work item will be | |
837 | * scheduled when someone transitions n_accesses to 0 - but if someone's | |
838 | * already done that, schedule it anyway. | |
839 | */ | |
840 | static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) | |
841 | { | |
842 | int n_accesses; | |
843 | bool unpinned; | |
844 | ||
845 | unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); | |
846 | ||
847 | /* Need to read the access count after unpinning */ | |
848 | n_accesses = atomic_read(&cookie->n_accesses); | |
849 | if (unpinned) | |
850 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | |
851 | n_accesses, fscache_access_cache_unpin); | |
852 | if (n_accesses == 0) | |
853 | fscache_queue_cookie(cookie, fscache_cookie_get_end_access); | |
854 | } | |
855 | ||
12bb21a2 DH |
856 | static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) |
857 | { | |
858 | fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); | |
859 | ||
860 | spin_lock(&cookie->lock); | |
861 | if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || | |
862 | time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || | |
863 | atomic_read(&cookie->n_active) > 0) { | |
864 | spin_unlock(&cookie->lock); | |
865 | fscache_stat(&fscache_n_cookies_lru_removed); | |
866 | } else { | |
867 | set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); | |
868 | spin_unlock(&cookie->lock); | |
869 | fscache_stat(&fscache_n_cookies_lru_expired); | |
870 | _debug("lru c=%x", cookie->debug_id); | |
871 | __fscache_withdraw_cookie(cookie); | |
872 | } | |
873 | ||
874 | fscache_put_cookie(cookie, fscache_cookie_put_lru); | |
875 | } | |
876 | ||
877 | static void fscache_cookie_lru_worker(struct work_struct *work) | |
878 | { | |
879 | struct fscache_cookie *cookie; | |
880 | unsigned long unused_at; | |
881 | ||
882 | spin_lock(&fscache_cookie_lru_lock); | |
883 | ||
884 | while (!list_empty(&fscache_cookie_lru)) { | |
885 | cookie = list_first_entry(&fscache_cookie_lru, | |
886 | struct fscache_cookie, commit_link); | |
887 | unused_at = cookie->unused_at + fscache_lru_cookie_timeout; | |
888 | if (time_before(jiffies, unused_at)) { | |
889 | timer_reduce(&fscache_cookie_lru_timer, unused_at); | |
890 | break; | |
891 | } | |
892 | ||
893 | list_del_init(&cookie->commit_link); | |
894 | fscache_stat_d(&fscache_n_cookies_lru); | |
895 | spin_unlock(&fscache_cookie_lru_lock); | |
896 | fscache_cookie_lru_do_one(cookie); | |
897 | spin_lock(&fscache_cookie_lru_lock); | |
898 | } | |
899 | ||
900 | spin_unlock(&fscache_cookie_lru_lock); | |
901 | } | |
902 | ||
903 | static void fscache_cookie_lru_timed_out(struct timer_list *timer) | |
904 | { | |
905 | queue_work(fscache_wq, &fscache_cookie_lru_work); | |
906 | } | |
907 | ||
908 | static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) | |
909 | { | |
910 | bool need_put = false; | |
911 | ||
912 | if (!list_empty(&cookie->commit_link)) { | |
913 | spin_lock(&fscache_cookie_lru_lock); | |
914 | if (!list_empty(&cookie->commit_link)) { | |
915 | list_del_init(&cookie->commit_link); | |
916 | fscache_stat_d(&fscache_n_cookies_lru); | |
917 | fscache_stat(&fscache_n_cookies_lru_dropped); | |
918 | need_put = true; | |
919 | } | |
920 | spin_unlock(&fscache_cookie_lru_lock); | |
921 | if (need_put) | |
922 | fscache_put_cookie(cookie, fscache_cookie_put_lru); | |
923 | } | |
924 | } | |
925 | ||
7f3283ab DH |
926 | /* |
927 | * Remove a cookie from the hash table. | |
928 | */ | |
929 | static void fscache_unhash_cookie(struct fscache_cookie *cookie) | |
930 | { | |
931 | struct hlist_bl_head *h; | |
932 | unsigned int bucket; | |
933 | ||
934 | bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); | |
935 | h = &fscache_cookie_hash[bucket]; | |
936 | ||
937 | hlist_bl_lock(h); | |
938 | hlist_bl_del(&cookie->hash_link); | |
939 | clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); | |
940 | hlist_bl_unlock(h); | |
5d00e426 | 941 | fscache_stat(&fscache_n_relinquishes_dropped); |
7f3283ab DH |
942 | } |
943 | ||
5d00e426 | 944 | static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) |
7f3283ab | 945 | { |
12bb21a2 | 946 | fscache_cookie_drop_from_lru(cookie); |
5d00e426 DH |
947 | __fscache_withdraw_cookie(cookie); |
948 | } | |
7f3283ab | 949 | |
5d00e426 DH |
950 | /** |
951 | * fscache_withdraw_cookie - Mark a cookie for withdrawal | |
952 | * @cookie: The cookie to be withdrawn. | |
953 | * | |
954 | * Allow the cache backend to withdraw the backing for a cookie for its own | |
955 | * reasons, even if that cookie is in active use. | |
956 | */ | |
957 | void fscache_withdraw_cookie(struct fscache_cookie *cookie) | |
958 | { | |
959 | set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); | |
960 | fscache_drop_withdraw_cookie(cookie); | |
7f3283ab | 961 | } |
5d00e426 | 962 | EXPORT_SYMBOL(fscache_withdraw_cookie); |
7f3283ab DH |
963 | |
964 | /* | |
965 | * Allow the netfs to release a cookie back to the cache. | |
966 | * - the object will be marked as recyclable on disk if retire is true | |
967 | */ | |
968 | void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) | |
969 | { | |
970 | fscache_stat(&fscache_n_relinquishes); | |
971 | if (retire) | |
972 | fscache_stat(&fscache_n_relinquishes_retire); | |
973 | ||
974 | _enter("c=%08x{%d},%d", | |
975 | cookie->debug_id, atomic_read(&cookie->n_active), retire); | |
976 | ||
977 | if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), | |
978 | "Cookie c=%x already relinquished\n", cookie->debug_id)) | |
979 | return; | |
980 | ||
981 | if (retire) | |
982 | set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); | |
983 | trace_fscache_relinquish(cookie, retire); | |
984 | ||
985 | ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); | |
986 | ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); | |
987 | atomic_dec(&cookie->volume->n_cookies); | |
988 | ||
5d00e426 DH |
989 | if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { |
990 | set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); | |
991 | fscache_drop_withdraw_cookie(cookie); | |
992 | } else { | |
993 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); | |
994 | fscache_unhash_cookie(cookie); | |
995 | } | |
7f3283ab DH |
996 | fscache_put_cookie(cookie, fscache_cookie_put_relinquish); |
997 | } | |
998 | EXPORT_SYMBOL(__fscache_relinquish_cookie); | |
999 | ||
1000 | /* | |
1001 | * Drop a reference to a cookie. | |
1002 | */ | |
1003 | void fscache_put_cookie(struct fscache_cookie *cookie, | |
1004 | enum fscache_cookie_trace where) | |
1005 | { | |
1006 | struct fscache_volume *volume = cookie->volume; | |
1007 | unsigned int cookie_debug_id = cookie->debug_id; | |
1008 | bool zero; | |
1009 | int ref; | |
1010 | ||
1011 | zero = __refcount_dec_and_test(&cookie->ref, &ref); | |
1012 | trace_fscache_cookie(cookie_debug_id, ref - 1, where); | |
1013 | if (zero) { | |
1014 | fscache_free_cookie(cookie); | |
1015 | fscache_put_volume(volume, fscache_volume_put_cookie); | |
1016 | } | |
1017 | } | |
1018 | EXPORT_SYMBOL(fscache_put_cookie); | |
1019 | ||
1020 | /* | |
1021 | * Get a reference to a cookie. | |
1022 | */ | |
1023 | struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, | |
1024 | enum fscache_cookie_trace where) | |
1025 | { | |
1026 | int ref; | |
1027 | ||
1028 | __refcount_inc(&cookie->ref, &ref); | |
1029 | trace_fscache_cookie(cookie->debug_id, ref + 1, where); | |
1030 | return cookie; | |
1031 | } | |
1032 | EXPORT_SYMBOL(fscache_get_cookie); | |
1033 | ||
d24af13e DH |
1034 | /* |
1035 | * Ask the cache to effect invalidation of a cookie. | |
1036 | */ | |
1037 | static void fscache_perform_invalidation(struct fscache_cookie *cookie) | |
1038 | { | |
1039 | if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) | |
1040 | fscache_caching_failed(cookie); | |
1041 | fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); | |
1042 | } | |
1043 | ||
1044 | /* | |
1045 | * Invalidate an object. | |
1046 | */ | |
1047 | void __fscache_invalidate(struct fscache_cookie *cookie, | |
1048 | const void *aux_data, loff_t new_size, | |
1049 | unsigned int flags) | |
1050 | { | |
1051 | bool is_caching; | |
1052 | ||
1053 | _enter("c=%x", cookie->debug_id); | |
1054 | ||
1055 | fscache_stat(&fscache_n_invalidates); | |
1056 | ||
1057 | if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), | |
1058 | "Trying to invalidate relinquished cookie\n")) | |
1059 | return; | |
1060 | ||
1061 | if ((flags & FSCACHE_INVAL_DIO_WRITE) && | |
1062 | test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) | |
1063 | return; | |
1064 | ||
1065 | spin_lock(&cookie->lock); | |
1066 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); | |
1067 | fscache_update_aux(cookie, aux_data, &new_size); | |
1068 | cookie->inval_counter++; | |
1069 | trace_fscache_invalidate(cookie, new_size); | |
1070 | ||
1071 | switch (cookie->state) { | |
1072 | case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ | |
1073 | default: | |
1074 | spin_unlock(&cookie->lock); | |
1075 | _leave(" [no %u]", cookie->state); | |
1076 | return; | |
1077 | ||
1078 | case FSCACHE_COOKIE_STATE_LOOKING_UP: | |
fb24771f JL |
1079 | if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) |
1080 | __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie); | |
85e4ea10 | 1081 | fallthrough; |
d24af13e DH |
1082 | case FSCACHE_COOKIE_STATE_CREATING: |
1083 | spin_unlock(&cookie->lock); | |
1084 | _leave(" [look %x]", cookie->inval_counter); | |
1085 | return; | |
1086 | ||
1087 | case FSCACHE_COOKIE_STATE_ACTIVE: | |
1088 | is_caching = fscache_begin_cookie_access( | |
1089 | cookie, fscache_access_invalidate_cookie); | |
1090 | if (is_caching) | |
1091 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); | |
1092 | spin_unlock(&cookie->lock); | |
1093 | wake_up_cookie_state(cookie); | |
1094 | ||
1095 | if (is_caching) | |
1096 | fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); | |
1097 | _leave(" [inv]"); | |
1098 | return; | |
1099 | } | |
1100 | } | |
1101 | EXPORT_SYMBOL(__fscache_invalidate); | |
1102 | ||
19517e53 | 1103 | #ifdef CONFIG_PROC_FS |
7f3283ab DH |
1104 | /* |
1105 | * Generate a list of extant cookies in /proc/fs/fscache/cookies | |
1106 | */ | |
1107 | static int fscache_cookies_seq_show(struct seq_file *m, void *v) | |
1108 | { | |
1109 | struct fscache_cookie *cookie; | |
1110 | unsigned int keylen = 0, auxlen = 0; | |
1111 | u8 *p; | |
1112 | ||
1113 | if (v == &fscache_cookies) { | |
1114 | seq_puts(m, | |
1115 | "COOKIE VOLUME REF ACT ACC S FL DEF \n" | |
1116 | "======== ======== === === === = == ================\n" | |
1117 | ); | |
1118 | return 0; | |
1119 | } | |
1120 | ||
1121 | cookie = list_entry(v, struct fscache_cookie, proc_link); | |
1122 | ||
1123 | seq_printf(m, | |
1124 | "%08x %08x %3d %3d %3d %c %02lx", | |
1125 | cookie->debug_id, | |
1126 | cookie->volume->debug_id, | |
1127 | refcount_read(&cookie->ref), | |
1128 | atomic_read(&cookie->n_active), | |
1129 | atomic_read(&cookie->n_accesses), | |
1130 | fscache_cookie_states[cookie->state], | |
1131 | cookie->flags); | |
1132 | ||
1133 | keylen = cookie->key_len; | |
1134 | auxlen = cookie->aux_len; | |
1135 | ||
1136 | if (keylen > 0 || auxlen > 0) { | |
1137 | seq_puts(m, " "); | |
1138 | p = keylen <= sizeof(cookie->inline_key) ? | |
1139 | cookie->inline_key : cookie->key; | |
1140 | for (; keylen > 0; keylen--) | |
1141 | seq_printf(m, "%02x", *p++); | |
1142 | if (auxlen > 0) { | |
1143 | seq_puts(m, ", "); | |
1144 | p = auxlen <= sizeof(cookie->inline_aux) ? | |
1145 | cookie->inline_aux : cookie->aux; | |
1146 | for (; auxlen > 0; auxlen--) | |
1147 | seq_printf(m, "%02x", *p++); | |
1148 | } | |
1149 | } | |
1150 | ||
1151 | seq_puts(m, "\n"); | |
1152 | return 0; | |
1153 | } | |
1154 | ||
1155 | static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) | |
1156 | __acquires(fscache_cookies_lock) | |
1157 | { | |
1158 | read_lock(&fscache_cookies_lock); | |
1159 | return seq_list_start_head(&fscache_cookies, *_pos); | |
1160 | } | |
1161 | ||
1162 | static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) | |
1163 | { | |
1164 | return seq_list_next(v, &fscache_cookies, _pos); | |
1165 | } | |
1166 | ||
1167 | static void fscache_cookies_seq_stop(struct seq_file *m, void *v) | |
1168 | __releases(rcu) | |
1169 | { | |
1170 | read_unlock(&fscache_cookies_lock); | |
1171 | } | |
1172 | ||
1173 | ||
1174 | const struct seq_operations fscache_cookies_seq_ops = { | |
1175 | .start = fscache_cookies_seq_start, | |
1176 | .next = fscache_cookies_seq_next, | |
1177 | .stop = fscache_cookies_seq_stop, | |
1178 | .show = fscache_cookies_seq_show, | |
1179 | }; | |
19517e53 | 1180 | #endif |