Commit | Line | Data |
---|---|---|
62ab6335 DH |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* Volume-level cache cookie handling. | |
3 | * | |
4 | * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. | |
5 | * Written by David Howells (dhowells@redhat.com) | |
6 | */ | |
7 | ||
8 | #define FSCACHE_DEBUG_LEVEL COOKIE | |
9 | #include <linux/export.h> | |
10 | #include <linux/slab.h> | |
11 | #include "internal.h" | |
12 | ||
13 | #define fscache_volume_hash_shift 10 | |
14 | static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; | |
15 | static atomic_t fscache_volume_debug_id; | |
16 | static LIST_HEAD(fscache_volumes); | |
17 | ||
bfa22da3 DH |
18 | static void fscache_create_volume_work(struct work_struct *work); |
19 | ||
62ab6335 DH |
20 | struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, |
21 | enum fscache_volume_trace where) | |
22 | { | |
23 | int ref; | |
24 | ||
25 | __refcount_inc(&volume->ref, &ref); | |
26 | trace_fscache_volume(volume->debug_id, ref + 1, where); | |
27 | return volume; | |
28 | } | |
29 | ||
30 | static void fscache_see_volume(struct fscache_volume *volume, | |
31 | enum fscache_volume_trace where) | |
32 | { | |
33 | int ref = refcount_read(&volume->ref); | |
34 | ||
35 | trace_fscache_volume(volume->debug_id, ref, where); | |
36 | } | |
37 | ||
e6acd329 DH |
38 | /* |
39 | * Pin the cache behind a volume so that we can access it. | |
40 | */ | |
41 | static void __fscache_begin_volume_access(struct fscache_volume *volume, | |
42 | struct fscache_cookie *cookie, | |
43 | enum fscache_access_trace why) | |
44 | { | |
45 | int n_accesses; | |
46 | ||
47 | n_accesses = atomic_inc_return(&volume->n_accesses); | |
48 | smp_mb__after_atomic(); | |
49 | trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, | |
50 | refcount_read(&volume->ref), | |
51 | n_accesses, why); | |
52 | } | |
53 | ||
54 | /** | |
55 | * fscache_begin_volume_access - Pin a cache so a volume can be accessed | |
56 | * @volume: The volume cookie | |
57 | * @cookie: A datafile cookie for a tracing reference (or NULL) | |
58 | * @why: An indication of the circumstances of the access for tracing | |
59 | * | |
60 | * Attempt to pin the cache to prevent it from going away whilst we're | |
61 | * accessing a volume and returns true if successful. This works as follows: | |
62 | * | |
63 | * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), | |
64 | * then we return false to indicate access was not permitted. | |
65 | * | |
66 | * (2) If the cache tests as live, then we increment the volume's n_accesses | |
67 | * count and then recheck the cache liveness, ending the access if it | |
68 | * ceased to be live. | |
69 | * | |
70 | * (3) When we end the access, we decrement the volume's n_accesses and wake | |
71 | * up the any waiters if it reaches 0. | |
72 | * | |
73 | * (4) Whilst the cache is caching, the volume's n_accesses is kept | |
74 | * artificially incremented to prevent wakeups from happening. | |
75 | * | |
76 | * (5) When the cache is taken offline, the state is changed to prevent new | |
77 | * accesses, the volume's n_accesses is decremented and we wait for it to | |
78 | * become 0. | |
79 | * | |
80 | * The datafile @cookie and the @why indicator are merely provided for tracing | |
81 | * purposes. | |
82 | */ | |
83 | bool fscache_begin_volume_access(struct fscache_volume *volume, | |
84 | struct fscache_cookie *cookie, | |
85 | enum fscache_access_trace why) | |
86 | { | |
87 | if (!fscache_cache_is_live(volume->cache)) | |
88 | return false; | |
89 | __fscache_begin_volume_access(volume, cookie, why); | |
90 | if (!fscache_cache_is_live(volume->cache)) { | |
91 | fscache_end_volume_access(volume, cookie, fscache_access_unlive); | |
92 | return false; | |
93 | } | |
94 | return true; | |
95 | } | |
96 | ||
97 | /** | |
98 | * fscache_end_volume_access - Unpin a cache at the end of an access. | |
99 | * @volume: The volume cookie | |
100 | * @cookie: A datafile cookie for a tracing reference (or NULL) | |
101 | * @why: An indication of the circumstances of the access for tracing | |
102 | * | |
103 | * Unpin a cache volume after we've accessed it. The datafile @cookie and the | |
104 | * @why indicator are merely provided for tracing purposes. | |
105 | */ | |
106 | void fscache_end_volume_access(struct fscache_volume *volume, | |
107 | struct fscache_cookie *cookie, | |
108 | enum fscache_access_trace why) | |
109 | { | |
110 | int n_accesses; | |
111 | ||
112 | smp_mb__before_atomic(); | |
113 | n_accesses = atomic_dec_return(&volume->n_accesses); | |
114 | trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, | |
115 | refcount_read(&volume->ref), | |
116 | n_accesses, why); | |
117 | if (n_accesses == 0) | |
118 | wake_up_var(&volume->n_accesses); | |
119 | } | |
120 | EXPORT_SYMBOL(fscache_end_volume_access); | |
121 | ||
62ab6335 DH |
122 | static bool fscache_volume_same(const struct fscache_volume *a, |
123 | const struct fscache_volume *b) | |
124 | { | |
125 | size_t klen; | |
126 | ||
127 | if (a->key_hash != b->key_hash || | |
128 | a->cache != b->cache || | |
129 | a->key[0] != b->key[0]) | |
130 | return false; | |
131 | ||
132 | klen = round_up(a->key[0] + 1, sizeof(__le32)); | |
133 | return memcmp(a->key, b->key, klen) == 0; | |
134 | } | |
135 | ||
136 | static bool fscache_is_acquire_pending(struct fscache_volume *volume) | |
137 | { | |
138 | return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); | |
139 | } | |
140 | ||
141 | static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, | |
142 | unsigned int collidee_debug_id) | |
143 | { | |
8226e37d HT |
144 | wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, |
145 | TASK_UNINTERRUPTIBLE, 20 * HZ); | |
bf17455b | 146 | if (fscache_is_acquire_pending(candidate)) { |
62ab6335 DH |
147 | pr_notice("Potential volume collision new=%08x old=%08x", |
148 | candidate->debug_id, collidee_debug_id); | |
149 | fscache_stat(&fscache_n_volumes_collision); | |
8226e37d HT |
150 | wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, |
151 | TASK_UNINTERRUPTIBLE); | |
62ab6335 DH |
152 | } |
153 | } | |
154 | ||
155 | /* | |
156 | * Attempt to insert the new volume into the hash. If there's a collision, we | |
157 | * wait for the old volume to complete if it's being relinquished and an error | |
158 | * otherwise. | |
159 | */ | |
160 | static bool fscache_hash_volume(struct fscache_volume *candidate) | |
161 | { | |
162 | struct fscache_volume *cursor; | |
163 | struct hlist_bl_head *h; | |
164 | struct hlist_bl_node *p; | |
165 | unsigned int bucket, collidee_debug_id = 0; | |
166 | ||
167 | bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); | |
168 | h = &fscache_volume_hash[bucket]; | |
169 | ||
170 | hlist_bl_lock(h); | |
171 | hlist_bl_for_each_entry(cursor, p, h, hash_link) { | |
172 | if (fscache_volume_same(candidate, cursor)) { | |
173 | if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) | |
174 | goto collision; | |
175 | fscache_see_volume(cursor, fscache_volume_get_hash_collision); | |
176 | set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); | |
177 | set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); | |
178 | collidee_debug_id = cursor->debug_id; | |
179 | break; | |
180 | } | |
181 | } | |
182 | ||
183 | hlist_bl_add_head(&candidate->hash_link, h); | |
184 | hlist_bl_unlock(h); | |
185 | ||
bf17455b | 186 | if (fscache_is_acquire_pending(candidate)) |
62ab6335 DH |
187 | fscache_wait_on_volume_collision(candidate, collidee_debug_id); |
188 | return true; | |
189 | ||
190 | collision: | |
191 | fscache_see_volume(cursor, fscache_volume_collision); | |
192 | hlist_bl_unlock(h); | |
193 | return false; | |
194 | } | |
195 | ||
196 | /* | |
197 | * Allocate and initialise a volume representation cookie. | |
198 | */ | |
199 | static struct fscache_volume *fscache_alloc_volume(const char *volume_key, | |
200 | const char *cache_name, | |
201 | const void *coherency_data, | |
202 | size_t coherency_len) | |
203 | { | |
204 | struct fscache_volume *volume; | |
205 | struct fscache_cache *cache; | |
206 | size_t klen, hlen; | |
9f0933ac DH |
207 | u8 *key; |
208 | ||
209 | klen = strlen(volume_key); | |
210 | if (klen > NAME_MAX) | |
211 | return NULL; | |
62ab6335 | 212 | |
32e15003 DH |
213 | if (!coherency_data) |
214 | coherency_len = 0; | |
215 | ||
62ab6335 DH |
216 | cache = fscache_lookup_cache(cache_name, false); |
217 | if (IS_ERR(cache)) | |
218 | return NULL; | |
219 | ||
32e15003 DH |
220 | volume = kzalloc(struct_size(volume, coherency, coherency_len), |
221 | GFP_KERNEL); | |
62ab6335 DH |
222 | if (!volume) |
223 | goto err_cache; | |
224 | ||
225 | volume->cache = cache; | |
32e15003 DH |
226 | volume->coherency_len = coherency_len; |
227 | if (coherency_data) | |
228 | memcpy(volume->coherency, coherency_data, coherency_len); | |
62ab6335 | 229 | INIT_LIST_HEAD(&volume->proc_link); |
bfa22da3 | 230 | INIT_WORK(&volume->work, fscache_create_volume_work); |
62ab6335 DH |
231 | refcount_set(&volume->ref, 1); |
232 | spin_lock_init(&volume->lock); | |
233 | ||
234 | /* Stick the length on the front of the key and pad it out to make | |
235 | * hashing easier. | |
236 | */ | |
62ab6335 DH |
237 | hlen = round_up(1 + klen + 1, sizeof(__le32)); |
238 | key = kzalloc(hlen, GFP_KERNEL); | |
239 | if (!key) | |
240 | goto err_vol; | |
241 | key[0] = klen; | |
242 | memcpy(key + 1, volume_key, klen); | |
243 | ||
244 | volume->key = key; | |
245 | volume->key_hash = fscache_hash(0, key, hlen); | |
246 | ||
247 | volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); | |
248 | down_write(&fscache_addremove_sem); | |
249 | atomic_inc(&cache->n_volumes); | |
250 | list_add_tail(&volume->proc_link, &fscache_volumes); | |
251 | fscache_see_volume(volume, fscache_volume_new_acquire); | |
252 | fscache_stat(&fscache_n_volumes); | |
253 | up_write(&fscache_addremove_sem); | |
254 | _leave(" = v=%x", volume->debug_id); | |
255 | return volume; | |
256 | ||
257 | err_vol: | |
258 | kfree(volume); | |
259 | err_cache: | |
260 | fscache_put_cache(cache, fscache_cache_put_alloc_volume); | |
261 | fscache_stat(&fscache_n_volumes_nomem); | |
262 | return NULL; | |
263 | } | |
264 | ||
bfa22da3 DH |
265 | /* |
266 | * Create a volume's representation on disk. Have a volume ref and a cache | |
267 | * access we have to release. | |
268 | */ | |
269 | static void fscache_create_volume_work(struct work_struct *work) | |
270 | { | |
271 | const struct fscache_cache_ops *ops; | |
272 | struct fscache_volume *volume = | |
273 | container_of(work, struct fscache_volume, work); | |
274 | ||
275 | fscache_see_volume(volume, fscache_volume_see_create_work); | |
276 | ||
277 | ops = volume->cache->ops; | |
278 | if (ops->acquire_volume) | |
279 | ops->acquire_volume(volume); | |
280 | fscache_end_cache_access(volume->cache, | |
281 | fscache_access_acquire_volume_end); | |
282 | ||
3288666c | 283 | clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags); |
bfa22da3 DH |
284 | fscache_put_volume(volume, fscache_volume_put_create_work); |
285 | } | |
286 | ||
287 | /* | |
288 | * Dispatch a worker thread to create a volume's representation on disk. | |
289 | */ | |
290 | void fscache_create_volume(struct fscache_volume *volume, bool wait) | |
291 | { | |
292 | if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) | |
293 | goto maybe_wait; | |
294 | if (volume->cache_priv) | |
295 | goto no_wait; /* We raced */ | |
296 | if (!fscache_begin_cache_access(volume->cache, | |
297 | fscache_access_acquire_volume)) | |
298 | goto no_wait; | |
299 | ||
300 | fscache_get_volume(volume, fscache_volume_get_create_work); | |
301 | if (!schedule_work(&volume->work)) | |
302 | fscache_put_volume(volume, fscache_volume_put_create_work); | |
303 | ||
304 | maybe_wait: | |
305 | if (wait) { | |
306 | fscache_see_volume(volume, fscache_volume_wait_create_work); | |
307 | wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, | |
308 | TASK_UNINTERRUPTIBLE); | |
309 | } | |
310 | return; | |
311 | no_wait: | |
312 | clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); | |
313 | wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); | |
314 | } | |
315 | ||
62ab6335 DH |
316 | /* |
317 | * Acquire a volume representation cookie and link it to a (proposed) cache. | |
318 | */ | |
319 | struct fscache_volume *__fscache_acquire_volume(const char *volume_key, | |
320 | const char *cache_name, | |
321 | const void *coherency_data, | |
322 | size_t coherency_len) | |
323 | { | |
324 | struct fscache_volume *volume; | |
325 | ||
326 | volume = fscache_alloc_volume(volume_key, cache_name, | |
327 | coherency_data, coherency_len); | |
328 | if (!volume) | |
329 | return ERR_PTR(-ENOMEM); | |
330 | ||
331 | if (!fscache_hash_volume(volume)) { | |
332 | fscache_put_volume(volume, fscache_volume_put_hash_collision); | |
333 | return ERR_PTR(-EBUSY); | |
334 | } | |
335 | ||
bfa22da3 | 336 | fscache_create_volume(volume, false); |
62ab6335 DH |
337 | return volume; |
338 | } | |
339 | EXPORT_SYMBOL(__fscache_acquire_volume); | |
340 | ||
341 | static void fscache_wake_pending_volume(struct fscache_volume *volume, | |
342 | struct hlist_bl_head *h) | |
343 | { | |
344 | struct fscache_volume *cursor; | |
345 | struct hlist_bl_node *p; | |
346 | ||
347 | hlist_bl_for_each_entry(cursor, p, h, hash_link) { | |
348 | if (fscache_volume_same(cursor, volume)) { | |
349 | fscache_see_volume(cursor, fscache_volume_see_hash_wake); | |
8226e37d HT |
350 | clear_and_wake_up_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, |
351 | &cursor->flags); | |
62ab6335 DH |
352 | return; |
353 | } | |
354 | } | |
355 | } | |
356 | ||
357 | /* | |
358 | * Remove a volume cookie from the hash table. | |
359 | */ | |
360 | static void fscache_unhash_volume(struct fscache_volume *volume) | |
361 | { | |
362 | struct hlist_bl_head *h; | |
363 | unsigned int bucket; | |
364 | ||
365 | bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); | |
366 | h = &fscache_volume_hash[bucket]; | |
367 | ||
368 | hlist_bl_lock(h); | |
369 | hlist_bl_del(&volume->hash_link); | |
370 | if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) | |
371 | fscache_wake_pending_volume(volume, h); | |
372 | hlist_bl_unlock(h); | |
373 | } | |
374 | ||
375 | /* | |
376 | * Drop a cache's volume attachments. | |
377 | */ | |
378 | static void fscache_free_volume(struct fscache_volume *volume) | |
379 | { | |
380 | struct fscache_cache *cache = volume->cache; | |
381 | ||
382 | if (volume->cache_priv) { | |
bfa22da3 DH |
383 | __fscache_begin_volume_access(volume, NULL, |
384 | fscache_access_relinquish_volume); | |
385 | if (volume->cache_priv) | |
386 | cache->ops->free_volume(volume); | |
387 | fscache_end_volume_access(volume, NULL, | |
388 | fscache_access_relinquish_volume_end); | |
62ab6335 DH |
389 | } |
390 | ||
391 | down_write(&fscache_addremove_sem); | |
392 | list_del_init(&volume->proc_link); | |
393 | atomic_dec(&volume->cache->n_volumes); | |
394 | up_write(&fscache_addremove_sem); | |
395 | ||
396 | if (!hlist_bl_unhashed(&volume->hash_link)) | |
397 | fscache_unhash_volume(volume); | |
398 | ||
399 | trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); | |
400 | kfree(volume->key); | |
401 | kfree(volume); | |
402 | fscache_stat_d(&fscache_n_volumes); | |
403 | fscache_put_cache(cache, fscache_cache_put_volume); | |
404 | } | |
405 | ||
406 | /* | |
407 | * Drop a reference to a volume cookie. | |
408 | */ | |
409 | void fscache_put_volume(struct fscache_volume *volume, | |
410 | enum fscache_volume_trace where) | |
411 | { | |
412 | if (volume) { | |
413 | unsigned int debug_id = volume->debug_id; | |
414 | bool zero; | |
415 | int ref; | |
416 | ||
417 | zero = __refcount_dec_and_test(&volume->ref, &ref); | |
418 | trace_fscache_volume(debug_id, ref - 1, where); | |
419 | if (zero) | |
420 | fscache_free_volume(volume); | |
421 | } | |
422 | } | |
423 | ||
424 | /* | |
425 | * Relinquish a volume representation cookie. | |
426 | */ | |
427 | void __fscache_relinquish_volume(struct fscache_volume *volume, | |
428 | const void *coherency_data, | |
429 | bool invalidate) | |
430 | { | |
431 | if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) | |
432 | return; | |
433 | ||
32e15003 | 434 | if (invalidate) { |
62ab6335 | 435 | set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); |
32e15003 DH |
436 | } else if (coherency_data) { |
437 | memcpy(volume->coherency, coherency_data, volume->coherency_len); | |
438 | } | |
62ab6335 DH |
439 | |
440 | fscache_put_volume(volume, fscache_volume_put_relinquish); | |
441 | } | |
442 | EXPORT_SYMBOL(__fscache_relinquish_volume); | |
443 | ||
bfa22da3 DH |
444 | /** |
445 | * fscache_withdraw_volume - Withdraw a volume from being cached | |
446 | * @volume: Volume cookie | |
447 | * | |
448 | * Withdraw a cache volume from service, waiting for all accesses to complete | |
449 | * before returning. | |
450 | */ | |
451 | void fscache_withdraw_volume(struct fscache_volume *volume) | |
452 | { | |
453 | int n_accesses; | |
454 | ||
455 | _debug("withdraw V=%x", volume->debug_id); | |
456 | ||
457 | /* Allow wakeups on dec-to-0 */ | |
458 | n_accesses = atomic_dec_return(&volume->n_accesses); | |
459 | trace_fscache_access_volume(volume->debug_id, 0, | |
460 | refcount_read(&volume->ref), | |
461 | n_accesses, fscache_access_cache_unpin); | |
462 | ||
463 | wait_var_event(&volume->n_accesses, | |
464 | atomic_read(&volume->n_accesses) == 0); | |
465 | } | |
466 | EXPORT_SYMBOL(fscache_withdraw_volume); | |
467 | ||
62ab6335 DH |
468 | #ifdef CONFIG_PROC_FS |
469 | /* | |
470 | * Generate a list of volumes in /proc/fs/fscache/volumes | |
471 | */ | |
472 | static int fscache_volumes_seq_show(struct seq_file *m, void *v) | |
473 | { | |
474 | struct fscache_volume *volume; | |
475 | ||
476 | if (v == &fscache_volumes) { | |
477 | seq_puts(m, | |
478 | "VOLUME REF nCOOK ACC FL CACHE KEY\n" | |
479 | "======== ===== ===== === == =============== ================\n"); | |
480 | return 0; | |
481 | } | |
482 | ||
483 | volume = list_entry(v, struct fscache_volume, proc_link); | |
484 | seq_printf(m, | |
485 | "%08x %5d %5d %3d %02lx %-15.15s %s\n", | |
486 | volume->debug_id, | |
487 | refcount_read(&volume->ref), | |
488 | atomic_read(&volume->n_cookies), | |
489 | atomic_read(&volume->n_accesses), | |
490 | volume->flags, | |
491 | volume->cache->name ?: "-", | |
492 | volume->key + 1); | |
493 | return 0; | |
494 | } | |
495 | ||
496 | static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) | |
497 | __acquires(&fscache_addremove_sem) | |
498 | { | |
499 | down_read(&fscache_addremove_sem); | |
500 | return seq_list_start_head(&fscache_volumes, *_pos); | |
501 | } | |
502 | ||
503 | static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) | |
504 | { | |
505 | return seq_list_next(v, &fscache_volumes, _pos); | |
506 | } | |
507 | ||
508 | static void fscache_volumes_seq_stop(struct seq_file *m, void *v) | |
509 | __releases(&fscache_addremove_sem) | |
510 | { | |
511 | up_read(&fscache_addremove_sem); | |
512 | } | |
513 | ||
514 | const struct seq_operations fscache_volumes_seq_ops = { | |
515 | .start = fscache_volumes_seq_start, | |
516 | .next = fscache_volumes_seq_next, | |
517 | .stop = fscache_volumes_seq_stop, | |
518 | .show = fscache_volumes_seq_show, | |
519 | }; | |
520 | #endif /* CONFIG_PROC_FS */ |