Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
955d0091 DH |
2 | /* netfs cookie management |
3 | * | |
4 | * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. | |
5 | * Written by David Howells (dhowells@redhat.com) | |
6 | * | |
ccc4fc3d DH |
7 | * See Documentation/filesystems/caching/netfs-api.txt for more information on |
8 | * the netfs API. | |
955d0091 DH |
9 | */ |
10 | ||
11 | #define FSCACHE_DEBUG_LEVEL COOKIE | |
12 | #include <linux/module.h> | |
13 | #include <linux/slab.h> | |
14 | #include "internal.h" | |
15 | ||
16 | struct kmem_cache *fscache_cookie_jar; | |
17 | ||
ccc4fc3d DH |
18 | static atomic_t fscache_object_debug_id = ATOMIC_INIT(0); |
19 | ||
ec0328e4 DH |
20 | #define fscache_cookie_hash_shift 15 |
21 | static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; | |
22 | ||
ee1235a9 DH |
23 | static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, |
24 | loff_t object_size); | |
ccc4fc3d DH |
25 | static int fscache_alloc_object(struct fscache_cache *cache, |
26 | struct fscache_cookie *cookie); | |
27 | static int fscache_attach_object(struct fscache_cookie *cookie, | |
28 | struct fscache_object *object); | |
29 | ||
ec0328e4 DH |
30 | static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) |
31 | { | |
32 | struct hlist_node *object; | |
33 | const u8 *k; | |
34 | unsigned loop; | |
35 | ||
36 | pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n", | |
37 | prefix, cookie, cookie->parent, cookie->flags, | |
38 | atomic_read(&cookie->n_children), | |
39 | atomic_read(&cookie->n_active)); | |
40 | pr_err("%c-cookie d=%p n=%p\n", | |
41 | prefix, cookie->def, cookie->netfs_data); | |
42 | ||
43 | object = READ_ONCE(cookie->backing_objects.first); | |
44 | if (object) | |
45 | pr_err("%c-cookie o=%p\n", | |
46 | prefix, hlist_entry(object, struct fscache_object, cookie_link)); | |
47 | ||
48 | pr_err("%c-key=[%u] '", prefix, cookie->key_len); | |
49 | k = (cookie->key_len <= sizeof(cookie->inline_key)) ? | |
50 | cookie->inline_key : cookie->key; | |
51 | for (loop = 0; loop < cookie->key_len; loop++) | |
52 | pr_cont("%02x", k[loop]); | |
53 | pr_cont("'\n"); | |
54 | } | |
55 | ||
56 | void fscache_free_cookie(struct fscache_cookie *cookie) | |
57 | { | |
58 | if (cookie) { | |
59 | BUG_ON(!hlist_empty(&cookie->backing_objects)); | |
60 | if (cookie->aux_len > sizeof(cookie->inline_aux)) | |
61 | kfree(cookie->aux); | |
62 | if (cookie->key_len > sizeof(cookie->inline_key)) | |
63 | kfree(cookie->key); | |
64 | kmem_cache_free(fscache_cookie_jar, cookie); | |
65 | } | |
66 | } | |
67 | ||
ec0328e4 | 68 | /* |
fa520c47 | 69 | * Set the index key in a cookie. The cookie struct has space for a 16-byte |
ec0328e4 DH |
70 | * key plus length and hash, but if that's not big enough, it's instead a |
71 | * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then | |
72 | * the key data. | |
73 | */ | |
74 | static int fscache_set_key(struct fscache_cookie *cookie, | |
75 | const void *index_key, size_t index_key_len) | |
76 | { | |
77 | unsigned long long h; | |
78 | u32 *buf; | |
fa520c47 | 79 | int bufs; |
ec0328e4 DH |
80 | int i; |
81 | ||
fa520c47 ES |
82 | bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf)); |
83 | ||
ec0328e4 | 84 | if (index_key_len > sizeof(cookie->inline_key)) { |
fa520c47 | 85 | buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL); |
ec0328e4 DH |
86 | if (!buf) |
87 | return -ENOMEM; | |
88 | cookie->key = buf; | |
89 | } else { | |
90 | buf = (u32 *)cookie->inline_key; | |
ec0328e4 DH |
91 | } |
92 | ||
93 | memcpy(buf, index_key, index_key_len); | |
94 | ||
95 | /* Calculate a hash and combine this with the length in the first word | |
96 | * or first half word | |
97 | */ | |
98 | h = (unsigned long)cookie->parent; | |
99 | h += index_key_len + cookie->type; | |
fa520c47 ES |
100 | |
101 | for (i = 0; i < bufs; i++) | |
ec0328e4 DH |
102 | h += buf[i]; |
103 | ||
104 | cookie->key_hash = h ^ (h >> 32); | |
105 | return 0; | |
106 | } | |
107 | ||
108 | static long fscache_compare_cookie(const struct fscache_cookie *a, | |
109 | const struct fscache_cookie *b) | |
110 | { | |
111 | const void *ka, *kb; | |
112 | ||
113 | if (a->key_hash != b->key_hash) | |
114 | return (long)a->key_hash - (long)b->key_hash; | |
115 | if (a->parent != b->parent) | |
116 | return (long)a->parent - (long)b->parent; | |
117 | if (a->key_len != b->key_len) | |
118 | return (long)a->key_len - (long)b->key_len; | |
119 | if (a->type != b->type) | |
120 | return (long)a->type - (long)b->type; | |
121 | ||
122 | if (a->key_len <= sizeof(a->inline_key)) { | |
123 | ka = &a->inline_key; | |
124 | kb = &b->inline_key; | |
125 | } else { | |
126 | ka = a->key; | |
127 | kb = b->key; | |
128 | } | |
129 | return memcmp(ka, kb, a->key_len); | |
130 | } | |
131 | ||
132 | /* | |
133 | * Allocate a cookie. | |
134 | */ | |
135 | struct fscache_cookie *fscache_alloc_cookie( | |
136 | struct fscache_cookie *parent, | |
137 | const struct fscache_cookie_def *def, | |
138 | const void *index_key, size_t index_key_len, | |
139 | const void *aux_data, size_t aux_data_len, | |
140 | void *netfs_data, | |
141 | loff_t object_size) | |
142 | { | |
143 | struct fscache_cookie *cookie; | |
144 | ||
145 | /* allocate and initialise a cookie */ | |
1ff22883 | 146 | cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); |
ec0328e4 DH |
147 | if (!cookie) |
148 | return NULL; | |
149 | ||
150 | cookie->key_len = index_key_len; | |
151 | cookie->aux_len = aux_data_len; | |
152 | ||
153 | if (fscache_set_key(cookie, index_key, index_key_len) < 0) | |
154 | goto nomem; | |
155 | ||
156 | if (cookie->aux_len <= sizeof(cookie->inline_aux)) { | |
157 | memcpy(cookie->inline_aux, aux_data, cookie->aux_len); | |
158 | } else { | |
159 | cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); | |
160 | if (!cookie->aux) | |
161 | goto nomem; | |
162 | } | |
163 | ||
164 | atomic_set(&cookie->usage, 1); | |
165 | atomic_set(&cookie->n_children, 0); | |
166 | ||
167 | /* We keep the active count elevated until relinquishment to prevent an | |
168 | * attempt to wake up every time the object operations queue quiesces. | |
169 | */ | |
170 | atomic_set(&cookie->n_active, 1); | |
171 | ||
172 | cookie->def = def; | |
173 | cookie->parent = parent; | |
174 | cookie->netfs_data = netfs_data; | |
175 | cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET); | |
176 | cookie->type = def->type; | |
1ff22883 DH |
177 | spin_lock_init(&cookie->lock); |
178 | spin_lock_init(&cookie->stores_lock); | |
179 | INIT_HLIST_HEAD(&cookie->backing_objects); | |
ec0328e4 DH |
180 | |
181 | /* radix tree insertion won't use the preallocation pool unless it's | |
182 | * told it may not wait */ | |
183 | INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM); | |
184 | return cookie; | |
185 | ||
186 | nomem: | |
187 | fscache_free_cookie(cookie); | |
188 | return NULL; | |
189 | } | |
190 | ||
191 | /* | |
192 | * Attempt to insert the new cookie into the hash. If there's a collision, we | |
193 | * return the old cookie if it's not in use and an error otherwise. | |
194 | */ | |
195 | struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate) | |
196 | { | |
197 | struct fscache_cookie *cursor; | |
198 | struct hlist_bl_head *h; | |
199 | struct hlist_bl_node *p; | |
200 | unsigned int bucket; | |
201 | ||
202 | bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); | |
203 | h = &fscache_cookie_hash[bucket]; | |
204 | ||
205 | hlist_bl_lock(h); | |
206 | hlist_bl_for_each_entry(cursor, p, h, hash_link) { | |
207 | if (fscache_compare_cookie(candidate, cursor) == 0) | |
208 | goto collision; | |
209 | } | |
210 | ||
211 | __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags); | |
212 | fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent); | |
213 | atomic_inc(&candidate->parent->n_children); | |
214 | hlist_bl_add_head(&candidate->hash_link, h); | |
215 | hlist_bl_unlock(h); | |
216 | return candidate; | |
217 | ||
218 | collision: | |
219 | if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) { | |
220 | trace_fscache_cookie(cursor, fscache_cookie_collision, | |
221 | atomic_read(&cursor->usage)); | |
222 | pr_err("Duplicate cookie detected\n"); | |
223 | fscache_print_cookie(cursor, 'O'); | |
224 | fscache_print_cookie(candidate, 'N'); | |
225 | hlist_bl_unlock(h); | |
226 | return NULL; | |
227 | } | |
228 | ||
229 | fscache_cookie_get(cursor, fscache_cookie_get_reacquire); | |
230 | hlist_bl_unlock(h); | |
231 | return cursor; | |
232 | } | |
233 | ||
ccc4fc3d DH |
234 | /* |
235 | * request a cookie to represent an object (index, datafile, xattr, etc) | |
236 | * - parent specifies the parent object | |
237 | * - the top level index cookie for each netfs is stored in the fscache_netfs | |
238 | * struct upon registration | |
239 | * - def points to the definition | |
240 | * - the netfs_data will be passed to the functions pointed to in *def | |
241 | * - all attached caches will be searched to see if they contain this object | |
242 | * - index objects aren't stored on disk until there's a dependent file that | |
243 | * needs storing | |
244 | * - other objects are stored in a selected cache immediately, and all the | |
245 | * indices forming the path to it are instantiated if necessary | |
246 | * - we never let on to the netfs about errors | |
247 | * - we may set a negative cookie pointer, but that's okay | |
248 | */ | |
249 | struct fscache_cookie *__fscache_acquire_cookie( | |
250 | struct fscache_cookie *parent, | |
251 | const struct fscache_cookie_def *def, | |
402cb8dd DH |
252 | const void *index_key, size_t index_key_len, |
253 | const void *aux_data, size_t aux_data_len, | |
94d30ae9 | 254 | void *netfs_data, |
ee1235a9 | 255 | loff_t object_size, |
94d30ae9 | 256 | bool enable) |
ccc4fc3d | 257 | { |
ec0328e4 | 258 | struct fscache_cookie *candidate, *cookie; |
ccc4fc3d DH |
259 | |
260 | BUG_ON(!def); | |
261 | ||
94d30ae9 | 262 | _enter("{%s},{%s},%p,%u", |
ccc4fc3d | 263 | parent ? (char *) parent->def->name : "<no-parent>", |
94d30ae9 | 264 | def->name, netfs_data, enable); |
ccc4fc3d | 265 | |
402cb8dd DH |
266 | if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) |
267 | return NULL; | |
268 | if (!aux_data || !aux_data_len) { | |
269 | aux_data = NULL; | |
270 | aux_data_len = 0; | |
271 | } | |
272 | ||
ccc4fc3d DH |
273 | fscache_stat(&fscache_n_acquires); |
274 | ||
275 | /* if there's no parent cookie, then we don't create one here either */ | |
276 | if (!parent) { | |
277 | fscache_stat(&fscache_n_acquires_null); | |
278 | _leave(" [no parent]"); | |
279 | return NULL; | |
280 | } | |
281 | ||
282 | /* validate the definition */ | |
ccc4fc3d DH |
283 | BUG_ON(!def->name[0]); |
284 | ||
285 | BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX && | |
402cb8dd | 286 | parent->type != FSCACHE_COOKIE_TYPE_INDEX); |
ccc4fc3d | 287 | |
ec0328e4 DH |
288 | candidate = fscache_alloc_cookie(parent, def, |
289 | index_key, index_key_len, | |
290 | aux_data, aux_data_len, | |
291 | netfs_data, object_size); | |
292 | if (!candidate) { | |
ccc4fc3d DH |
293 | fscache_stat(&fscache_n_acquires_oom); |
294 | _leave(" [ENOMEM]"); | |
295 | return NULL; | |
296 | } | |
297 | ||
ec0328e4 DH |
298 | cookie = fscache_hash_cookie(candidate); |
299 | if (!cookie) { | |
300 | trace_fscache_cookie(candidate, fscache_cookie_discard, 1); | |
301 | goto out; | |
402cb8dd DH |
302 | } |
303 | ||
ec0328e4 DH |
304 | if (cookie == candidate) |
305 | candidate = NULL; | |
ccc4fc3d | 306 | |
402cb8dd | 307 | switch (cookie->type) { |
ccc4fc3d DH |
308 | case FSCACHE_COOKIE_TYPE_INDEX: |
309 | fscache_stat(&fscache_n_cookie_index); | |
310 | break; | |
311 | case FSCACHE_COOKIE_TYPE_DATAFILE: | |
312 | fscache_stat(&fscache_n_cookie_data); | |
313 | break; | |
314 | default: | |
315 | fscache_stat(&fscache_n_cookie_special); | |
316 | break; | |
317 | } | |
318 | ||
a18feb55 DH |
319 | trace_fscache_acquire(cookie); |
320 | ||
94d30ae9 DH |
321 | if (enable) { |
322 | /* if the object is an index then we need do nothing more here | |
323 | * - we create indices on disk when we need them as an index | |
324 | * may exist in multiple caches */ | |
402cb8dd | 325 | if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { |
ee1235a9 | 326 | if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) { |
94d30ae9 DH |
327 | set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); |
328 | } else { | |
329 | atomic_dec(&parent->n_children); | |
a18feb55 DH |
330 | fscache_cookie_put(cookie, |
331 | fscache_cookie_put_acquire_nobufs); | |
94d30ae9 DH |
332 | fscache_stat(&fscache_n_acquires_nobufs); |
333 | _leave(" = NULL"); | |
334 | return NULL; | |
335 | } | |
336 | } else { | |
337 | set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); | |
ccc4fc3d DH |
338 | } |
339 | } | |
340 | ||
341 | fscache_stat(&fscache_n_acquires_ok); | |
402cb8dd | 342 | |
ec0328e4 DH |
343 | out: |
344 | fscache_free_cookie(candidate); | |
345 | return cookie; | |
ccc4fc3d DH |
346 | } |
347 | EXPORT_SYMBOL(__fscache_acquire_cookie); | |
348 | ||
94d30ae9 DH |
349 | /* |
350 | * Enable a cookie to permit it to accept new operations. | |
351 | */ | |
352 | void __fscache_enable_cookie(struct fscache_cookie *cookie, | |
402cb8dd | 353 | const void *aux_data, |
ee1235a9 | 354 | loff_t object_size, |
94d30ae9 DH |
355 | bool (*can_enable)(void *data), |
356 | void *data) | |
357 | { | |
358 | _enter("%p", cookie); | |
359 | ||
a18feb55 DH |
360 | trace_fscache_enable(cookie); |
361 | ||
94d30ae9 | 362 | wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, |
74316201 | 363 | TASK_UNINTERRUPTIBLE); |
94d30ae9 | 364 | |
402cb8dd DH |
365 | fscache_update_aux(cookie, aux_data); |
366 | ||
94d30ae9 DH |
367 | if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) |
368 | goto out_unlock; | |
369 | ||
370 | if (can_enable && !can_enable(data)) { | |
371 | /* The netfs decided it didn't want to enable after all */ | |
402cb8dd | 372 | } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) { |
94d30ae9 DH |
373 | /* Wait for outstanding disablement to complete */ |
374 | __fscache_wait_on_invalidate(cookie); | |
375 | ||
ee1235a9 | 376 | if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) |
94d30ae9 DH |
377 | set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); |
378 | } else { | |
379 | set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags); | |
380 | } | |
381 | ||
382 | out_unlock: | |
383 | clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags); | |
384 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK); | |
385 | } | |
386 | EXPORT_SYMBOL(__fscache_enable_cookie); | |
387 | ||
ccc4fc3d DH |
388 | /* |
389 | * acquire a non-index cookie | |
390 | * - this must make sure the index chain is instantiated and instantiate the | |
391 | * object representation too | |
392 | */ | |
ee1235a9 DH |
393 | static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie, |
394 | loff_t object_size) | |
ccc4fc3d DH |
395 | { |
396 | struct fscache_object *object; | |
397 | struct fscache_cache *cache; | |
ccc4fc3d DH |
398 | int ret; |
399 | ||
400 | _enter(""); | |
401 | ||
94d30ae9 | 402 | set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags); |
ccc4fc3d DH |
403 | |
404 | /* now we need to see whether the backing objects for this cookie yet | |
405 | * exist, if not there'll be nothing to search */ | |
406 | down_read(&fscache_addremove_sem); | |
407 | ||
408 | if (list_empty(&fscache_cache_list)) { | |
409 | up_read(&fscache_addremove_sem); | |
410 | _leave(" = 0 [no caches]"); | |
411 | return 0; | |
412 | } | |
413 | ||
414 | /* select a cache in which to store the object */ | |
415 | cache = fscache_select_cache_for_object(cookie->parent); | |
416 | if (!cache) { | |
417 | up_read(&fscache_addremove_sem); | |
418 | fscache_stat(&fscache_n_acquires_no_cache); | |
419 | _leave(" = -ENOMEDIUM [no cache]"); | |
420 | return -ENOMEDIUM; | |
421 | } | |
422 | ||
423 | _debug("cache %s", cache->tag->name); | |
424 | ||
94d30ae9 | 425 | set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags); |
ccc4fc3d DH |
426 | |
427 | /* ask the cache to allocate objects for this cookie and its parent | |
428 | * chain */ | |
429 | ret = fscache_alloc_object(cache, cookie); | |
430 | if (ret < 0) { | |
431 | up_read(&fscache_addremove_sem); | |
432 | _leave(" = %d", ret); | |
433 | return ret; | |
434 | } | |
435 | ||
ccc4fc3d DH |
436 | spin_lock(&cookie->lock); |
437 | if (hlist_empty(&cookie->backing_objects)) { | |
438 | spin_unlock(&cookie->lock); | |
439 | goto unavailable; | |
440 | } | |
441 | ||
442 | object = hlist_entry(cookie->backing_objects.first, | |
443 | struct fscache_object, cookie_link); | |
444 | ||
ee1235a9 | 445 | fscache_set_store_limit(object, object_size); |
ccc4fc3d DH |
446 | |
447 | /* initiate the process of looking up all the objects in the chain | |
448 | * (done by fscache_initialise_object()) */ | |
caaef690 | 449 | fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD); |
ccc4fc3d DH |
450 | |
451 | spin_unlock(&cookie->lock); | |
452 | ||
453 | /* we may be required to wait for lookup to complete at this point */ | |
454 | if (!fscache_defer_lookup) { | |
455 | _debug("non-deferred lookup %p", &cookie->flags); | |
456 | wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP, | |
74316201 | 457 | TASK_UNINTERRUPTIBLE); |
ccc4fc3d DH |
458 | _debug("complete"); |
459 | if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags)) | |
460 | goto unavailable; | |
461 | } | |
462 | ||
463 | up_read(&fscache_addremove_sem); | |
464 | _leave(" = 0 [deferred]"); | |
465 | return 0; | |
466 | ||
467 | unavailable: | |
468 | up_read(&fscache_addremove_sem); | |
469 | _leave(" = -ENOBUFS"); | |
470 | return -ENOBUFS; | |
471 | } | |
472 | ||
473 | /* | |
474 | * recursively allocate cache object records for a cookie/cache combination | |
475 | * - caller must be holding the addremove sem | |
476 | */ | |
477 | static int fscache_alloc_object(struct fscache_cache *cache, | |
478 | struct fscache_cookie *cookie) | |
479 | { | |
480 | struct fscache_object *object; | |
ccc4fc3d DH |
481 | int ret; |
482 | ||
483 | _enter("%p,%p{%s}", cache, cookie, cookie->def->name); | |
484 | ||
485 | spin_lock(&cookie->lock); | |
b67bfe0d | 486 | hlist_for_each_entry(object, &cookie->backing_objects, |
ccc4fc3d DH |
487 | cookie_link) { |
488 | if (object->cache == cache) | |
489 | goto object_already_extant; | |
490 | } | |
491 | spin_unlock(&cookie->lock); | |
492 | ||
493 | /* ask the cache to allocate an object (we may end up with duplicate | |
494 | * objects at this stage, but we sort that out later) */ | |
52bd75fd | 495 | fscache_stat(&fscache_n_cop_alloc_object); |
ccc4fc3d | 496 | object = cache->ops->alloc_object(cache, cookie); |
52bd75fd | 497 | fscache_stat_d(&fscache_n_cop_alloc_object); |
ccc4fc3d DH |
498 | if (IS_ERR(object)) { |
499 | fscache_stat(&fscache_n_object_no_alloc); | |
500 | ret = PTR_ERR(object); | |
501 | goto error; | |
502 | } | |
503 | ||
f29507ce | 504 | ASSERTCMP(object->cookie, ==, cookie); |
ccc4fc3d DH |
505 | fscache_stat(&fscache_n_object_alloc); |
506 | ||
507 | object->debug_id = atomic_inc_return(&fscache_object_debug_id); | |
508 | ||
509 | _debug("ALLOC OBJ%x: %s {%lx}", | |
510 | object->debug_id, cookie->def->name, object->events); | |
511 | ||
512 | ret = fscache_alloc_object(cache, cookie->parent); | |
513 | if (ret < 0) | |
514 | goto error_put; | |
515 | ||
516 | /* only attach if we managed to allocate all we needed, otherwise | |
517 | * discard the object we just allocated and instead use the one | |
518 | * attached to the cookie */ | |
52bd75fd DH |
519 | if (fscache_attach_object(cookie, object) < 0) { |
520 | fscache_stat(&fscache_n_cop_put_object); | |
a18feb55 | 521 | cache->ops->put_object(object, fscache_obj_put_attach_fail); |
52bd75fd DH |
522 | fscache_stat_d(&fscache_n_cop_put_object); |
523 | } | |
ccc4fc3d DH |
524 | |
525 | _leave(" = 0"); | |
526 | return 0; | |
527 | ||
528 | object_already_extant: | |
529 | ret = -ENOBUFS; | |
87021526 DH |
530 | if (fscache_object_is_dying(object) || |
531 | fscache_cache_is_broken(object)) { | |
ccc4fc3d DH |
532 | spin_unlock(&cookie->lock); |
533 | goto error; | |
534 | } | |
535 | spin_unlock(&cookie->lock); | |
536 | _leave(" = 0 [found]"); | |
537 | return 0; | |
538 | ||
539 | error_put: | |
52bd75fd | 540 | fscache_stat(&fscache_n_cop_put_object); |
a18feb55 | 541 | cache->ops->put_object(object, fscache_obj_put_alloc_fail); |
52bd75fd | 542 | fscache_stat_d(&fscache_n_cop_put_object); |
ccc4fc3d DH |
543 | error: |
544 | _leave(" = %d", ret); | |
545 | return ret; | |
546 | } | |
547 | ||
548 | /* | |
549 | * attach a cache object to a cookie | |
550 | */ | |
551 | static int fscache_attach_object(struct fscache_cookie *cookie, | |
552 | struct fscache_object *object) | |
553 | { | |
554 | struct fscache_object *p; | |
555 | struct fscache_cache *cache = object->cache; | |
ccc4fc3d DH |
556 | int ret; |
557 | ||
558 | _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id); | |
559 | ||
f29507ce KKM |
560 | ASSERTCMP(object->cookie, ==, cookie); |
561 | ||
ccc4fc3d DH |
562 | spin_lock(&cookie->lock); |
563 | ||
564 | /* there may be multiple initial creations of this object, but we only | |
565 | * want one */ | |
566 | ret = -EEXIST; | |
b67bfe0d | 567 | hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) { |
ccc4fc3d | 568 | if (p->cache == object->cache) { |
493f7bc1 | 569 | if (fscache_object_is_dying(p)) |
ccc4fc3d DH |
570 | ret = -ENOBUFS; |
571 | goto cant_attach_object; | |
572 | } | |
573 | } | |
574 | ||
575 | /* pin the parent object */ | |
576 | spin_lock_nested(&cookie->parent->lock, 1); | |
b67bfe0d | 577 | hlist_for_each_entry(p, &cookie->parent->backing_objects, |
ccc4fc3d DH |
578 | cookie_link) { |
579 | if (p->cache == object->cache) { | |
493f7bc1 | 580 | if (fscache_object_is_dying(p)) { |
ccc4fc3d DH |
581 | ret = -ENOBUFS; |
582 | spin_unlock(&cookie->parent->lock); | |
583 | goto cant_attach_object; | |
584 | } | |
585 | object->parent = p; | |
586 | spin_lock(&p->lock); | |
587 | p->n_children++; | |
588 | spin_unlock(&p->lock); | |
589 | break; | |
590 | } | |
591 | } | |
592 | spin_unlock(&cookie->parent->lock); | |
593 | ||
594 | /* attach to the cache's object list */ | |
595 | if (list_empty(&object->cache_link)) { | |
596 | spin_lock(&cache->object_list_lock); | |
597 | list_add(&object->cache_link, &cache->object_list); | |
598 | spin_unlock(&cache->object_list_lock); | |
599 | } | |
600 | ||
f29507ce | 601 | /* Attach to the cookie. The object already has a ref on it. */ |
ccc4fc3d | 602 | hlist_add_head(&object->cookie_link, &cookie->backing_objects); |
4fbf4291 DH |
603 | |
604 | fscache_objlist_add(object); | |
ccc4fc3d DH |
605 | ret = 0; |
606 | ||
607 | cant_attach_object: | |
608 | spin_unlock(&cookie->lock); | |
609 | _leave(" = %d", ret); | |
610 | return ret; | |
611 | } | |
612 | ||
ef778e7a DH |
613 | /* |
614 | * Invalidate an object. Callable with spinlocks held. | |
615 | */ | |
616 | void __fscache_invalidate(struct fscache_cookie *cookie) | |
617 | { | |
618 | struct fscache_object *object; | |
619 | ||
620 | _enter("{%s}", cookie->def->name); | |
621 | ||
622 | fscache_stat(&fscache_n_invalidates); | |
623 | ||
624 | /* Only permit invalidation of data files. Invalidating an index will | |
625 | * require the caller to release all its attachments to the tree rooted | |
626 | * there, and if it's doing that, it may as well just retire the | |
627 | * cookie. | |
628 | */ | |
402cb8dd | 629 | ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); |
ef778e7a DH |
630 | |
631 | /* If there's an object, we tell the object state machine to handle the | |
632 | * invalidation on our behalf, otherwise there's nothing to do. | |
633 | */ | |
634 | if (!hlist_empty(&cookie->backing_objects)) { | |
635 | spin_lock(&cookie->lock); | |
636 | ||
94d30ae9 DH |
637 | if (fscache_cookie_enabled(cookie) && |
638 | !hlist_empty(&cookie->backing_objects) && | |
ef778e7a DH |
639 | !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING, |
640 | &cookie->flags)) { | |
641 | object = hlist_entry(cookie->backing_objects.first, | |
642 | struct fscache_object, | |
643 | cookie_link); | |
493f7bc1 | 644 | if (fscache_object_is_live(object)) |
ef778e7a DH |
645 | fscache_raise_event( |
646 | object, FSCACHE_OBJECT_EV_INVALIDATE); | |
647 | } | |
648 | ||
649 | spin_unlock(&cookie->lock); | |
650 | } | |
651 | ||
652 | _leave(""); | |
653 | } | |
654 | EXPORT_SYMBOL(__fscache_invalidate); | |
655 | ||
656 | /* | |
657 | * Wait for object invalidation to complete. | |
658 | */ | |
659 | void __fscache_wait_on_invalidate(struct fscache_cookie *cookie) | |
660 | { | |
661 | _enter("%p", cookie); | |
662 | ||
663 | wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING, | |
ef778e7a DH |
664 | TASK_UNINTERRUPTIBLE); |
665 | ||
666 | _leave(""); | |
667 | } | |
668 | EXPORT_SYMBOL(__fscache_wait_on_invalidate); | |
669 | ||
ccc4fc3d DH |
670 | /* |
671 | * update the index entries backing a cookie | |
672 | */ | |
402cb8dd | 673 | void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data) |
ccc4fc3d DH |
674 | { |
675 | struct fscache_object *object; | |
ccc4fc3d DH |
676 | |
677 | fscache_stat(&fscache_n_updates); | |
678 | ||
679 | if (!cookie) { | |
680 | fscache_stat(&fscache_n_updates_null); | |
681 | _leave(" [no cookie]"); | |
682 | return; | |
683 | } | |
684 | ||
685 | _enter("{%s}", cookie->def->name); | |
686 | ||
ccc4fc3d DH |
687 | spin_lock(&cookie->lock); |
688 | ||
402cb8dd DH |
689 | fscache_update_aux(cookie, aux_data); |
690 | ||
94d30ae9 DH |
691 | if (fscache_cookie_enabled(cookie)) { |
692 | /* update the index entry on disk in each cache backing this | |
693 | * cookie. | |
694 | */ | |
695 | hlist_for_each_entry(object, | |
696 | &cookie->backing_objects, cookie_link) { | |
697 | fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE); | |
698 | } | |
ccc4fc3d DH |
699 | } |
700 | ||
701 | spin_unlock(&cookie->lock); | |
702 | _leave(""); | |
703 | } | |
704 | EXPORT_SYMBOL(__fscache_update_cookie); | |
705 | ||
706 | /* | |
94d30ae9 | 707 | * Disable a cookie to stop it from accepting new requests from the netfs. |
ccc4fc3d | 708 | */ |
402cb8dd DH |
709 | void __fscache_disable_cookie(struct fscache_cookie *cookie, |
710 | const void *aux_data, | |
711 | bool invalidate) | |
ccc4fc3d | 712 | { |
ccc4fc3d | 713 | struct fscache_object *object; |
94d30ae9 | 714 | bool awaken = false; |
ccc4fc3d | 715 | |
94d30ae9 | 716 | _enter("%p,%u", cookie, invalidate); |
1362729b | 717 | |
a18feb55 DH |
718 | trace_fscache_disable(cookie); |
719 | ||
1362729b | 720 | ASSERTCMP(atomic_read(&cookie->n_active), >, 0); |
ccc4fc3d DH |
721 | |
722 | if (atomic_read(&cookie->n_children) != 0) { | |
36dfd116 | 723 | pr_err("Cookie '%s' still has children\n", |
ccc4fc3d DH |
724 | cookie->def->name); |
725 | BUG(); | |
726 | } | |
727 | ||
94d30ae9 | 728 | wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK, |
74316201 | 729 | TASK_UNINTERRUPTIBLE); |
402cb8dd DH |
730 | |
731 | fscache_update_aux(cookie, aux_data); | |
732 | ||
94d30ae9 DH |
733 | if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags)) |
734 | goto out_unlock_enable; | |
735 | ||
736 | /* If the cookie is being invalidated, wait for that to complete first | |
737 | * so that we can reuse the flag. | |
738 | */ | |
739 | __fscache_wait_on_invalidate(cookie); | |
740 | ||
741 | /* Dispose of the backing objects */ | |
742 | set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags); | |
ccc4fc3d | 743 | |
ccc4fc3d | 744 | spin_lock(&cookie->lock); |
94d30ae9 DH |
745 | if (!hlist_empty(&cookie->backing_objects)) { |
746 | hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) { | |
747 | if (invalidate) | |
748 | set_bit(FSCACHE_OBJECT_RETIRED, &object->flags); | |
6bdded59 | 749 | clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); |
94d30ae9 DH |
750 | fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL); |
751 | } | |
752 | } else { | |
753 | if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) | |
754 | awaken = true; | |
ccc4fc3d | 755 | } |
1362729b | 756 | spin_unlock(&cookie->lock); |
94d30ae9 DH |
757 | if (awaken) |
758 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING); | |
ccc4fc3d | 759 | |
1362729b | 760 | /* Wait for cessation of activity requiring access to the netfs (when |
94d30ae9 DH |
761 | * n_active reaches 0). This makes sure outstanding reads and writes |
762 | * have completed. | |
1362729b | 763 | */ |
dc5d4afb PZ |
764 | if (!atomic_dec_and_test(&cookie->n_active)) { |
765 | wait_var_event(&cookie->n_active, | |
766 | !atomic_read(&cookie->n_active)); | |
767 | } | |
1362729b | 768 | |
6bdded59 | 769 | /* Make sure any pending writes are cancelled. */ |
402cb8dd | 770 | if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) |
6bdded59 DH |
771 | fscache_invalidate_writes(cookie); |
772 | ||
94d30ae9 DH |
773 | /* Reset the cookie state if it wasn't relinquished */ |
774 | if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) { | |
775 | atomic_inc(&cookie->n_active); | |
776 | set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); | |
777 | } | |
778 | ||
779 | out_unlock_enable: | |
780 | clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags); | |
781 | wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK); | |
782 | _leave(""); | |
783 | } | |
784 | EXPORT_SYMBOL(__fscache_disable_cookie); | |
785 | ||
786 | /* | |
787 | * release a cookie back to the cache | |
788 | * - the object will be marked as recyclable on disk if retire is true | |
789 | * - all dependents of this cookie must have already been unregistered | |
790 | * (indices/files/pages) | |
791 | */ | |
402cb8dd DH |
792 | void __fscache_relinquish_cookie(struct fscache_cookie *cookie, |
793 | const void *aux_data, | |
794 | bool retire) | |
94d30ae9 DH |
795 | { |
796 | fscache_stat(&fscache_n_relinquishes); | |
797 | if (retire) | |
798 | fscache_stat(&fscache_n_relinquishes_retire); | |
799 | ||
800 | if (!cookie) { | |
801 | fscache_stat(&fscache_n_relinquishes_null); | |
802 | _leave(" [no cookie]"); | |
803 | return; | |
804 | } | |
805 | ||
806 | _enter("%p{%s,%p,%d},%d", | |
807 | cookie, cookie->def->name, cookie->netfs_data, | |
808 | atomic_read(&cookie->n_active), retire); | |
809 | ||
a18feb55 DH |
810 | trace_fscache_relinquish(cookie, retire); |
811 | ||
94d30ae9 | 812 | /* No further netfs-accessing operations on this cookie permitted */ |
d0fb31ec DH |
813 | if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) |
814 | BUG(); | |
94d30ae9 | 815 | |
402cb8dd | 816 | __fscache_disable_cookie(cookie, aux_data, retire); |
94d30ae9 | 817 | |
1362729b | 818 | /* Clear pointers back to the netfs */ |
7e311a20 DH |
819 | cookie->netfs_data = NULL; |
820 | cookie->def = NULL; | |
e5a95541 | 821 | BUG_ON(!radix_tree_empty(&cookie->stores)); |
ccc4fc3d DH |
822 | |
823 | if (cookie->parent) { | |
824 | ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0); | |
825 | ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0); | |
826 | atomic_dec(&cookie->parent->n_children); | |
827 | } | |
828 | ||
1362729b | 829 | /* Dispose of the netfs's link to the cookie */ |
ccc4fc3d | 830 | ASSERTCMP(atomic_read(&cookie->usage), >, 0); |
a18feb55 | 831 | fscache_cookie_put(cookie, fscache_cookie_put_relinquish); |
ccc4fc3d DH |
832 | |
833 | _leave(""); | |
834 | } | |
835 | EXPORT_SYMBOL(__fscache_relinquish_cookie); | |
836 | ||
ec0328e4 DH |
837 | /* |
838 | * Remove a cookie from the hash table. | |
839 | */ | |
840 | static void fscache_unhash_cookie(struct fscache_cookie *cookie) | |
841 | { | |
842 | struct hlist_bl_head *h; | |
843 | unsigned int bucket; | |
844 | ||
845 | bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); | |
846 | h = &fscache_cookie_hash[bucket]; | |
847 | ||
848 | hlist_bl_lock(h); | |
849 | hlist_bl_del(&cookie->hash_link); | |
850 | hlist_bl_unlock(h); | |
851 | } | |
852 | ||
955d0091 | 853 | /* |
a18feb55 | 854 | * Drop a reference to a cookie. |
955d0091 | 855 | */ |
a18feb55 DH |
856 | void fscache_cookie_put(struct fscache_cookie *cookie, |
857 | enum fscache_cookie_trace where) | |
955d0091 DH |
858 | { |
859 | struct fscache_cookie *parent; | |
a18feb55 | 860 | int usage; |
955d0091 DH |
861 | |
862 | _enter("%p", cookie); | |
863 | ||
a18feb55 DH |
864 | do { |
865 | usage = atomic_dec_return(&cookie->usage); | |
866 | trace_fscache_cookie(cookie, where, usage); | |
867 | ||
868 | if (usage > 0) | |
869 | return; | |
870 | BUG_ON(usage < 0); | |
871 | ||
955d0091 | 872 | parent = cookie->parent; |
ec0328e4 DH |
873 | fscache_unhash_cookie(cookie); |
874 | fscache_free_cookie(cookie); | |
955d0091 | 875 | |
955d0091 | 876 | cookie = parent; |
a18feb55 DH |
877 | where = fscache_cookie_put_parent; |
878 | } while (cookie); | |
955d0091 DH |
879 | |
880 | _leave(""); | |
881 | } | |
da9803bc DH |
882 | |
883 | /* | |
884 | * check the consistency between the netfs inode and the backing cache | |
885 | * | |
886 | * NOTE: it only serves no-index type | |
887 | */ | |
402cb8dd DH |
888 | int __fscache_check_consistency(struct fscache_cookie *cookie, |
889 | const void *aux_data) | |
da9803bc DH |
890 | { |
891 | struct fscache_operation *op; | |
892 | struct fscache_object *object; | |
8fb883f3 | 893 | bool wake_cookie = false; |
da9803bc DH |
894 | int ret; |
895 | ||
896 | _enter("%p,", cookie); | |
897 | ||
402cb8dd | 898 | ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE); |
da9803bc DH |
899 | |
900 | if (fscache_wait_for_deferred_lookup(cookie) < 0) | |
901 | return -ERESTARTSYS; | |
902 | ||
903 | if (hlist_empty(&cookie->backing_objects)) | |
904 | return 0; | |
905 | ||
906 | op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY); | |
907 | if (!op) | |
908 | return -ENOMEM; | |
909 | ||
08c2e3d0 | 910 | fscache_operation_init(cookie, op, NULL, NULL, NULL); |
da9803bc | 911 | op->flags = FSCACHE_OP_MYTHREAD | |
9c89d629 MT |
912 | (1 << FSCACHE_OP_WAITING) | |
913 | (1 << FSCACHE_OP_UNUSE_COOKIE); | |
08c2e3d0 | 914 | trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency); |
da9803bc DH |
915 | |
916 | spin_lock(&cookie->lock); | |
917 | ||
402cb8dd DH |
918 | fscache_update_aux(cookie, aux_data); |
919 | ||
94d30ae9 DH |
920 | if (!fscache_cookie_enabled(cookie) || |
921 | hlist_empty(&cookie->backing_objects)) | |
da9803bc DH |
922 | goto inconsistent; |
923 | object = hlist_entry(cookie->backing_objects.first, | |
924 | struct fscache_object, cookie_link); | |
925 | if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) | |
926 | goto inconsistent; | |
927 | ||
928 | op->debug_id = atomic_inc_return(&fscache_op_debug_id); | |
929 | ||
8fb883f3 | 930 | __fscache_use_cookie(cookie); |
da9803bc DH |
931 | if (fscache_submit_op(object, op) < 0) |
932 | goto submit_failed; | |
933 | ||
934 | /* the work queue now carries its own ref on the object */ | |
935 | spin_unlock(&cookie->lock); | |
936 | ||
d3b97ca4 | 937 | ret = fscache_wait_for_operation_activation(object, op, NULL, NULL); |
da9803bc DH |
938 | if (ret == 0) { |
939 | /* ask the cache to honour the operation */ | |
940 | ret = object->cache->ops->check_consistency(op); | |
941 | fscache_op_complete(op, false); | |
942 | } else if (ret == -ENOBUFS) { | |
943 | ret = 0; | |
944 | } | |
945 | ||
946 | fscache_put_operation(op); | |
947 | _leave(" = %d", ret); | |
948 | return ret; | |
949 | ||
950 | submit_failed: | |
8fb883f3 | 951 | wake_cookie = __fscache_unuse_cookie(cookie); |
da9803bc DH |
952 | inconsistent: |
953 | spin_unlock(&cookie->lock); | |
8fb883f3 DH |
954 | if (wake_cookie) |
955 | __fscache_wake_unused_cookie(cookie); | |
da9803bc DH |
956 | kfree(op); |
957 | _leave(" = -ESTALE"); | |
958 | return -ESTALE; | |
959 | } | |
960 | EXPORT_SYMBOL(__fscache_check_consistency); |