Commit | Line | Data |
---|---|---|
06b3db1b DH |
1 | /* Internal definitions for FS-Cache |
2 | * | |
3 | * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved. | |
4 | * Written by David Howells (dhowells@redhat.com) | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | /* | |
13 | * Lock order, in the order in which multiple locks should be obtained: | |
14 | * - fscache_addremove_sem | |
15 | * - cookie->lock | |
16 | * - cookie->parent->lock | |
17 | * - cache->object_list_lock | |
18 | * - object->lock | |
19 | * - object->parent->lock | |
1bccf513 | 20 | * - cookie->stores_lock |
06b3db1b DH |
21 | * - fscache_thread_lock |
22 | * | |
23 | */ | |
24 | ||
36dfd116 FF |
25 | #ifdef pr_fmt |
26 | #undef pr_fmt | |
27 | #endif | |
28 | ||
29 | #define pr_fmt(fmt) "FS-Cache: " fmt | |
30 | ||
06b3db1b DH |
31 | #include <linux/fscache-cache.h> |
32 | #include <linux/sched.h> | |
33 | ||
34 | #define FSCACHE_MIN_THREADS 4 | |
35 | #define FSCACHE_MAX_THREADS 32 | |
36 | ||
0e04d4ce | 37 | /* |
348ca102 | 38 | * cache.c |
0e04d4ce DH |
39 | */ |
40 | extern struct list_head fscache_cache_list; | |
41 | extern struct rw_semaphore fscache_addremove_sem; | |
42 | ||
43 | extern struct fscache_cache *fscache_select_cache_for_object( | |
44 | struct fscache_cookie *); | |
45 | ||
955d0091 | 46 | /* |
348ca102 | 47 | * cookie.c |
955d0091 DH |
48 | */ |
49 | extern struct kmem_cache *fscache_cookie_jar; | |
50 | ||
51 | extern void fscache_cookie_init_once(void *); | |
52 | extern void __fscache_cookie_put(struct fscache_cookie *); | |
53 | ||
a6891645 | 54 | /* |
348ca102 | 55 | * fsdef.c |
a6891645 DH |
56 | */ |
57 | extern struct fscache_cookie fscache_fsdef_index; | |
58 | extern struct fscache_cookie_def fscache_fsdef_netfs_def; | |
59 | ||
7394daa8 | 60 | /* |
348ca102 | 61 | * histogram.c |
7394daa8 DH |
62 | */ |
63 | #ifdef CONFIG_FSCACHE_HISTOGRAM | |
64 | extern atomic_t fscache_obj_instantiate_histogram[HZ]; | |
65 | extern atomic_t fscache_objs_histogram[HZ]; | |
66 | extern atomic_t fscache_ops_histogram[HZ]; | |
67 | extern atomic_t fscache_retrieval_delay_histogram[HZ]; | |
68 | extern atomic_t fscache_retrieval_histogram[HZ]; | |
69 | ||
70 | static inline void fscache_hist(atomic_t histogram[], unsigned long start_jif) | |
71 | { | |
72 | unsigned long jif = jiffies - start_jif; | |
73 | if (jif >= HZ) | |
74 | jif = HZ - 1; | |
75 | atomic_inc(&histogram[jif]); | |
76 | } | |
77 | ||
78 | extern const struct file_operations fscache_histogram_fops; | |
79 | ||
80 | #else | |
81 | #define fscache_hist(hist, start_jif) do {} while (0) | |
82 | #endif | |
83 | ||
06b3db1b | 84 | /* |
348ca102 | 85 | * main.c |
06b3db1b DH |
86 | */ |
87 | extern unsigned fscache_defer_lookup; | |
88 | extern unsigned fscache_defer_create; | |
89 | extern unsigned fscache_debug; | |
90 | extern struct kobject *fscache_root; | |
8b8edefa | 91 | extern struct workqueue_struct *fscache_object_wq; |
8af7c124 | 92 | extern struct workqueue_struct *fscache_op_wq; |
8b8edefa TH |
93 | DECLARE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait); |
94 | ||
95 | static inline bool fscache_object_congested(void) | |
96 | { | |
97 | return workqueue_congested(WORK_CPU_UNBOUND, fscache_object_wq); | |
98 | } | |
06b3db1b | 99 | |
1362729b | 100 | extern int fscache_wait_atomic_t(atomic_t *); |
2868cbea | 101 | |
36c95590 | 102 | /* |
348ca102 | 103 | * object.c |
36c95590 | 104 | */ |
36c95590 DH |
105 | extern void fscache_enqueue_object(struct fscache_object *); |
106 | ||
4fbf4291 DH |
107 | /* |
108 | * object-list.c | |
109 | */ | |
110 | #ifdef CONFIG_FSCACHE_OBJECT_LIST | |
111 | extern const struct file_operations fscache_objlist_fops; | |
112 | ||
113 | extern void fscache_objlist_add(struct fscache_object *); | |
1362729b | 114 | extern void fscache_objlist_remove(struct fscache_object *); |
4fbf4291 DH |
115 | #else |
116 | #define fscache_objlist_add(object) do {} while(0) | |
1362729b | 117 | #define fscache_objlist_remove(object) do {} while(0) |
4fbf4291 DH |
118 | #endif |
119 | ||
36c95590 | 120 | /* |
348ca102 | 121 | * operation.c |
36c95590 | 122 | */ |
952efe7b DH |
123 | extern int fscache_submit_exclusive_op(struct fscache_object *, |
124 | struct fscache_operation *); | |
125 | extern int fscache_submit_op(struct fscache_object *, | |
126 | struct fscache_operation *); | |
91c7fbbf DH |
127 | extern int fscache_cancel_op(struct fscache_operation *, |
128 | void (*)(struct fscache_operation *)); | |
ef778e7a | 129 | extern void fscache_cancel_all_ops(struct fscache_object *); |
952efe7b DH |
130 | extern void fscache_abort_object(struct fscache_object *); |
131 | extern void fscache_start_operations(struct fscache_object *); | |
132 | extern void fscache_operation_gc(struct work_struct *); | |
36c95590 | 133 | |
ef778e7a DH |
134 | /* |
135 | * page.c | |
136 | */ | |
da9803bc DH |
137 | extern int fscache_wait_for_deferred_lookup(struct fscache_cookie *); |
138 | extern int fscache_wait_for_operation_activation(struct fscache_object *, | |
139 | struct fscache_operation *, | |
140 | atomic_t *, | |
141 | atomic_t *, | |
142 | void (*)(struct fscache_operation *)); | |
ef778e7a DH |
143 | extern void fscache_invalidate_writes(struct fscache_cookie *); |
144 | ||
7394daa8 | 145 | /* |
348ca102 | 146 | * proc.c |
7394daa8 DH |
147 | */ |
148 | #ifdef CONFIG_PROC_FS | |
149 | extern int __init fscache_proc_init(void); | |
150 | extern void fscache_proc_cleanup(void); | |
151 | #else | |
152 | #define fscache_proc_init() (0) | |
153 | #define fscache_proc_cleanup() do {} while (0) | |
154 | #endif | |
155 | ||
156 | /* | |
348ca102 | 157 | * stats.c |
7394daa8 DH |
158 | */ |
159 | #ifdef CONFIG_FSCACHE_STATS | |
160 | extern atomic_t fscache_n_ops_processed[FSCACHE_MAX_THREADS]; | |
161 | extern atomic_t fscache_n_objs_processed[FSCACHE_MAX_THREADS]; | |
162 | ||
163 | extern atomic_t fscache_n_op_pend; | |
164 | extern atomic_t fscache_n_op_run; | |
165 | extern atomic_t fscache_n_op_enqueue; | |
166 | extern atomic_t fscache_n_op_deferred_release; | |
167 | extern atomic_t fscache_n_op_release; | |
168 | extern atomic_t fscache_n_op_gc; | |
5753c441 | 169 | extern atomic_t fscache_n_op_cancelled; |
e3d4d28b | 170 | extern atomic_t fscache_n_op_rejected; |
7394daa8 DH |
171 | |
172 | extern atomic_t fscache_n_attr_changed; | |
173 | extern atomic_t fscache_n_attr_changed_ok; | |
174 | extern atomic_t fscache_n_attr_changed_nobufs; | |
175 | extern atomic_t fscache_n_attr_changed_nomem; | |
176 | extern atomic_t fscache_n_attr_changed_calls; | |
177 | ||
178 | extern atomic_t fscache_n_allocs; | |
179 | extern atomic_t fscache_n_allocs_ok; | |
180 | extern atomic_t fscache_n_allocs_wait; | |
181 | extern atomic_t fscache_n_allocs_nobufs; | |
5753c441 | 182 | extern atomic_t fscache_n_allocs_intr; |
60d543ca | 183 | extern atomic_t fscache_n_allocs_object_dead; |
7394daa8 DH |
184 | extern atomic_t fscache_n_alloc_ops; |
185 | extern atomic_t fscache_n_alloc_op_waits; | |
186 | ||
187 | extern atomic_t fscache_n_retrievals; | |
188 | extern atomic_t fscache_n_retrievals_ok; | |
189 | extern atomic_t fscache_n_retrievals_wait; | |
190 | extern atomic_t fscache_n_retrievals_nodata; | |
191 | extern atomic_t fscache_n_retrievals_nobufs; | |
192 | extern atomic_t fscache_n_retrievals_intr; | |
193 | extern atomic_t fscache_n_retrievals_nomem; | |
60d543ca | 194 | extern atomic_t fscache_n_retrievals_object_dead; |
7394daa8 DH |
195 | extern atomic_t fscache_n_retrieval_ops; |
196 | extern atomic_t fscache_n_retrieval_op_waits; | |
197 | ||
198 | extern atomic_t fscache_n_stores; | |
199 | extern atomic_t fscache_n_stores_ok; | |
200 | extern atomic_t fscache_n_stores_again; | |
201 | extern atomic_t fscache_n_stores_nobufs; | |
202 | extern atomic_t fscache_n_stores_oom; | |
203 | extern atomic_t fscache_n_store_ops; | |
204 | extern atomic_t fscache_n_store_calls; | |
1bccf513 DH |
205 | extern atomic_t fscache_n_store_pages; |
206 | extern atomic_t fscache_n_store_radix_deletes; | |
207 | extern atomic_t fscache_n_store_pages_over_limit; | |
7394daa8 | 208 | |
201a1542 DH |
209 | extern atomic_t fscache_n_store_vmscan_not_storing; |
210 | extern atomic_t fscache_n_store_vmscan_gone; | |
211 | extern atomic_t fscache_n_store_vmscan_busy; | |
212 | extern atomic_t fscache_n_store_vmscan_cancelled; | |
8c209ce7 | 213 | extern atomic_t fscache_n_store_vmscan_wait; |
201a1542 | 214 | |
7394daa8 DH |
215 | extern atomic_t fscache_n_marks; |
216 | extern atomic_t fscache_n_uncaches; | |
217 | ||
218 | extern atomic_t fscache_n_acquires; | |
219 | extern atomic_t fscache_n_acquires_null; | |
220 | extern atomic_t fscache_n_acquires_no_cache; | |
221 | extern atomic_t fscache_n_acquires_ok; | |
222 | extern atomic_t fscache_n_acquires_nobufs; | |
223 | extern atomic_t fscache_n_acquires_oom; | |
224 | ||
ef778e7a DH |
225 | extern atomic_t fscache_n_invalidates; |
226 | extern atomic_t fscache_n_invalidates_run; | |
227 | ||
7394daa8 DH |
228 | extern atomic_t fscache_n_updates; |
229 | extern atomic_t fscache_n_updates_null; | |
230 | extern atomic_t fscache_n_updates_run; | |
231 | ||
232 | extern atomic_t fscache_n_relinquishes; | |
233 | extern atomic_t fscache_n_relinquishes_null; | |
234 | extern atomic_t fscache_n_relinquishes_waitcrt; | |
2175bb06 | 235 | extern atomic_t fscache_n_relinquishes_retire; |
7394daa8 DH |
236 | |
237 | extern atomic_t fscache_n_cookie_index; | |
238 | extern atomic_t fscache_n_cookie_data; | |
239 | extern atomic_t fscache_n_cookie_special; | |
240 | ||
241 | extern atomic_t fscache_n_object_alloc; | |
242 | extern atomic_t fscache_n_object_no_alloc; | |
243 | extern atomic_t fscache_n_object_lookups; | |
244 | extern atomic_t fscache_n_object_lookups_negative; | |
245 | extern atomic_t fscache_n_object_lookups_positive; | |
fee096de | 246 | extern atomic_t fscache_n_object_lookups_timed_out; |
7394daa8 DH |
247 | extern atomic_t fscache_n_object_created; |
248 | extern atomic_t fscache_n_object_avail; | |
249 | extern atomic_t fscache_n_object_dead; | |
250 | ||
251 | extern atomic_t fscache_n_checkaux_none; | |
252 | extern atomic_t fscache_n_checkaux_okay; | |
253 | extern atomic_t fscache_n_checkaux_update; | |
254 | extern atomic_t fscache_n_checkaux_obsolete; | |
255 | ||
52bd75fd DH |
256 | extern atomic_t fscache_n_cop_alloc_object; |
257 | extern atomic_t fscache_n_cop_lookup_object; | |
258 | extern atomic_t fscache_n_cop_lookup_complete; | |
259 | extern atomic_t fscache_n_cop_grab_object; | |
ef778e7a | 260 | extern atomic_t fscache_n_cop_invalidate_object; |
52bd75fd DH |
261 | extern atomic_t fscache_n_cop_update_object; |
262 | extern atomic_t fscache_n_cop_drop_object; | |
263 | extern atomic_t fscache_n_cop_put_object; | |
264 | extern atomic_t fscache_n_cop_sync_cache; | |
265 | extern atomic_t fscache_n_cop_attr_changed; | |
266 | extern atomic_t fscache_n_cop_read_or_alloc_page; | |
267 | extern atomic_t fscache_n_cop_read_or_alloc_pages; | |
268 | extern atomic_t fscache_n_cop_allocate_page; | |
269 | extern atomic_t fscache_n_cop_allocate_pages; | |
270 | extern atomic_t fscache_n_cop_write_page; | |
271 | extern atomic_t fscache_n_cop_uncache_page; | |
272 | extern atomic_t fscache_n_cop_dissociate_pages; | |
273 | ||
182d919b DH |
274 | extern atomic_t fscache_n_cache_no_space_reject; |
275 | extern atomic_t fscache_n_cache_stale_objects; | |
276 | extern atomic_t fscache_n_cache_retired_objects; | |
277 | extern atomic_t fscache_n_cache_culled_objects; | |
278 | ||
7394daa8 DH |
279 | static inline void fscache_stat(atomic_t *stat) |
280 | { | |
281 | atomic_inc(stat); | |
282 | } | |
283 | ||
52bd75fd DH |
284 | static inline void fscache_stat_d(atomic_t *stat) |
285 | { | |
286 | atomic_dec(stat); | |
287 | } | |
288 | ||
60d543ca DH |
289 | #define __fscache_stat(stat) (stat) |
290 | ||
7394daa8 DH |
291 | extern const struct file_operations fscache_stats_fops; |
292 | #else | |
293 | ||
60d543ca | 294 | #define __fscache_stat(stat) (NULL) |
7394daa8 | 295 | #define fscache_stat(stat) do {} while (0) |
4fa9f4ed | 296 | #define fscache_stat_d(stat) do {} while (0) |
7394daa8 DH |
297 | #endif |
298 | ||
0e04d4ce DH |
299 | /* |
300 | * raise an event on an object | |
301 | * - if the event is not masked for that object, then the object is | |
302 | * queued for attention by the thread pool. | |
303 | */ | |
304 | static inline void fscache_raise_event(struct fscache_object *object, | |
305 | unsigned event) | |
306 | { | |
8d76349d | 307 | BUG_ON(event >= NR_FSCACHE_OBJECT_EVENTS); |
caaef690 DH |
308 | #if 0 |
309 | printk("*** fscache_raise_event(OBJ%d{%lx},%x)\n", | |
310 | object->debug_id, object->event_mask, (1 << event)); | |
311 | #endif | |
36c95590 DH |
312 | if (!test_and_set_bit(event, &object->events) && |
313 | test_bit(event, &object->event_mask)) | |
314 | fscache_enqueue_object(object); | |
315 | } | |
316 | ||
317 | /* | |
318 | * drop a reference to a cookie | |
319 | */ | |
320 | static inline void fscache_cookie_put(struct fscache_cookie *cookie) | |
321 | { | |
322 | BUG_ON(atomic_read(&cookie->usage) <= 0); | |
323 | if (atomic_dec_and_test(&cookie->usage)) | |
324 | __fscache_cookie_put(cookie); | |
0e04d4ce DH |
325 | } |
326 | ||
b5108822 DH |
327 | /* |
328 | * get an extra reference to a netfs retrieval context | |
329 | */ | |
330 | static inline | |
331 | void *fscache_get_context(struct fscache_cookie *cookie, void *context) | |
332 | { | |
333 | if (cookie->def->get_context) | |
334 | cookie->def->get_context(cookie->netfs_data, context); | |
335 | return context; | |
336 | } | |
337 | ||
338 | /* | |
339 | * release a reference to a netfs retrieval context | |
340 | */ | |
341 | static inline | |
342 | void fscache_put_context(struct fscache_cookie *cookie, void *context) | |
343 | { | |
344 | if (cookie->def->put_context) | |
345 | cookie->def->put_context(cookie->netfs_data, context); | |
346 | } | |
347 | ||
06b3db1b DH |
348 | /*****************************************************************************/ |
349 | /* | |
350 | * debug tracing | |
351 | */ | |
352 | #define dbgprintk(FMT, ...) \ | |
353 | printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) | |
354 | ||
06b3db1b DH |
355 | #define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) |
356 | #define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) | |
357 | #define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) | |
358 | ||
12fdff3f | 359 | #define kjournal(FMT, ...) no_printk(FMT, ##__VA_ARGS__) |
06b3db1b DH |
360 | |
361 | #ifdef __KDEBUG | |
362 | #define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) | |
363 | #define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) | |
364 | #define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) | |
365 | ||
366 | #elif defined(CONFIG_FSCACHE_DEBUG) | |
367 | #define _enter(FMT, ...) \ | |
368 | do { \ | |
369 | if (__do_kdebug(ENTER)) \ | |
370 | kenter(FMT, ##__VA_ARGS__); \ | |
371 | } while (0) | |
372 | ||
373 | #define _leave(FMT, ...) \ | |
374 | do { \ | |
375 | if (__do_kdebug(LEAVE)) \ | |
376 | kleave(FMT, ##__VA_ARGS__); \ | |
377 | } while (0) | |
378 | ||
379 | #define _debug(FMT, ...) \ | |
380 | do { \ | |
381 | if (__do_kdebug(DEBUG)) \ | |
382 | kdebug(FMT, ##__VA_ARGS__); \ | |
383 | } while (0) | |
384 | ||
385 | #else | |
12fdff3f DH |
386 | #define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) |
387 | #define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) | |
388 | #define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) | |
06b3db1b DH |
389 | #endif |
390 | ||
391 | /* | |
392 | * determine whether a particular optional debugging point should be logged | |
393 | * - we need to go through three steps to persuade cpp to correctly join the | |
394 | * shorthand in FSCACHE_DEBUG_LEVEL with its prefix | |
395 | */ | |
396 | #define ____do_kdebug(LEVEL, POINT) \ | |
397 | unlikely((fscache_debug & \ | |
398 | (FSCACHE_POINT_##POINT << (FSCACHE_DEBUG_ ## LEVEL * 3)))) | |
399 | #define ___do_kdebug(LEVEL, POINT) \ | |
400 | ____do_kdebug(LEVEL, POINT) | |
401 | #define __do_kdebug(POINT) \ | |
402 | ___do_kdebug(FSCACHE_DEBUG_LEVEL, POINT) | |
403 | ||
404 | #define FSCACHE_DEBUG_CACHE 0 | |
405 | #define FSCACHE_DEBUG_COOKIE 1 | |
406 | #define FSCACHE_DEBUG_PAGE 2 | |
407 | #define FSCACHE_DEBUG_OPERATION 3 | |
408 | ||
409 | #define FSCACHE_POINT_ENTER 1 | |
410 | #define FSCACHE_POINT_LEAVE 2 | |
411 | #define FSCACHE_POINT_DEBUG 4 | |
412 | ||
413 | #ifndef FSCACHE_DEBUG_LEVEL | |
414 | #define FSCACHE_DEBUG_LEVEL CACHE | |
415 | #endif | |
416 | ||
417 | /* | |
418 | * assertions | |
419 | */ | |
420 | #if 1 /* defined(__KDEBUGALL) */ | |
421 | ||
422 | #define ASSERT(X) \ | |
423 | do { \ | |
424 | if (unlikely(!(X))) { \ | |
36dfd116 FF |
425 | pr_err("\n"); \ |
426 | pr_err("Assertion failed\n"); \ | |
06b3db1b DH |
427 | BUG(); \ |
428 | } \ | |
429 | } while (0) | |
430 | ||
431 | #define ASSERTCMP(X, OP, Y) \ | |
432 | do { \ | |
433 | if (unlikely(!((X) OP (Y)))) { \ | |
36dfd116 FF |
434 | pr_err("\n"); \ |
435 | pr_err("Assertion failed\n"); \ | |
436 | pr_err("%lx " #OP " %lx is false\n", \ | |
06b3db1b DH |
437 | (unsigned long)(X), (unsigned long)(Y)); \ |
438 | BUG(); \ | |
439 | } \ | |
440 | } while (0) | |
441 | ||
442 | #define ASSERTIF(C, X) \ | |
443 | do { \ | |
444 | if (unlikely((C) && !(X))) { \ | |
36dfd116 FF |
445 | pr_err("\n"); \ |
446 | pr_err("Assertion failed\n"); \ | |
06b3db1b DH |
447 | BUG(); \ |
448 | } \ | |
449 | } while (0) | |
450 | ||
451 | #define ASSERTIFCMP(C, X, OP, Y) \ | |
452 | do { \ | |
453 | if (unlikely((C) && !((X) OP (Y)))) { \ | |
36dfd116 FF |
454 | pr_err("\n"); \ |
455 | pr_err("Assertion failed\n"); \ | |
456 | pr_err("%lx " #OP " %lx is false\n", \ | |
06b3db1b DH |
457 | (unsigned long)(X), (unsigned long)(Y)); \ |
458 | BUG(); \ | |
459 | } \ | |
460 | } while (0) | |
461 | ||
462 | #else | |
463 | ||
464 | #define ASSERT(X) do {} while (0) | |
465 | #define ASSERTCMP(X, OP, Y) do {} while (0) | |
466 | #define ASSERTIF(C, X) do {} while (0) | |
467 | #define ASSERTIFCMP(C, X, OP, Y) do {} while (0) | |
468 | ||
469 | #endif /* assert or not */ |