fs/nfsd/filecache.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * The NFSD open file cache.
   4  *
   5  * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
   6  *
   7  * An nfsd_file object is a per-file collection of open state that binds
   8  * together:
   9  *   - a struct file *
  10  *   - a user credential
  11  *   - a network namespace
  12  *   - a read-ahead context
  13  *   - monitoring for writeback errors
  14  *
  15  * nfsd_file objects are reference-counted. Consumers acquire a new
  16  * object via the nfsd_file_acquire API. They manage their interest in
  17  * the acquired object, and hence the object's reference count, via
  18  * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file
  19  * object:
  20  *
  21  *  * non-garbage-collected: When a consumer wants to precisely control
  22  *    the lifetime of a file's open state, it acquires a non-garbage-
  23  *    collected nfsd_file. The final nfsd_file_put releases the open
  24  *    state immediately.
  25  *
  26  *  * garbage-collected: When a consumer does not control the lifetime
  27  *    of open state, it acquires a garbage-collected nfsd_file. The
  28  *    final nfsd_file_put allows the open state to linger for a period
  29  *    during which it may be re-used.
  30  */
  31
  32 #include <linux/hash.h>
  33 #include <linux/slab.h>
  34 #include <linux/file.h>
  35 #include <linux/pagemap.h>
  36 #include <linux/sched.h>
  37 #include <linux/list_lru.h>
  38 #include <linux/fsnotify_backend.h>
  39 #include <linux/fsnotify.h>
  40 #include <linux/seq_file.h>
  41 #include <linux/rhashtable.h>
  42
  43 #include "vfs.h"
  44 #include "nfsd.h"
  45 #include "nfsfh.h"
  46 #include "netns.h"
  47 #include "filecache.h"
  48 #include "trace.h"
  49
  50 #define NFSD_LAUNDRETTE_DELAY                (2 * HZ)
  51
  52 #define NFSD_FILE_CACHE_UP                   (0)
  53
  54 /* We only care about NFSD_MAY_READ/WRITE for this cache */
  55 #define NFSD_FILE_MAY_MASK      (NFSD_MAY_READ|NFSD_MAY_WRITE)
  56
  57 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
  58 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
  59 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
  60 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
  61 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
  62
  63 struct nfsd_fcache_disposal {
  64         struct work_struct work;
  65         spinlock_t lock;
  66         struct list_head freeme;
  67 };
  68
  69 static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
  70
  71 static struct kmem_cache                *nfsd_file_slab;
  72 static struct kmem_cache                *nfsd_file_mark_slab;
  73 static struct list_lru                  nfsd_file_lru;
  74 static unsigned long                    nfsd_file_flags;
  75 static struct fsnotify_group            *nfsd_file_fsnotify_group;
  76 static struct delayed_work              nfsd_filecache_laundrette;
  77 static struct rhashtable                nfsd_file_rhash_tbl
  78                                                 ____cacheline_aligned_in_smp;
  79
  80 enum nfsd_file_lookup_type {
  81         NFSD_FILE_KEY_INODE,
  82         NFSD_FILE_KEY_FULL,
  83 };
  84
  85 struct nfsd_file_lookup_key {
  86         struct inode                    *inode;
  87         struct net                      *net;
  88         const struct cred               *cred;
  89         unsigned char                   need;
  90         bool                            gc;
  91         enum nfsd_file_lookup_type      type;
  92 };
  93
  94 /*
  95  * The returned hash value is based solely on the address of an in-code
  96  * inode, a pointer to a slab-allocated object. The entropy in such a
  97  * pointer is concentrated in its middle bits.
  98  */
  99 static u32 nfsd_file_inode_hash(const struct inode *inode, u32 seed)
 100 {
 101         unsigned long ptr = (unsigned long)inode;
 102         u32 k;
 103
 104         k = ptr >> L1_CACHE_SHIFT;
 105         k &= 0x00ffffff;
 106         return jhash2(&k, 1, seed);
 107 }
 108
 109 /**
 110  * nfsd_file_key_hashfn - Compute the hash value of a lookup key
 111  * @data: key on which to compute the hash value
 112  * @len: rhash table's key_len parameter (unused)
 113  * @seed: rhash table's random seed of the day
 114  *
 115  * Return value:
 116  *   Computed 32-bit hash value
 117  */
 118 static u32 nfsd_file_key_hashfn(const void *data, u32 len, u32 seed)
 119 {
 120         const struct nfsd_file_lookup_key *key = data;
 121
 122         return nfsd_file_inode_hash(key->inode, seed);
 123 }
 124
 125 /**
 126  * nfsd_file_obj_hashfn - Compute the hash value of an nfsd_file
 127  * @data: object on which to compute the hash value
 128  * @len: rhash table's key_len parameter (unused)
 129  * @seed: rhash table's random seed of the day
 130  *
 131  * Return value:
 132  *   Computed 32-bit hash value
 133  */
 134 static u32 nfsd_file_obj_hashfn(const void *data, u32 len, u32 seed)
 135 {
 136         const struct nfsd_file *nf = data;
 137
 138         return nfsd_file_inode_hash(nf->nf_inode, seed);
 139 }
 140
 141 static bool
 142 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
 143 {
 144         int i;
 145
 146         if (!uid_eq(c1->fsuid, c2->fsuid))
 147                 return false;
 148         if (!gid_eq(c1->fsgid, c2->fsgid))
 149                 return false;
 150         if (c1->group_info == NULL || c2->group_info == NULL)
 151                 return c1->group_info == c2->group_info;
 152         if (c1->group_info->ngroups != c2->group_info->ngroups)
 153                 return false;
 154         for (i = 0; i < c1->group_info->ngroups; i++) {
 155                 if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
 156                         return false;
 157         }
 158         return true;
 159 }
 160
 161 /**
 162  * nfsd_file_obj_cmpfn - Match a cache item against search criteria
 163  * @arg: search criteria
 164  * @ptr: cache item to check
 165  *
 166  * Return values:
 167  *   %0 - Item matches search criteria
 168  *   %1 - Item does not match search criteria
 169  */
 170 static int nfsd_file_obj_cmpfn(struct rhashtable_compare_arg *arg,
 171                                const void *ptr)
 172 {
 173         const struct nfsd_file_lookup_key *key = arg->key;
 174         const struct nfsd_file *nf = ptr;
 175
 176         switch (key->type) {
 177         case NFSD_FILE_KEY_INODE:
 178                 if (nf->nf_inode != key->inode)
 179                         return 1;
 180                 break;
 181         case NFSD_FILE_KEY_FULL:
 182                 if (nf->nf_inode != key->inode)
 183                         return 1;
 184                 if (nf->nf_may != key->need)
 185                         return 1;
 186                 if (nf->nf_net != key->net)
 187                         return 1;
 188                 if (!nfsd_match_cred(nf->nf_cred, key->cred))
 189                         return 1;
 190                 if (!!test_bit(NFSD_FILE_GC, &nf->nf_flags) != key->gc)
 191                         return 1;
 192                 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
 193                         return 1;
 194                 break;
 195         }
 196         return 0;
 197 }
 198
 199 static const struct rhashtable_params nfsd_file_rhash_params = {
 200         .key_len                = sizeof_field(struct nfsd_file, nf_inode),
 201         .key_offset             = offsetof(struct nfsd_file, nf_inode),
 202         .head_offset            = offsetof(struct nfsd_file, nf_rhash),
 203         .hashfn                 = nfsd_file_key_hashfn,
 204         .obj_hashfn             = nfsd_file_obj_hashfn,
 205         .obj_cmpfn              = nfsd_file_obj_cmpfn,
 206         /* Reduce resizing churn on light workloads */
 207         .min_size               = 512,          /* buckets */
 208         .automatic_shrinking    = true,
 209 };
 210
 211 static void
 212 nfsd_file_schedule_laundrette(void)
 213 {
 214         if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags))
 215                 queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
 216                                    NFSD_LAUNDRETTE_DELAY);
 217 }
 218
 219 static void
 220 nfsd_file_slab_free(struct rcu_head *rcu)
 221 {
 222         struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
 223
 224         put_cred(nf->nf_cred);
 225         kmem_cache_free(nfsd_file_slab, nf);
 226 }
 227
 228 static void
 229 nfsd_file_mark_free(struct fsnotify_mark *mark)
 230 {
 231         struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
 232                                                   nfm_mark);
 233
 234         kmem_cache_free(nfsd_file_mark_slab, nfm);
 235 }
 236
 237 static struct nfsd_file_mark *
 238 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
 239 {
 240         if (!refcount_inc_not_zero(&nfm->nfm_ref))
 241                 return NULL;
 242         return nfm;
 243 }
 244
 245 static void
 246 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
 247 {
 248         if (refcount_dec_and_test(&nfm->nfm_ref)) {
 249                 fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
 250                 fsnotify_put_mark(&nfm->nfm_mark);
 251         }
 252 }
 253
 254 static struct nfsd_file_mark *
 255 nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
 256 {
 257         int                     err;
 258         struct fsnotify_mark    *mark;
 259         struct nfsd_file_mark   *nfm = NULL, *new;
 260
 261         do {
 262                 fsnotify_group_lock(nfsd_file_fsnotify_group);
 263                 mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
 264                                           nfsd_file_fsnotify_group);
 265                 if (mark) {
 266                         nfm = nfsd_file_mark_get(container_of(mark,
 267                                                  struct nfsd_file_mark,
 268                                                  nfm_mark));
 269                         fsnotify_group_unlock(nfsd_file_fsnotify_group);
 270                         if (nfm) {
 271                                 fsnotify_put_mark(mark);
 272                                 break;
 273                         }
 274                         /* Avoid soft lockup race with nfsd_file_mark_put() */
 275                         fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
 276                         fsnotify_put_mark(mark);
 277                 } else {
 278                         fsnotify_group_unlock(nfsd_file_fsnotify_group);
 279                 }
 280
 281                 /* allocate a new nfm */
 282                 new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
 283                 if (!new)
 284                         return NULL;
 285                 fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
 286                 new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
 287                 refcount_set(&new->nfm_ref, 1);
 288
 289                 err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
 290
 291                 /*
 292                  * If the add was successful, then return the object.
 293                  * Otherwise, we need to put the reference we hold on the
 294                  * nfm_mark. The fsnotify code will take a reference and put
 295                  * it on failure, so we can't just free it directly. It's also
 296                  * not safe to call fsnotify_destroy_mark on it as the
 297                  * mark->group will be NULL. Thus, we can't let the nfm_ref
 298                  * counter drive the destruction at this point.
 299                  */
 300                 if (likely(!err))
 301                         nfm = new;
 302                 else
 303                         fsnotify_put_mark(&new->nfm_mark);
 304         } while (unlikely(err == -EEXIST));
 305
 306         return nfm;
 307 }
 308
 309 static struct nfsd_file *
 310 nfsd_file_alloc(struct nfsd_file_lookup_key *key, unsigned int may)
 311 {
 312         struct nfsd_file *nf;
 313
 314         nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
 315         if (nf) {
 316                 INIT_LIST_HEAD(&nf->nf_lru);
 317                 nf->nf_birthtime = ktime_get();
 318                 nf->nf_file = NULL;
 319                 nf->nf_cred = get_current_cred();
 320                 nf->nf_net = key->net;
 321                 nf->nf_flags = 0;
 322                 __set_bit(NFSD_FILE_HASHED, &nf->nf_flags);
 323                 __set_bit(NFSD_FILE_PENDING, &nf->nf_flags);
 324                 if (key->gc)
 325                         __set_bit(NFSD_FILE_GC, &nf->nf_flags);
 326                 nf->nf_inode = key->inode;
 327                 refcount_set(&nf->nf_ref, 1);
 328                 nf->nf_may = key->need;
 329                 nf->nf_mark = NULL;
 330         }
 331         return nf;
 332 }
 333
 334 /**
 335  * nfsd_file_check_write_error - check for writeback errors on a file
 336  * @nf: nfsd_file to check for writeback errors
 337  *
 338  * Check whether a nfsd_file has an unseen error. Reset the write
 339  * verifier if so.
 340  */
 341 static void
 342 nfsd_file_check_write_error(struct nfsd_file *nf)
 343 {
 344         struct file *file = nf->nf_file;
 345
 346         if ((file->f_mode & FMODE_WRITE) &&
 347             filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
 348                 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
 349 }
 350
 351 static void
 352 nfsd_file_hash_remove(struct nfsd_file *nf)
 353 {
 354         trace_nfsd_file_unhash(nf);
 355         rhashtable_remove_fast(&nfsd_file_rhash_tbl, &nf->nf_rhash,
 356                                nfsd_file_rhash_params);
 357 }
 358
 359 static bool
 360 nfsd_file_unhash(struct nfsd_file *nf)
 361 {
 362         if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
 363                 nfsd_file_hash_remove(nf);
 364                 return true;
 365         }
 366         return false;
 367 }
 368
 369 static void
 370 nfsd_file_free(struct nfsd_file *nf)
 371 {
 372         s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
 373
 374         trace_nfsd_file_free(nf);
 375
 376         this_cpu_inc(nfsd_file_releases);
 377         this_cpu_add(nfsd_file_total_age, age);
 378
 379         nfsd_file_unhash(nf);
 380         if (nf->nf_mark)
 381                 nfsd_file_mark_put(nf->nf_mark);
 382         if (nf->nf_file) {
 383                 get_file(nf->nf_file);
 384                 filp_close(nf->nf_file, NULL);
 385                 nfsd_file_check_write_error(nf);
 386                 fput(nf->nf_file);
 387         }
 388
 389         /*
 390          * If this item is still linked via nf_lru, that's a bug.
 391          * WARN and leak it to preserve system stability.
 392          */
 393         if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
 394                 return;
 395
 396         call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
 397 }
 398
 399 static bool
 400 nfsd_file_check_writeback(struct nfsd_file *nf)
 401 {
 402         struct file *file = nf->nf_file;
 403         struct address_space *mapping;
 404
 405         if (!file || !(file->f_mode & FMODE_WRITE))
 406                 return false;
 407         mapping = file->f_mapping;
 408         return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
 409                 mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
 410 }
 411
 412 static bool nfsd_file_lru_add(struct nfsd_file *nf)
 413 {
 414         set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
 415         if (list_lru_add(&nfsd_file_lru, &nf->nf_lru)) {
 416                 trace_nfsd_file_lru_add(nf);
 417                 return true;
 418         }
 419         return false;
 420 }
 421
 422 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
 423 {
 424         if (list_lru_del(&nfsd_file_lru, &nf->nf_lru)) {
 425                 trace_nfsd_file_lru_del(nf);
 426                 return true;
 427         }
 428         return false;
 429 }
 430
 431 struct nfsd_file *
 432 nfsd_file_get(struct nfsd_file *nf)
 433 {
 434         if (nf && refcount_inc_not_zero(&nf->nf_ref))
 435                 return nf;
 436         return NULL;
 437 }
 438
 439 /**
 440  * nfsd_file_put - put the reference to a nfsd_file
 441  * @nf: nfsd_file of which to put the reference
 442  *
 443  * Put a reference to a nfsd_file. In the non-GC case, we just put the
 444  * reference immediately. In the GC case, if the reference would be
 445  * the last one, the put it on the LRU instead to be cleaned up later.
 446  */
 447 void
 448 nfsd_file_put(struct nfsd_file *nf)
 449 {
 450         might_sleep();
 451         trace_nfsd_file_put(nf);
 452
 453         if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
 454             test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
 455                 /*
 456                  * If this is the last reference (nf_ref == 1), then try to
 457                  * transfer it to the LRU.
 458                  */
 459                 if (refcount_dec_not_one(&nf->nf_ref))
 460                         return;
 461
 462                 /* Try to add it to the LRU.  If that fails, decrement. */
 463                 if (nfsd_file_lru_add(nf)) {
 464                         /* If it's still hashed, we're done */
 465                         if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
 466                                 nfsd_file_schedule_laundrette();
 467                                 return;
 468                         }
 469
 470                         /*
 471                          * We're racing with unhashing, so try to remove it from
 472                          * the LRU. If removal fails, then someone else already
 473                          * has our reference.
 474                          */
 475                         if (!nfsd_file_lru_remove(nf))
 476                                 return;
 477                 }
 478         }
 479         if (refcount_dec_and_test(&nf->nf_ref))
 480                 nfsd_file_free(nf);
 481 }
 482
 483 static void
 484 nfsd_file_dispose_list(struct list_head *dispose)
 485 {
 486         struct nfsd_file *nf;
 487
 488         while (!list_empty(dispose)) {
 489                 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
 490                 list_del_init(&nf->nf_lru);
 491                 nfsd_file_free(nf);
 492         }
 493 }
 494
 495 static void
 496 nfsd_file_list_remove_disposal(struct list_head *dst,
 497                 struct nfsd_fcache_disposal *l)
 498 {
 499         spin_lock(&l->lock);
 500         list_splice_init(&l->freeme, dst);
 501         spin_unlock(&l->lock);
 502 }
 503
 504 static void
 505 nfsd_file_list_add_disposal(struct list_head *files, struct net *net)
 506 {
 507         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 508         struct nfsd_fcache_disposal *l = nn->fcache_disposal;
 509
 510         spin_lock(&l->lock);
 511         list_splice_tail_init(files, &l->freeme);
 512         spin_unlock(&l->lock);
 513         queue_work(nfsd_filecache_wq, &l->work);
 514 }
 515
 516 static void
 517 nfsd_file_list_add_pernet(struct list_head *dst, struct list_head *src,
 518                 struct net *net)
 519 {
 520         struct nfsd_file *nf, *tmp;
 521
 522         list_for_each_entry_safe(nf, tmp, src, nf_lru) {
 523                 if (nf->nf_net == net)
 524                         list_move_tail(&nf->nf_lru, dst);
 525         }
 526 }
 527
 528 static void
 529 nfsd_file_dispose_list_delayed(struct list_head *dispose)
 530 {
 531         LIST_HEAD(list);
 532         struct nfsd_file *nf;
 533
 534         while(!list_empty(dispose)) {
 535                 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
 536                 nfsd_file_list_add_pernet(&list, dispose, nf->nf_net);
 537                 nfsd_file_list_add_disposal(&list, nf->nf_net);
 538         }
 539 }
 540
 541 /**
 542  * nfsd_file_lru_cb - Examine an entry on the LRU list
 543  * @item: LRU entry to examine
 544  * @lru: controlling LRU
 545  * @lock: LRU list lock (unused)
 546  * @arg: dispose list
 547  *
 548  * Return values:
 549  *   %LRU_REMOVED: @item was removed from the LRU
 550  *   %LRU_ROTATE: @item is to be moved to the LRU tail
 551  *   %LRU_SKIP: @item cannot be evicted
 552  */
 553 static enum lru_status
 554 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
 555                  spinlock_t *lock, void *arg)
 556         __releases(lock)
 557         __acquires(lock)
 558 {
 559         struct list_head *head = arg;
 560         struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
 561
 562         /* We should only be dealing with GC entries here */
 563         WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
 564
 565         /*
 566          * Don't throw out files that are still undergoing I/O or
 567          * that have uncleared errors pending.
 568          */
 569         if (nfsd_file_check_writeback(nf)) {
 570                 trace_nfsd_file_gc_writeback(nf);
 571                 return LRU_SKIP;
 572         }
 573
 574         /* If it was recently added to the list, skip it */
 575         if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
 576                 trace_nfsd_file_gc_referenced(nf);
 577                 return LRU_ROTATE;
 578         }
 579
 580         /*
 581          * Put the reference held on behalf of the LRU. If it wasn't the last
 582          * one, then just remove it from the LRU and ignore it.
 583          */
 584         if (!refcount_dec_and_test(&nf->nf_ref)) {
 585                 trace_nfsd_file_gc_in_use(nf);
 586                 list_lru_isolate(lru, &nf->nf_lru);
 587                 return LRU_REMOVED;
 588         }
 589
 590         /* Refcount went to zero. Unhash it and queue it to the dispose list */
 591         nfsd_file_unhash(nf);
 592         list_lru_isolate_move(lru, &nf->nf_lru, head);
 593         this_cpu_inc(nfsd_file_evictions);
 594         trace_nfsd_file_gc_disposed(nf);
 595         return LRU_REMOVED;
 596 }
 597
 598 static void
 599 nfsd_file_gc(void)
 600 {
 601         LIST_HEAD(dispose);
 602         unsigned long ret;
 603
 604         ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
 605                             &dispose, list_lru_count(&nfsd_file_lru));
 606         trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
 607         nfsd_file_dispose_list_delayed(&dispose);
 608 }
 609
 610 static void
 611 nfsd_file_gc_worker(struct work_struct *work)
 612 {
 613         nfsd_file_gc();
 614         if (list_lru_count(&nfsd_file_lru))
 615                 nfsd_file_schedule_laundrette();
 616 }
 617
 618 static unsigned long
 619 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
 620 {
 621         return list_lru_count(&nfsd_file_lru);
 622 }
 623
 624 static unsigned long
 625 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
 626 {
 627         LIST_HEAD(dispose);
 628         unsigned long ret;
 629
 630         ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
 631                                    nfsd_file_lru_cb, &dispose);
 632         trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
 633         nfsd_file_dispose_list_delayed(&dispose);
 634         return ret;
 635 }
 636
 637 static struct shrinker  nfsd_file_shrinker = {
 638         .scan_objects = nfsd_file_lru_scan,
 639         .count_objects = nfsd_file_lru_count,
 640         .seeks = 1,
 641 };
 642
 643 /**
 644  * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
 645  * @nf: nfsd_file to attempt to queue
 646  * @dispose: private list to queue successfully-put objects
 647  *
 648  * Unhash an nfsd_file, try to get a reference to it, and then put that
 649  * reference. If it's the last reference, queue it to the dispose list.
 650  */
 651 static void
 652 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
 653         __must_hold(RCU)
 654 {
 655         int decrement = 1;
 656
 657         /* If we raced with someone else unhashing, ignore it */
 658         if (!nfsd_file_unhash(nf))
 659                 return;
 660
 661         /* If we can't get a reference, ignore it */
 662         if (!nfsd_file_get(nf))
 663                 return;
 664
 665         /* Extra decrement if we remove from the LRU */
 666         if (nfsd_file_lru_remove(nf))
 667                 ++decrement;
 668
 669         /* If refcount goes to 0, then put on the dispose list */
 670         if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
 671                 list_add(&nf->nf_lru, dispose);
 672                 trace_nfsd_file_closing(nf);
 673         }
 674 }
 675
 676 /**
 677  * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
 678  * @inode:   inode on which to close out nfsd_files
 679  * @dispose: list on which to gather nfsd_files to close out
 680  *
 681  * An nfsd_file represents a struct file being held open on behalf of nfsd. An
 682  * open file however can block other activity (such as leases), or cause
 683  * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
 684  *
 685  * This function is intended to find open nfsd_files when this sort of
 686  * conflicting access occurs and then attempt to close those files out.
 687  *
 688  * Populates the dispose list with entries that have already had their
 689  * refcounts go to zero. The actual free of an nfsd_file can be expensive,
 690  * so we leave it up to the caller whether it wants to wait or not.
 691  */
 692 static void
 693 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
 694 {
 695         struct nfsd_file_lookup_key key = {
 696                 .type   = NFSD_FILE_KEY_INODE,
 697                 .inode  = inode,
 698         };
 699         struct nfsd_file *nf;
 700
 701         rcu_read_lock();
 702         do {
 703                 nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
 704                                        nfsd_file_rhash_params);
 705                 if (!nf)
 706                         break;
 707                 nfsd_file_cond_queue(nf, dispose);
 708         } while (1);
 709         rcu_read_unlock();
 710 }
 711
 712 /**
 713  * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
 714  * @inode: inode of the file to attempt to remove
 715  *
 716  * Close out any open nfsd_files that can be reaped for @inode. The
 717  * actual freeing is deferred to the dispose_list_delayed infrastructure.
 718  *
 719  * This is used by the fsnotify callbacks and setlease notifier.
 720  */
 721 static void
 722 nfsd_file_close_inode(struct inode *inode)
 723 {
 724         LIST_HEAD(dispose);
 725
 726         nfsd_file_queue_for_close(inode, &dispose);
 727         nfsd_file_dispose_list_delayed(&dispose);
 728 }
 729
 730 /**
 731  * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
 732  * @inode: inode of the file to attempt to remove
 733  *
 734  * Close out any open nfsd_files that can be reaped for @inode. The
 735  * nfsd_files are closed out synchronously.
 736  *
 737  * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
 738  * when reexporting NFS.
 739  */
 740 void
 741 nfsd_file_close_inode_sync(struct inode *inode)
 742 {
 743         struct nfsd_file *nf;
 744         LIST_HEAD(dispose);
 745
 746         trace_nfsd_file_close(inode);
 747
 748         nfsd_file_queue_for_close(inode, &dispose);
 749         while (!list_empty(&dispose)) {
 750                 nf = list_first_entry(&dispose, struct nfsd_file, nf_lru);
 751                 list_del_init(&nf->nf_lru);
 752                 nfsd_file_free(nf);
 753         }
 754         flush_delayed_fput();
 755 }
 756
 757 /**
 758  * nfsd_file_delayed_close - close unused nfsd_files
 759  * @work: dummy
 760  *
 761  * Walk the LRU list and destroy any entries that have not been used since
 762  * the last scan.
 763  */
 764 static void
 765 nfsd_file_delayed_close(struct work_struct *work)
 766 {
 767         LIST_HEAD(head);
 768         struct nfsd_fcache_disposal *l = container_of(work,
 769                         struct nfsd_fcache_disposal, work);
 770
 771         nfsd_file_list_remove_disposal(&head, l);
 772         nfsd_file_dispose_list(&head);
 773 }
 774
 775 static int
 776 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
 777                             void *data)
 778 {
 779         struct file_lock *fl = data;
 780
 781         /* Only close files for F_SETLEASE leases */
 782         if (fl->fl_flags & FL_LEASE)
 783                 nfsd_file_close_inode(file_inode(fl->fl_file));
 784         return 0;
 785 }
 786
 787 static struct notifier_block nfsd_file_lease_notifier = {
 788         .notifier_call = nfsd_file_lease_notifier_call,
 789 };
 790
 791 static int
 792 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
 793                                 struct inode *inode, struct inode *dir,
 794                                 const struct qstr *name, u32 cookie)
 795 {
 796         if (WARN_ON_ONCE(!inode))
 797                 return 0;
 798
 799         trace_nfsd_file_fsnotify_handle_event(inode, mask);
 800
 801         /* Should be no marks on non-regular files */
 802         if (!S_ISREG(inode->i_mode)) {
 803                 WARN_ON_ONCE(1);
 804                 return 0;
 805         }
 806
 807         /* don't close files if this was not the last link */
 808         if (mask & FS_ATTRIB) {
 809                 if (inode->i_nlink)
 810                         return 0;
 811         }
 812
 813         nfsd_file_close_inode(inode);
 814         return 0;
 815 }
 816
 817
 818 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
 819         .handle_inode_event = nfsd_file_fsnotify_handle_event,
 820         .free_mark = nfsd_file_mark_free,
 821 };
 822
 823 int
 824 nfsd_file_cache_init(void)
 825 {
 826         int ret;
 827
 828         lockdep_assert_held(&nfsd_mutex);
 829         if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
 830                 return 0;
 831
 832         ret = rhashtable_init(&nfsd_file_rhash_tbl, &nfsd_file_rhash_params);
 833         if (ret)
 834                 return ret;
 835
 836         ret = -ENOMEM;
 837         nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
 838         if (!nfsd_filecache_wq)
 839                 goto out;
 840
 841         nfsd_file_slab = kmem_cache_create("nfsd_file",
 842                                 sizeof(struct nfsd_file), 0, 0, NULL);
 843         if (!nfsd_file_slab) {
 844                 pr_err("nfsd: unable to create nfsd_file_slab\n");
 845                 goto out_err;
 846         }
 847
 848         nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
 849                                         sizeof(struct nfsd_file_mark), 0, 0, NULL);
 850         if (!nfsd_file_mark_slab) {
 851                 pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
 852                 goto out_err;
 853         }
 854
 855
 856         ret = list_lru_init(&nfsd_file_lru);
 857         if (ret) {
 858                 pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
 859                 goto out_err;
 860         }
 861
 862         ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
 863         if (ret) {
 864                 pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
 865                 goto out_lru;
 866         }
 867
 868         ret = lease_register_notifier(&nfsd_file_lease_notifier);
 869         if (ret) {
 870                 pr_err("nfsd: unable to register lease notifier: %d\n", ret);
 871                 goto out_shrinker;
 872         }
 873
 874         nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
 875                                                         FSNOTIFY_GROUP_NOFS);
 876         if (IS_ERR(nfsd_file_fsnotify_group)) {
 877                 pr_err("nfsd: unable to create fsnotify group: %ld\n",
 878                         PTR_ERR(nfsd_file_fsnotify_group));
 879                 ret = PTR_ERR(nfsd_file_fsnotify_group);
 880                 nfsd_file_fsnotify_group = NULL;
 881                 goto out_notifier;
 882         }
 883
 884         INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
 885 out:
 886         return ret;
 887 out_notifier:
 888         lease_unregister_notifier(&nfsd_file_lease_notifier);
 889 out_shrinker:
 890         unregister_shrinker(&nfsd_file_shrinker);
 891 out_lru:
 892         list_lru_destroy(&nfsd_file_lru);
 893 out_err:
 894         kmem_cache_destroy(nfsd_file_slab);
 895         nfsd_file_slab = NULL;
 896         kmem_cache_destroy(nfsd_file_mark_slab);
 897         nfsd_file_mark_slab = NULL;
 898         destroy_workqueue(nfsd_filecache_wq);
 899         nfsd_filecache_wq = NULL;
 900         rhashtable_destroy(&nfsd_file_rhash_tbl);
 901         goto out;
 902 }
 903
 904 /**
 905  * __nfsd_file_cache_purge: clean out the cache for shutdown
 906  * @net: net-namespace to shut down the cache (may be NULL)
 907  *
 908  * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
 909  * then close out everything. Called when an nfsd instance is being shut down.
 910  */
 911 static void
 912 __nfsd_file_cache_purge(struct net *net)
 913 {
 914         struct rhashtable_iter iter;
 915         struct nfsd_file *nf;
 916         LIST_HEAD(dispose);
 917
 918         rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter);
 919         do {
 920                 rhashtable_walk_start(&iter);
 921
 922                 nf = rhashtable_walk_next(&iter);
 923                 while (!IS_ERR_OR_NULL(nf)) {
 924                         if (!net || nf->nf_net == net)
 925                                 nfsd_file_cond_queue(nf, &dispose);
 926                         nf = rhashtable_walk_next(&iter);
 927                 }
 928
 929                 rhashtable_walk_stop(&iter);
 930         } while (nf == ERR_PTR(-EAGAIN));
 931         rhashtable_walk_exit(&iter);
 932
 933         nfsd_file_dispose_list(&dispose);
 934 }
 935
 936 static struct nfsd_fcache_disposal *
 937 nfsd_alloc_fcache_disposal(void)
 938 {
 939         struct nfsd_fcache_disposal *l;
 940
 941         l = kmalloc(sizeof(*l), GFP_KERNEL);
 942         if (!l)
 943                 return NULL;
 944         INIT_WORK(&l->work, nfsd_file_delayed_close);
 945         spin_lock_init(&l->lock);
 946         INIT_LIST_HEAD(&l->freeme);
 947         return l;
 948 }
 949
 950 static void
 951 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
 952 {
 953         cancel_work_sync(&l->work);
 954         nfsd_file_dispose_list(&l->freeme);
 955         kfree(l);
 956 }
 957
 958 static void
 959 nfsd_free_fcache_disposal_net(struct net *net)
 960 {
 961         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 962         struct nfsd_fcache_disposal *l = nn->fcache_disposal;
 963
 964         nfsd_free_fcache_disposal(l);
 965 }
 966
 967 int
 968 nfsd_file_cache_start_net(struct net *net)
 969 {
 970         struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 971
 972         nn->fcache_disposal = nfsd_alloc_fcache_disposal();
 973         return nn->fcache_disposal ? 0 : -ENOMEM;
 974 }
 975
 976 /**
 977  * nfsd_file_cache_purge - Remove all cache items associated with @net
 978  * @net: target net namespace
 979  *
 980  */
 981 void
 982 nfsd_file_cache_purge(struct net *net)
 983 {
 984         lockdep_assert_held(&nfsd_mutex);
 985         if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
 986                 __nfsd_file_cache_purge(net);
 987 }
 988
 989 void
 990 nfsd_file_cache_shutdown_net(struct net *net)
 991 {
 992         nfsd_file_cache_purge(net);
 993         nfsd_free_fcache_disposal_net(net);
 994 }
 995
 996 void
 997 nfsd_file_cache_shutdown(void)
 998 {
 999         int i;
1000
1001         lockdep_assert_held(&nfsd_mutex);
1002         if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
1003                 return;
1004
1005         lease_unregister_notifier(&nfsd_file_lease_notifier);
1006         unregister_shrinker(&nfsd_file_shrinker);
1007         /*
1008          * make sure all callers of nfsd_file_lru_cb are done before
1009          * calling nfsd_file_cache_purge
1010          */
1011         cancel_delayed_work_sync(&nfsd_filecache_laundrette);
1012         __nfsd_file_cache_purge(NULL);
1013         list_lru_destroy(&nfsd_file_lru);
1014         rcu_barrier();
1015         fsnotify_put_group(nfsd_file_fsnotify_group);
1016         nfsd_file_fsnotify_group = NULL;
1017         kmem_cache_destroy(nfsd_file_slab);
1018         nfsd_file_slab = NULL;
1019         fsnotify_wait_marks_destroyed();
1020         kmem_cache_destroy(nfsd_file_mark_slab);
1021         nfsd_file_mark_slab = NULL;
1022         destroy_workqueue(nfsd_filecache_wq);
1023         nfsd_filecache_wq = NULL;
1024         rhashtable_destroy(&nfsd_file_rhash_tbl);
1025
1026         for_each_possible_cpu(i) {
1027                 per_cpu(nfsd_file_cache_hits, i) = 0;
1028                 per_cpu(nfsd_file_acquisitions, i) = 0;
1029                 per_cpu(nfsd_file_releases, i) = 0;
1030                 per_cpu(nfsd_file_total_age, i) = 0;
1031                 per_cpu(nfsd_file_evictions, i) = 0;
1032         }
1033 }
1034
1035 /**
1036  * nfsd_file_is_cached - are there any cached open files for this inode?
1037  * @inode: inode to check
1038  *
1039  * The lookup matches inodes in all net namespaces and is atomic wrt
1040  * nfsd_file_acquire().
1041  *
1042  * Return values:
1043  *   %true: filecache contains at least one file matching this inode
1044  *   %false: filecache contains no files matching this inode
1045  */
1046 bool
1047 nfsd_file_is_cached(struct inode *inode)
1048 {
1049         struct nfsd_file_lookup_key key = {
1050                 .type   = NFSD_FILE_KEY_INODE,
1051                 .inode  = inode,
1052         };
1053         bool ret = false;
1054
1055         if (rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
1056                                    nfsd_file_rhash_params) != NULL)
1057                 ret = true;
1058         trace_nfsd_file_is_cached(inode, (int)ret);
1059         return ret;
1060 }
1061
1062 static __be32
1063 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1064                      unsigned int may_flags, struct file *file,
1065                      struct nfsd_file **pnf, bool want_gc)
1066 {
1067         struct nfsd_file_lookup_key key = {
1068                 .type   = NFSD_FILE_KEY_FULL,
1069                 .need   = may_flags & NFSD_FILE_MAY_MASK,
1070                 .net    = SVC_NET(rqstp),
1071                 .gc     = want_gc,
1072         };
1073         bool open_retry = true;
1074         struct nfsd_file *nf;
1075         __be32 status;
1076         int ret;
1077
1078         status = fh_verify(rqstp, fhp, S_IFREG,
1079                                 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1080         if (status != nfs_ok)
1081                 return status;
1082         key.inode = d_inode(fhp->fh_dentry);
1083         key.cred = get_current_cred();
1084
1085 retry:
1086         rcu_read_lock();
1087         nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
1088                                nfsd_file_rhash_params);
1089         nf = nfsd_file_get(nf);
1090         rcu_read_unlock();
1091
1092         if (nf) {
1093                 if (nfsd_file_lru_remove(nf))
1094                         WARN_ON_ONCE(refcount_dec_and_test(&nf->nf_ref));
1095                 goto wait_for_construction;
1096         }
1097
1098         nf = nfsd_file_alloc(&key, may_flags);
1099         if (!nf) {
1100                 status = nfserr_jukebox;
1101                 goto out_status;
1102         }
1103
1104         ret = rhashtable_lookup_insert_key(&nfsd_file_rhash_tbl,
1105                                            &key, &nf->nf_rhash,
1106                                            nfsd_file_rhash_params);
1107         if (likely(ret == 0))
1108                 goto open_file;
1109
1110         nfsd_file_slab_free(&nf->nf_rcu);
1111         nf = NULL;
1112         if (ret == -EEXIST)
1113                 goto retry;
1114         trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, ret);
1115         status = nfserr_jukebox;
1116         goto out_status;
1117
1118 wait_for_construction:
1119         wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1120
1121         /* Did construction of this file fail? */
1122         if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1123                 trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf);
1124                 if (!open_retry) {
1125                         status = nfserr_jukebox;
1126                         goto out;
1127                 }
1128                 open_retry = false;
1129                 if (refcount_dec_and_test(&nf->nf_ref))
1130                         nfsd_file_free(nf);
1131                 goto retry;
1132         }
1133
1134         this_cpu_inc(nfsd_file_cache_hits);
1135
1136         status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1137 out:
1138         if (status == nfs_ok) {
1139                 this_cpu_inc(nfsd_file_acquisitions);
1140                 nfsd_file_check_write_error(nf);
1141                 *pnf = nf;
1142         } else {
1143                 if (refcount_dec_and_test(&nf->nf_ref))
1144                         nfsd_file_free(nf);
1145                 nf = NULL;
1146         }
1147
1148 out_status:
1149         put_cred(key.cred);
1150         trace_nfsd_file_acquire(rqstp, key.inode, may_flags, nf, status);
1151         return status;
1152
1153 open_file:
1154         trace_nfsd_file_alloc(nf);
1155         nf->nf_mark = nfsd_file_mark_find_or_create(nf, key.inode);
1156         if (nf->nf_mark) {
1157                 if (file) {
1158                         get_file(file);
1159                         nf->nf_file = file;
1160                         status = nfs_ok;
1161                         trace_nfsd_file_opened(nf, status);
1162                 } else {
1163                         status = nfsd_open_verified(rqstp, fhp, may_flags,
1164                                                     &nf->nf_file);
1165                         trace_nfsd_file_open(nf, status);
1166                 }
1167         } else
1168                 status = nfserr_jukebox;
1169         /*
1170          * If construction failed, or we raced with a call to unlink()
1171          * then unhash.
1172          */
1173         if (status == nfs_ok && key.inode->i_nlink == 0)
1174                 status = nfserr_jukebox;
1175         if (status != nfs_ok)
1176                 nfsd_file_unhash(nf);
1177         clear_bit_unlock(NFSD_FILE_PENDING, &nf->nf_flags);
1178         smp_mb__after_atomic();
1179         wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
1180         goto out;
1181 }
1182
1183 /**
1184  * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1185  * @rqstp: the RPC transaction being executed
1186  * @fhp: the NFS filehandle of the file to be opened
1187  * @may_flags: NFSD_MAY_ settings for the file
1188  * @pnf: OUT: new or found "struct nfsd_file" object
1189  *
1190  * The nfsd_file object returned by this API is reference-counted
1191  * and garbage-collected. The object is retained for a few
1192  * seconds after the final nfsd_file_put() in case the caller
1193  * wants to re-use it.
1194  *
1195  * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1196  * network byte order is returned.
1197  */
1198 __be32
1199 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1200                      unsigned int may_flags, struct nfsd_file **pnf)
1201 {
1202         return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, true);
1203 }
1204
1205 /**
1206  * nfsd_file_acquire - Get a struct nfsd_file with an open file
1207  * @rqstp: the RPC transaction being executed
1208  * @fhp: the NFS filehandle of the file to be opened
1209  * @may_flags: NFSD_MAY_ settings for the file
1210  * @pnf: OUT: new or found "struct nfsd_file" object
1211  *
1212  * The nfsd_file_object returned by this API is reference-counted
1213  * but not garbage-collected. The object is unhashed after the
1214  * final nfsd_file_put().
1215  *
1216  * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1217  * network byte order is returned.
1218  */
1219 __be32
1220 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1221                   unsigned int may_flags, struct nfsd_file **pnf)
1222 {
1223         return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, false);
1224 }
1225
1226 /**
1227  * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1228  * @rqstp: the RPC transaction being executed
1229  * @fhp: the NFS filehandle of the file just created
1230  * @may_flags: NFSD_MAY_ settings for the file
1231  * @file: cached, already-open file (may be NULL)
1232  * @pnf: OUT: new or found "struct nfsd_file" object
1233  *
1234  * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1235  * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1236  * opening a new one.
1237  *
1238  * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1239  * network byte order is returned.
1240  */
1241 __be32
1242 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1243                          unsigned int may_flags, struct file *file,
1244                          struct nfsd_file **pnf)
1245 {
1246         return nfsd_file_do_acquire(rqstp, fhp, may_flags, file, pnf, false);
1247 }
1248
1249 /*
1250  * Note that fields may be added, removed or reordered in the future. Programs
1251  * scraping this file for info should test the labels to ensure they're
1252  * getting the correct field.
1253  */
1254 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1255 {
1256         unsigned long releases = 0, evictions = 0;
1257         unsigned long hits = 0, acquisitions = 0;
1258         unsigned int i, count = 0, buckets = 0;
1259         unsigned long lru = 0, total_age = 0;
1260
1261         /* Serialize with server shutdown */
1262         mutex_lock(&nfsd_mutex);
1263         if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1264                 struct bucket_table *tbl;
1265                 struct rhashtable *ht;
1266
1267                 lru = list_lru_count(&nfsd_file_lru);
1268
1269                 rcu_read_lock();
1270                 ht = &nfsd_file_rhash_tbl;
1271                 count = atomic_read(&ht->nelems);
1272                 tbl = rht_dereference_rcu(ht->tbl, ht);
1273                 buckets = tbl->size;
1274                 rcu_read_unlock();
1275         }
1276         mutex_unlock(&nfsd_mutex);
1277
1278         for_each_possible_cpu(i) {
1279                 hits += per_cpu(nfsd_file_cache_hits, i);
1280                 acquisitions += per_cpu(nfsd_file_acquisitions, i);
1281                 releases += per_cpu(nfsd_file_releases, i);
1282                 total_age += per_cpu(nfsd_file_total_age, i);
1283                 evictions += per_cpu(nfsd_file_evictions, i);
1284         }
1285
1286         seq_printf(m, "total entries: %u\n", count);
1287         seq_printf(m, "hash buckets:  %u\n", buckets);
1288         seq_printf(m, "lru entries:   %lu\n", lru);
1289         seq_printf(m, "cache hits:    %lu\n", hits);
1290         seq_printf(m, "acquisitions:  %lu\n", acquisitions);
1291         seq_printf(m, "releases:      %lu\n", releases);
1292         seq_printf(m, "evictions:     %lu\n", evictions);
1293         if (releases)
1294                 seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1295         else
1296                 seq_printf(m, "mean age (ms): -\n");
1297         return 0;
1298 }