fs/fscache/page.c

   1 /* Cache page management and data I/O routines
   2  *
   3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #define FSCACHE_DEBUG_LEVEL PAGE
  13 #include <linux/module.h>
  14 #include <linux/fscache-cache.h>
  15 #include <linux/buffer_head.h>
  16 #include <linux/pagevec.h>
  17 #include <linux/slab.h>
  18 #include "internal.h"
  19
  20 /*
  21  * check to see if a page is being written to the cache
  22  */
  23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  24 {
  25         void *val;
  26
  27         rcu_read_lock();
  28         val = radix_tree_lookup(&cookie->stores, page->index);
  29         rcu_read_unlock();
  30
  31         return val != NULL;
  32 }
  33 EXPORT_SYMBOL(__fscache_check_page_write);
  34
  35 /*
  36  * wait for a page to finish being written to the cache
  37  */
  38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  39 {
  40         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  41
  42         wait_event(*wq, !__fscache_check_page_write(cookie, page));
  43 }
  44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
  45
  46 /*
  47  * wait for a page to finish being written to the cache. Put a timeout here
  48  * since we might be called recursively via parent fs.
  49  */
  50 static
  51 bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)
  52 {
  53         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  54
  55         return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),
  56                                   HZ);
  57 }
  58
  59 /*
  60  * decide whether a page can be released, possibly by cancelling a store to it
  61  * - we're allowed to sleep if __GFP_WAIT is flagged
  62  */
  63 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  64                                   struct page *page,
  65                                   gfp_t gfp)
  66 {
  67         struct page *xpage;
  68         void *val;
  69
  70         _enter("%p,%p,%x", cookie, page, gfp);
  71
  72 try_again:
  73         rcu_read_lock();
  74         val = radix_tree_lookup(&cookie->stores, page->index);
  75         if (!val) {
  76                 rcu_read_unlock();
  77                 fscache_stat(&fscache_n_store_vmscan_not_storing);
  78                 __fscache_uncache_page(cookie, page);
  79                 return true;
  80         }
  81
  82         /* see if the page is actually undergoing storage - if so we can't get
  83          * rid of it till the cache has finished with it */
  84         if (radix_tree_tag_get(&cookie->stores, page->index,
  85                                FSCACHE_COOKIE_STORING_TAG)) {
  86                 rcu_read_unlock();
  87                 goto page_busy;
  88         }
  89
  90         /* the page is pending storage, so we attempt to cancel the store and
  91          * discard the store request so that the page can be reclaimed */
  92         spin_lock(&cookie->stores_lock);
  93         rcu_read_unlock();
  94
  95         if (radix_tree_tag_get(&cookie->stores, page->index,
  96                                FSCACHE_COOKIE_STORING_TAG)) {
  97                 /* the page started to undergo storage whilst we were looking,
  98                  * so now we can only wait or return */
  99                 spin_unlock(&cookie->stores_lock);
 100                 goto page_busy;
 101         }
 102
 103         xpage = radix_tree_delete(&cookie->stores, page->index);
 104         spin_unlock(&cookie->stores_lock);
 105
 106         if (xpage) {
 107                 fscache_stat(&fscache_n_store_vmscan_cancelled);
 108                 fscache_stat(&fscache_n_store_radix_deletes);
 109                 ASSERTCMP(xpage, ==, page);
 110         } else {
 111                 fscache_stat(&fscache_n_store_vmscan_gone);
 112         }
 113
 114         wake_up_bit(&cookie->flags, 0);
 115         if (xpage)
 116                 page_cache_release(xpage);
 117         __fscache_uncache_page(cookie, page);
 118         return true;
 119
 120 page_busy:
 121         /* We will wait here if we're allowed to, but that could deadlock the
 122          * allocator as the work threads writing to the cache may all end up
 123          * sleeping on memory allocation, so we may need to impose a timeout
 124          * too. */
 125         if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
 126                 fscache_stat(&fscache_n_store_vmscan_busy);
 127                 return false;
 128         }
 129
 130         fscache_stat(&fscache_n_store_vmscan_wait);
 131         if (!release_page_wait_timeout(cookie, page))
 132                 _debug("fscache writeout timeout page: %p{%lx}",
 133                         page, page->index);
 134
 135         gfp &= ~__GFP_WAIT;
 136         goto try_again;
 137 }
 138 EXPORT_SYMBOL(__fscache_maybe_release_page);
 139
 140 /*
 141  * note that a page has finished being written to the cache
 142  */
 143 static void fscache_end_page_write(struct fscache_object *object,
 144                                    struct page *page)
 145 {
 146         struct fscache_cookie *cookie;
 147         struct page *xpage = NULL;
 148
 149         spin_lock(&object->lock);
 150         cookie = object->cookie;
 151         if (cookie) {
 152                 /* delete the page from the tree if it is now no longer
 153                  * pending */
 154                 spin_lock(&cookie->stores_lock);
 155                 radix_tree_tag_clear(&cookie->stores, page->index,
 156                                      FSCACHE_COOKIE_STORING_TAG);
 157                 if (!radix_tree_tag_get(&cookie->stores, page->index,
 158                                         FSCACHE_COOKIE_PENDING_TAG)) {
 159                         fscache_stat(&fscache_n_store_radix_deletes);
 160                         xpage = radix_tree_delete(&cookie->stores, page->index);
 161                 }
 162                 spin_unlock(&cookie->stores_lock);
 163                 wake_up_bit(&cookie->flags, 0);
 164         }
 165         spin_unlock(&object->lock);
 166         if (xpage)
 167                 page_cache_release(xpage);
 168 }
 169
 170 /*
 171  * actually apply the changed attributes to a cache object
 172  */
 173 static void fscache_attr_changed_op(struct fscache_operation *op)
 174 {
 175         struct fscache_object *object = op->object;
 176         int ret;
 177
 178         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 179
 180         fscache_stat(&fscache_n_attr_changed_calls);
 181
 182         if (fscache_object_is_active(object)) {
 183                 fscache_stat(&fscache_n_cop_attr_changed);
 184                 ret = object->cache->ops->attr_changed(object);
 185                 fscache_stat_d(&fscache_n_cop_attr_changed);
 186                 if (ret < 0)
 187                         fscache_abort_object(object);
 188         }
 189
 190         fscache_op_complete(op, true);
 191         _leave("");
 192 }
 193
 194 /*
 195  * notification that the attributes on an object have changed
 196  */
 197 int __fscache_attr_changed(struct fscache_cookie *cookie)
 198 {
 199         struct fscache_operation *op;
 200         struct fscache_object *object;
 201         bool wake_cookie;
 202
 203         _enter("%p", cookie);
 204
 205         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 206
 207         fscache_stat(&fscache_n_attr_changed);
 208
 209         op = kzalloc(sizeof(*op), GFP_KERNEL);
 210         if (!op) {
 211                 fscache_stat(&fscache_n_attr_changed_nomem);
 212                 _leave(" = -ENOMEM");
 213                 return -ENOMEM;
 214         }
 215
 216         fscache_operation_init(op, fscache_attr_changed_op, NULL);
 217         op->flags = FSCACHE_OP_ASYNC |
 218                 (1 << FSCACHE_OP_EXCLUSIVE) |
 219                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 220
 221         spin_lock(&cookie->lock);
 222
 223         if (!fscache_cookie_enabled(cookie) ||
 224             hlist_empty(&cookie->backing_objects))
 225                 goto nobufs;
 226         object = hlist_entry(cookie->backing_objects.first,
 227                              struct fscache_object, cookie_link);
 228
 229         __fscache_use_cookie(cookie);
 230         if (fscache_submit_exclusive_op(object, op) < 0)
 231                 goto nobufs;
 232         spin_unlock(&cookie->lock);
 233         fscache_stat(&fscache_n_attr_changed_ok);
 234         fscache_put_operation(op);
 235         _leave(" = 0");
 236         return 0;
 237
 238 nobufs:
 239         wake_cookie = __fscache_unuse_cookie(cookie);
 240         spin_unlock(&cookie->lock);
 241         kfree(op);
 242         if (wake_cookie)
 243                 __fscache_wake_unused_cookie(cookie);
 244         fscache_stat(&fscache_n_attr_changed_nobufs);
 245         _leave(" = %d", -ENOBUFS);
 246         return -ENOBUFS;
 247 }
 248 EXPORT_SYMBOL(__fscache_attr_changed);
 249
 250 /*
 251  * release a retrieval op reference
 252  */
 253 static void fscache_release_retrieval_op(struct fscache_operation *_op)
 254 {
 255         struct fscache_retrieval *op =
 256                 container_of(_op, struct fscache_retrieval, op);
 257
 258         _enter("{OP%x}", op->op.debug_id);
 259
 260         ASSERTCMP(atomic_read(&op->n_pages), ==, 0);
 261
 262         fscache_hist(fscache_retrieval_histogram, op->start_time);
 263         if (op->context)
 264                 fscache_put_context(op->op.object->cookie, op->context);
 265
 266         _leave("");
 267 }
 268
 269 /*
 270  * allocate a retrieval op
 271  */
 272 static struct fscache_retrieval *fscache_alloc_retrieval(
 273         struct fscache_cookie *cookie,
 274         struct address_space *mapping,
 275         fscache_rw_complete_t end_io_func,
 276         void *context)
 277 {
 278         struct fscache_retrieval *op;
 279
 280         /* allocate a retrieval operation and attempt to submit it */
 281         op = kzalloc(sizeof(*op), GFP_NOIO);
 282         if (!op) {
 283                 fscache_stat(&fscache_n_retrievals_nomem);
 284                 return NULL;
 285         }
 286
 287         fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
 288         op->op.flags    = FSCACHE_OP_MYTHREAD |
 289                 (1UL << FSCACHE_OP_WAITING) |
 290                 (1UL << FSCACHE_OP_UNUSE_COOKIE);
 291         op->mapping     = mapping;
 292         op->end_io_func = end_io_func;
 293         op->context     = context;
 294         op->start_time  = jiffies;
 295         INIT_LIST_HEAD(&op->to_do);
 296         return op;
 297 }
 298
 299 /*
 300  * wait for a deferred lookup to complete
 301  */
 302 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 303 {
 304         unsigned long jif;
 305
 306         _enter("");
 307
 308         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 309                 _leave(" = 0 [imm]");
 310                 return 0;
 311         }
 312
 313         fscache_stat(&fscache_n_retrievals_wait);
 314
 315         jif = jiffies;
 316         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 317                         TASK_INTERRUPTIBLE) != 0) {
 318                 fscache_stat(&fscache_n_retrievals_intr);
 319                 _leave(" = -ERESTARTSYS");
 320                 return -ERESTARTSYS;
 321         }
 322
 323         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 324
 325         smp_rmb();
 326         fscache_hist(fscache_retrieval_delay_histogram, jif);
 327         _leave(" = 0 [dly]");
 328         return 0;
 329 }
 330
 331 /*
 332  * Handle cancellation of a pending retrieval op
 333  */
 334 static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
 335 {
 336         struct fscache_retrieval *op =
 337                 container_of(_op, struct fscache_retrieval, op);
 338
 339         atomic_set(&op->n_pages, 0);
 340 }
 341
 342 /*
 343  * wait for an object to become active (or dead)
 344  */
 345 int fscache_wait_for_operation_activation(struct fscache_object *object,
 346                                           struct fscache_operation *op,
 347                                           atomic_t *stat_op_waits,
 348                                           atomic_t *stat_object_dead,
 349                                           void (*do_cancel)(struct fscache_operation *))
 350 {
 351         int ret;
 352
 353         if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
 354                 goto check_if_dead;
 355
 356         _debug(">>> WT");
 357         if (stat_op_waits)
 358                 fscache_stat(stat_op_waits);
 359         if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 360                         TASK_INTERRUPTIBLE) != 0) {
 361                 ret = fscache_cancel_op(op, do_cancel);
 362                 if (ret == 0)
 363                         return -ERESTARTSYS;
 364
 365                 /* it's been removed from the pending queue by another party,
 366                  * so we should get to run shortly */
 367                 wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 368                             TASK_UNINTERRUPTIBLE);
 369         }
 370         _debug("<<< GO");
 371
 372 check_if_dead:
 373         if (op->state == FSCACHE_OP_ST_CANCELLED) {
 374                 if (stat_object_dead)
 375                         fscache_stat(stat_object_dead);
 376                 _leave(" = -ENOBUFS [cancelled]");
 377                 return -ENOBUFS;
 378         }
 379         if (unlikely(fscache_object_is_dead(object))) {
 380                 pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
 381                 fscache_cancel_op(op, do_cancel);
 382                 if (stat_object_dead)
 383                         fscache_stat(stat_object_dead);
 384                 return -ENOBUFS;
 385         }
 386         return 0;
 387 }
 388
 389 /*
 390  * read a page from the cache or allocate a block in which to store it
 391  * - we return:
 392  *   -ENOMEM    - out of memory, nothing done
 393  *   -ERESTARTSYS - interrupted
 394  *   -ENOBUFS   - no backing object available in which to cache the block
 395  *   -ENODATA   - no data available in the backing object for this block
 396  *   0          - dispatched a read - it'll call end_io_func() when finished
 397  */
 398 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 399                                  struct page *page,
 400                                  fscache_rw_complete_t end_io_func,
 401                                  void *context,
 402                                  gfp_t gfp)
 403 {
 404         struct fscache_retrieval *op;
 405         struct fscache_object *object;
 406         bool wake_cookie = false;
 407         int ret;
 408
 409         _enter("%p,%p,,,", cookie, page);
 410
 411         fscache_stat(&fscache_n_retrievals);
 412
 413         if (hlist_empty(&cookie->backing_objects))
 414                 goto nobufs;
 415
 416         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 417                 _leave(" = -ENOBUFS [invalidating]");
 418                 return -ENOBUFS;
 419         }
 420
 421         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 422         ASSERTCMP(page, !=, NULL);
 423
 424         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 425                 return -ERESTARTSYS;
 426
 427         op = fscache_alloc_retrieval(cookie, page->mapping,
 428                                      end_io_func, context);
 429         if (!op) {
 430                 _leave(" = -ENOMEM");
 431                 return -ENOMEM;
 432         }
 433         atomic_set(&op->n_pages, 1);
 434
 435         spin_lock(&cookie->lock);
 436
 437         if (!fscache_cookie_enabled(cookie) ||
 438             hlist_empty(&cookie->backing_objects))
 439                 goto nobufs_unlock;
 440         object = hlist_entry(cookie->backing_objects.first,
 441                              struct fscache_object, cookie_link);
 442
 443         ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
 444
 445         __fscache_use_cookie(cookie);
 446         atomic_inc(&object->n_reads);
 447         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 448
 449         if (fscache_submit_op(object, &op->op) < 0)
 450                 goto nobufs_unlock_dec;
 451         spin_unlock(&cookie->lock);
 452
 453         fscache_stat(&fscache_n_retrieval_ops);
 454
 455         /* pin the netfs read context in case we need to do the actual netfs
 456          * read because we've encountered a cache read failure */
 457         fscache_get_context(object->cookie, op->context);
 458
 459         /* we wait for the operation to become active, and then process it
 460          * *here*, in this thread, and not in the thread pool */
 461         ret = fscache_wait_for_operation_activation(
 462                 object, &op->op,
 463                 __fscache_stat(&fscache_n_retrieval_op_waits),
 464                 __fscache_stat(&fscache_n_retrievals_object_dead),
 465                 fscache_do_cancel_retrieval);
 466         if (ret < 0)
 467                 goto error;
 468
 469         /* ask the cache to honour the operation */
 470         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 471                 fscache_stat(&fscache_n_cop_allocate_page);
 472                 ret = object->cache->ops->allocate_page(op, page, gfp);
 473                 fscache_stat_d(&fscache_n_cop_allocate_page);
 474                 if (ret == 0)
 475                         ret = -ENODATA;
 476         } else {
 477                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
 478                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 479                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 480         }
 481
 482 error:
 483         if (ret == -ENOMEM)
 484                 fscache_stat(&fscache_n_retrievals_nomem);
 485         else if (ret == -ERESTARTSYS)
 486                 fscache_stat(&fscache_n_retrievals_intr);
 487         else if (ret == -ENODATA)
 488                 fscache_stat(&fscache_n_retrievals_nodata);
 489         else if (ret < 0)
 490                 fscache_stat(&fscache_n_retrievals_nobufs);
 491         else
 492                 fscache_stat(&fscache_n_retrievals_ok);
 493
 494         fscache_put_retrieval(op);
 495         _leave(" = %d", ret);
 496         return ret;
 497
 498 nobufs_unlock_dec:
 499         atomic_dec(&object->n_reads);
 500         wake_cookie = __fscache_unuse_cookie(cookie);
 501 nobufs_unlock:
 502         spin_unlock(&cookie->lock);
 503         if (wake_cookie)
 504                 __fscache_wake_unused_cookie(cookie);
 505         kfree(op);
 506 nobufs:
 507         fscache_stat(&fscache_n_retrievals_nobufs);
 508         _leave(" = -ENOBUFS");
 509         return -ENOBUFS;
 510 }
 511 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 512
 513 /*
 514  * read a list of page from the cache or allocate a block in which to store
 515  * them
 516  * - we return:
 517  *   -ENOMEM    - out of memory, some pages may be being read
 518  *   -ERESTARTSYS - interrupted, some pages may be being read
 519  *   -ENOBUFS   - no backing object or space available in which to cache any
 520  *                pages not being read
 521  *   -ENODATA   - no data available in the backing object for some or all of
 522  *                the pages
 523  *   0          - dispatched a read on all pages
 524  *
 525  * end_io_func() will be called for each page read from the cache as it is
 526  * finishes being read
 527  *
 528  * any pages for which a read is dispatched will be removed from pages and
 529  * nr_pages
 530  */
 531 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 532                                   struct address_space *mapping,
 533                                   struct list_head *pages,
 534                                   unsigned *nr_pages,
 535                                   fscache_rw_complete_t end_io_func,
 536                                   void *context,
 537                                   gfp_t gfp)
 538 {
 539         struct fscache_retrieval *op;
 540         struct fscache_object *object;
 541         bool wake_cookie = false;
 542         int ret;
 543
 544         _enter("%p,,%d,,,", cookie, *nr_pages);
 545
 546         fscache_stat(&fscache_n_retrievals);
 547
 548         if (hlist_empty(&cookie->backing_objects))
 549                 goto nobufs;
 550
 551         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 552                 _leave(" = -ENOBUFS [invalidating]");
 553                 return -ENOBUFS;
 554         }
 555
 556         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 557         ASSERTCMP(*nr_pages, >, 0);
 558         ASSERT(!list_empty(pages));
 559
 560         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 561                 return -ERESTARTSYS;
 562
 563         op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
 564         if (!op)
 565                 return -ENOMEM;
 566         atomic_set(&op->n_pages, *nr_pages);
 567
 568         spin_lock(&cookie->lock);
 569
 570         if (!fscache_cookie_enabled(cookie) ||
 571             hlist_empty(&cookie->backing_objects))
 572                 goto nobufs_unlock;
 573         object = hlist_entry(cookie->backing_objects.first,
 574                              struct fscache_object, cookie_link);
 575
 576         __fscache_use_cookie(cookie);
 577         atomic_inc(&object->n_reads);
 578         __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 579
 580         if (fscache_submit_op(object, &op->op) < 0)
 581                 goto nobufs_unlock_dec;
 582         spin_unlock(&cookie->lock);
 583
 584         fscache_stat(&fscache_n_retrieval_ops);
 585
 586         /* pin the netfs read context in case we need to do the actual netfs
 587          * read because we've encountered a cache read failure */
 588         fscache_get_context(object->cookie, op->context);
 589
 590         /* we wait for the operation to become active, and then process it
 591          * *here*, in this thread, and not in the thread pool */
 592         ret = fscache_wait_for_operation_activation(
 593                 object, &op->op,
 594                 __fscache_stat(&fscache_n_retrieval_op_waits),
 595                 __fscache_stat(&fscache_n_retrievals_object_dead),
 596                 fscache_do_cancel_retrieval);
 597         if (ret < 0)
 598                 goto error;
 599
 600         /* ask the cache to honour the operation */
 601         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 602                 fscache_stat(&fscache_n_cop_allocate_pages);
 603                 ret = object->cache->ops->allocate_pages(
 604                         op, pages, nr_pages, gfp);
 605                 fscache_stat_d(&fscache_n_cop_allocate_pages);
 606         } else {
 607                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 608                 ret = object->cache->ops->read_or_alloc_pages(
 609                         op, pages, nr_pages, gfp);
 610                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 611         }
 612
 613 error:
 614         if (ret == -ENOMEM)
 615                 fscache_stat(&fscache_n_retrievals_nomem);
 616         else if (ret == -ERESTARTSYS)
 617                 fscache_stat(&fscache_n_retrievals_intr);
 618         else if (ret == -ENODATA)
 619                 fscache_stat(&fscache_n_retrievals_nodata);
 620         else if (ret < 0)
 621                 fscache_stat(&fscache_n_retrievals_nobufs);
 622         else
 623                 fscache_stat(&fscache_n_retrievals_ok);
 624
 625         fscache_put_retrieval(op);
 626         _leave(" = %d", ret);
 627         return ret;
 628
 629 nobufs_unlock_dec:
 630         atomic_dec(&object->n_reads);
 631         wake_cookie = __fscache_unuse_cookie(cookie);
 632 nobufs_unlock:
 633         spin_unlock(&cookie->lock);
 634         kfree(op);
 635         if (wake_cookie)
 636                 __fscache_wake_unused_cookie(cookie);
 637 nobufs:
 638         fscache_stat(&fscache_n_retrievals_nobufs);
 639         _leave(" = -ENOBUFS");
 640         return -ENOBUFS;
 641 }
 642 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 643
 644 /*
 645  * allocate a block in the cache on which to store a page
 646  * - we return:
 647  *   -ENOMEM    - out of memory, nothing done
 648  *   -ERESTARTSYS - interrupted
 649  *   -ENOBUFS   - no backing object available in which to cache the block
 650  *   0          - block allocated
 651  */
 652 int __fscache_alloc_page(struct fscache_cookie *cookie,
 653                          struct page *page,
 654                          gfp_t gfp)
 655 {
 656         struct fscache_retrieval *op;
 657         struct fscache_object *object;
 658         bool wake_cookie = false;
 659         int ret;
 660
 661         _enter("%p,%p,,,", cookie, page);
 662
 663         fscache_stat(&fscache_n_allocs);
 664
 665         if (hlist_empty(&cookie->backing_objects))
 666                 goto nobufs;
 667
 668         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 669         ASSERTCMP(page, !=, NULL);
 670
 671         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 672                 _leave(" = -ENOBUFS [invalidating]");
 673                 return -ENOBUFS;
 674         }
 675
 676         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 677                 return -ERESTARTSYS;
 678
 679         op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
 680         if (!op)
 681                 return -ENOMEM;
 682         atomic_set(&op->n_pages, 1);
 683
 684         spin_lock(&cookie->lock);
 685
 686         if (!fscache_cookie_enabled(cookie) ||
 687             hlist_empty(&cookie->backing_objects))
 688                 goto nobufs_unlock;
 689         object = hlist_entry(cookie->backing_objects.first,
 690                              struct fscache_object, cookie_link);
 691
 692         __fscache_use_cookie(cookie);
 693         if (fscache_submit_op(object, &op->op) < 0)
 694                 goto nobufs_unlock_dec;
 695         spin_unlock(&cookie->lock);
 696
 697         fscache_stat(&fscache_n_alloc_ops);
 698
 699         ret = fscache_wait_for_operation_activation(
 700                 object, &op->op,
 701                 __fscache_stat(&fscache_n_alloc_op_waits),
 702                 __fscache_stat(&fscache_n_allocs_object_dead),
 703                 fscache_do_cancel_retrieval);
 704         if (ret < 0)
 705                 goto error;
 706
 707         /* ask the cache to honour the operation */
 708         fscache_stat(&fscache_n_cop_allocate_page);
 709         ret = object->cache->ops->allocate_page(op, page, gfp);
 710         fscache_stat_d(&fscache_n_cop_allocate_page);
 711
 712 error:
 713         if (ret == -ERESTARTSYS)
 714                 fscache_stat(&fscache_n_allocs_intr);
 715         else if (ret < 0)
 716                 fscache_stat(&fscache_n_allocs_nobufs);
 717         else
 718                 fscache_stat(&fscache_n_allocs_ok);
 719
 720         fscache_put_retrieval(op);
 721         _leave(" = %d", ret);
 722         return ret;
 723
 724 nobufs_unlock_dec:
 725         wake_cookie = __fscache_unuse_cookie(cookie);
 726 nobufs_unlock:
 727         spin_unlock(&cookie->lock);
 728         kfree(op);
 729         if (wake_cookie)
 730                 __fscache_wake_unused_cookie(cookie);
 731 nobufs:
 732         fscache_stat(&fscache_n_allocs_nobufs);
 733         _leave(" = -ENOBUFS");
 734         return -ENOBUFS;
 735 }
 736 EXPORT_SYMBOL(__fscache_alloc_page);
 737
 738 /*
 739  * Unmark pages allocate in the readahead code path (via:
 740  * fscache_readpages_or_alloc) after delegating to the base filesystem
 741  */
 742 void __fscache_readpages_cancel(struct fscache_cookie *cookie,
 743                                 struct list_head *pages)
 744 {
 745         struct page *page;
 746
 747         list_for_each_entry(page, pages, lru) {
 748                 if (PageFsCache(page))
 749                         __fscache_uncache_page(cookie, page);
 750         }
 751 }
 752 EXPORT_SYMBOL(__fscache_readpages_cancel);
 753
 754 /*
 755  * release a write op reference
 756  */
 757 static void fscache_release_write_op(struct fscache_operation *_op)
 758 {
 759         _enter("{OP%x}", _op->debug_id);
 760 }
 761
 762 /*
 763  * perform the background storage of a page into the cache
 764  */
 765 static void fscache_write_op(struct fscache_operation *_op)
 766 {
 767         struct fscache_storage *op =
 768                 container_of(_op, struct fscache_storage, op);
 769         struct fscache_object *object = op->op.object;
 770         struct fscache_cookie *cookie;
 771         struct page *page;
 772         unsigned n;
 773         void *results[1];
 774         int ret;
 775
 776         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 777
 778         spin_lock(&object->lock);
 779         cookie = object->cookie;
 780
 781         if (!fscache_object_is_active(object)) {
 782                 /* If we get here, then the on-disk cache object likely longer
 783                  * exists, so we should just cancel this write operation.
 784                  */
 785                 spin_unlock(&object->lock);
 786                 fscache_op_complete(&op->op, false);
 787                 _leave(" [inactive]");
 788                 return;
 789         }
 790
 791         if (!cookie) {
 792                 /* If we get here, then the cookie belonging to the object was
 793                  * detached, probably by the cookie being withdrawn due to
 794                  * memory pressure, which means that the pages we might write
 795                  * to the cache from no longer exist - therefore, we can just
 796                  * cancel this write operation.
 797                  */
 798                 spin_unlock(&object->lock);
 799                 fscache_op_complete(&op->op, false);
 800                 _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
 801                        _op->flags, _op->state, object->state->short_name,
 802                        object->flags);
 803                 return;
 804         }
 805
 806         spin_lock(&cookie->stores_lock);
 807
 808         fscache_stat(&fscache_n_store_calls);
 809
 810         /* find a page to store */
 811         page = NULL;
 812         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 813                                        FSCACHE_COOKIE_PENDING_TAG);
 814         if (n != 1)
 815                 goto superseded;
 816         page = results[0];
 817         _debug("gang %d [%lx]", n, page->index);
 818         if (page->index > op->store_limit) {
 819                 fscache_stat(&fscache_n_store_pages_over_limit);
 820                 goto superseded;
 821         }
 822
 823         radix_tree_tag_set(&cookie->stores, page->index,
 824                            FSCACHE_COOKIE_STORING_TAG);
 825         radix_tree_tag_clear(&cookie->stores, page->index,
 826                              FSCACHE_COOKIE_PENDING_TAG);
 827
 828         spin_unlock(&cookie->stores_lock);
 829         spin_unlock(&object->lock);
 830
 831         fscache_stat(&fscache_n_store_pages);
 832         fscache_stat(&fscache_n_cop_write_page);
 833         ret = object->cache->ops->write_page(op, page);
 834         fscache_stat_d(&fscache_n_cop_write_page);
 835         fscache_end_page_write(object, page);
 836         if (ret < 0) {
 837                 fscache_abort_object(object);
 838                 fscache_op_complete(&op->op, true);
 839         } else {
 840                 fscache_enqueue_operation(&op->op);
 841         }
 842
 843         _leave("");
 844         return;
 845
 846 superseded:
 847         /* this writer is going away and there aren't any more things to
 848          * write */
 849         _debug("cease");
 850         spin_unlock(&cookie->stores_lock);
 851         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 852         spin_unlock(&object->lock);
 853         fscache_op_complete(&op->op, true);
 854         _leave("");
 855 }
 856
 857 /*
 858  * Clear the pages pending writing for invalidation
 859  */
 860 void fscache_invalidate_writes(struct fscache_cookie *cookie)
 861 {
 862         struct page *page;
 863         void *results[16];
 864         int n, i;
 865
 866         _enter("");
 867
 868         for (;;) {
 869                 spin_lock(&cookie->stores_lock);
 870                 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
 871                                                ARRAY_SIZE(results),
 872                                                FSCACHE_COOKIE_PENDING_TAG);
 873                 if (n == 0) {
 874                         spin_unlock(&cookie->stores_lock);
 875                         break;
 876                 }
 877
 878                 for (i = n - 1; i >= 0; i--) {
 879                         page = results[i];
 880                         radix_tree_delete(&cookie->stores, page->index);
 881                 }
 882
 883                 spin_unlock(&cookie->stores_lock);
 884
 885                 for (i = n - 1; i >= 0; i--)
 886                         page_cache_release(results[i]);
 887         }
 888
 889         _leave("");
 890 }
 891
 892 /*
 893  * request a page be stored in the cache
 894  * - returns:
 895  *   -ENOMEM    - out of memory, nothing done
 896  *   -ENOBUFS   - no backing object available in which to cache the page
 897  *   0          - dispatched a write - it'll call end_io_func() when finished
 898  *
 899  * if the cookie still has a backing object at this point, that object can be
 900  * in one of a few states with respect to storage processing:
 901  *
 902  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 903  *      set)
 904  *
 905  *      (a) no writes yet
 906  *
 907  *      (b) writes deferred till post-creation (mark page for writing and
 908  *          return immediately)
 909  *
 910  *  (2) negative lookup, object created, initial fill being made from netfs
 911  *
 912  *      (a) fill point not yet reached this page (mark page for writing and
 913  *          return)
 914  *
 915  *      (b) fill point passed this page (queue op to store this page)
 916  *
 917  *  (3) object extant (queue op to store this page)
 918  *
 919  * any other state is invalid
 920  */
 921 int __fscache_write_page(struct fscache_cookie *cookie,
 922                          struct page *page,
 923                          gfp_t gfp)
 924 {
 925         struct fscache_storage *op;
 926         struct fscache_object *object;
 927         bool wake_cookie = false;
 928         int ret;
 929
 930         _enter("%p,%x,", cookie, (u32) page->flags);
 931
 932         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 933         ASSERT(PageFsCache(page));
 934
 935         fscache_stat(&fscache_n_stores);
 936
 937         if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 938                 _leave(" = -ENOBUFS [invalidating]");
 939                 return -ENOBUFS;
 940         }
 941
 942         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 943         if (!op)
 944                 goto nomem;
 945
 946         fscache_operation_init(&op->op, fscache_write_op,
 947                                fscache_release_write_op);
 948         op->op.flags = FSCACHE_OP_ASYNC |
 949                 (1 << FSCACHE_OP_WAITING) |
 950                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 951
 952         ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
 953         if (ret < 0)
 954                 goto nomem_free;
 955
 956         ret = -ENOBUFS;
 957         spin_lock(&cookie->lock);
 958
 959         if (!fscache_cookie_enabled(cookie) ||
 960             hlist_empty(&cookie->backing_objects))
 961                 goto nobufs;
 962         object = hlist_entry(cookie->backing_objects.first,
 963                              struct fscache_object, cookie_link);
 964         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 965                 goto nobufs;
 966
 967         /* add the page to the pending-storage radix tree on the backing
 968          * object */
 969         spin_lock(&object->lock);
 970         spin_lock(&cookie->stores_lock);
 971
 972         _debug("store limit %llx", (unsigned long long) object->store_limit);
 973
 974         ret = radix_tree_insert(&cookie->stores, page->index, page);
 975         if (ret < 0) {
 976                 if (ret == -EEXIST)
 977                         goto already_queued;
 978                 _debug("insert failed %d", ret);
 979                 goto nobufs_unlock_obj;
 980         }
 981
 982         radix_tree_tag_set(&cookie->stores, page->index,
 983                            FSCACHE_COOKIE_PENDING_TAG);
 984         page_cache_get(page);
 985
 986         /* we only want one writer at a time, but we do need to queue new
 987          * writers after exclusive ops */
 988         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 989                 goto already_pending;
 990
 991         spin_unlock(&cookie->stores_lock);
 992         spin_unlock(&object->lock);
 993
 994         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 995         op->store_limit = object->store_limit;
 996
 997         __fscache_use_cookie(cookie);
 998         if (fscache_submit_op(object, &op->op) < 0)
 999                 goto submit_failed;
1000
1001         spin_unlock(&cookie->lock);
1002         radix_tree_preload_end();
1003         fscache_stat(&fscache_n_store_ops);
1004         fscache_stat(&fscache_n_stores_ok);
1005
1006         /* the work queue now carries its own ref on the object */
1007         fscache_put_operation(&op->op);
1008         _leave(" = 0");
1009         return 0;
1010
1011 already_queued:
1012         fscache_stat(&fscache_n_stores_again);
1013 already_pending:
1014         spin_unlock(&cookie->stores_lock);
1015         spin_unlock(&object->lock);
1016         spin_unlock(&cookie->lock);
1017         radix_tree_preload_end();
1018         kfree(op);
1019         fscache_stat(&fscache_n_stores_ok);
1020         _leave(" = 0");
1021         return 0;
1022
1023 submit_failed:
1024         spin_lock(&cookie->stores_lock);
1025         radix_tree_delete(&cookie->stores, page->index);
1026         spin_unlock(&cookie->stores_lock);
1027         wake_cookie = __fscache_unuse_cookie(cookie);
1028         page_cache_release(page);
1029         ret = -ENOBUFS;
1030         goto nobufs;
1031
1032 nobufs_unlock_obj:
1033         spin_unlock(&cookie->stores_lock);
1034         spin_unlock(&object->lock);
1035 nobufs:
1036         spin_unlock(&cookie->lock);
1037         radix_tree_preload_end();
1038         kfree(op);
1039         if (wake_cookie)
1040                 __fscache_wake_unused_cookie(cookie);
1041         fscache_stat(&fscache_n_stores_nobufs);
1042         _leave(" = -ENOBUFS");
1043         return -ENOBUFS;
1044
1045 nomem_free:
1046         kfree(op);
1047 nomem:
1048         fscache_stat(&fscache_n_stores_oom);
1049         _leave(" = -ENOMEM");
1050         return -ENOMEM;
1051 }
1052 EXPORT_SYMBOL(__fscache_write_page);
1053
1054 /*
1055  * remove a page from the cache
1056  */
1057 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1058 {
1059         struct fscache_object *object;
1060
1061         _enter(",%p", page);
1062
1063         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1064         ASSERTCMP(page, !=, NULL);
1065
1066         fscache_stat(&fscache_n_uncaches);
1067
1068         /* cache withdrawal may beat us to it */
1069         if (!PageFsCache(page))
1070                 goto done;
1071
1072         /* get the object */
1073         spin_lock(&cookie->lock);
1074
1075         if (hlist_empty(&cookie->backing_objects)) {
1076                 ClearPageFsCache(page);
1077                 goto done_unlock;
1078         }
1079
1080         object = hlist_entry(cookie->backing_objects.first,
1081                              struct fscache_object, cookie_link);
1082
1083         /* there might now be stuff on disk we could read */
1084         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1085
1086         /* only invoke the cache backend if we managed to mark the page
1087          * uncached here; this deals with synchronisation vs withdrawal */
1088         if (TestClearPageFsCache(page) &&
1089             object->cache->ops->uncache_page) {
1090                 /* the cache backend releases the cookie lock */
1091                 fscache_stat(&fscache_n_cop_uncache_page);
1092                 object->cache->ops->uncache_page(object, page);
1093                 fscache_stat_d(&fscache_n_cop_uncache_page);
1094                 goto done;
1095         }
1096
1097 done_unlock:
1098         spin_unlock(&cookie->lock);
1099 done:
1100         _leave("");
1101 }
1102 EXPORT_SYMBOL(__fscache_uncache_page);
1103
1104 /**
1105  * fscache_mark_page_cached - Mark a page as being cached
1106  * @op: The retrieval op pages are being marked for
1107  * @page: The page to be marked
1108  *
1109  * Mark a netfs page as being cached.  After this is called, the netfs
1110  * must call fscache_uncache_page() to remove the mark.
1111  */
1112 void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1113 {
1114         struct fscache_cookie *cookie = op->op.object->cookie;
1115
1116 #ifdef CONFIG_FSCACHE_STATS
1117         atomic_inc(&fscache_n_marks);
1118 #endif
1119
1120         _debug("- mark %p{%lx}", page, page->index);
1121         if (TestSetPageFsCache(page)) {
1122                 static bool once_only;
1123                 if (!once_only) {
1124                         once_only = true;
1125                         pr_warn("Cookie type %s marked page %lx multiple times\n",
1126                                 cookie->def->name, page->index);
1127                 }
1128         }
1129
1130         if (cookie->def->mark_page_cached)
1131                 cookie->def->mark_page_cached(cookie->netfs_data,
1132                                               op->mapping, page);
1133 }
1134 EXPORT_SYMBOL(fscache_mark_page_cached);
1135
1136 /**
1137  * fscache_mark_pages_cached - Mark pages as being cached
1138  * @op: The retrieval op pages are being marked for
1139  * @pagevec: The pages to be marked
1140  *
1141  * Mark a bunch of netfs pages as being cached.  After this is called,
1142  * the netfs must call fscache_uncache_page() to remove the mark.
1143  */
1144 void fscache_mark_pages_cached(struct fscache_retrieval *op,
1145                                struct pagevec *pagevec)
1146 {
1147         unsigned long loop;
1148
1149         for (loop = 0; loop < pagevec->nr; loop++)
1150                 fscache_mark_page_cached(op, pagevec->pages[loop]);
1151
1152         pagevec_reinit(pagevec);
1153 }
1154 EXPORT_SYMBOL(fscache_mark_pages_cached);
1155
1156 /*
1157  * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1158  * to be associated with the given cookie.
1159  */
1160 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1161                                        struct inode *inode)
1162 {
1163         struct address_space *mapping = inode->i_mapping;
1164         struct pagevec pvec;
1165         pgoff_t next;
1166         int i;
1167
1168         _enter("%p,%p", cookie, inode);
1169
1170         if (!mapping || mapping->nrpages == 0) {
1171                 _leave(" [no pages]");
1172                 return;
1173         }
1174
1175         pagevec_init(&pvec, 0);
1176         next = 0;
1177         do {
1178                 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
1179                         break;
1180                 for (i = 0; i < pagevec_count(&pvec); i++) {
1181                         struct page *page = pvec.pages[i];
1182                         next = page->index;
1183                         if (PageFsCache(page)) {
1184                                 __fscache_wait_on_page_write(cookie, page);
1185                                 __fscache_uncache_page(cookie, page);
1186                         }
1187                 }
1188                 pagevec_release(&pvec);
1189                 cond_resched();
1190         } while (++next);
1191
1192         _leave("");
1193 }
1194 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);