filemap: allocate mapping_min_order folios in the page cache

filemap_create_folio() and do_read_cache_folio() were always allocating
folio of order 0. __filemap_get_folio was trying to allocate higher
order folios when fgp_flags had higher order hint set but it will default
to order 0 folio if higher order memory allocation fails.

Supporting mapping_min_order implies that we guarantee each folio in the
page cache has at least an order of mapping_min_order. When adding new
folios to the page cache we must also ensure the index used is aligned to
the mapping_min_order as the page cache requires the index to be aligned
to the order of the folio.

Co-developed-by: Luis Chamberlain <mcgrof@kernel.org>
Signed-off-by: Luis Chamberlain <mcgrof@kernel.org>
Signed-off-by: Pankaj Raghav <p.raghav@samsung.com>
Link: https://lore.kernel.org/r/20240822135018.1931258-3-kernel@pankajraghav.com
Tested-by: David Howells <dhowells@redhat.com>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Reviewed-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Daniel Gomez <da.gomez@samsung.com>
Reviewed-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Christian Brauner <brauner@kernel.org>

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index c60025bb584c5d0ae9d1c57bbff4512b60e38032..4cc170949e9c09e3b6b561d60cc1b6edca528ba6 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -448,6 +448,26 @@ mapping_min_folio_order(const struct address_space *mapping)
        return (mapping->flags & AS_FOLIO_ORDER_MIN_MASK) >> AS_FOLIO_ORDER_MIN;
 }
 
+static inline unsigned long
+mapping_min_folio_nrpages(struct address_space *mapping)
+{
+       return 1UL << mapping_min_folio_order(mapping);
+}
+
+/**
+ * mapping_align_index() - Align index for this mapping.
+ * @mapping: The address_space.
+ *
+ * The index of a folio must be naturally aligned.  If you are adding a
+ * new folio to the page cache and need to know what index to give it,
+ * call this function.
+ */
+static inline pgoff_t mapping_align_index(struct address_space *mapping,
+                                         pgoff_t index)
+{
+       return round_down(index, mapping_min_folio_nrpages(mapping));
+}
+
 /*
  * Large folio support currently depends on THP.  These dependencies are
  * being worked on but are not yet fixed.

diff --git a/mm/filemap.c b/mm/filemap.c
index ad5e4a848070e9e23953f7385ec75b5069277a7d..d27e9ac54309def8f48e364e10fc55d54a0ca724 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -859,6 +859,8 @@ noinline int __filemap_add_folio(struct address_space *mapping,
 
        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
        VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio);
+       VM_BUG_ON_FOLIO(folio_order(folio) < mapping_min_folio_order(mapping),
+                       folio);
        mapping_set_update(&xas, mapping);
 
        VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);
@@ -1919,8 +1921,10 @@ repeat:
                folio_wait_stable(folio);
 no_page:
        if (!folio && (fgp_flags & FGP_CREAT)) {
-               unsigned order = FGF_GET_ORDER(fgp_flags);
+               unsigned int min_order = mapping_min_folio_order(mapping);
+               unsigned int order = max(min_order, FGF_GET_ORDER(fgp_flags));
                int err;
+               index = mapping_align_index(mapping, index);
 
                if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
                        gfp |= __GFP_WRITE;
@@ -1943,7 +1947,7 @@ no_page:
                        gfp_t alloc_gfp = gfp;
 
                        err = -ENOMEM;
-                       if (order > 0)
+                       if (order > min_order)
                                alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
                        folio = filemap_alloc_folio(alloc_gfp, order);
                        if (!folio)
@@ -1958,7 +1962,7 @@ no_page:
                                break;
                        folio_put(folio);
                        folio = NULL;
-               } while (order-- > 0);
+               } while (order-- > min_order);
 
                if (err == -EEXIST)
                        goto repeat;
@@ -2447,13 +2451,15 @@ unlock_mapping:
 }
 
 static int filemap_create_folio(struct file *file,
-               struct address_space *mapping, pgoff_t index,
+               struct address_space *mapping, loff_t pos,
                struct folio_batch *fbatch)
 {
        struct folio *folio;
        int error;
+       unsigned int min_order = mapping_min_folio_order(mapping);
+       pgoff_t index;
 
-       folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0);
+       folio = filemap_alloc_folio(mapping_gfp_mask(mapping), min_order);
        if (!folio)
                return -ENOMEM;
 
@@ -2471,6 +2477,7 @@ static int filemap_create_folio(struct file *file,
         * well to keep locking rules simple.
         */
        filemap_invalidate_lock_shared(mapping);
+       index = (pos >> (PAGE_SHIFT + min_order)) << min_order;
        error = filemap_add_folio(mapping, folio, index,
                        mapping_gfp_constraint(mapping, GFP_KERNEL));
        if (error == -EEXIST)
@@ -2531,8 +2538,7 @@ retry:
        if (!folio_batch_count(fbatch)) {
                if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))
                        return -EAGAIN;
-               err = filemap_create_folio(filp, mapping,
-                               iocb->ki_pos >> PAGE_SHIFT, fbatch);
+               err = filemap_create_folio(filp, mapping, iocb->ki_pos, fbatch);
                if (err == AOP_TRUNCATED_PAGE)
                        goto retry;
                return err;
@@ -3748,9 +3754,11 @@ static struct folio *do_read_cache_folio(struct address_space *mapping,
 repeat:
        folio = filemap_get_folio(mapping, index);
        if (IS_ERR(folio)) {
-               folio = filemap_alloc_folio(gfp, 0);
+               folio = filemap_alloc_folio(gfp,
+                                           mapping_min_folio_order(mapping));
                if (!folio)
                        return ERR_PTR(-ENOMEM);
+               index = mapping_align_index(mapping, index);
                err = filemap_add_folio(mapping, folio, index, gfp);
                if (unlikely(err)) {
                        folio_put(folio);