mm/gup: decrement head page once for group of subpages

[linux-block.git] / mm / gup.c

diff --git a/mm/gup.c b/mm/gup.c
index 5856f68467b309a6cee087eb57eee2f0db5078c0..de1b75ef44da604362056aad6e305e3edd782df4 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -265,20 +265,15 @@ void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
                                 bool make_dirty)
 {
        unsigned long index;
-
-       /*
-        * TODO: this can be optimized for huge pages: if a series of pages is
-        * physically contiguous and part of the same compound page, then a
-        * single operation to the head page should suffice.
-        */
+       struct page *head;
+       unsigned int ntails;
 
        if (!make_dirty) {
                unpin_user_pages(pages, npages);
                return;
        }
 
-       for (index = 0; index < npages; index++) {
-               struct page *page = compound_head(pages[index]);
+       for_each_compound_head(index, pages, npages, head, ntails) {
                /*
                 * Checking PageDirty at this point may race with
                 * clear_page_dirty_for_io(), but that's OK. Two key
@@ -299,9 +294,9 @@ void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
                 * written back, so it gets written back again in the
                 * next writeback cycle. This is harmless.
                 */
-               if (!PageDirty(page))
-                       set_page_dirty_lock(page);
-               unpin_user_page(page);
+               if (!PageDirty(head))
+                       set_page_dirty_lock(head);
+               put_compound_head(head, ntails, FOLL_PIN);
        }
 }
 EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
@@ -318,6 +313,8 @@ EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
 void unpin_user_pages(struct page **pages, unsigned long npages)
 {
        unsigned long index;
+       struct page *head;
+       unsigned int ntails;
 
        /*
         * If this WARN_ON() fires, then the system *might* be leaking pages (by
@@ -326,13 +323,9 @@ void unpin_user_pages(struct page **pages, unsigned long npages)
         */
        if (WARN_ON(IS_ERR_VALUE(npages)))
                return;
-       /*
-        * TODO: this can be optimized for huge pages: if a series of pages is
-        * physically contiguous and part of the same compound page, then a
-        * single operation to the head page should suffice.
-        */
-       for (index = 0; index < npages; index++)
-               unpin_user_page(pages[index]);
+
+       for_each_compound_head(index, pages, npages, head, ntails)
+               put_compound_head(head, ntails, FOLL_PIN);
 }
 EXPORT_SYMBOL(unpin_user_pages);