iomap: support RWF_UNCACHED for buffered writes

This adds support for RWF_UNCACHED for file systems using iomap to
perform buffered writes. We use the generic infrastructure for this,
by tracking pages we created and calling write_drop_cached_pages()
to issue writeback and prune those pages.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/iomap/apply.c b/fs/iomap/apply.c
index e76148db03b8f76f8ca72cca4a12b72310ebc8bb..f2f87fb412d56b133b8d4550228fb9ad1f6ec0bb 100644 (file)
--- a/fs/iomap/apply.c
+++ b/fs/iomap/apply.c
@@ -24,7 +24,10 @@ loff_t
 iomap_apply(struct iomap_data *data, const struct iomap_ops *ops,
            iomap_actor_t actor)
 {
-       struct iomap iomap = { .type = IOMAP_HOLE };
+       struct iomap iomap = {
+               .type           = IOMAP_HOLE,
+               .page_list      = LIST_HEAD_INIT(iomap.page_list)
+       };
        struct iomap srcmap = { .type = IOMAP_HOLE };
        loff_t written = 0, ret;
        u64 end;
@@ -92,5 +95,8 @@ iomap_apply(struct iomap_data *data, const struct iomap_ops *ops,
                                     data->flags, &iomap);
        }
 
+       if (!list_empty(&iomap.page_list))
+               uncached_write_pages(data->inode->i_mapping, &iomap.page_list);
+
        return written ? written : ret;
 }

diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index 0a1a195ed1cc258be603551f0b5bccc7ca4549a4..681607807d8021c55e04d53b45d85bf4a80a173a 100644 (file)
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -582,6 +582,7 @@ EXPORT_SYMBOL_GPL(iomap_migrate_page);
 
 enum {
        IOMAP_WRITE_F_UNSHARE           = (1 << 0),
+       IOMAP_WRITE_F_UNCACHED          = (1 << 1),
 };
 
 static void
@@ -659,6 +660,7 @@ iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags,
                struct page **pagep, struct iomap *iomap, struct iomap *srcmap)
 {
        const struct iomap_page_ops *page_ops = iomap->page_ops;
+       unsigned aop_flags;
        struct page *page;
        int status = 0;
 
@@ -675,8 +677,11 @@ iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags,
                        return status;
        }
 
+       aop_flags = AOP_FLAG_NOFS;
+       if (flags & IOMAP_WRITE_F_UNCACHED)
+               aop_flags |= AOP_FLAG_UNCACHED;
        page = grab_cache_page_write_begin(inode->i_mapping, pos >> PAGE_SHIFT,
-                       AOP_FLAG_NOFS);
+                                               aop_flags);
        if (!page) {
                status = -ENOMEM;
                goto out_no_page;
@@ -820,9 +825,13 @@ iomap_write_actor(const struct iomap_data *data, struct iomap *iomap,
        struct iov_iter *i = data->priv;
        loff_t length = data->len;
        loff_t pos = data->pos;
+       unsigned flags = 0;
        long status = 0;
        ssize_t written = 0;
 
+       if (data->flags & IOMAP_UNCACHED)
+               flags |= IOMAP_WRITE_F_UNCACHED;
+
        do {
                struct page *page;
                unsigned long offset;   /* Offset into pagecache page */
@@ -851,8 +860,8 @@ again:
                        break;
                }
 
-               status = iomap_write_begin(inode, pos, bytes, 0, &page, iomap,
-                               srcmap);
+               status = iomap_write_begin(inode, pos, bytes, flags,
+                                               &page, iomap, srcmap);
                if (unlikely(status))
                        break;
 
@@ -889,7 +898,10 @@ again:
                written += copied;
                length -= copied;
 
-               balance_dirty_pages_ratelimited(inode->i_mapping);
+               if (!PagePrivio(page))
+                       balance_dirty_pages_ratelimited(inode->i_mapping);
+               else
+                       list_add_tail(&page->lru, &iomap->page_list);
        } while (iov_iter_count(i) && length);
 
        return written ? written : status;
@@ -907,6 +919,9 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *iter,
        };
        loff_t ret = 0, written = 0;
 
+       if (iocb->ki_flags & IOCB_UNCACHED)
+               data.flags |= IOMAP_UNCACHED;
+
        while (iov_iter_count(iter)) {
                data.len = iov_iter_count(iter);
                ret = iomap_apply(&data, ops, iomap_write_actor);

diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h
index 6dc227b8c47ecc9d1dd60d46f4ead59fa34d8b84..63c771e3eef5479c1d9015f42a0fa178ee326c9c 100644 (file)
--- a/fs/iomap/trace.h
+++ b/fs/iomap/trace.h
@@ -93,7 +93,8 @@ DEFINE_PAGE_EVENT(iomap_invalidatepage);
        { IOMAP_REPORT,         "REPORT" }, \
        { IOMAP_FAULT,          "FAULT" }, \
        { IOMAP_DIRECT,         "DIRECT" }, \
-       { IOMAP_NOWAIT,         "NOWAIT" }
+       { IOMAP_NOWAIT,         "NOWAIT" }, \
+       { IOMAP_UNCACHED,       "UNCACHED" }
 
 #define IOMAP_F_FLAGS_STRINGS \
        { IOMAP_F_NEW,          "NEW" }, \
@@ -101,6 +102,7 @@ DEFINE_PAGE_EVENT(iomap_invalidatepage);
        { IOMAP_F_SHARED,       "SHARED" }, \
        { IOMAP_F_MERGED,       "MERGED" }, \
        { IOMAP_F_BUFFER_HEAD,  "BH" }, \
+       { IOMAP_F_PAGE_CREATE,  "PAGE_CREATE" }, \
        { IOMAP_F_SIZE_CHANGED, "SIZE_CHANGED" }
 
 DECLARE_EVENT_CLASS(iomap_class,

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 5ea5fc167524b215ed4a954a3d45c3155892c111..36b96c98ddd488f0e675c7f4101cad3a45b7c7b9 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -3107,6 +3107,8 @@ extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *);
 extern ssize_t generic_perform_write(struct file *, struct iov_iter *,
                                     struct kiocb *);
 
+extern void uncached_write_pages(struct address_space *, struct list_head *);
+
 ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
                rwf_t flags);
 ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,

diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 30f40145a9e9b57fe0e458f9a926ab029944ea6a..259848d2dfd94d023eed5abbb957f6c8f36bf9aa 100644 (file)
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -48,12 +48,16 @@ struct vm_fault;
  *
  * IOMAP_F_BUFFER_HEAD indicates that the file system requires the use of
  * buffer heads for this mapping.
+ *
+ * IOMAP_F_PAGE_CREATE indicates that pages had to be allocated to satisfy
+ * this operation.
  */
 #define IOMAP_F_NEW            0x01
 #define IOMAP_F_DIRTY          0x02
 #define IOMAP_F_SHARED         0x04
 #define IOMAP_F_MERGED         0x08
 #define IOMAP_F_BUFFER_HEAD    0x10
+#define IOMAP_F_PAGE_CREATE    0x20
 
 /*
  * Flags set by the core iomap code during operations:
@@ -87,6 +91,7 @@ struct iomap {
        void                    *inline_data;
        void                    *private; /* filesystem private */
        const struct iomap_page_ops *page_ops;
+       struct list_head        page_list;
 };
 
 static inline sector_t
@@ -121,6 +126,7 @@ struct iomap_page_ops {
 #define IOMAP_FAULT            (1 << 3) /* mapping for page fault */
 #define IOMAP_DIRECT           (1 << 4) /* direct I/O */
 #define IOMAP_NOWAIT           (1 << 5) /* do not block */
+#define IOMAP_UNCACHED         (1 << 6) /* uncached IO */
 
 struct iomap_ops {
        /*

diff --git a/mm/filemap.c b/mm/filemap.c
index 42d9457afdf45fe68e039f8a516ccf1f401977f1..5d04ee3260023bbd04115875d8ff1059e6108d9a 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -3268,23 +3268,67 @@ EXPORT_SYMBOL(generic_file_direct_write);
 struct page *grab_cache_page_write_begin(struct address_space *mapping,
                                        pgoff_t index, unsigned flags)
 {
-       struct page *page;
+       gfp_t gfp = mapping_gfp_mask(mapping);
        int fgp_flags = FGP_LOCK|FGP_WRITE;
+       struct page *page;
 
        if (flags & AOP_FLAG_NOFS)
                fgp_flags |= FGP_NOFS;
        if (!(flags & AOP_FLAG_UNCACHED))
                fgp_flags |= FGP_CREAT;
-
-       page = pagecache_get_page(mapping, index, fgp_flags,
-                       mapping_gfp_mask(mapping));
-       if (page)
+       page = pagecache_get_page(mapping, index, fgp_flags, gfp);
+       if (!page && (flags & AOP_FLAG_UNCACHED)) {
+               if (flags & AOP_FLAG_NOFS)
+                       gfp &= ~__GFP_FS;
+               page = __page_cache_alloc(gfp);
+               if (page) {
+                       page->mapping = mapping;
+                       page->index = index;
+                       __SetPageLocked(page);
+                       __SetPagePrivio(page);
+                       get_page(page);
+               }
+       } else if (page)
                wait_for_stable_page(page);
 
        return page;
 }
 EXPORT_SYMBOL(grab_cache_page_write_begin);
 
+void uncached_write_pages(struct address_space *mapping,
+                         struct list_head *wb_list)
+{
+       struct writeback_control wbc = {
+               .sync_mode      = WB_SYNC_ALL,
+               .for_sync       = 1,
+       };
+       struct blk_plug plug;
+       struct page *page;
+
+       wbc_attach_fdatawrite_inode(&wbc, mapping->host);
+       blk_start_plug(&plug);
+
+       list_for_each_entry(page, wb_list, lru) {
+               lock_page(page);
+               wbc.nr_to_write = 1;
+               wbc.pages_skipped = 0;
+               test_clear_page_writeback(page);
+               mapping->a_ops->writepage(page, &wbc);
+               if (wbc.pages_skipped)
+                       printk("wp done: skipped %ld\n", wbc.pages_skipped);
+       }
+       while (!list_empty(wb_list)) {
+               page = list_first_entry(wb_list, struct page, lru);
+               list_del(&page->lru);
+               wait_on_page_writeback(page);
+               page->mapping = NULL;
+               put_page(page);
+       }
+
+       blk_finish_plug(&plug);
+       wbc_detach_inode(&wbc);
+}
+
 ssize_t generic_perform_write(struct file *file,
                                struct iov_iter *i, struct kiocb *iocb)
 {