#include <trace/events/writeback.h>
+static inline loff_t folio_end_pos(struct folio *folio)
+{
+ return folio_pos(folio) + folio_size(folio);
+}
+
+static inline size_t folio_sectors(struct folio *folio)
+{
+ return PAGE_SECTORS << folio_order(folio);
+}
+
+static inline loff_t folio_sector(struct folio *folio)
+{
+ return folio_pos(folio) >> 9;
+}
+
+static inline loff_t folio_end_sector(struct folio *folio)
+{
+ return folio_end_pos(folio) >> 9;
+}
+
struct nocow_flush {
struct closure *cl;
struct bch_dev *ca;
struct bch_folio_sector s[PAGE_SECTORS];
};
-static inline struct bch_folio *__bch2_folio(struct page *page)
+static inline struct bch_folio *__bch2_folio(struct folio *folio)
{
- return page_has_private(page)
- ? (struct bch_folio *) page_private(page)
+ return folio_has_private(folio)
+ ? (struct bch_folio *) folio_get_private(folio)
: NULL;
}
-static inline struct bch_folio *bch2_folio(struct page *page)
+static inline struct bch_folio *bch2_folio(struct folio *folio)
{
- EBUG_ON(!PageLocked(page));
+ EBUG_ON(!folio_test_locked(folio));
- return __bch2_folio(page);
+ return __bch2_folio(folio);
}
-/* for newly allocated pages: */
-static void __bch2_folio_release(struct page *page)
+/* for newly allocated folios: */
+static void __bch2_folio_release(struct folio *folio)
{
- kfree(detach_page_private(page));
+ kfree(folio_detach_private(folio));
}
-static void bch2_folio_release(struct page *page)
+static void bch2_folio_release(struct folio *folio)
{
- EBUG_ON(!PageLocked(page));
- __bch2_folio_release(page);
+ EBUG_ON(!folio_test_locked(folio));
+ __bch2_folio_release(folio);
}
-/* for newly allocated pages: */
-static struct bch_folio *__bch2_folio_create(struct page *page,
- gfp_t gfp)
+/* for newly allocated folios: */
+static struct bch_folio *__bch2_folio_create(struct folio *folio, gfp_t gfp)
{
struct bch_folio *s;
return NULL;
spin_lock_init(&s->lock);
- attach_page_private(page, s);
+ folio_attach_private(folio, s);
return s;
}
-static struct bch_folio *bch2_folio_create(struct page *page,
- gfp_t gfp)
+static struct bch_folio *bch2_folio_create(struct folio *folio, gfp_t gfp)
{
- return bch2_folio(page) ?: __bch2_folio_create(page, gfp);
+ return bch2_folio(folio) ?: __bch2_folio_create(folio, gfp);
}
static unsigned bkey_to_sector_state(struct bkey_s_c k)
return SECTOR_UNALLOCATED;
}
-static void __bch2_folio_set(struct page *page,
+static void __bch2_folio_set(struct folio *folio,
unsigned pg_offset, unsigned pg_len,
unsigned nr_ptrs, unsigned state)
{
- struct bch_folio *s = bch2_folio_create(page, __GFP_NOFAIL);
+ struct bch_folio *s = bch2_folio_create(folio, __GFP_NOFAIL);
unsigned i;
BUG_ON(pg_offset >= PAGE_SECTORS);
* extents btree:
*/
static int bch2_folio_set(struct bch_fs *c, subvol_inum inum,
- struct page **pages, unsigned nr_pages)
+ struct folio **folios, unsigned nr_folios)
{
struct btree_trans trans;
struct btree_iter iter;
struct bkey_s_c k;
- u64 offset = pages[0]->index << PAGE_SECTORS_SHIFT;
- unsigned pg_idx = 0;
+ u64 offset = folio_sector(folios[0]);
+ unsigned folio_idx = 0;
u32 snapshot;
int ret;
unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k);
unsigned state = bkey_to_sector_state(k);
- while (pg_idx < nr_pages) {
- struct page *page = pages[pg_idx];
- u64 pg_start = page->index << PAGE_SECTORS_SHIFT;
- u64 pg_end = (page->index + 1) << PAGE_SECTORS_SHIFT;
- unsigned pg_offset = max(bkey_start_offset(k.k), pg_start) - pg_start;
- unsigned pg_len = min(k.k->p.offset, pg_end) - pg_offset - pg_start;
+ while (folio_idx < nr_folios) {
+ struct folio *folio = folios[folio_idx];
+ u64 folio_start = folio_sector(folio);
+ u64 folio_end = folio_end_sector(folio);
+ unsigned folio_offset = max(bkey_start_offset(k.k), folio_start) - folio_start;
+ unsigned folio_len = min(k.k->p.offset, folio_end) - folio_offset - folio_start;
- BUG_ON(k.k->p.offset < pg_start);
- BUG_ON(bkey_start_offset(k.k) > pg_end);
+ BUG_ON(k.k->p.offset < folio_start);
+ BUG_ON(bkey_start_offset(k.k) > folio_end);
- if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate)
- __bch2_folio_set(page, pg_offset, pg_len, nr_ptrs, state);
+ if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate)
+ __bch2_folio_set(folio, folio_offset, folio_len, nr_ptrs, state);
- if (k.k->p.offset < pg_end)
+ if (k.k->p.offset < folio_end)
break;
- pg_idx++;
+ folio_idx++;
}
- if (pg_idx == nr_pages)
+ if (folio_idx == nr_folios)
break;
}
unsigned state = bkey_to_sector_state(k);
bio_for_each_segment(bv, bio, iter)
- __bch2_folio_set(bv.bv_page, bv.bv_offset >> 9,
- bv.bv_len >> 9, nr_ptrs, state);
+ __bch2_folio_set(page_folio(bv.bv_page), bv.bv_offset >> 9,
+ bv.bv_len >> 9, nr_ptrs, state);
}
static void mark_pagecache_unallocated(struct bch_inode_info *inode,
&index, end_index, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
- u64 pg_start = folio->index << PAGE_SECTORS_SHIFT;
- u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
- unsigned pg_offset = max(start, pg_start) - pg_start;
- unsigned pg_len = min(end, pg_end) - pg_offset - pg_start;
+ u64 folio_start = folio->index << PAGE_SECTORS_SHIFT;
+ u64 folio_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
+ unsigned folio_offset = max(start, folio_start) - folio_start;
+ unsigned folio_len = min(end, folio_end) - folio_offset - folio_start;
struct bch_folio *s;
- BUG_ON(end <= pg_start);
- BUG_ON(pg_offset >= PAGE_SECTORS);
- BUG_ON(pg_offset + pg_len > PAGE_SECTORS);
+ BUG_ON(end <= folio_start);
+ BUG_ON(folio_offset >= PAGE_SECTORS);
+ BUG_ON(folio_offset + folio_len > PAGE_SECTORS);
folio_lock(folio);
- s = bch2_folio(&folio->page);
+ s = bch2_folio(folio);
if (s) {
spin_lock(&s->lock);
- for (j = pg_offset; j < pg_offset + pg_len; j++)
+ for (j = folio_offset; j < folio_offset + folio_len; j++)
s->s[j].nr_replicas = 0;
spin_unlock(&s->lock);
}
&index, end_index, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct folio *folio = fbatch.folios[i];
- u64 pg_start = folio->index << PAGE_SECTORS_SHIFT;
- u64 pg_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
- unsigned pg_offset = max(start, pg_start) - pg_start;
- unsigned pg_len = min(end, pg_end) - pg_offset - pg_start;
+ u64 folio_start = folio->index << PAGE_SECTORS_SHIFT;
+ u64 folio_end = (folio->index + 1) << PAGE_SECTORS_SHIFT;
+ unsigned folio_offset = max(start, folio_start) - folio_start;
+ unsigned folio_len = min(end, folio_end) - folio_offset - folio_start;
struct bch_folio *s;
- BUG_ON(end <= pg_start);
- BUG_ON(pg_offset >= PAGE_SECTORS);
- BUG_ON(pg_offset + pg_len > PAGE_SECTORS);
+ BUG_ON(end <= folio_start);
+ BUG_ON(folio_offset >= PAGE_SECTORS);
+ BUG_ON(folio_offset + folio_len > PAGE_SECTORS);
folio_lock(folio);
- s = bch2_folio(&folio->page);
+ s = bch2_folio(folio);
if (s) {
spin_lock(&s->lock);
- for (j = pg_offset; j < pg_offset + pg_len; j++)
+ for (j = folio_offset; j < folio_offset + folio_len; j++)
switch (s->s[j].state) {
case SECTOR_UNALLOCATED:
s->s[j].state = SECTOR_RESERVED;
s->replicas_reserved);
}
-static int bch2_get_page_disk_reservation(struct bch_fs *c,
+static int bch2_get_folio_disk_reservation(struct bch_fs *c,
struct bch_inode_info *inode,
- struct page *page, bool check_enospc)
+ struct folio *folio, bool check_enospc)
{
- struct bch_folio *s = bch2_folio_create(page, 0);
+ struct bch_folio *s = bch2_folio_create(folio, 0);
unsigned nr_replicas = inode_nr_replicas(c, inode);
struct disk_reservation disk_res = { 0 };
unsigned i, disk_res_sectors = 0;
return 0;
}
-struct bch2_page_reservation {
+struct bch2_folio_reservation {
struct disk_reservation disk;
struct quota_res quota;
};
-static void bch2_page_reservation_init(struct bch_fs *c,
+static void bch2_folio_reservation_init(struct bch_fs *c,
struct bch_inode_info *inode,
- struct bch2_page_reservation *res)
+ struct bch2_folio_reservation *res)
{
memset(res, 0, sizeof(*res));
res->disk.nr_replicas = inode_nr_replicas(c, inode);
}
-static void bch2_page_reservation_put(struct bch_fs *c,
+static void bch2_folio_reservation_put(struct bch_fs *c,
struct bch_inode_info *inode,
- struct bch2_page_reservation *res)
+ struct bch2_folio_reservation *res)
{
bch2_disk_reservation_put(c, &res->disk);
bch2_quota_reservation_put(c, inode, &res->quota);
}
-static int bch2_page_reservation_get(struct bch_fs *c,
- struct bch_inode_info *inode, struct page *page,
- struct bch2_page_reservation *res,
+static int bch2_folio_reservation_get(struct bch_fs *c,
+ struct bch_inode_info *inode,
+ struct folio *folio,
+ struct bch2_folio_reservation *res,
unsigned offset, unsigned len)
{
- struct bch_folio *s = bch2_folio_create(page, 0);
+ struct bch_folio *s = bch2_folio_create(folio, 0);
unsigned i, disk_sectors = 0, quota_sectors = 0;
int ret;
return 0;
}
-static void bch2_clear_page_bits(struct page *page)
+static void bch2_clear_folio_bits(struct folio *folio)
{
- struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
+ struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
- struct bch_folio *s = bch2_folio(page);
+ struct bch_folio *s = bch2_folio(folio);
struct disk_reservation disk_res = { 0 };
int i, dirty_sectors = 0;
if (!s)
return;
- EBUG_ON(!PageLocked(page));
- EBUG_ON(PageWriteback(page));
+ EBUG_ON(!folio_test_locked(folio));
+ EBUG_ON(folio_test_writeback(folio));
for (i = 0; i < ARRAY_SIZE(s->s); i++) {
disk_res.sectors += s->s[i].replicas_reserved;
i_sectors_acct(c, inode, NULL, dirty_sectors);
- bch2_folio_release(page);
+ bch2_folio_release(folio);
}
-static void bch2_set_page_dirty(struct bch_fs *c,
- struct bch_inode_info *inode, struct page *page,
- struct bch2_page_reservation *res,
+static void bch2_set_folio_dirty(struct bch_fs *c,
+ struct bch_inode_info *inode,
+ struct folio *folio,
+ struct bch2_folio_reservation *res,
unsigned offset, unsigned len)
{
- struct bch_folio *s = bch2_folio(page);
+ struct bch_folio *s = bch2_folio(folio);
unsigned i, dirty_sectors = 0;
- WARN_ON((u64) page_offset(page) + offset + len >
+ WARN_ON((u64) folio_pos(folio) + offset + len >
round_up((u64) i_size_read(&inode->v), block_bytes(c)));
spin_lock(&s->lock);
i_sectors_acct(c, inode, &res->quota, dirty_sectors);
- if (!PageDirty(page))
- filemap_dirty_folio(inode->v.i_mapping, page_folio(page));
+ if (!folio_test_dirty(folio))
+ filemap_dirty_folio(inode->v.i_mapping, folio);
}
vm_fault_t bch2_page_fault(struct vm_fault *vmf)
vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
{
- struct page *page = vmf->page;
+ struct folio *folio = page_folio(vmf->page);
struct file *file = vmf->vma->vm_file;
struct bch_inode_info *inode = file_bch_inode(file);
struct address_space *mapping = file->f_mapping;
struct bch_fs *c = inode->v.i_sb->s_fs_info;
- struct bch2_page_reservation res;
+ struct bch2_folio_reservation res;
unsigned len;
loff_t isize;
int ret;
- bch2_page_reservation_init(c, inode, &res);
+ bch2_folio_reservation_init(c, inode, &res);
sb_start_pagefault(inode->v.i_sb);
file_update_time(file);
*/
bch2_pagecache_add_get(inode);
- lock_page(page);
+ folio_lock(folio);
isize = i_size_read(&inode->v);
- if (page->mapping != mapping || page_offset(page) >= isize) {
- unlock_page(page);
+ if (folio->mapping != mapping || folio_pos(folio) >= isize) {
+ folio_unlock(folio);
ret = VM_FAULT_NOPAGE;
goto out;
}
- len = min_t(loff_t, PAGE_SIZE, isize - page_offset(page));
+ len = min_t(loff_t, PAGE_SIZE, isize - folio_pos(folio));
- if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) {
- if (bch2_folio_set(c, inode_inum(inode), &page, 1)) {
- unlock_page(page);
+ if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate) {
+ if (bch2_folio_set(c, inode_inum(inode), &folio, 1)) {
+ folio_unlock(folio);
ret = VM_FAULT_SIGBUS;
goto out;
}
}
- if (bch2_page_reservation_get(c, inode, page, &res, 0, len)) {
- unlock_page(page);
+ if (bch2_folio_reservation_get(c, inode, folio, &res, 0, len)) {
+ folio_unlock(folio);
ret = VM_FAULT_SIGBUS;
goto out;
}
- bch2_set_page_dirty(c, inode, page, &res, 0, len);
- bch2_page_reservation_put(c, inode, &res);
+ bch2_set_folio_dirty(c, inode, folio, &res, 0, len);
+ bch2_folio_reservation_put(c, inode, &res);
- wait_for_stable_page(page);
+ folio_wait_stable(folio);
ret = VM_FAULT_LOCKED;
out:
bch2_pagecache_add_put(inode);
if (offset || length < folio_size(folio))
return;
- bch2_clear_page_bits(&folio->page);
+ bch2_clear_folio_bits(folio);
}
bool bch2_release_folio(struct folio *folio, gfp_t gfp_mask)
if (folio_test_dirty(folio) || folio_test_writeback(folio))
return false;
- bch2_clear_page_bits(&folio->page);
+ bch2_clear_folio_bits(folio);
return true;
}
static void bch2_readpages_end_io(struct bio *bio)
{
- struct bvec_iter_all iter;
- struct bio_vec *bv;
-
- bio_for_each_segment_all(bv, bio, iter) {
- struct page *page = bv->bv_page;
+ struct folio_iter fi;
+ bio_for_each_folio_all(fi, bio) {
if (!bio->bi_status) {
- SetPageUptodate(page);
+ folio_mark_uptodate(fi.folio);
} else {
- ClearPageUptodate(page);
- SetPageError(page);
+ folio_clear_uptodate(fi.folio);
+ folio_set_error(fi.folio);
}
- unlock_page(page);
+ folio_unlock(fi.folio);
}
bio_put(bio);
nr_pages = __readahead_batch(ractl, iter->pages, nr_pages);
for (i = 0; i < nr_pages; i++) {
- __bch2_folio_create(iter->pages[i], __GFP_NOFAIL);
+ __bch2_folio_create(page_folio(iter->pages[i]), __GFP_NOFAIL);
put_page(iter->pages[i]);
}
return 0;
}
-static inline struct page *readpage_iter_next(struct readpages_iter *iter)
+static inline struct folio *readpage_iter_next(struct readpages_iter *iter)
{
if (iter->idx >= iter->nr_pages)
return NULL;
EBUG_ON(iter->pages[iter->idx]->index != iter->offset + iter->idx);
- return iter->pages[iter->idx];
+ return page_folio(iter->pages[iter->idx]);
}
static bool extent_partial_reads_expensive(struct bkey_s_c k)
{
while (bio_sectors(bio) < sectors_this_extent &&
bio->bi_vcnt < bio->bi_max_vecs) {
- pgoff_t page_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
- struct page *page = readpage_iter_next(iter);
+ pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT;
+ struct folio *folio = readpage_iter_next(iter);
int ret;
- if (page) {
- if (iter->offset + iter->idx != page_offset)
+ if (folio) {
+ if (iter->offset + iter->idx != folio_offset)
break;
iter->idx++;
if (!get_more)
break;
- page = xa_load(&iter->mapping->i_pages, page_offset);
- if (page && !xa_is_value(page))
+ folio = xa_load(&iter->mapping->i_pages, folio_offset);
+ if (folio && !xa_is_value(folio))
break;
- page = __page_cache_alloc(readahead_gfp_mask(iter->mapping));
- if (!page)
+ folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0);
+ if (!folio)
break;
- if (!__bch2_folio_create(page, 0)) {
- put_page(page);
+ if (!__bch2_folio_create(folio, 0)) {
+ folio_put(folio);
break;
}
- ret = add_to_page_cache_lru(page, iter->mapping,
- page_offset, GFP_NOFS);
+ ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_NOFS);
if (ret) {
- __bch2_folio_release(page);
- put_page(page);
+ __bch2_folio_release(folio);
+ folio_put(folio);
break;
}
- put_page(page);
+ folio_put(folio);
}
- BUG_ON(!bio_add_page(bio, page, PAGE_SIZE, 0));
+ BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0));
}
}
struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch_io_opts opts;
struct btree_trans trans;
- struct page *page;
+ struct folio *folio;
struct readpages_iter readpages_iter;
int ret;
bch2_pagecache_add_get(inode);
- while ((page = readpage_iter_next(&readpages_iter))) {
+ while ((folio = readpage_iter_next(&readpages_iter))) {
pgoff_t index = readpages_iter.offset + readpages_iter.idx;
unsigned n = min_t(unsigned,
readpages_iter.nr_pages -
rbio->bio.bi_iter.bi_sector = (sector_t) index << PAGE_SECTORS_SHIFT;
rbio->bio.bi_end_io = bch2_readpages_end_io;
- BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0));
+ BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
bchfs_read(&trans, rbio, inode_inum(inode),
&readpages_iter);
kfree(readpages_iter.pages);
}
-static void __bchfs_readpage(struct bch_fs *c, struct bch_read_bio *rbio,
- subvol_inum inum, struct page *page)
+static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio,
+ subvol_inum inum, struct folio *folio)
{
struct btree_trans trans;
- bch2_folio_create(page, __GFP_NOFAIL);
+ bch2_folio_create(folio, __GFP_NOFAIL);
rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC;
- rbio->bio.bi_iter.bi_sector =
- (sector_t) page->index << PAGE_SECTORS_SHIFT;
- BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0));
+ rbio->bio.bi_iter.bi_sector = folio_sector(folio);
+ BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0));
bch2_trans_init(&trans, c, 0, 0);
bchfs_read(&trans, rbio, inum, NULL);
bch2_trans_exit(&trans);
}
-static void bch2_read_single_page_end_io(struct bio *bio)
+static void bch2_read_single_folio_end_io(struct bio *bio)
{
complete(bio->bi_private);
}
-static int bch2_read_single_page(struct page *page,
- struct address_space *mapping)
+static int bch2_read_single_folio(struct folio *folio,
+ struct address_space *mapping)
{
struct bch_inode_info *inode = to_bch_ei(mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_NOFS, &c->bio_read),
opts);
rbio->bio.bi_private = &done;
- rbio->bio.bi_end_io = bch2_read_single_page_end_io;
+ rbio->bio.bi_end_io = bch2_read_single_folio_end_io;
- __bchfs_readpage(c, rbio, inode_inum(inode), page);
+ __bchfs_readfolio(c, rbio, inode_inum(inode), folio);
wait_for_completion(&done);
ret = blk_status_to_errno(rbio->bio.bi_status);
if (ret < 0)
return ret;
- SetPageUptodate(page);
+ folio_mark_uptodate(folio);
return 0;
}
int bch2_read_folio(struct file *file, struct folio *folio)
{
- struct page *page = &folio->page;
int ret;
- ret = bch2_read_single_page(page, page->mapping);
+ ret = bch2_read_single_folio(folio, folio->mapping);
folio_unlock(folio);
return bch2_err_class(ret);
}
SetPageError(bvec->bv_page);
mapping_set_error(bvec->bv_page->mapping, -EIO);
- s = __bch2_folio(bvec->bv_page);
+ s = __bch2_folio(page_folio(bvec->bv_page));
spin_lock(&s->lock);
for (i = 0; i < PAGE_SECTORS; i++)
s->s[i].nr_replicas = 0;
bio_for_each_segment_all(bvec, bio, iter) {
struct bch_folio *s;
- s = __bch2_folio(bvec->bv_page);
+ s = __bch2_folio(page_folio(bvec->bv_page));
spin_lock(&s->lock);
for (i = 0; i < PAGE_SECTORS; i++)
s->s[i].nr_replicas = 0;
i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta);
bio_for_each_segment_all(bvec, bio, iter) {
- struct bch_folio *s = __bch2_folio(bvec->bv_page);
+ struct folio *folio = page_folio(bvec->bv_page);
+ struct bch_folio *s = __bch2_folio(folio);
if (atomic_dec_and_test(&s->write_count))
- end_page_writeback(bvec->bv_page);
+ folio_end_writeback(folio);
}
bio_put(&io->op.wbio.bio);
struct writeback_control *wbc,
void *data)
{
- struct page *page = &folio->page;
- struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
+ struct bch_inode_info *inode = to_bch_ei(folio->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch_writepage_state *w = data;
struct bch_folio *s, orig;
- unsigned i, offset, nr_replicas_this_write = U32_MAX;
+ unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX;
loff_t i_size = i_size_read(&inode->v);
pgoff_t end_index = i_size >> PAGE_SHIFT;
int ret;
- EBUG_ON(!PageUptodate(page));
+ EBUG_ON(!folio_test_uptodate(folio));
- /* Is the page fully inside i_size? */
- if (page->index < end_index)
+ /* Is the folio fully inside i_size? */
+ if (folio->index < end_index)
goto do_io;
- /* Is the page fully outside i_size? (truncate in progress) */
+ /* Is the folio fully outside i_size? (truncate in progress) */
offset = i_size & (PAGE_SIZE - 1);
- if (page->index > end_index || !offset) {
- unlock_page(page);
+ if (folio->index > end_index || !offset) {
+ folio_unlock(folio);
return 0;
}
/*
- * The page straddles i_size. It must be zeroed out on each and every
+ * The folio straddles i_size. It must be zeroed out on each and every
* writepage invocation because it may be mmapped. "A file is mapped
- * in multiples of the page size. For a file that is not a multiple of
- * the page size, the remaining memory is zeroed when mapped, and
+ * in multiples of the folio size. For a file that is not a multiple of
+ * the folio size, the remaining memory is zeroed when mapped, and
* writes to that region are not written out to the file."
*/
- zero_user_segment(page, offset, PAGE_SIZE);
+ folio_zero_segment(folio, offset, folio_size(folio));
do_io:
- s = bch2_folio_create(page, __GFP_NOFAIL);
+ f_sectors = folio_sectors(folio);
+ s = bch2_folio_create(folio, __GFP_NOFAIL);
/*
* Things get really hairy with errors during writeback:
*/
- ret = bch2_get_page_disk_reservation(c, inode, page, false);
+ ret = bch2_get_folio_disk_reservation(c, inode, folio, false);
BUG_ON(ret);
/* Before unlocking the page, get copy of reservations: */
orig = *s;
spin_unlock(&s->lock);
- for (i = 0; i < PAGE_SECTORS; i++) {
+ for (i = 0; i < f_sectors; i++) {
if (s->s[i].state < SECTOR_DIRTY)
continue;
s->s[i].replicas_reserved);
}
- for (i = 0; i < PAGE_SECTORS; i++) {
+ for (i = 0; i < f_sectors; i++) {
if (s->s[i].state < SECTOR_DIRTY)
continue;
BUG_ON(atomic_read(&s->write_count));
atomic_set(&s->write_count, 1);
- BUG_ON(PageWriteback(page));
- set_page_writeback(page);
+ BUG_ON(folio_test_writeback(folio));
+ folio_start_writeback(folio);
- unlock_page(page);
+ folio_unlock(folio);
offset = 0;
while (1) {
unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0;
u64 sector;
- while (offset < PAGE_SECTORS &&
+ while (offset < f_sectors &&
orig.s[offset].state < SECTOR_DIRTY)
offset++;
- if (offset == PAGE_SECTORS)
+ if (offset == f_sectors)
break;
- while (offset + sectors < PAGE_SECTORS &&
+ while (offset + sectors < f_sectors &&
orig.s[offset + sectors].state >= SECTOR_DIRTY) {
reserved_sectors += orig.s[offset + sectors].replicas_reserved;
dirty_sectors += orig.s[offset + sectors].state == SECTOR_DIRTY;
}
BUG_ON(!sectors);
- sector = ((u64) page->index << PAGE_SECTORS_SHIFT) + offset;
+ sector = folio_sector(folio) + offset;
if (w->io &&
(w->io->op.res.nr_replicas != nr_replicas_this_write ||
atomic_inc(&s->write_count);
BUG_ON(inode != w->io->inode);
- BUG_ON(!bio_add_page(&w->io->op.wbio.bio, page,
+ BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio,
sectors << 9, offset << 9));
/* Check for writing past i_size: */
}
if (atomic_dec_and_test(&s->write_count))
- end_page_writeback(page);
+ folio_end_writeback(folio);
return 0;
}
{
struct bch_inode_info *inode = to_bch_ei(mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
- struct bch2_page_reservation *res;
+ struct bch2_folio_reservation *res;
pgoff_t index = pos >> PAGE_SHIFT;
unsigned offset = pos & (PAGE_SIZE - 1);
- struct page *page;
+ struct folio *folio;
int ret = -ENOMEM;
res = kmalloc(sizeof(*res), GFP_KERNEL);
if (!res)
return -ENOMEM;
- bch2_page_reservation_init(c, inode, res);
+ bch2_folio_reservation_init(c, inode, res);
*fsdata = res;
bch2_pagecache_add_get(inode);
- page = grab_cache_page_write_begin(mapping, index);
- if (!page)
+ folio = __filemap_get_folio(mapping, index,
+ FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
+ mapping_gfp_mask(mapping));
+ if (!folio)
goto err_unlock;
- if (PageUptodate(page))
+ if (folio_test_uptodate(folio))
goto out;
- /* If we're writing entire page, don't need to read it in first: */
- if (len == PAGE_SIZE)
+ /* If we're writing entire folio, don't need to read it in first: */
+ if (len == folio_size(folio))
goto out;
if (!offset && pos + len >= inode->v.i_size) {
- zero_user_segment(page, len, PAGE_SIZE);
- flush_dcache_page(page);
+ folio_zero_segment(folio, len, folio_size(folio));
+ flush_dcache_folio(folio);
goto out;
}
if (index > inode->v.i_size >> PAGE_SHIFT) {
- zero_user_segments(page, 0, offset, offset + len, PAGE_SIZE);
- flush_dcache_page(page);
+ folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio));
+ flush_dcache_folio(folio);
goto out;
}
readpage:
- ret = bch2_read_single_page(page, mapping);
+ ret = bch2_read_single_folio(folio, mapping);
if (ret)
goto err;
out:
- if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) {
- ret = bch2_folio_set(c, inode_inum(inode), &page, 1);
+ if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate) {
+ ret = bch2_folio_set(c, inode_inum(inode), &folio, 1);
if (ret)
goto err;
}
- ret = bch2_page_reservation_get(c, inode, page, res, offset, len);
+ ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len);
if (ret) {
- if (!PageUptodate(page)) {
+ if (!folio_test_uptodate(folio)) {
/*
- * If the page hasn't been read in, we won't know if we
+ * If the folio hasn't been read in, we won't know if we
* actually need a reservation - we don't actually need
- * to read here, we just need to check if the page is
+ * to read here, we just need to check if the folio is
* fully backed by uncompressed data:
*/
goto readpage;
goto err;
}
- *pagep = page;
+ *pagep = &folio->page;
return 0;
err:
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
*pagep = NULL;
err_unlock:
bch2_pagecache_add_put(inode);
{
struct bch_inode_info *inode = to_bch_ei(mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
- struct bch2_page_reservation *res = fsdata;
+ struct bch2_folio_reservation *res = fsdata;
+ struct folio *folio = page_folio(page);
unsigned offset = pos & (PAGE_SIZE - 1);
lockdep_assert_held(&inode->v.i_rwsem);
- if (unlikely(copied < len && !PageUptodate(page))) {
+ if (unlikely(copied < len && !folio_test_uptodate(folio))) {
/*
- * The page needs to be read in, but that would destroy
+ * The folio needs to be read in, but that would destroy
* our partial write - simplest thing is to just force
* userspace to redo the write:
*/
- zero_user(page, 0, PAGE_SIZE);
- flush_dcache_page(page);
+ folio_zero_range(folio, 0, folio_size(folio));
+ flush_dcache_folio(folio);
copied = 0;
}
spin_unlock(&inode->v.i_lock);
if (copied) {
- if (!PageUptodate(page))
- SetPageUptodate(page);
+ if (!folio_test_uptodate(folio))
+ folio_mark_uptodate(folio);
- bch2_set_page_dirty(c, inode, page, res, offset, copied);
+ bch2_set_folio_dirty(c, inode, folio, res, offset, copied);
inode->ei_last_dirtied = (unsigned long) current;
}
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
bch2_pagecache_add_put(inode);
- bch2_page_reservation_put(c, inode, res);
+ bch2_folio_reservation_put(c, inode, res);
kfree(res);
return copied;
loff_t pos, unsigned len)
{
struct bch_fs *c = inode->v.i_sb->s_fs_info;
- struct page *pages[WRITE_BATCH_PAGES];
- struct bch2_page_reservation res;
+ struct folio *folios[WRITE_BATCH_PAGES];
+ struct bch2_folio_reservation res;
unsigned long index = pos >> PAGE_SHIFT;
unsigned offset = pos & (PAGE_SIZE - 1);
- unsigned nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
+ unsigned nr_folios = DIV_ROUND_UP(offset + len, PAGE_SIZE);
unsigned i, reserved = 0, set_dirty = 0;
- unsigned copied = 0, nr_pages_copied = 0;
+ unsigned copied = 0, nr_folios_copied = 0;
int ret = 0;
BUG_ON(!len);
- BUG_ON(nr_pages > ARRAY_SIZE(pages));
+ BUG_ON(nr_folios > ARRAY_SIZE(folios));
- bch2_page_reservation_init(c, inode, &res);
+ bch2_folio_reservation_init(c, inode, &res);
- for (i = 0; i < nr_pages; i++) {
- pages[i] = grab_cache_page_write_begin(mapping, index + i);
- if (!pages[i]) {
- nr_pages = i;
+ for (i = 0; i < nr_folios; i++) {
+ folios[i] = __filemap_get_folio(mapping, index + i,
+ FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE,
+ mapping_gfp_mask(mapping));
+ if (!folios[i]) {
+ nr_folios = i;
if (!i) {
ret = -ENOMEM;
goto out;
}
len = min_t(unsigned, len,
- nr_pages * PAGE_SIZE - offset);
+ nr_folios * PAGE_SIZE - offset);
break;
}
}
- if (offset && !PageUptodate(pages[0])) {
- ret = bch2_read_single_page(pages[0], mapping);
+ if (offset && !folio_test_uptodate(folios[0])) {
+ ret = bch2_read_single_folio(folios[0], mapping);
if (ret)
goto out;
}
if ((pos + len) & (PAGE_SIZE - 1) &&
- !PageUptodate(pages[nr_pages - 1])) {
- if ((index + nr_pages - 1) << PAGE_SHIFT >= inode->v.i_size) {
- zero_user(pages[nr_pages - 1], 0, PAGE_SIZE);
+ !folio_test_uptodate(folios[nr_folios - 1])) {
+ if ((index + nr_folios - 1) << PAGE_SHIFT >= inode->v.i_size) {
+ folio_zero_range(folios[nr_folios - 1], 0,
+ folio_size(folios[nr_folios - 1]));
} else {
- ret = bch2_read_single_page(pages[nr_pages - 1], mapping);
+ ret = bch2_read_single_folio(folios[nr_folios - 1], mapping);
if (ret)
goto out;
}
while (reserved < len) {
unsigned i = (offset + reserved) >> PAGE_SHIFT;
- struct page *page = pages[i];
- unsigned pg_offset = (offset + reserved) & (PAGE_SIZE - 1);
- unsigned pg_len = min_t(unsigned, len - reserved,
- PAGE_SIZE - pg_offset);
+ struct folio *folio = folios[i];
+ unsigned folio_offset = (offset + reserved) & (PAGE_SIZE - 1);
+ unsigned folio_len = min_t(unsigned, len - reserved,
+ PAGE_SIZE - folio_offset);
- if (!bch2_folio_create(page, __GFP_NOFAIL)->uptodate) {
+ if (!bch2_folio_create(folio, __GFP_NOFAIL)->uptodate) {
ret = bch2_folio_set(c, inode_inum(inode),
- pages + i, nr_pages - i);
+ folios + i, nr_folios - i);
if (ret)
goto out;
}
* we aren't completely out of disk space - we don't do that
* yet:
*/
- ret = bch2_page_reservation_get(c, inode, page, &res,
- pg_offset, pg_len);
+ ret = bch2_folio_reservation_get(c, inode, folio, &res,
+ folio_offset, folio_len);
if (unlikely(ret)) {
if (!reserved)
goto out;
break;
}
- reserved += pg_len;
+ reserved += folio_len;
}
if (mapping_writably_mapped(mapping))
- for (i = 0; i < nr_pages; i++)
- flush_dcache_page(pages[i]);
+ for (i = 0; i < nr_folios; i++)
+ flush_dcache_folio(folios[i]);
while (copied < reserved) {
- struct page *page = pages[(offset + copied) >> PAGE_SHIFT];
- unsigned pg_offset = (offset + copied) & (PAGE_SIZE - 1);
- unsigned pg_len = min_t(unsigned, reserved - copied,
- PAGE_SIZE - pg_offset);
- unsigned pg_copied = copy_page_from_iter_atomic(page,
- pg_offset, pg_len, iter);
-
- if (!pg_copied)
+ struct folio *folio = folios[(offset + copied) >> PAGE_SHIFT];
+ unsigned folio_offset = (offset + copied) & (PAGE_SIZE - 1);
+ unsigned folio_len = min_t(unsigned, reserved - copied,
+ PAGE_SIZE - folio_offset);
+ unsigned folio_copied = copy_page_from_iter_atomic(&folio->page,
+ folio_offset, folio_len, iter);
+
+ if (!folio_copied)
break;
- if (!PageUptodate(page) &&
- pg_copied != PAGE_SIZE &&
- pos + copied + pg_copied < inode->v.i_size) {
- zero_user(page, 0, PAGE_SIZE);
+ if (!folio_test_uptodate(folio) &&
+ folio_copied != PAGE_SIZE &&
+ pos + copied + folio_copied < inode->v.i_size) {
+ folio_zero_range(folio, 0, folio_size(folio));
break;
}
- flush_dcache_page(page);
- copied += pg_copied;
+ flush_dcache_folio(folio);
+ copied += folio_copied;
- if (pg_copied != pg_len)
+ if (folio_copied != folio_len)
break;
}
spin_unlock(&inode->v.i_lock);
while (set_dirty < copied) {
- struct page *page = pages[(offset + set_dirty) >> PAGE_SHIFT];
- unsigned pg_offset = (offset + set_dirty) & (PAGE_SIZE - 1);
- unsigned pg_len = min_t(unsigned, copied - set_dirty,
- PAGE_SIZE - pg_offset);
+ struct folio *folio = folios[(offset + set_dirty) >> PAGE_SHIFT];
+ unsigned folio_offset = (offset + set_dirty) & (PAGE_SIZE - 1);
+ unsigned folio_len = min_t(unsigned, copied - set_dirty,
+ PAGE_SIZE - folio_offset);
- if (!PageUptodate(page))
- SetPageUptodate(page);
+ if (!folio_test_uptodate(folio))
+ folio_mark_uptodate(folio);
- bch2_set_page_dirty(c, inode, page, &res, pg_offset, pg_len);
- unlock_page(page);
- put_page(page);
+ bch2_set_folio_dirty(c, inode, folio, &res, folio_offset, folio_len);
+ folio_unlock(folio);
+ folio_put(folio);
- set_dirty += pg_len;
+ set_dirty += folio_len;
}
- nr_pages_copied = DIV_ROUND_UP(offset + copied, PAGE_SIZE);
+ nr_folios_copied = DIV_ROUND_UP(offset + copied, PAGE_SIZE);
inode->ei_last_dirtied = (unsigned long) current;
out:
- for (i = nr_pages_copied; i < nr_pages; i++) {
- unlock_page(pages[i]);
- put_page(pages[i]);
+ for (i = nr_folios_copied; i < nr_folios; i++) {
+ folio_unlock(folios[i]);
+ folio_put(folios[i]);
}
- bch2_page_reservation_put(c, inode, &res);
+ bch2_folio_reservation_put(c, inode, &res);
return copied ?: ret;
}
unsigned start_offset = start & (PAGE_SIZE - 1);
unsigned end_offset = ((end - 1) & (PAGE_SIZE - 1)) + 1;
unsigned i;
- struct page *page;
+ struct folio *folio;
s64 i_sectors_delta = 0;
int ret = 0;
if (index << PAGE_SHIFT >= inode->v.i_size)
return 0;
- page = find_lock_page(mapping, index);
- if (!page) {
+ folio = filemap_lock_folio(mapping, index);
+ if (!folio) {
/*
* XXX: we're doing two index lookups when we end up reading the
- * page
+ * folio
*/
ret = range_has_data(c, inode->ei_subvol,
POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)),
if (ret <= 0)
return ret;
- page = find_or_create_page(mapping, index, GFP_KERNEL);
- if (unlikely(!page)) {
+ folio = __filemap_get_folio(mapping, index,
+ FGP_LOCK|FGP_CREAT, GFP_KERNEL);
+ if (unlikely(!folio)) {
ret = -ENOMEM;
goto out;
}
}
- s = bch2_folio_create(page, 0);
+ s = bch2_folio_create(folio, 0);
if (!s) {
ret = -ENOMEM;
goto unlock;
}
- if (!PageUptodate(page)) {
- ret = bch2_read_single_page(page, mapping);
+ if (!folio_test_uptodate(folio)) {
+ ret = bch2_read_single_folio(folio, mapping);
if (ret)
goto unlock;
}
i_sectors_acct(c, inode, NULL, i_sectors_delta);
/*
- * Caller needs to know whether this page will be written out by
+ * Caller needs to know whether this folio will be written out by
* writeback - doing an i_size update if necessary - or whether it will
* be responsible for the i_size update:
*/
ret = s->s[(min_t(u64, inode->v.i_size - (index << PAGE_SHIFT),
PAGE_SIZE) - 1) >> 9].state >= SECTOR_DIRTY;
- zero_user_segment(page, start_offset, end_offset);
+ folio_zero_segment(folio, start_offset, end_offset);
/*
* Bit of a hack - we don't want truncate to fail due to -ENOSPC.
*
- * XXX: because we aren't currently tracking whether the page has actual
+ * XXX: because we aren't currently tracking whether the folio has actual
* data in it (vs. just 0s, or only partially written) this wrong. ick.
*/
- BUG_ON(bch2_get_page_disk_reservation(c, inode, page, false));
+ BUG_ON(bch2_get_folio_disk_reservation(c, inode, folio, false));
/*
* This removes any writeable userspace mappings; we need to force
* .page_mkwrite to be called again before any mmapped writes, to
* redirty the full page:
*/
- page_mkclean(page);
- filemap_dirty_folio(mapping, page_folio(page));
+ folio_mkclean(folio);
+ filemap_dirty_folio(mapping, folio);
unlock:
- unlock_page(page);
- put_page(page);
+ folio_unlock(folio);
+ folio_put(folio);
out:
return ret;
}
static int folio_data_offset(struct folio *folio, unsigned offset)
{
- struct bch_folio *s = bch2_folio(&folio->page);
+ struct bch_folio *s = bch2_folio(folio);
unsigned i;
if (s)
return vfs_setpos(file, next_data, MAX_LFS_FILESIZE);
}
-static int __page_hole_offset(struct page *page, unsigned offset)
+static int __folio_hole_offset(struct folio *folio, unsigned offset)
{
- struct bch_folio *s = bch2_folio(page);
+ struct bch_folio *s = bch2_folio(folio);
unsigned i;
if (!s)
return -1;
}
-static loff_t page_hole_offset(struct address_space *mapping, loff_t offset)
+static loff_t folio_hole_offset(struct address_space *mapping, loff_t offset)
{
pgoff_t index = offset >> PAGE_SHIFT;
- struct page *page;
- int pg_offset;
+ struct folio *folio;
+ int folio_offset;
loff_t ret = -1;
- page = find_lock_page(mapping, index);
- if (!page)
+ folio = filemap_lock_folio(mapping, index);
+ if (!folio)
return offset;
- pg_offset = __page_hole_offset(page, offset & (PAGE_SIZE - 1));
- if (pg_offset >= 0)
- ret = ((loff_t) index << PAGE_SHIFT) + pg_offset;
-
- unlock_page(page);
+ folio_offset = __folio_hole_offset(folio, offset & (folio_size(folio) - 1));
+ if (folio_offset >= 0)
+ ret = folio_pos(folio) + folio_offset;
+ folio_unlock(folio);
return ret;
}
loff_t offset = start_offset, hole;
while (offset < end_offset) {
- hole = page_hole_offset(mapping, offset);
+ hole = folio_hole_offset(mapping, offset);
if (hole >= 0 && hole <= end_offset)
return max(start_offset, hole);
projects / linux-block.git / commitdiff