[linux-block.git] / fs / erofs / fscache.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2022, Alibaba Cloud
 */
#include <linux/fscache.h>
#include "internal.h"

static struct netfs_io_request *erofs_fscache_alloc_request(struct address_space *mapping,
					     loff_t start, size_t len)
{
	struct netfs_io_request *rreq;

	rreq = kzalloc(sizeof(struct netfs_io_request), GFP_KERNEL);
	if (!rreq)
		return ERR_PTR(-ENOMEM);

	rreq->start	= start;
	rreq->len	= len;
	rreq->mapping	= mapping;
	rreq->inode	= mapping->host;
	INIT_LIST_HEAD(&rreq->subrequests);
	refcount_set(&rreq->ref, 1);
	return rreq;
}

static void erofs_fscache_put_request(struct netfs_io_request *rreq)
{
	if (!refcount_dec_and_test(&rreq->ref))
		return;
	if (rreq->cache_resources.ops)
		rreq->cache_resources.ops->end_operation(&rreq->cache_resources);
	kfree(rreq);
}

static void erofs_fscache_put_subrequest(struct netfs_io_subrequest *subreq)
{
	if (!refcount_dec_and_test(&subreq->ref))
		return;
	erofs_fscache_put_request(subreq->rreq);
	kfree(subreq);
}

static void erofs_fscache_clear_subrequests(struct netfs_io_request *rreq)
{
	struct netfs_io_subrequest *subreq;

	while (!list_empty(&rreq->subrequests)) {
		subreq = list_first_entry(&rreq->subrequests,
				struct netfs_io_subrequest, rreq_link);
		list_del(&subreq->rreq_link);
		erofs_fscache_put_subrequest(subreq);
	}
}

static void erofs_fscache_rreq_unlock_folios(struct netfs_io_request *rreq)
{
	struct netfs_io_subrequest *subreq;
	struct folio *folio;
	unsigned int iopos = 0;
	pgoff_t start_page = rreq->start / PAGE_SIZE;
	pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
	bool subreq_failed = false;

	XA_STATE(xas, &rreq->mapping->i_pages, start_page);

	subreq = list_first_entry(&rreq->subrequests,
				  struct netfs_io_subrequest, rreq_link);
	subreq_failed = (subreq->error < 0);

	rcu_read_lock();
	xas_for_each(&xas, folio, last_page) {
		unsigned int pgpos =
			(folio_index(folio) - start_page) * PAGE_SIZE;
		unsigned int pgend = pgpos + folio_size(folio);
		bool pg_failed = false;

		for (;;) {
			if (!subreq) {
				pg_failed = true;
				break;
			}

			pg_failed |= subreq_failed;
			if (pgend < iopos + subreq->len)
				break;

			iopos += subreq->len;
			if (!list_is_last(&subreq->rreq_link,
					  &rreq->subrequests)) {
				subreq = list_next_entry(subreq, rreq_link);
				subreq_failed = (subreq->error < 0);
			} else {
				subreq = NULL;
				subreq_failed = false;
			}
			if (pgend == iopos)
				break;
		}

		if (!pg_failed)
			folio_mark_uptodate(folio);

		folio_unlock(folio);
	}
	rcu_read_unlock();
}

static void erofs_fscache_rreq_complete(struct netfs_io_request *rreq)
{
	erofs_fscache_rreq_unlock_folios(rreq);
	erofs_fscache_clear_subrequests(rreq);
	erofs_fscache_put_request(rreq);
}

static void erofc_fscache_subreq_complete(void *priv,
		ssize_t transferred_or_error, bool was_async)
{
	struct netfs_io_subrequest *subreq = priv;
	struct netfs_io_request *rreq = subreq->rreq;

	if (IS_ERR_VALUE(transferred_or_error))
		subreq->error = transferred_or_error;

	if (atomic_dec_and_test(&rreq->nr_outstanding))
		erofs_fscache_rreq_complete(rreq);

	erofs_fscache_put_subrequest(subreq);
}

/*
 * Read data from fscache and fill the read data into page cache described by
 * @rreq, which shall be both aligned with PAGE_SIZE. @pstart describes
 * the start physical address in the cache file.
 */
static int erofs_fscache_read_folios_async(struct fscache_cookie *cookie,
				struct netfs_io_request *rreq, loff_t pstart)
{
	enum netfs_io_source source;
	struct super_block *sb = rreq->mapping->host->i_sb;
	struct netfs_io_subrequest *subreq;
	struct netfs_cache_resources *cres = &rreq->cache_resources;
	struct iov_iter iter;
	loff_t start = rreq->start;
	size_t len = rreq->len;
	size_t done = 0;
	int ret;

	atomic_set(&rreq->nr_outstanding, 1);

	ret = fscache_begin_read_operation(cres, cookie);
	if (ret)
		goto out;

	while (done < len) {
		subreq = kzalloc(sizeof(struct netfs_io_subrequest),
				 GFP_KERNEL);
		if (subreq) {
			INIT_LIST_HEAD(&subreq->rreq_link);
			refcount_set(&subreq->ref, 2);
			subreq->rreq = rreq;
			refcount_inc(&rreq->ref);
		} else {
			ret = -ENOMEM;
			goto out;
		}

		subreq->start = pstart + done;
		subreq->len	=  len - done;
		subreq->flags = 1 << NETFS_SREQ_ONDEMAND;

		list_add_tail(&subreq->rreq_link, &rreq->subrequests);

		source = cres->ops->prepare_read(subreq, LLONG_MAX);
		if (WARN_ON(subreq->len == 0))
			source = NETFS_INVALID_READ;
		if (source != NETFS_READ_FROM_CACHE) {
			erofs_err(sb, "failed to fscache prepare_read (source %d)",
				  source);
			ret = -EIO;
			subreq->error = ret;
			erofs_fscache_put_subrequest(subreq);
			goto out;
		}

		atomic_inc(&rreq->nr_outstanding);

		iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages,
				start + done, subreq->len);

		ret = fscache_read(cres, subreq->start, &iter,
				   NETFS_READ_HOLE_FAIL,
				   erofc_fscache_subreq_complete, subreq);
		if (ret == -EIOCBQUEUED)
			ret = 0;
		if (ret) {
			erofs_err(sb, "failed to fscache_read (ret %d)", ret);
			goto out;
		}

		done += subreq->len;
	}
out:
	if (atomic_dec_and_test(&rreq->nr_outstanding))
		erofs_fscache_rreq_complete(rreq);

	return ret;
}

static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio)
{
	int ret;
	struct super_block *sb = folio_mapping(folio)->host->i_sb;
	struct netfs_io_request *rreq;
	struct erofs_map_dev mdev = {
		.m_deviceid = 0,
		.m_pa = folio_pos(folio),
	};

	ret = erofs_map_dev(sb, &mdev);
	if (ret)
		goto out;

	rreq = erofs_fscache_alloc_request(folio_mapping(folio),
				folio_pos(folio), folio_size(folio));
	if (IS_ERR(rreq))
		goto out;

	return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
				rreq, mdev.m_pa);
out:
	folio_unlock(folio);
	return ret;
}

static int erofs_fscache_read_folio_inline(struct folio *folio,
					 struct erofs_map_blocks *map)
{
	struct super_block *sb = folio_mapping(folio)->host->i_sb;
	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
	erofs_blk_t blknr;
	size_t offset, len;
	void *src, *dst;

	/* For tail packing layout, the offset may be non-zero. */
	offset = erofs_blkoff(map->m_pa);
	blknr = erofs_blknr(map->m_pa);
	len = map->m_llen;

	src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP);
	if (IS_ERR(src))
		return PTR_ERR(src);

	dst = kmap_local_folio(folio, 0);
	memcpy(dst, src + offset, len);
	memset(dst + len, 0, PAGE_SIZE - len);
	kunmap_local(dst);

	erofs_put_metabuf(&buf);
	return 0;
}

static int erofs_fscache_read_folio(struct file *file, struct folio *folio)
{
	struct inode *inode = folio_mapping(folio)->host;
	struct super_block *sb = inode->i_sb;
	struct erofs_map_blocks map;
	struct erofs_map_dev mdev;
	struct netfs_io_request *rreq;
	erofs_off_t pos;
	loff_t pstart;
	int ret;

	DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ);

	pos = folio_pos(folio);
	map.m_la = pos;

	ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
	if (ret)
		goto out_unlock;

	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
		folio_zero_range(folio, 0, folio_size(folio));
		goto out_uptodate;
	}

	if (map.m_flags & EROFS_MAP_META) {
		ret = erofs_fscache_read_folio_inline(folio, &map);
		goto out_uptodate;
	}

	mdev = (struct erofs_map_dev) {
		.m_deviceid = map.m_deviceid,
		.m_pa = map.m_pa,
	};

	ret = erofs_map_dev(sb, &mdev);
	if (ret)
		goto out_unlock;


	rreq = erofs_fscache_alloc_request(folio_mapping(folio),
				folio_pos(folio), folio_size(folio));
	if (IS_ERR(rreq))
		goto out_unlock;

	pstart = mdev.m_pa + (pos - map.m_la);
	return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
				rreq, pstart);

out_uptodate:
	if (!ret)
		folio_mark_uptodate(folio);
out_unlock:
	folio_unlock(folio);
	return ret;
}

static void erofs_fscache_advance_folios(struct readahead_control *rac,
					 size_t len, bool unlock)
{
	while (len) {
		struct folio *folio = readahead_folio(rac);
		len -= folio_size(folio);
		if (unlock) {
			folio_mark_uptodate(folio);
			folio_unlock(folio);
		}
	}
}

static void erofs_fscache_readahead(struct readahead_control *rac)
{
	struct inode *inode = rac->mapping->host;
	struct super_block *sb = inode->i_sb;
	size_t len, count, done = 0;
	erofs_off_t pos;
	loff_t start, offset;
	int ret;

	if (!readahead_count(rac))
		return;

	start = readahead_pos(rac);
	len = readahead_length(rac);

	do {
		struct erofs_map_blocks map;
		struct erofs_map_dev mdev;
		struct netfs_io_request *rreq;

		pos = start + done;
		map.m_la = pos;

		ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
		if (ret)
			return;

		offset = start + done;
		count = min_t(size_t, map.m_llen - (pos - map.m_la),
			      len - done);

		if (!(map.m_flags & EROFS_MAP_MAPPED)) {
			struct iov_iter iter;

			iov_iter_xarray(&iter, READ, &rac->mapping->i_pages,
					offset, count);
			iov_iter_zero(count, &iter);

			erofs_fscache_advance_folios(rac, count, true);
			ret = count;
			continue;
		}

		if (map.m_flags & EROFS_MAP_META) {
			struct folio *folio = readahead_folio(rac);

			ret = erofs_fscache_read_folio_inline(folio, &map);
			if (!ret) {
				folio_mark_uptodate(folio);
				ret = folio_size(folio);
			}

			folio_unlock(folio);
			continue;
		}

		mdev = (struct erofs_map_dev) {
			.m_deviceid = map.m_deviceid,
			.m_pa = map.m_pa,
		};
		ret = erofs_map_dev(sb, &mdev);
		if (ret)
			return;

		rreq = erofs_fscache_alloc_request(rac->mapping, offset, count);
		if (IS_ERR(rreq))
			return;
		/*
		 * Drop the ref of folios here. Unlock them in
		 * rreq_unlock_folios() when rreq complete.
		 */
		erofs_fscache_advance_folios(rac, count, false);
		ret = erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
					rreq, mdev.m_pa + (pos - map.m_la));
		if (!ret)
			ret = count;
	} while (ret > 0 && ((done += ret) < len));
}

static const struct address_space_operations erofs_fscache_meta_aops = {
	.read_folio = erofs_fscache_meta_read_folio,
};

const struct address_space_operations erofs_fscache_access_aops = {
	.read_folio = erofs_fscache_read_folio,
	.readahead = erofs_fscache_readahead,
};

int erofs_fscache_register_cookie(struct super_block *sb,
				  struct erofs_fscache **fscache,
				  char *name, bool need_inode)
{
	struct fscache_volume *volume = EROFS_SB(sb)->volume;
	struct erofs_fscache *ctx;
	struct fscache_cookie *cookie;
	int ret;

	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx)
		return -ENOMEM;

	cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE,
					name, strlen(name), NULL, 0, 0);
	if (!cookie) {
		erofs_err(sb, "failed to get cookie for %s", name);
		ret = -EINVAL;
		goto err;
	}

	fscache_use_cookie(cookie, false);
	ctx->cookie = cookie;

	if (need_inode) {
		struct inode *const inode = new_inode(sb);

		if (!inode) {
			erofs_err(sb, "failed to get anon inode for %s", name);
			ret = -ENOMEM;
			goto err_cookie;
		}

		set_nlink(inode, 1);
		inode->i_size = OFFSET_MAX;
		inode->i_mapping->a_ops = &erofs_fscache_meta_aops;
		mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);

		ctx->inode = inode;
	}

	*fscache = ctx;
	return 0;

err_cookie:
	fscache_unuse_cookie(ctx->cookie, NULL, NULL);
	fscache_relinquish_cookie(ctx->cookie, false);
	ctx->cookie = NULL;
err:
	kfree(ctx);
	return ret;
}

void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache)
{
	struct erofs_fscache *ctx = *fscache;

	if (!ctx)
		return;

	fscache_unuse_cookie(ctx->cookie, NULL, NULL);
	fscache_relinquish_cookie(ctx->cookie, false);
	ctx->cookie = NULL;

	iput(ctx->inode);
	ctx->inode = NULL;

	kfree(ctx);
	*fscache = NULL;
}

int erofs_fscache_register_fs(struct super_block *sb)
{
	struct erofs_sb_info *sbi = EROFS_SB(sb);
	struct fscache_volume *volume;
	char *name;
	int ret = 0;

	name = kasprintf(GFP_KERNEL, "erofs,%s", sbi->opt.fsid);
	if (!name)
		return -ENOMEM;

	volume = fscache_acquire_volume(name, NULL, NULL, 0);
	if (IS_ERR_OR_NULL(volume)) {
		erofs_err(sb, "failed to register volume for %s", name);
		ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP;
		volume = NULL;
	}

	sbi->volume = volume;
	kfree(name);
	return ret;
}

void erofs_fscache_unregister_fs(struct super_block *sb)
{
	struct erofs_sb_info *sbi = EROFS_SB(sb);

	fscache_relinquish_volume(sbi->volume, NULL, false);
	sbi->volume = NULL;
}
Commit	Line	Data
c6be2bd0 JX	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/*
	3	* Copyright (C) 2022, Alibaba Cloud
	4	*/
	5	#include <linux/fscache.h>
	6	#include "internal.h"
	7
d435d532 XY	8	static struct netfs_io_request erofs_fscache_alloc_request(struct address_space mapping,
	9	loff_t start, size_t len)
	10	{
	11	struct netfs_io_request *rreq;
	12
	13	rreq = kzalloc(sizeof(struct netfs_io_request), GFP_KERNEL);
	14	if (!rreq)
	15	return ERR_PTR(-ENOMEM);
	16
	17	rreq->start = start;
	18	rreq->len = len;
	19	rreq->mapping = mapping;
b5cb79dc	20	rreq->inode = mapping->host;
d435d532 XY	21	INIT_LIST_HEAD(&rreq->subrequests);
	22	refcount_set(&rreq->ref, 1);
	23	return rreq;
	24	}
	25
	26	static void erofs_fscache_put_request(struct netfs_io_request *rreq)
	27	{
	28	if (!refcount_dec_and_test(&rreq->ref))
	29	return;
	30	if (rreq->cache_resources.ops)
	31	rreq->cache_resources.ops->end_operation(&rreq->cache_resources);
	32	kfree(rreq);
	33	}
	34
	35	static void erofs_fscache_put_subrequest(struct netfs_io_subrequest *subreq)
	36	{
	37	if (!refcount_dec_and_test(&subreq->ref))
	38	return;
	39	erofs_fscache_put_request(subreq->rreq);
	40	kfree(subreq);
	41	}
	42
	43	static void erofs_fscache_clear_subrequests(struct netfs_io_request *rreq)
	44	{
	45	struct netfs_io_subrequest *subreq;
	46
	47	while (!list_empty(&rreq->subrequests)) {
	48	subreq = list_first_entry(&rreq->subrequests,
	49	struct netfs_io_subrequest, rreq_link);
	50	list_del(&subreq->rreq_link);
	51	erofs_fscache_put_subrequest(subreq);
	52	}
	53	}
	54
	55	static void erofs_fscache_rreq_unlock_folios(struct netfs_io_request *rreq)
	56	{
	57	struct netfs_io_subrequest *subreq;
	58	struct folio *folio;
	59	unsigned int iopos = 0;
	60	pgoff_t start_page = rreq->start / PAGE_SIZE;
	61	pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
	62	bool subreq_failed = false;
	63
	64	XA_STATE(xas, &rreq->mapping->i_pages, start_page);
	65
	66	subreq = list_first_entry(&rreq->subrequests,
	67	struct netfs_io_subrequest, rreq_link);
	68	subreq_failed = (subreq->error < 0);
	69
	70	rcu_read_lock();
	71	xas_for_each(&xas, folio, last_page) {
	72	unsigned int pgpos =
	73	(folio_index(folio) - start_page) * PAGE_SIZE;
	74	unsigned int pgend = pgpos + folio_size(folio);
	75	bool pg_failed = false;
	76
	77	for (;;) {
	78	if (!subreq) {
	79	pg_failed = true;
	80	break;
	81	}
	82
	83	pg_failed \|= subreq_failed;
	84	if (pgend < iopos + subreq->len)
85	break;
86
87	iopos += subreq->len;
88	if (!list_is_last(&subreq->rreq_link,
89	&rreq->subrequests)) {
90	subreq = list_next_entry(subreq, rreq_link);
91	subreq_failed = (subreq->error < 0);
92	} else {
93	subreq = NULL;
94	subreq_failed = false;
95	}
96	if (pgend == iopos)
97	break;
98	}
99
100	if (!pg_failed)
101	folio_mark_uptodate(folio);
102
103	folio_unlock(folio);
104	}
105	rcu_read_unlock();
106	}
107
108	static void erofs_fscache_rreq_complete(struct netfs_io_request *rreq)
109	{
110	erofs_fscache_rreq_unlock_folios(rreq);
111	erofs_fscache_clear_subrequests(rreq);
112	erofs_fscache_put_request(rreq);
113	}
114
115	static void erofc_fscache_subreq_complete(void *priv,
116	ssize_t transferred_or_error, bool was_async)
117	{
118	struct netfs_io_subrequest *subreq = priv;
119	struct netfs_io_request *rreq = subreq->rreq;
120
121	if (IS_ERR_VALUE(transferred_or_error))
122	subreq->error = transferred_or_error;
123
124	if (atomic_dec_and_test(&rreq->nr_outstanding))
125	erofs_fscache_rreq_complete(rreq);
126
127	erofs_fscache_put_subrequest(subreq);
128	}
129
ec00b5e2 JX	130	/*
ec00b5e2 JX	131	* Read data from fscache and fill the read data into page cache described by
d435d532	132	* @rreq, which shall be both aligned with PAGE_SIZE. @pstart describes
ec00b5e2 JX	133	* the start physical address in the cache file.
ec00b5e2 JX	134	*/
d435d532 XY	135	static int erofs_fscache_read_folios_async(struct fscache_cookie *cookie,
d435d532 XY	136	struct netfs_io_request *rreq, loff_t pstart)
ec00b5e2 JX	137	{
ec00b5e2 JX	138	enum netfs_io_source source;
d435d532 XY	139	struct super_block *sb = rreq->mapping->host->i_sb;
	140	struct netfs_io_subrequest *subreq;
	141	struct netfs_cache_resources *cres = &rreq->cache_resources;
ec00b5e2	142	struct iov_iter iter;
d435d532 XY	143	loff_t start = rreq->start;
d435d532 XY	144	size_t len = rreq->len;
ec00b5e2 JX	145	size_t done = 0;
	146	int ret;
	147
d435d532 XY	148	atomic_set(&rreq->nr_outstanding, 1);
d435d532 XY	149
ec00b5e2 JX	150	ret = fscache_begin_read_operation(cres, cookie);
ec00b5e2 JX	151	if (ret)
d435d532	152	goto out;
ec00b5e2 JX	153
ec00b5e2 JX	154	while (done < len) {
d435d532 XY	155	subreq = kzalloc(sizeof(struct netfs_io_subrequest),
	156	GFP_KERNEL);
	157	if (subreq) {
	158	INIT_LIST_HEAD(&subreq->rreq_link);
	159	refcount_set(&subreq->ref, 2);
	160	subreq->rreq = rreq;
	161	refcount_inc(&rreq->ref);
	162	} else {
	163	ret = -ENOMEM;
	164	goto out;
	165	}
	166
	167	subreq->start = pstart + done;
	168	subreq->len = len - done;
	169	subreq->flags = 1 << NETFS_SREQ_ONDEMAND;
	170
	171	list_add_tail(&subreq->rreq_link, &rreq->subrequests);
ec00b5e2	172
d435d532 XY	173	source = cres->ops->prepare_read(subreq, LLONG_MAX);
d435d532 XY	174	if (WARN_ON(subreq->len == 0))
ec00b5e2 JX	175	source = NETFS_INVALID_READ;
	176	if (source != NETFS_READ_FROM_CACHE) {
	177	erofs_err(sb, "failed to fscache prepare_read (source %d)",
	178	source);
	179	ret = -EIO;
d435d532 XY	180	subreq->error = ret;
d435d532 XY	181	erofs_fscache_put_subrequest(subreq);
ec00b5e2 JX	182	goto out;
	183	}
	184
d435d532 XY	185	atomic_inc(&rreq->nr_outstanding);
	186
	187	iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages,
	188	start + done, subreq->len);
	189
	190	ret = fscache_read(cres, subreq->start, &iter,
	191	NETFS_READ_HOLE_FAIL,
	192	erofc_fscache_subreq_complete, subreq);
	193	if (ret == -EIOCBQUEUED)
	194	ret = 0;
ec00b5e2 JX	195	if (ret) {
	196	erofs_err(sb, "failed to fscache_read (ret %d)", ret);
	197	goto out;
	198	}
	199
d435d532	200	done += subreq->len;
ec00b5e2 JX	201	}
ec00b5e2 JX	202	out:
d435d532 XY	203	if (atomic_dec_and_test(&rreq->nr_outstanding))
	204	erofs_fscache_rreq_complete(rreq);
	205
ec00b5e2 JX	206	return ret;
	207	}
	208
fdaf9a58	209	static int erofs_fscache_meta_read_folio(struct file data, struct folio folio)
5375e7c8 JX	210	{
5375e7c8 JX	211	int ret;
5375e7c8	212	struct super_block *sb = folio_mapping(folio)->host->i_sb;
d435d532	213	struct netfs_io_request *rreq;
5375e7c8 JX	214	struct erofs_map_dev mdev = {
	215	.m_deviceid = 0,
	216	.m_pa = folio_pos(folio),
	217	};
	218
	219	ret = erofs_map_dev(sb, &mdev);
	220	if (ret)
	221	goto out;
	222
d435d532 XY	223	rreq = erofs_fscache_alloc_request(folio_mapping(folio),
	224	folio_pos(folio), folio_size(folio));
	225	if (IS_ERR(rreq))
	226	goto out;
	227
	228	return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
	229	rreq, mdev.m_pa);
5375e7c8 JX	230	out:
	231	folio_unlock(folio);
	232	return ret;
	233	}
	234
fdaf9a58	235	static int erofs_fscache_read_folio_inline(struct folio *folio,
bd735bda JX	236	struct erofs_map_blocks *map)
	237	{
	238	struct super_block *sb = folio_mapping(folio)->host->i_sb;
	239	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
	240	erofs_blk_t blknr;
	241	size_t offset, len;
	242	void src, dst;
	243
	244	/* For tail packing layout, the offset may be non-zero. */
	245	offset = erofs_blkoff(map->m_pa);
	246	blknr = erofs_blknr(map->m_pa);
	247	len = map->m_llen;
	248
	249	src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP);
	250	if (IS_ERR(src))
	251	return PTR_ERR(src);
	252
	253	dst = kmap_local_folio(folio, 0);
	254	memcpy(dst, src + offset, len);
	255	memset(dst + len, 0, PAGE_SIZE - len);
	256	kunmap_local(dst);
	257
	258	erofs_put_metabuf(&buf);
	259	return 0;
	260	}
	261
fdaf9a58	262	static int erofs_fscache_read_folio(struct file file, struct folio folio)
1442b02b	263	{
1442b02b JX	264	struct inode *inode = folio_mapping(folio)->host;
	265	struct super_block *sb = inode->i_sb;
	266	struct erofs_map_blocks map;
	267	struct erofs_map_dev mdev;
d435d532	268	struct netfs_io_request *rreq;
1442b02b JX	269	erofs_off_t pos;
	270	loff_t pstart;
	271	int ret;
	272
	273	DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ);
	274
	275	pos = folio_pos(folio);
	276	map.m_la = pos;
	277
	278	ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
	279	if (ret)
	280	goto out_unlock;
	281
	282	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
	283	folio_zero_range(folio, 0, folio_size(folio));
	284	goto out_uptodate;
	285	}
	286
bd735bda	287	if (map.m_flags & EROFS_MAP_META) {
fdaf9a58	288	ret = erofs_fscache_read_folio_inline(folio, &map);
bd735bda JX	289	goto out_uptodate;
	290	}
	291
1442b02b JX	292	mdev = (struct erofs_map_dev) {
	293	.m_deviceid = map.m_deviceid,
	294	.m_pa = map.m_pa,
	295	};
	296
	297	ret = erofs_map_dev(sb, &mdev);
	298	if (ret)
	299	goto out_unlock;
	300
d435d532 XY	301
	302	rreq = erofs_fscache_alloc_request(folio_mapping(folio),
	303	folio_pos(folio), folio_size(folio));
	304	if (IS_ERR(rreq))
	305	goto out_unlock;
	306
1442b02b	307	pstart = mdev.m_pa + (pos - map.m_la);
d435d532 XY	308	return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
d435d532 XY	309	rreq, pstart);
1442b02b JX	310
	311	out_uptodate:
	312	if (!ret)
	313	folio_mark_uptodate(folio);
	314	out_unlock:
	315	folio_unlock(folio);
	316	return ret;
	317	}
	318
d435d532 XY	319	static void erofs_fscache_advance_folios(struct readahead_control *rac,
d435d532 XY	320	size_t len, bool unlock)
c665b394 JX	321	{
	322	while (len) {
	323	struct folio *folio = readahead_folio(rac);
c665b394	324	len -= folio_size(folio);
d435d532 XY	325	if (unlock) {
	326	folio_mark_uptodate(folio);
	327	folio_unlock(folio);
	328	}
c665b394 JX	329	}
	330	}
	331
	332	static void erofs_fscache_readahead(struct readahead_control *rac)
	333	{
	334	struct inode *inode = rac->mapping->host;
	335	struct super_block *sb = inode->i_sb;
	336	size_t len, count, done = 0;
	337	erofs_off_t pos;
	338	loff_t start, offset;
	339	int ret;
	340
	341	if (!readahead_count(rac))
	342	return;
	343
	344	start = readahead_pos(rac);
	345	len = readahead_length(rac);
	346
	347	do {
	348	struct erofs_map_blocks map;
	349	struct erofs_map_dev mdev;
d435d532	350	struct netfs_io_request *rreq;
c665b394 JX	351
	352	pos = start + done;
	353	map.m_la = pos;
	354
	355	ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
	356	if (ret)
	357	return;
	358
	359	offset = start + done;
	360	count = min_t(size_t, map.m_llen - (pos - map.m_la),
	361	len - done);
	362
	363	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
	364	struct iov_iter iter;
	365
	366	iov_iter_xarray(&iter, READ, &rac->mapping->i_pages,
	367	offset, count);
	368	iov_iter_zero(count, &iter);
	369
d435d532	370	erofs_fscache_advance_folios(rac, count, true);
c665b394 JX	371	ret = count;
	372	continue;
	373	}
	374
	375	if (map.m_flags & EROFS_MAP_META) {
	376	struct folio *folio = readahead_folio(rac);
	377
fdaf9a58	378	ret = erofs_fscache_read_folio_inline(folio, &map);
c665b394 JX	379	if (!ret) {
	380	folio_mark_uptodate(folio);
	381	ret = folio_size(folio);
	382	}
	383
	384	folio_unlock(folio);
	385	continue;
	386	}
	387
	388	mdev = (struct erofs_map_dev) {
	389	.m_deviceid = map.m_deviceid,
	390	.m_pa = map.m_pa,
	391	};
	392	ret = erofs_map_dev(sb, &mdev);
	393	if (ret)
	394	return;
	395
d435d532 XY	396	rreq = erofs_fscache_alloc_request(rac->mapping, offset, count);
	397	if (IS_ERR(rreq))
	398	return;
c665b394	399	/*
d435d532 XY	400	* Drop the ref of folios here. Unlock them in
d435d532 XY	401	* rreq_unlock_folios() when rreq complete.
c665b394	402	*/
d435d532 XY	403	erofs_fscache_advance_folios(rac, count, false);
	404	ret = erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
	405	rreq, mdev.m_pa + (pos - map.m_la));
	406	if (!ret)
c665b394	407	ret = count;
c665b394 JX	408	} while (ret > 0 && ((done += ret) < len));
	409	}
	410
3c265d7d	411	static const struct address_space_operations erofs_fscache_meta_aops = {
fdaf9a58	412	.read_folio = erofs_fscache_meta_read_folio,
3c265d7d JX	413	};
3c265d7d JX	414
1442b02b	415	const struct address_space_operations erofs_fscache_access_aops = {
fdaf9a58	416	.read_folio = erofs_fscache_read_folio,
c665b394	417	.readahead = erofs_fscache_readahead,
1442b02b JX	418	};
1442b02b JX	419
b02c602f	420	int erofs_fscache_register_cookie(struct super_block *sb,
3c265d7d JX	421	struct erofs_fscache **fscache,
3c265d7d JX	422	char *name, bool need_inode)
b02c602f JX	423	{
	424	struct fscache_volume *volume = EROFS_SB(sb)->volume;
	425	struct erofs_fscache *ctx;
	426	struct fscache_cookie *cookie;
3c265d7d	427	int ret;
b02c602f JX	428
	429	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	430	if (!ctx)
	431	return -ENOMEM;
	432
	433	cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE,
	434	name, strlen(name), NULL, 0, 0);
	435	if (!cookie) {
	436	erofs_err(sb, "failed to get cookie for %s", name);
3c265d7d JX	437	ret = -EINVAL;
3c265d7d JX	438	goto err;
b02c602f JX	439	}
	440
	441	fscache_use_cookie(cookie, false);
	442	ctx->cookie = cookie;
	443
3c265d7d JX	444	if (need_inode) {
	445	struct inode *const inode = new_inode(sb);
	446
	447	if (!inode) {
	448	erofs_err(sb, "failed to get anon inode for %s", name);
	449	ret = -ENOMEM;
	450	goto err_cookie;
	451	}
	452
	453	set_nlink(inode, 1);
	454	inode->i_size = OFFSET_MAX;
	455	inode->i_mapping->a_ops = &erofs_fscache_meta_aops;
	456	mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
	457
	458	ctx->inode = inode;
	459	}
	460
b02c602f JX	461	*fscache = ctx;
b02c602f JX	462	return 0;
3c265d7d JX	463
	464	err_cookie:
	465	fscache_unuse_cookie(ctx->cookie, NULL, NULL);
	466	fscache_relinquish_cookie(ctx->cookie, false);
	467	ctx->cookie = NULL;
	468	err:
	469	kfree(ctx);
	470	return ret;
b02c602f JX	471	}
	472
	473	void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache)
	474	{
	475	struct erofs_fscache ctx = fscache;
	476
	477	if (!ctx)
	478	return;
	479
	480	fscache_unuse_cookie(ctx->cookie, NULL, NULL);
	481	fscache_relinquish_cookie(ctx->cookie, false);
	482	ctx->cookie = NULL;
	483
3c265d7d JX	484	iput(ctx->inode);
	485	ctx->inode = NULL;
	486
b02c602f JX	487	kfree(ctx);
	488	*fscache = NULL;
	489	}
	490
c6be2bd0 JX	491	int erofs_fscache_register_fs(struct super_block *sb)
	492	{
	493	struct erofs_sb_info *sbi = EROFS_SB(sb);
	494	struct fscache_volume *volume;
	495	char *name;
	496	int ret = 0;
	497
	498	name = kasprintf(GFP_KERNEL, "erofs,%s", sbi->opt.fsid);
	499	if (!name)
	500	return -ENOMEM;
	501
	502	volume = fscache_acquire_volume(name, NULL, NULL, 0);
	503	if (IS_ERR_OR_NULL(volume)) {
	504	erofs_err(sb, "failed to register volume for %s", name);
	505	ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP;
	506	volume = NULL;
	507	}
	508
	509	sbi->volume = volume;
	510	kfree(name);
	511	return ret;
	512	}
	513
	514	void erofs_fscache_unregister_fs(struct super_block *sb)
	515	{
	516	struct erofs_sb_info *sbi = EROFS_SB(sb);
	517
	518	fscache_relinquish_volume(sbi->volume, NULL, false);
	519	sbi->volume = NULL;
	520	}