[linux-block.git] / fs / cachefiles / io.c

// SPDX-License-Identifier: GPL-2.0-or-later
/* kiocb-using read/write
 *
 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/mount.h>
#include <linux/slab.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/falloc.h>
#include <linux/sched/mm.h>
#include <trace/events/fscache.h>
#include "internal.h"

struct cachefiles_kiocb {
	struct kiocb		iocb;
	refcount_t		ki_refcnt;
	loff_t			start;
	union {
		size_t		skipped;
		size_t		len;
	};
	struct cachefiles_object *object;
	netfs_io_terminated_t	term_func;
	void			*term_func_priv;
	bool			was_async;
	unsigned int		inval_counter;	/* Copy of cookie->inval_counter */
	u64			b_writing;
};

static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
{
	if (refcount_dec_and_test(&ki->ki_refcnt)) {
		cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);
		fput(ki->iocb.ki_filp);
		kfree(ki);
	}
}

/*
 * Handle completion of a read from the cache.
 */
static void cachefiles_read_complete(struct kiocb *iocb, long ret)
{
	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
	struct inode *inode = file_inode(ki->iocb.ki_filp);

	_enter("%ld", ret);

	if (ret < 0)
		trace_cachefiles_io_error(ki->object, inode, ret,
					  cachefiles_trace_read_error);

	if (ki->term_func) {
		if (ret >= 0) {
			if (ki->object->cookie->inval_counter == ki->inval_counter)
				ki->skipped += ret;
			else
				ret = -ESTALE;
		}

		ki->term_func(ki->term_func_priv, ret, ki->was_async);
	}

	cachefiles_put_kiocb(ki);
}

/*
 * Initiate a read from the cache.
 */
static int cachefiles_read(struct netfs_cache_resources *cres,
			   loff_t start_pos,
			   struct iov_iter *iter,
			   enum netfs_read_from_hole read_hole,
			   netfs_io_terminated_t term_func,
			   void *term_func_priv)
{
	struct cachefiles_object *object;
	struct cachefiles_kiocb *ki;
	struct file *file;
	unsigned int old_nofs;
	ssize_t ret = -ENOBUFS;
	size_t len = iov_iter_count(iter), skipped = 0;

	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
		goto presubmission_error;

	fscache_count_read();
	object = cachefiles_cres_object(cres);
	file = cachefiles_cres_file(cres);

	_enter("%pD,%li,%llx,%zx/%llx",
	       file, file_inode(file)->i_ino, start_pos, len,
	       i_size_read(file_inode(file)));

	/* If the caller asked us to seek for data before doing the read, then
	 * we should do that now.  If we find a gap, we fill it with zeros.
	 */
	if (read_hole != NETFS_READ_HOLE_IGNORE) {
		loff_t off = start_pos, off2;

		off2 = cachefiles_inject_read_error();
		if (off2 == 0)
			off2 = vfs_llseek(file, off, SEEK_DATA);
		if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
			skipped = 0;
			ret = off2;
			goto presubmission_error;
		}

		if (off2 == -ENXIO || off2 >= start_pos + len) {
			/* The region is beyond the EOF or there's no more data
			 * in the region, so clear the rest of the buffer and
			 * return success.
			 */
			ret = -ENODATA;
			if (read_hole == NETFS_READ_HOLE_FAIL)
				goto presubmission_error;

			iov_iter_zero(len, iter);
			skipped = len;
			ret = 0;
			goto presubmission_error;
		}

		skipped = off2 - off;
		iov_iter_zero(skipped, iter);
	}

	ret = -ENOMEM;
	ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
	if (!ki)
		goto presubmission_error;

	refcount_set(&ki->ki_refcnt, 2);
	ki->iocb.ki_filp	= file;
	ki->iocb.ki_pos		= start_pos + skipped;
	ki->iocb.ki_flags	= IOCB_DIRECT;
	ki->iocb.ki_ioprio	= get_current_ioprio();
	ki->skipped		= skipped;
	ki->object		= object;
	ki->inval_counter	= cres->inval_counter;
	ki->term_func		= term_func;
	ki->term_func_priv	= term_func_priv;
	ki->was_async		= true;

	if (ki->term_func)
		ki->iocb.ki_complete = cachefiles_read_complete;

	get_file(ki->iocb.ki_filp);
	cachefiles_grab_object(object, cachefiles_obj_get_ioreq);

	trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);
	old_nofs = memalloc_nofs_save();
	ret = cachefiles_inject_read_error();
	if (ret == 0)
		ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
	memalloc_nofs_restore(old_nofs);
	switch (ret) {
	case -EIOCBQUEUED:
		goto in_progress;

	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/* There's no easy way to restart the syscall since other AIO's
		 * may be already running. Just fail this IO with EINTR.
		 */
		ret = -EINTR;
		fallthrough;
	default:
		ki->was_async = false;
		cachefiles_read_complete(&ki->iocb, ret);
		if (ret > 0)
			ret = 0;
		break;
	}

in_progress:
	cachefiles_put_kiocb(ki);
	_leave(" = %zd", ret);
	return ret;

presubmission_error:
	if (term_func)
		term_func(term_func_priv, ret < 0 ? ret : skipped, false);
	return ret;
}

/*
 * Query the occupancy of the cache in a region, returning where the next chunk
 * of data starts and how long it is.
 */
static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
				      loff_t start, size_t len, size_t granularity,
				      loff_t *_data_start, size_t *_data_len)
{
	struct cachefiles_object *object;
	struct file *file;
	loff_t off, off2;

	*_data_start = -1;
	*_data_len = 0;

	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
		return -ENOBUFS;

	object = cachefiles_cres_object(cres);
	file = cachefiles_cres_file(cres);
	granularity = max_t(size_t, object->volume->cache->bsize, granularity);

	_enter("%pD,%li,%llx,%zx/%llx",
	       file, file_inode(file)->i_ino, start, len,
	       i_size_read(file_inode(file)));

	off = cachefiles_inject_read_error();
	if (off == 0)
		off = vfs_llseek(file, start, SEEK_DATA);
	if (off == -ENXIO)
		return -ENODATA; /* Beyond EOF */
	if (off < 0 && off >= (loff_t)-MAX_ERRNO)
		return -ENOBUFS; /* Error. */
	if (round_up(off, granularity) >= start + len)
		return -ENODATA; /* No data in range */

	off2 = cachefiles_inject_read_error();
	if (off2 == 0)
		off2 = vfs_llseek(file, off, SEEK_HOLE);
	if (off2 == -ENXIO)
		return -ENODATA; /* Beyond EOF */
	if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
		return -ENOBUFS; /* Error. */

	/* Round away partial blocks */
	off = round_up(off, granularity);
	off2 = round_down(off2, granularity);
	if (off2 <= off)
		return -ENODATA;

	*_data_start = off;
	if (off2 > start + len)
		*_data_len = len;
	else
		*_data_len = off2 - off;
	return 0;
}

/*
 * Handle completion of a write to the cache.
 */
static void cachefiles_write_complete(struct kiocb *iocb, long ret)
{
	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
	struct cachefiles_object *object = ki->object;
	struct inode *inode = file_inode(ki->iocb.ki_filp);

	_enter("%ld", ret);

	/* Tell lockdep we inherited freeze protection from submission thread */
	__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
	__sb_end_write(inode->i_sb, SB_FREEZE_WRITE);

	if (ret < 0)
		trace_cachefiles_io_error(object, inode, ret,
					  cachefiles_trace_write_error);

	atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);
	set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags);
	if (ki->term_func)
		ki->term_func(ki->term_func_priv, ret, ki->was_async);
	cachefiles_put_kiocb(ki);
}

/*
 * Initiate a write to the cache.
 */
int __cachefiles_write(struct cachefiles_object *object,
		       struct file *file,
		       loff_t start_pos,
		       struct iov_iter *iter,
		       netfs_io_terminated_t term_func,
		       void *term_func_priv)
{
	struct cachefiles_cache *cache;
	struct cachefiles_kiocb *ki;
	struct inode *inode;
	unsigned int old_nofs;
	ssize_t ret;
	size_t len = iov_iter_count(iter);

	fscache_count_write();
	cache = object->volume->cache;

	_enter("%pD,%li,%llx,%zx/%llx",
	       file, file_inode(file)->i_ino, start_pos, len,
	       i_size_read(file_inode(file)));

	ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
	if (!ki) {
		if (term_func)
			term_func(term_func_priv, -ENOMEM, false);
		return -ENOMEM;
	}

	refcount_set(&ki->ki_refcnt, 2);
	ki->iocb.ki_filp	= file;
	ki->iocb.ki_pos		= start_pos;
	ki->iocb.ki_flags	= IOCB_DIRECT | IOCB_WRITE;
	ki->iocb.ki_ioprio	= get_current_ioprio();
	ki->object		= object;
	ki->start		= start_pos;
	ki->len			= len;
	ki->term_func		= term_func;
	ki->term_func_priv	= term_func_priv;
	ki->was_async		= true;
	ki->b_writing		= (len + (1 << cache->bshift) - 1) >> cache->bshift;

	if (ki->term_func)
		ki->iocb.ki_complete = cachefiles_write_complete;
	atomic_long_add(ki->b_writing, &cache->b_writing);

	/* Open-code file_start_write here to grab freeze protection, which
	 * will be released by another thread in aio_complete_rw().  Fool
	 * lockdep by telling it the lock got released so that it doesn't
	 * complain about the held lock when we return to userspace.
	 */
	inode = file_inode(file);
	__sb_start_write(inode->i_sb, SB_FREEZE_WRITE);
	__sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);

	get_file(ki->iocb.ki_filp);
	cachefiles_grab_object(object, cachefiles_obj_get_ioreq);

	trace_cachefiles_write(object, inode, ki->iocb.ki_pos, len);
	old_nofs = memalloc_nofs_save();
	ret = cachefiles_inject_write_error();
	if (ret == 0)
		ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
	memalloc_nofs_restore(old_nofs);
	switch (ret) {
	case -EIOCBQUEUED:
		goto in_progress;

	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/* There's no easy way to restart the syscall since other AIO's
		 * may be already running. Just fail this IO with EINTR.
		 */
		ret = -EINTR;
		fallthrough;
	default:
		ki->was_async = false;
		cachefiles_write_complete(&ki->iocb, ret);
		if (ret > 0)
			ret = 0;
		break;
	}

in_progress:
	cachefiles_put_kiocb(ki);
	_leave(" = %zd", ret);
	return ret;
}

static int cachefiles_write(struct netfs_cache_resources *cres,
			    loff_t start_pos,
			    struct iov_iter *iter,
			    netfs_io_terminated_t term_func,
			    void *term_func_priv)
{
	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
		if (term_func)
			term_func(term_func_priv, -ENOBUFS, false);
		return -ENOBUFS;
	}

	return __cachefiles_write(cachefiles_cres_object(cres),
				  cachefiles_cres_file(cres),
				  start_pos, iter,
				  term_func, term_func_priv);
}

static inline enum netfs_io_source
cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
			   loff_t start, size_t *_len, loff_t i_size,
			   unsigned long *_flags, ino_t netfs_ino)
{
	enum cachefiles_prepare_read_trace why;
	struct cachefiles_object *object = NULL;
	struct cachefiles_cache *cache;
	struct fscache_cookie *cookie = fscache_cres_cookie(cres);
	const struct cred *saved_cred;
	struct file *file = cachefiles_cres_file(cres);
	enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
	size_t len = *_len;
	loff_t off, to;
	ino_t ino = file ? file_inode(file)->i_ino : 0;
	int rc;

	_enter("%zx @%llx/%llx", len, start, i_size);

	if (start >= i_size) {
		ret = NETFS_FILL_WITH_ZEROES;
		why = cachefiles_trace_read_after_eof;
		goto out_no_object;
	}

	if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
		__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
		why = cachefiles_trace_read_no_data;
		if (!test_bit(NETFS_SREQ_ONDEMAND, _flags))
			goto out_no_object;
	}

	/* The object and the file may be being created in the background. */
	if (!file) {
		why = cachefiles_trace_read_no_file;
		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
			goto out_no_object;
		file = cachefiles_cres_file(cres);
		if (!file)
			goto out_no_object;
		ino = file_inode(file)->i_ino;
	}

	object = cachefiles_cres_object(cres);
	cache = object->volume->cache;
	cachefiles_begin_secure(cache, &saved_cred);
retry:
	off = cachefiles_inject_read_error();
	if (off == 0)
		off = vfs_llseek(file, start, SEEK_DATA);
	if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
		if (off == (loff_t)-ENXIO) {
			why = cachefiles_trace_read_seek_nxio;
			goto download_and_store;
		}
		trace_cachefiles_io_error(object, file_inode(file), off,
					  cachefiles_trace_seek_error);
		why = cachefiles_trace_read_seek_error;
		goto out;
	}

	if (off >= start + len) {
		why = cachefiles_trace_read_found_hole;
		goto download_and_store;
	}

	if (off > start) {
		off = round_up(off, cache->bsize);
		len = off - start;
		*_len = len;
		why = cachefiles_trace_read_found_part;
		goto download_and_store;
	}

	to = cachefiles_inject_read_error();
	if (to == 0)
		to = vfs_llseek(file, start, SEEK_HOLE);
	if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
		trace_cachefiles_io_error(object, file_inode(file), to,
					  cachefiles_trace_seek_error);
		why = cachefiles_trace_read_seek_error;
		goto out;
	}

	if (to < start + len) {
		if (start + len >= i_size)
			to = round_up(to, cache->bsize);
		else
			to = round_down(to, cache->bsize);
		len = to - start;
		*_len = len;
	}

	why = cachefiles_trace_read_have_data;
	ret = NETFS_READ_FROM_CACHE;
	goto out;

download_and_store:
	__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
	if (test_bit(NETFS_SREQ_ONDEMAND, _flags)) {
		rc = cachefiles_ondemand_read(object, start, len);
		if (!rc) {
			__clear_bit(NETFS_SREQ_ONDEMAND, _flags);
			goto retry;
		}
		ret = NETFS_INVALID_READ;
	}
out:
	cachefiles_end_secure(cache, saved_cred);
out_no_object:
	trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
	return ret;
}

/*
 * Prepare a read operation, shortening it to a cached/uncached
 * boundary as appropriate.
 */
static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
						    loff_t i_size)
{
	return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
					  subreq->start, &subreq->len, i_size,
					  &subreq->flags, subreq->rreq->inode->i_ino);
}

/*
 * Prepare an on-demand read operation, shortening it to a cached/uncached
 * boundary as appropriate.
 */
static enum netfs_io_source
cachefiles_prepare_ondemand_read(struct netfs_cache_resources *cres,
				 loff_t start, size_t *_len, loff_t i_size,
				 unsigned long *_flags, ino_t ino)
{
	return cachefiles_do_prepare_read(cres, start, _len, i_size, _flags, ino);
}

/*
 * Prepare for a write to occur.
 */
int __cachefiles_prepare_write(struct cachefiles_object *object,
			       struct file *file,
			       loff_t *_start, size_t *_len,
			       bool no_space_allocated_yet)
{
	struct cachefiles_cache *cache = object->volume->cache;
	loff_t start = *_start, pos;
	size_t len = *_len, down;
	int ret;

	/* Round to DIO size */
	down = start - round_down(start, PAGE_SIZE);
	*_start = start - down;
	*_len = round_up(down + len, PAGE_SIZE);

	/* We need to work out whether there's sufficient disk space to perform
	 * the write - but we can skip that check if we have space already
	 * allocated.
	 */
	if (no_space_allocated_yet)
		goto check_space;

	pos = cachefiles_inject_read_error();
	if (pos == 0)
		pos = vfs_llseek(file, *_start, SEEK_DATA);
	if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
		if (pos == -ENXIO)
			goto check_space; /* Unallocated tail */
		trace_cachefiles_io_error(object, file_inode(file), pos,
					  cachefiles_trace_seek_error);
		return pos;
	}
	if ((u64)pos >= (u64)*_start + *_len)
		goto check_space; /* Unallocated region */

	/* We have a block that's at least partially filled - if we're low on
	 * space, we need to see if it's fully allocated.  If it's not, we may
	 * want to cull it.
	 */
	if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
				 cachefiles_has_space_check) == 0)
		return 0; /* Enough space to simply overwrite the whole block */

	pos = cachefiles_inject_read_error();
	if (pos == 0)
		pos = vfs_llseek(file, *_start, SEEK_HOLE);
	if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
		trace_cachefiles_io_error(object, file_inode(file), pos,
					  cachefiles_trace_seek_error);
		return pos;
	}
	if ((u64)pos >= (u64)*_start + *_len)
		return 0; /* Fully allocated */

	/* Partially allocated, but insufficient space: cull. */
	fscache_count_no_write_space();
	ret = cachefiles_inject_remove_error();
	if (ret == 0)
		ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
				    *_start, *_len);
	if (ret < 0) {
		trace_cachefiles_io_error(object, file_inode(file), ret,
					  cachefiles_trace_fallocate_error);
		cachefiles_io_error_obj(object,
					"CacheFiles: fallocate failed (%d)\n", ret);
		ret = -EIO;
	}

	return ret;

check_space:
	return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
				    cachefiles_has_space_for_write);
}

static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
				    loff_t *_start, size_t *_len, loff_t i_size,
				    bool no_space_allocated_yet)
{
	struct cachefiles_object *object = cachefiles_cres_object(cres);
	struct cachefiles_cache *cache = object->volume->cache;
	const struct cred *saved_cred;
	int ret;

	if (!cachefiles_cres_file(cres)) {
		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
			return -ENOBUFS;
		if (!cachefiles_cres_file(cres))
			return -ENOBUFS;
	}

	cachefiles_begin_secure(cache, &saved_cred);
	ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
					 _start, _len,
					 no_space_allocated_yet);
	cachefiles_end_secure(cache, saved_cred);
	return ret;
}

/*
 * Clean up an operation.
 */
static void cachefiles_end_operation(struct netfs_cache_resources *cres)
{
	struct file *file = cachefiles_cres_file(cres);

	if (file)
		fput(file);
	fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end);
}

static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
	.end_operation		= cachefiles_end_operation,
	.read			= cachefiles_read,
	.write			= cachefiles_write,
	.prepare_read		= cachefiles_prepare_read,
	.prepare_write		= cachefiles_prepare_write,
	.prepare_ondemand_read	= cachefiles_prepare_ondemand_read,
	.query_occupancy	= cachefiles_query_occupancy,
};

/*
 * Open the cache file when beginning a cache operation.
 */
bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
				enum fscache_want_state want_state)
{
	struct cachefiles_object *object = cachefiles_cres_object(cres);

	if (!cachefiles_cres_file(cres)) {
		cres->ops = &cachefiles_netfs_cache_ops;
		if (object->file) {
			spin_lock(&object->lock);
			if (!cres->cache_priv2 && object->file)
				cres->cache_priv2 = get_file(object->file);
			spin_unlock(&object->lock);
		}
	}

	if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) {
		pr_err("failed to get cres->file\n");
		return false;
	}

	return true;
}
Commit	Line	Data
287fd611 DH	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/* kiocb-using read/write
	3	*
	4	* Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
	5	* Written by David Howells (dhowells@redhat.com)
	6	*/
	7
	8	#include <linux/mount.h>
	9	#include <linux/slab.h>
	10	#include <linux/file.h>
	11	#include <linux/uio.h>
	12	#include <linux/falloc.h>
	13	#include <linux/sched/mm.h>
	14	#include <trace/events/fscache.h>
	15	#include "internal.h"
	16
047487c9 DH	17	struct cachefiles_kiocb {
	18	struct kiocb iocb;
	19	refcount_t ki_refcnt;
	20	loff_t start;
	21	union {
	22	size_t skipped;
	23	size_t len;
	24	};
	25	struct cachefiles_object *object;
	26	netfs_io_terminated_t term_func;
	27	void *term_func_priv;
	28	bool was_async;
	29	unsigned int inval_counter; /* Copy of cookie->inval_counter */
	30	u64 b_writing;
	31	};
	32
	33	static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
	34	{
	35	if (refcount_dec_and_test(&ki->ki_refcnt)) {
	36	cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);
	37	fput(ki->iocb.ki_filp);
	38	kfree(ki);
	39	}
	40	}
	41
	42	/*
	43	* Handle completion of a read from the cache.
	44	*/
	45	static void cachefiles_read_complete(struct kiocb *iocb, long ret)
	46	{
	47	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
	48	struct inode *inode = file_inode(ki->iocb.ki_filp);
	49
	50	_enter("%ld", ret);
	51
	52	if (ret < 0)
	53	trace_cachefiles_io_error(ki->object, inode, ret,
	54	cachefiles_trace_read_error);
	55
	56	if (ki->term_func) {
	57	if (ret >= 0) {
	58	if (ki->object->cookie->inval_counter == ki->inval_counter)
	59	ki->skipped += ret;
	60	else
	61	ret = -ESTALE;
	62	}
	63
	64	ki->term_func(ki->term_func_priv, ret, ki->was_async);
	65	}
	66
	67	cachefiles_put_kiocb(ki);
	68	}
	69
	70	/*
	71	* Initiate a read from the cache.
	72	*/
	73	static int cachefiles_read(struct netfs_cache_resources *cres,
	74	loff_t start_pos,
	75	struct iov_iter *iter,
	76	enum netfs_read_from_hole read_hole,
	77	netfs_io_terminated_t term_func,
	78	void *term_func_priv)
	79	{
	80	struct cachefiles_object *object;
81	struct cachefiles_kiocb *ki;
82	struct file *file;
83	unsigned int old_nofs;
84	ssize_t ret = -ENOBUFS;
85	size_t len = iov_iter_count(iter), skipped = 0;
86
87	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
88	goto presubmission_error;
89
90	fscache_count_read();
91	object = cachefiles_cres_object(cres);
92	file = cachefiles_cres_file(cres);
93
94	_enter("%pD,%li,%llx,%zx/%llx",
95	file, file_inode(file)->i_ino, start_pos, len,
96	i_size_read(file_inode(file)));
97
98	/* If the caller asked us to seek for data before doing the read, then
99	* we should do that now. If we find a gap, we fill it with zeros.
100	*/
101	if (read_hole != NETFS_READ_HOLE_IGNORE) {
102	loff_t off = start_pos, off2;
103
104	off2 = cachefiles_inject_read_error();
105	if (off2 == 0)
106	off2 = vfs_llseek(file, off, SEEK_DATA);
107	if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
108	skipped = 0;
109	ret = off2;
110	goto presubmission_error;
111	}
112
113	if (off2 == -ENXIO \|\| off2 >= start_pos + len) {
114	/* The region is beyond the EOF or there's no more data
115	* in the region, so clear the rest of the buffer and
116	* return success.
117	*/
118	ret = -ENODATA;
119	if (read_hole == NETFS_READ_HOLE_FAIL)
120	goto presubmission_error;
121
122	iov_iter_zero(len, iter);
123	skipped = len;
124	ret = 0;
125	goto presubmission_error;
126	}
127
128	skipped = off2 - off;
129	iov_iter_zero(skipped, iter);
130	}
131
132	ret = -ENOMEM;
133	ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
134	if (!ki)
135	goto presubmission_error;
136
137	refcount_set(&ki->ki_refcnt, 2);
138	ki->iocb.ki_filp = file;
139	ki->iocb.ki_pos = start_pos + skipped;
140	ki->iocb.ki_flags = IOCB_DIRECT;
047487c9 DH	141	ki->iocb.ki_ioprio = get_current_ioprio();
	142	ki->skipped = skipped;
	143	ki->object = object;
	144	ki->inval_counter = cres->inval_counter;
	145	ki->term_func = term_func;
	146	ki->term_func_priv = term_func_priv;
	147	ki->was_async = true;
	148
	149	if (ki->term_func)
	150	ki->iocb.ki_complete = cachefiles_read_complete;
	151
	152	get_file(ki->iocb.ki_filp);
	153	cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
	154
	155	trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);
	156	old_nofs = memalloc_nofs_save();
	157	ret = cachefiles_inject_read_error();
	158	if (ret == 0)
	159	ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
	160	memalloc_nofs_restore(old_nofs);
	161	switch (ret) {
	162	case -EIOCBQUEUED:
	163	goto in_progress;
	164
	165	case -ERESTARTSYS:
	166	case -ERESTARTNOINTR:
	167	case -ERESTARTNOHAND:
	168	case -ERESTART_RESTARTBLOCK:
	169	/* There's no easy way to restart the syscall since other AIO's
	170	* may be already running. Just fail this IO with EINTR.
	171	*/
	172	ret = -EINTR;
	173	fallthrough;
	174	default:
	175	ki->was_async = false;
	176	cachefiles_read_complete(&ki->iocb, ret);
	177	if (ret > 0)
	178	ret = 0;
	179	break;
	180	}
	181
	182	in_progress:
	183	cachefiles_put_kiocb(ki);
	184	_leave(" = %zd", ret);
	185	return ret;
	186
	187	presubmission_error:
	188	if (term_func)
	189	term_func(term_func_priv, ret < 0 ? ret : skipped, false);
	190	return ret;
	191	}
	192
bee9f655 DH	193	/*
	194	* Query the occupancy of the cache in a region, returning where the next chunk
	195	* of data starts and how long it is.
	196	*/
	197	static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
	198	loff_t start, size_t len, size_t granularity,
	199	loff_t _data_start, size_t _data_len)
	200	{
	201	struct cachefiles_object *object;
	202	struct file *file;
	203	loff_t off, off2;
	204
	205	*_data_start = -1;
	206	*_data_len = 0;
	207
	208	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
	209	return -ENOBUFS;
	210
	211	object = cachefiles_cres_object(cres);
	212	file = cachefiles_cres_file(cres);
	213	granularity = max_t(size_t, object->volume->cache->bsize, granularity);
	214
	215	_enter("%pD,%li,%llx,%zx/%llx",
	216	file, file_inode(file)->i_ino, start, len,
	217	i_size_read(file_inode(file)));
	218
	219	off = cachefiles_inject_read_error();
	220	if (off == 0)
	221	off = vfs_llseek(file, start, SEEK_DATA);
	222	if (off == -ENXIO)
	223	return -ENODATA; /* Beyond EOF */
	224	if (off < 0 && off >= (loff_t)-MAX_ERRNO)
	225	return -ENOBUFS; /* Error. */
	226	if (round_up(off, granularity) >= start + len)
	227	return -ENODATA; /* No data in range */
	228
	229	off2 = cachefiles_inject_read_error();
	230	if (off2 == 0)
	231	off2 = vfs_llseek(file, off, SEEK_HOLE);
	232	if (off2 == -ENXIO)
	233	return -ENODATA; /* Beyond EOF */
	234	if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
	235	return -ENOBUFS; /* Error. */
	236
	237	/* Round away partial blocks */
	238	off = round_up(off, granularity);
	239	off2 = round_down(off2, granularity);
	240	if (off2 <= off)
	241	return -ENODATA;
	242
	243	*_data_start = off;
	244	if (off2 > start + len)
	245	*_data_len = len;
	246	else
	247	*_data_len = off2 - off;
	248	return 0;
	249	}
	250
047487c9 DH	251	/*
	252	* Handle completion of a write to the cache.
	253	*/
	254	static void cachefiles_write_complete(struct kiocb *iocb, long ret)
	255	{
	256	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
	257	struct cachefiles_object *object = ki->object;
	258	struct inode *inode = file_inode(ki->iocb.ki_filp);
	259
	260	_enter("%ld", ret);
	261
	262	/* Tell lockdep we inherited freeze protection from submission thread */
	263	__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
	264	__sb_end_write(inode->i_sb, SB_FREEZE_WRITE);
	265
	266	if (ret < 0)
	267	trace_cachefiles_io_error(object, inode, ret,
	268	cachefiles_trace_write_error);
	269
	270	atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);
	271	set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags);
	272	if (ki->term_func)
	273	ki->term_func(ki->term_func_priv, ret, ki->was_async);
	274	cachefiles_put_kiocb(ki);
	275	}
	276
	277	/*
	278	* Initiate a write to the cache.
	279	*/
a06fac15 JX	280	int __cachefiles_write(struct cachefiles_object *object,
	281	struct file *file,
	282	loff_t start_pos,
	283	struct iov_iter *iter,
	284	netfs_io_terminated_t term_func,
	285	void *term_func_priv)
047487c9	286	{
047487c9 DH	287	struct cachefiles_cache *cache;
	288	struct cachefiles_kiocb *ki;
	289	struct inode *inode;
047487c9	290	unsigned int old_nofs;
a06fac15	291	ssize_t ret;
047487c9 DH	292	size_t len = iov_iter_count(iter);
047487c9 DH	293
047487c9	294	fscache_count_write();
047487c9	295	cache = object->volume->cache;
047487c9 DH	296
	297	_enter("%pD,%li,%llx,%zx/%llx",
	298	file, file_inode(file)->i_ino, start_pos, len,
	299	i_size_read(file_inode(file)));
	300
047487c9	301	ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
a06fac15 JX	302	if (!ki) {
	303	if (term_func)
	304	term_func(term_func_priv, -ENOMEM, false);
	305	return -ENOMEM;
	306	}
047487c9 DH	307
	308	refcount_set(&ki->ki_refcnt, 2);
	309	ki->iocb.ki_filp = file;
	310	ki->iocb.ki_pos = start_pos;
	311	ki->iocb.ki_flags = IOCB_DIRECT \| IOCB_WRITE;
047487c9 DH	312	ki->iocb.ki_ioprio = get_current_ioprio();
047487c9 DH	313	ki->object = object;
047487c9 DH	314	ki->start = start_pos;
	315	ki->len = len;
	316	ki->term_func = term_func;
	317	ki->term_func_priv = term_func_priv;
	318	ki->was_async = true;
5638b067	319	ki->b_writing = (len + (1 << cache->bshift) - 1) >> cache->bshift;
047487c9 DH	320
	321	if (ki->term_func)
	322	ki->iocb.ki_complete = cachefiles_write_complete;
	323	atomic_long_add(ki->b_writing, &cache->b_writing);
	324
	325	/* Open-code file_start_write here to grab freeze protection, which
	326	* will be released by another thread in aio_complete_rw(). Fool
	327	* lockdep by telling it the lock got released so that it doesn't
	328	* complain about the held lock when we return to userspace.
	329	*/
	330	inode = file_inode(file);
	331	__sb_start_write(inode->i_sb, SB_FREEZE_WRITE);
	332	__sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);
	333
	334	get_file(ki->iocb.ki_filp);
	335	cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
	336
	337	trace_cachefiles_write(object, inode, ki->iocb.ki_pos, len);
	338	old_nofs = memalloc_nofs_save();
	339	ret = cachefiles_inject_write_error();
	340	if (ret == 0)
	341	ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
	342	memalloc_nofs_restore(old_nofs);
	343	switch (ret) {
	344	case -EIOCBQUEUED:
	345	goto in_progress;
	346
	347	case -ERESTARTSYS:
	348	case -ERESTARTNOINTR:
	349	case -ERESTARTNOHAND:
	350	case -ERESTART_RESTARTBLOCK:
	351	/* There's no easy way to restart the syscall since other AIO's
	352	* may be already running. Just fail this IO with EINTR.
	353	*/
	354	ret = -EINTR;
	355	fallthrough;
	356	default:
	357	ki->was_async = false;
	358	cachefiles_write_complete(&ki->iocb, ret);
	359	if (ret > 0)
	360	ret = 0;
	361	break;
	362	}
	363
	364	in_progress:
	365	cachefiles_put_kiocb(ki);
	366	_leave(" = %zd", ret);
	367	return ret;
a06fac15	368	}
047487c9	369
a06fac15 JX	370	static int cachefiles_write(struct netfs_cache_resources *cres,
	371	loff_t start_pos,
	372	struct iov_iter *iter,
	373	netfs_io_terminated_t term_func,
	374	void *term_func_priv)
	375	{
	376	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
	377	if (term_func)
	378	term_func(term_func_priv, -ENOBUFS, false);
	379	return -ENOBUFS;
	380	}
	381
	382	return __cachefiles_write(cachefiles_cres_object(cres),
	383	cachefiles_cres_file(cres),
	384	start_pos, iter,
	385	term_func, term_func_priv);
047487c9 DH	386	}
047487c9 DH	387
86692475 JX	388	static inline enum netfs_io_source
	389	cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
	390	loff_t start, size_t *_len, loff_t i_size,
	391	unsigned long *_flags, ino_t netfs_ino)
047487c9 DH	392	{
047487c9 DH	393	enum cachefiles_prepare_read_trace why;
86692475	394	struct cachefiles_object *object = NULL;
047487c9 DH	395	struct cachefiles_cache *cache;
	396	struct fscache_cookie *cookie = fscache_cres_cookie(cres);
	397	const struct cred *saved_cred;
	398	struct file *file = cachefiles_cres_file(cres);
6a19114b	399	enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
86692475	400	size_t len = *_len;
047487c9 DH	401	loff_t off, to;
047487c9 DH	402	ino_t ino = file ? file_inode(file)->i_ino : 0;
9032b6e8	403	int rc;
047487c9	404
86692475	405	_enter("%zx @%llx/%llx", len, start, i_size);
047487c9	406
86692475	407	if (start >= i_size) {
047487c9 DH	408	ret = NETFS_FILL_WITH_ZEROES;
	409	why = cachefiles_trace_read_after_eof;
	410	goto out_no_object;
	411	}
	412
	413	if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
86692475	414	__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
047487c9	415	why = cachefiles_trace_read_no_data;
86692475	416	if (!test_bit(NETFS_SREQ_ONDEMAND, _flags))
9032b6e8	417	goto out_no_object;
047487c9 DH	418	}
	419
	420	/* The object and the file may be being created in the background. */
	421	if (!file) {
	422	why = cachefiles_trace_read_no_file;
	423	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
	424	goto out_no_object;
	425	file = cachefiles_cres_file(cres);
	426	if (!file)
	427	goto out_no_object;
	428	ino = file_inode(file)->i_ino;
	429	}
	430
	431	object = cachefiles_cres_object(cres);
	432	cache = object->volume->cache;
	433	cachefiles_begin_secure(cache, &saved_cred);
9032b6e8	434	retry:
047487c9 DH	435	off = cachefiles_inject_read_error();
047487c9 DH	436	if (off == 0)
86692475	437	off = vfs_llseek(file, start, SEEK_DATA);
047487c9 DH	438	if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
	439	if (off == (loff_t)-ENXIO) {
	440	why = cachefiles_trace_read_seek_nxio;
	441	goto download_and_store;
	442	}
	443	trace_cachefiles_io_error(object, file_inode(file), off,
	444	cachefiles_trace_seek_error);
	445	why = cachefiles_trace_read_seek_error;
	446	goto out;
	447	}
	448
86692475	449	if (off >= start + len) {
047487c9 DH	450	why = cachefiles_trace_read_found_hole;
	451	goto download_and_store;
	452	}
	453
86692475	454	if (off > start) {
047487c9	455	off = round_up(off, cache->bsize);
86692475 JX	456	len = off - start;
86692475 JX	457	*_len = len;
047487c9 DH	458	why = cachefiles_trace_read_found_part;
	459	goto download_and_store;
	460	}
	461
	462	to = cachefiles_inject_read_error();
	463	if (to == 0)
86692475	464	to = vfs_llseek(file, start, SEEK_HOLE);
047487c9 DH	465	if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
	466	trace_cachefiles_io_error(object, file_inode(file), to,
	467	cachefiles_trace_seek_error);
	468	why = cachefiles_trace_read_seek_error;
	469	goto out;
	470	}
	471
86692475 JX	472	if (to < start + len) {
86692475 JX	473	if (start + len >= i_size)
047487c9 DH	474	to = round_up(to, cache->bsize);
	475	else
	476	to = round_down(to, cache->bsize);
86692475 JX	477	len = to - start;
86692475 JX	478	*_len = len;
047487c9 DH	479	}
	480
	481	why = cachefiles_trace_read_have_data;
	482	ret = NETFS_READ_FROM_CACHE;
	483	goto out;
	484
	485	download_and_store:
86692475 JX	486	__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
	487	if (test_bit(NETFS_SREQ_ONDEMAND, _flags)) {
	488	rc = cachefiles_ondemand_read(object, start, len);
9032b6e8	489	if (!rc) {
86692475	490	__clear_bit(NETFS_SREQ_ONDEMAND, _flags);
9032b6e8 JX	491	goto retry;
	492	}
	493	ret = NETFS_INVALID_READ;
	494	}
047487c9 DH	495	out:
	496	cachefiles_end_secure(cache, saved_cred);
	497	out_no_object:
86692475	498	trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
047487c9 DH	499	return ret;
	500	}
	501
86692475 JX	502	/*
	503	* Prepare a read operation, shortening it to a cached/uncached
	504	* boundary as appropriate.
	505	*/
	506	static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
	507	loff_t i_size)
	508	{
	509	return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
	510	subreq->start, &subreq->len, i_size,
	511	&subreq->flags, subreq->rreq->inode->i_ino);
	512	}
	513
	514	/*
	515	* Prepare an on-demand read operation, shortening it to a cached/uncached
	516	* boundary as appropriate.
	517	*/
	518	static enum netfs_io_source
	519	cachefiles_prepare_ondemand_read(struct netfs_cache_resources *cres,
	520	loff_t start, size_t *_len, loff_t i_size,
	521	unsigned long *_flags, ino_t ino)
	522	{
	523	return cachefiles_do_prepare_read(cres, start, _len, i_size, _flags, ino);
	524	}
	525
047487c9 DH	526	/*
	527	* Prepare for a write to occur.
	528	*/
a06fac15 JX	529	int __cachefiles_prepare_write(struct cachefiles_object *object,
	530	struct file *file,
	531	loff_t _start, size_t _len,
	532	bool no_space_allocated_yet)
047487c9	533	{
047487c9	534	struct cachefiles_cache *cache = object->volume->cache;
047487c9 DH	535	loff_t start = *_start, pos;
	536	size_t len = *_len, down;
	537	int ret;
	538
	539	/* Round to DIO size */
	540	down = start - round_down(start, PAGE_SIZE);
	541	*_start = start - down;
	542	*_len = round_up(down + len, PAGE_SIZE);
	543
	544	/* We need to work out whether there's sufficient disk space to perform
	545	* the write - but we can skip that check if we have space already
	546	* allocated.
	547	*/
	548	if (no_space_allocated_yet)
	549	goto check_space;
	550
	551	pos = cachefiles_inject_read_error();
	552	if (pos == 0)
	553	pos = vfs_llseek(file, *_start, SEEK_DATA);
	554	if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
	555	if (pos == -ENXIO)
	556	goto check_space; /* Unallocated tail */
	557	trace_cachefiles_io_error(object, file_inode(file), pos,
	558	cachefiles_trace_seek_error);
	559	return pos;
	560	}
	561	if ((u64)pos >= (u64)_start + _len)
	562	goto check_space; /* Unallocated region */
	563
	564	/* We have a block that's at least partially filled - if we're low on
	565	* space, we need to see if it's fully allocated. If it's not, we may
	566	* want to cull it.
	567	*/
3929eca7 DH	568	if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
3929eca7 DH	569	cachefiles_has_space_check) == 0)
047487c9 DH	570	return 0; /* Enough space to simply overwrite the whole block */
	571
	572	pos = cachefiles_inject_read_error();
	573	if (pos == 0)
	574	pos = vfs_llseek(file, *_start, SEEK_HOLE);
	575	if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
	576	trace_cachefiles_io_error(object, file_inode(file), pos,
	577	cachefiles_trace_seek_error);
	578	return pos;
	579	}
	580	if ((u64)pos >= (u64)_start + _len)
	581	return 0; /* Fully allocated */
	582
	583	/* Partially allocated, but insufficient space: cull. */
3929eca7	584	fscache_count_no_write_space();
047487c9 DH	585	ret = cachefiles_inject_remove_error();
	586	if (ret == 0)
	587	ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE,
	588	_start, _len);
	589	if (ret < 0) {
	590	trace_cachefiles_io_error(object, file_inode(file), ret,
	591	cachefiles_trace_fallocate_error);
	592	cachefiles_io_error_obj(object,
	593	"CacheFiles: fallocate failed (%d)\n", ret);
	594	ret = -EIO;
	595	}
	596
	597	return ret;
	598
	599	check_space:
3929eca7 DH	600	return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
3929eca7 DH	601	cachefiles_has_space_for_write);
047487c9 DH	602	}
	603
	604	static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
	605	loff_t _start, size_t _len, loff_t i_size,
	606	bool no_space_allocated_yet)
	607	{
	608	struct cachefiles_object *object = cachefiles_cres_object(cres);
	609	struct cachefiles_cache *cache = object->volume->cache;
	610	const struct cred *saved_cred;
	611	int ret;
	612
	613	if (!cachefiles_cres_file(cres)) {
	614	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
	615	return -ENOBUFS;
	616	if (!cachefiles_cres_file(cres))
	617	return -ENOBUFS;
	618	}
	619
	620	cachefiles_begin_secure(cache, &saved_cred);
a06fac15 JX	621	ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
a06fac15 JX	622	_start, _len,
047487c9 DH	623	no_space_allocated_yet);
	624	cachefiles_end_secure(cache, saved_cred);
	625	return ret;
	626	}
	627
287fd611 DH	628	/*
	629	* Clean up an operation.
	630	*/
	631	static void cachefiles_end_operation(struct netfs_cache_resources *cres)
	632	{
	633	struct file *file = cachefiles_cres_file(cres);
	634
	635	if (file)
	636	fput(file);
	637	fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end);
	638	}
	639
	640	static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
	641	.end_operation = cachefiles_end_operation,
047487c9 DH	642	.read = cachefiles_read,
	643	.write = cachefiles_write,
	644	.prepare_read = cachefiles_prepare_read,
	645	.prepare_write = cachefiles_prepare_write,
86692475	646	.prepare_ondemand_read = cachefiles_prepare_ondemand_read,
bee9f655	647	.query_occupancy = cachefiles_query_occupancy,
287fd611 DH	648	};
	649
	650	/*
	651	* Open the cache file when beginning a cache operation.
	652	*/
	653	bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
	654	enum fscache_want_state want_state)
	655	{
	656	struct cachefiles_object *object = cachefiles_cres_object(cres);
	657
	658	if (!cachefiles_cres_file(cres)) {
	659	cres->ops = &cachefiles_netfs_cache_ops;
	660	if (object->file) {
	661	spin_lock(&object->lock);
	662	if (!cres->cache_priv2 && object->file)
	663	cres->cache_priv2 = get_file(object->file);
	664	spin_unlock(&object->lock);
	665	}
	666	}
	667
	668	if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) {
	669	pr_err("failed to get cres->file\n");
	670	return false;
	671	}
	672
	673	return true;
	674	}