[linux-2.6-block.git] / fs / nfs / pagelist.c

/*
 * linux/fs/nfs/pagelist.c
 *
 * A set of helper functions for managing NFS read and write requests.
 * The main purpose of these routines is to provide support for the
 * coalescing of several requests into a single RPC call.
 *
 * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
 *
 */

#include <linux/slab.h>
#include <linux/file.h>
#include <linux/sched.h>
#include <linux/sunrpc/clnt.h>
#include <linux/nfs.h>
#include <linux/nfs3.h>
#include <linux/nfs4.h>
#include <linux/nfs_page.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/export.h>

#include "internal.h"
#include "pnfs.h"

static struct kmem_cache *nfs_page_cachep;

static inline struct nfs_page *
nfs_page_alloc(void)
{
	struct nfs_page	*p = kmem_cache_zalloc(nfs_page_cachep, GFP_KERNEL);
	if (p)
		INIT_LIST_HEAD(&p->wb_list);
	return p;
}

static inline void
nfs_page_free(struct nfs_page *p)
{
	kmem_cache_free(nfs_page_cachep, p);
}

/**
 * nfs_create_request - Create an NFS read/write request.
 * @ctx: open context to use
 * @inode: inode to which the request is attached
 * @page: page to write
 * @offset: starting offset within the page for the write
 * @count: number of bytes to read/write
 *
 * The page must be locked by the caller. This makes sure we never
 * create two different requests for the same page.
 * User should ensure it is safe to sleep in this function.
 */
struct nfs_page *
nfs_create_request(struct nfs_open_context *ctx, struct inode *inode,
		   struct page *page,
		   unsigned int offset, unsigned int count)
{
	struct nfs_page		*req;

	/* try to allocate the request struct */
	req = nfs_page_alloc();
	if (req == NULL)
		return ERR_PTR(-ENOMEM);

	/* get lock context early so we can deal with alloc failures */
	req->wb_lock_context = nfs_get_lock_context(ctx);
	if (req->wb_lock_context == NULL) {
		nfs_page_free(req);
		return ERR_PTR(-ENOMEM);
	}

	/* Initialize the request struct. Initially, we assume a
	 * long write-back delay. This will be adjusted in
	 * update_nfs_request below if the region is not locked. */
	req->wb_page    = page;
	atomic_set(&req->wb_complete, 0);
	req->wb_index	= page->index;
	page_cache_get(page);
	BUG_ON(PagePrivate(page));
	BUG_ON(!PageLocked(page));
	BUG_ON(page->mapping->host != inode);
	req->wb_offset  = offset;
	req->wb_pgbase	= offset;
	req->wb_bytes   = count;
	req->wb_context = get_nfs_open_context(ctx);
	kref_init(&req->wb_kref);
	return req;
}

/**
 * nfs_unlock_request - Unlock request and wake up sleepers.
 * @req:
 */
void nfs_unlock_request(struct nfs_page *req)
{
	if (!NFS_WBACK_BUSY(req)) {
		printk(KERN_ERR "NFS: Invalid unlock attempted\n");
		BUG();
	}
	smp_mb__before_clear_bit();
	clear_bit(PG_BUSY, &req->wb_flags);
	smp_mb__after_clear_bit();
	wake_up_bit(&req->wb_flags, PG_BUSY);
	nfs_release_request(req);
}

/*
 * nfs_clear_request - Free up all resources allocated to the request
 * @req:
 *
 * Release page and open context resources associated with a read/write
 * request after it has completed.
 */
static void nfs_clear_request(struct nfs_page *req)
{
	struct page *page = req->wb_page;
	struct nfs_open_context *ctx = req->wb_context;
	struct nfs_lock_context *l_ctx = req->wb_lock_context;

	if (page != NULL) {
		page_cache_release(page);
		req->wb_page = NULL;
	}
	if (l_ctx != NULL) {
		nfs_put_lock_context(l_ctx);
		req->wb_lock_context = NULL;
	}
	if (ctx != NULL) {
		put_nfs_open_context(ctx);
		req->wb_context = NULL;
	}
}


/**
 * nfs_release_request - Release the count on an NFS read/write request
 * @req: request to release
 *
 * Note: Should never be called with the spinlock held!
 */
static void nfs_free_request(struct kref *kref)
{
	struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);

	/* Release struct file and open context */
	nfs_clear_request(req);
	nfs_page_free(req);
}

void nfs_release_request(struct nfs_page *req)
{
	kref_put(&req->wb_kref, nfs_free_request);
}

static int nfs_wait_bit_uninterruptible(void *word)
{
	io_schedule();
	return 0;
}

/**
 * nfs_wait_on_request - Wait for a request to complete.
 * @req: request to wait upon.
 *
 * Interruptible by fatal signals only.
 * The user is responsible for holding a count on the request.
 */
int
nfs_wait_on_request(struct nfs_page *req)
{
	return wait_on_bit(&req->wb_flags, PG_BUSY,
			nfs_wait_bit_uninterruptible,
			TASK_UNINTERRUPTIBLE);
}

bool nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, struct nfs_page *prev, struct nfs_page *req)
{
	/*
	 * FIXME: ideally we should be able to coalesce all requests
	 * that are not block boundary aligned, but currently this
	 * is problematic for the case of bsize < PAGE_CACHE_SIZE,
	 * since nfs_flush_multi and nfs_pagein_multi assume you
	 * can have only one struct nfs_page.
	 */
	if (desc->pg_bsize < PAGE_SIZE)
		return 0;

	return desc->pg_count + req->wb_bytes <= desc->pg_bsize;
}
EXPORT_SYMBOL_GPL(nfs_generic_pg_test);

/**
 * nfs_pageio_init - initialise a page io descriptor
 * @desc: pointer to descriptor
 * @inode: pointer to inode
 * @doio: pointer to io function
 * @bsize: io block size
 * @io_flags: extra parameters for the io function
 */
void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
		     struct inode *inode,
		     const struct nfs_pageio_ops *pg_ops,
		     size_t bsize,
		     int io_flags)
{
	INIT_LIST_HEAD(&desc->pg_list);
	desc->pg_bytes_written = 0;
	desc->pg_count = 0;
	desc->pg_bsize = bsize;
	desc->pg_base = 0;
	desc->pg_moreio = 0;
	desc->pg_recoalesce = 0;
	desc->pg_inode = inode;
	desc->pg_ops = pg_ops;
	desc->pg_ioflags = io_flags;
	desc->pg_error = 0;
	desc->pg_lseg = NULL;
}

/**
 * nfs_can_coalesce_requests - test two requests for compatibility
 * @prev: pointer to nfs_page
 * @req: pointer to nfs_page
 *
 * The nfs_page structures 'prev' and 'req' are compared to ensure that the
 * page data area they describe is contiguous, and that their RPC
 * credentials, NFSv4 open state, and lockowners are the same.
 *
 * Return 'true' if this is the case, else return 'false'.
 */
static bool nfs_can_coalesce_requests(struct nfs_page *prev,
				      struct nfs_page *req,
				      struct nfs_pageio_descriptor *pgio)
{
	if (req->wb_context->cred != prev->wb_context->cred)
		return false;
	if (req->wb_lock_context->lockowner != prev->wb_lock_context->lockowner)
		return false;
	if (req->wb_context->state != prev->wb_context->state)
		return false;
	if (req->wb_index != (prev->wb_index + 1))
		return false;
	if (req->wb_pgbase != 0)
		return false;
	if (prev->wb_pgbase + prev->wb_bytes != PAGE_CACHE_SIZE)
		return false;
	return pgio->pg_ops->pg_test(pgio, prev, req);
}

/**
 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
 * @desc: destination io descriptor
 * @req: request
 *
 * Returns true if the request 'req' was successfully coalesced into the
 * existing list of pages 'desc'.
 */
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
				     struct nfs_page *req)
{
	if (desc->pg_count != 0) {
		struct nfs_page *prev;

		prev = nfs_list_entry(desc->pg_list.prev);
		if (!nfs_can_coalesce_requests(prev, req, desc))
			return 0;
	} else {
		if (desc->pg_ops->pg_init)
			desc->pg_ops->pg_init(desc, req);
		desc->pg_base = req->wb_pgbase;
	}
	nfs_list_remove_request(req);
	nfs_list_add_request(req, &desc->pg_list);
	desc->pg_count += req->wb_bytes;
	return 1;
}

/*
 * Helper for nfs_pageio_add_request and nfs_pageio_complete
 */
static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
{
	if (!list_empty(&desc->pg_list)) {
		int error = desc->pg_ops->pg_doio(desc);
		if (error < 0)
			desc->pg_error = error;
		else
			desc->pg_bytes_written += desc->pg_count;
	}
	if (list_empty(&desc->pg_list)) {
		desc->pg_count = 0;
		desc->pg_base = 0;
	}
}

/**
 * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
 * @desc: destination io descriptor
 * @req: request
 *
 * Returns true if the request 'req' was successfully coalesced into the
 * existing list of pages 'desc'.
 */
static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
			   struct nfs_page *req)
{
	while (!nfs_pageio_do_add_request(desc, req)) {
		desc->pg_moreio = 1;
		nfs_pageio_doio(desc);
		if (desc->pg_error < 0)
			return 0;
		desc->pg_moreio = 0;
		if (desc->pg_recoalesce)
			return 0;
	}
	return 1;
}

static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
{
	LIST_HEAD(head);

	do {
		list_splice_init(&desc->pg_list, &head);
		desc->pg_bytes_written -= desc->pg_count;
		desc->pg_count = 0;
		desc->pg_base = 0;
		desc->pg_recoalesce = 0;

		while (!list_empty(&head)) {
			struct nfs_page *req;

			req = list_first_entry(&head, struct nfs_page, wb_list);
			nfs_list_remove_request(req);
			if (__nfs_pageio_add_request(desc, req))
				continue;
			if (desc->pg_error < 0)
				return 0;
			break;
		}
	} while (desc->pg_recoalesce);
	return 1;
}

int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
		struct nfs_page *req)
{
	int ret;

	do {
		ret = __nfs_pageio_add_request(desc, req);
		if (ret)
			break;
		if (desc->pg_error < 0)
			break;
		ret = nfs_do_recoalesce(desc);
	} while (ret);
	return ret;
}

/**
 * nfs_pageio_complete - Complete I/O on an nfs_pageio_descriptor
 * @desc: pointer to io descriptor
 */
void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
{
	for (;;) {
		nfs_pageio_doio(desc);
		if (!desc->pg_recoalesce)
			break;
		if (!nfs_do_recoalesce(desc))
			break;
	}
}

/**
 * nfs_pageio_cond_complete - Conditional I/O completion
 * @desc: pointer to io descriptor
 * @index: page index
 *
 * It is important to ensure that processes don't try to take locks
 * on non-contiguous ranges of pages as that might deadlock. This
 * function should be called before attempting to wait on a locked
 * nfs_page. It will complete the I/O if the page index 'index'
 * is not contiguous with the existing list of pages in 'desc'.
 */
void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
{
	if (!list_empty(&desc->pg_list)) {
		struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev);
		if (index != prev->wb_index + 1)
			nfs_pageio_complete(desc);
	}
}

int __init nfs_init_nfspagecache(void)
{
	nfs_page_cachep = kmem_cache_create("nfs_page",
					    sizeof(struct nfs_page),
					    0, SLAB_HWCACHE_ALIGN,
					    NULL);
	if (nfs_page_cachep == NULL)
		return -ENOMEM;

	return 0;
}

void nfs_destroy_nfspagecache(void)
{
	kmem_cache_destroy(nfs_page_cachep);
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/fs/nfs/pagelist.c
	3	*
	4	* A set of helper functions for managing NFS read and write requests.
	5	* The main purpose of these routines is to provide support for the
	6	* coalescing of several requests into a single RPC call.
	7	*
	8	* Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
	9	*
	10	*/
	11
1da177e4 LT	12	#include <linux/slab.h>
1da177e4 LT	13	#include <linux/file.h>
e8edc6e0	14	#include <linux/sched.h>
1da177e4	15	#include <linux/sunrpc/clnt.h>
1313e603	16	#include <linux/nfs.h>
1da177e4 LT	17	#include <linux/nfs3.h>
	18	#include <linux/nfs4.h>
	19	#include <linux/nfs_page.h>
	20	#include <linux/nfs_fs.h>
	21	#include <linux/nfs_mount.h>
afeacc8c	22	#include <linux/export.h>
1da177e4	23
8d5658c9	24	#include "internal.h"
bae724ef	25	#include "pnfs.h"
8d5658c9	26
e18b890b	27	static struct kmem_cache *nfs_page_cachep;
1da177e4 LT	28
	29	static inline struct nfs_page *
	30	nfs_page_alloc(void)
	31	{
72895b1a JJ	32	struct nfs_page *p = kmem_cache_zalloc(nfs_page_cachep, GFP_KERNEL);
72895b1a JJ	33	if (p)
1da177e4	34	INIT_LIST_HEAD(&p->wb_list);
1da177e4 LT	35	return p;
	36	}
	37
	38	static inline void
	39	nfs_page_free(struct nfs_page *p)
	40	{
	41	kmem_cache_free(nfs_page_cachep, p);
	42	}
	43
	44	/**
	45	* nfs_create_request - Create an NFS read/write request.
c02f557d	46	* @ctx: open context to use
1da177e4 LT	47	* @inode: inode to which the request is attached
	48	* @page: page to write
	49	* @offset: starting offset within the page for the write
	50	* @count: number of bytes to read/write
	51	*
	52	* The page must be locked by the caller. This makes sure we never
a19b89ca	53	* create two different requests for the same page.
1da177e4 LT	54	* User should ensure it is safe to sleep in this function.
	55	*/
	56	struct nfs_page *
	57	nfs_create_request(struct nfs_open_context ctx, struct inode inode,
	58	struct page *page,
	59	unsigned int offset, unsigned int count)
	60	{
1da177e4 LT	61	struct nfs_page *req;
1da177e4 LT	62
18eb8842 TM	63	/* try to allocate the request struct */
	64	req = nfs_page_alloc();
	65	if (req == NULL)
	66	return ERR_PTR(-ENOMEM);
1da177e4	67
015f0212 JL	68	/* get lock context early so we can deal with alloc failures */
	69	req->wb_lock_context = nfs_get_lock_context(ctx);
	70	if (req->wb_lock_context == NULL) {
	71	nfs_page_free(req);
	72	return ERR_PTR(-ENOMEM);
	73	}
	74
1da177e4 LT	75	/* Initialize the request struct. Initially, we assume a
	76	* long write-back delay. This will be adjusted in
	77	* update_nfs_request below if the region is not locked. */
	78	req->wb_page = page;
	79	atomic_set(&req->wb_complete, 0);
	80	req->wb_index = page->index;
	81	page_cache_get(page);
cd52ed35 TM	82	BUG_ON(PagePrivate(page));
	83	BUG_ON(!PageLocked(page));
	84	BUG_ON(page->mapping->host != inode);
1da177e4 LT	85	req->wb_offset = offset;
	86	req->wb_pgbase = offset;
	87	req->wb_bytes = count;
1da177e4	88	req->wb_context = get_nfs_open_context(ctx);
c03b4024	89	kref_init(&req->wb_kref);
1da177e4 LT	90	return req;
	91	}
	92
	93	/**
	94	* nfs_unlock_request - Unlock request and wake up sleepers.
	95	* @req:
	96	*/
	97	void nfs_unlock_request(struct nfs_page *req)
	98	{
	99	if (!NFS_WBACK_BUSY(req)) {
	100	printk(KERN_ERR "NFS: Invalid unlock attempted\n");
	101	BUG();
	102	}
	103	smp_mb__before_clear_bit();
	104	clear_bit(PG_BUSY, &req->wb_flags);
	105	smp_mb__after_clear_bit();
464a98bd	106	wake_up_bit(&req->wb_flags, PG_BUSY);
1da177e4 LT	107	nfs_release_request(req);
	108	}
	109
4d65c520	110	/*
1da177e4 LT	111	* nfs_clear_request - Free up all resources allocated to the request
	112	* @req:
	113	*
bb6fbc45 TM	114	* Release page and open context resources associated with a read/write
bb6fbc45 TM	115	* request after it has completed.
1da177e4	116	*/
4d65c520	117	static void nfs_clear_request(struct nfs_page *req)
1da177e4	118	{
cd52ed35	119	struct page *page = req->wb_page;
bb6fbc45	120	struct nfs_open_context *ctx = req->wb_context;
f11ac8db	121	struct nfs_lock_context *l_ctx = req->wb_lock_context;
bb6fbc45	122
cd52ed35	123	if (page != NULL) {
cd52ed35	124	page_cache_release(page);
1da177e4 LT	125	req->wb_page = NULL;
1da177e4 LT	126	}
f11ac8db TM	127	if (l_ctx != NULL) {
	128	nfs_put_lock_context(l_ctx);
	129	req->wb_lock_context = NULL;
	130	}
bb6fbc45 TM	131	if (ctx != NULL) {
	132	put_nfs_open_context(ctx);
	133	req->wb_context = NULL;
	134	}
1da177e4 LT	135	}
	136
	137
	138	/**
	139	* nfs_release_request - Release the count on an NFS read/write request
	140	* @req: request to release
	141	*
	142	* Note: Should never be called with the spinlock held!
	143	*/
c03b4024	144	static void nfs_free_request(struct kref *kref)
1da177e4	145	{
c03b4024	146	struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
1da177e4	147
bb6fbc45	148	/* Release struct file and open context */
1da177e4	149	nfs_clear_request(req);
1da177e4 LT	150	nfs_page_free(req);
	151	}
	152
c03b4024 TM	153	void nfs_release_request(struct nfs_page *req)
	154	{
	155	kref_put(&req->wb_kref, nfs_free_request);
	156	}
	157
9f557cd8 TM	158	static int nfs_wait_bit_uninterruptible(void *word)
	159	{
	160	io_schedule();
	161	return 0;
	162	}
	163
1da177e4 LT	164	/**
	165	* nfs_wait_on_request - Wait for a request to complete.
	166	* @req: request to wait upon.
	167	*
150030b7	168	* Interruptible by fatal signals only.
1da177e4 LT	169	* The user is responsible for holding a count on the request.
	170	*/
	171	int
	172	nfs_wait_on_request(struct nfs_page *req)
	173	{
9f557cd8 TM	174	return wait_on_bit(&req->wb_flags, PG_BUSY,
	175	nfs_wait_bit_uninterruptible,
	176	TASK_UNINTERRUPTIBLE);
1da177e4 LT	177	}
1da177e4 LT	178
19345cb2	179	bool nfs_generic_pg_test(struct nfs_pageio_descriptor desc, struct nfs_page prev, struct nfs_page *req)
5b36c7dc BH	180	{
	181	/*
	182	* FIXME: ideally we should be able to coalesce all requests
	183	* that are not block boundary aligned, but currently this
	184	* is problematic for the case of bsize < PAGE_CACHE_SIZE,
	185	* since nfs_flush_multi and nfs_pagein_multi assume you
	186	* can have only one struct nfs_page.
	187	*/
	188	if (desc->pg_bsize < PAGE_SIZE)
	189	return 0;
	190
	191	return desc->pg_count + req->wb_bytes <= desc->pg_bsize;
	192	}
19345cb2	193	EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
5b36c7dc	194
1da177e4	195	/**
d8a5ad75 TM	196	* nfs_pageio_init - initialise a page io descriptor
d8a5ad75 TM	197	* @desc: pointer to descriptor
bcb71bba TM	198	* @inode: pointer to inode
	199	* @doio: pointer to io function
	200	* @bsize: io block size
	201	* @io_flags: extra parameters for the io function
d8a5ad75	202	*/
bcb71bba TM	203	void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
bcb71bba TM	204	struct inode *inode,
1751c363	205	const struct nfs_pageio_ops *pg_ops,
84dde76c	206	size_t bsize,
bcb71bba	207	int io_flags)
d8a5ad75 TM	208	{
d8a5ad75 TM	209	INIT_LIST_HEAD(&desc->pg_list);
bcb71bba	210	desc->pg_bytes_written = 0;
d8a5ad75 TM	211	desc->pg_count = 0;
	212	desc->pg_bsize = bsize;
	213	desc->pg_base = 0;
b31268ac	214	desc->pg_moreio = 0;
d9156f9f	215	desc->pg_recoalesce = 0;
bcb71bba	216	desc->pg_inode = inode;
1751c363	217	desc->pg_ops = pg_ops;
bcb71bba TM	218	desc->pg_ioflags = io_flags;
bcb71bba TM	219	desc->pg_error = 0;
94ad1c80	220	desc->pg_lseg = NULL;
d8a5ad75 TM	221	}
	222
	223	/**
	224	* nfs_can_coalesce_requests - test two requests for compatibility
	225	* @prev: pointer to nfs_page
	226	* @req: pointer to nfs_page
	227	*
	228	* The nfs_page structures 'prev' and 'req' are compared to ensure that the
	229	* page data area they describe is contiguous, and that their RPC
	230	* credentials, NFSv4 open state, and lockowners are the same.
	231	*
	232	* Return 'true' if this is the case, else return 'false'.
	233	*/
18ad0a9f BH	234	static bool nfs_can_coalesce_requests(struct nfs_page *prev,
	235	struct nfs_page *req,
	236	struct nfs_pageio_descriptor *pgio)
d8a5ad75 TM	237	{
d8a5ad75 TM	238	if (req->wb_context->cred != prev->wb_context->cred)
18ad0a9f	239	return false;
f11ac8db	240	if (req->wb_lock_context->lockowner != prev->wb_lock_context->lockowner)
18ad0a9f	241	return false;
d8a5ad75	242	if (req->wb_context->state != prev->wb_context->state)
18ad0a9f	243	return false;
d8a5ad75	244	if (req->wb_index != (prev->wb_index + 1))
18ad0a9f	245	return false;
d8a5ad75	246	if (req->wb_pgbase != 0)
18ad0a9f	247	return false;
d8a5ad75	248	if (prev->wb_pgbase + prev->wb_bytes != PAGE_CACHE_SIZE)
18ad0a9f	249	return false;
1751c363	250	return pgio->pg_ops->pg_test(pgio, prev, req);
d8a5ad75 TM	251	}
	252
	253	/**
bcb71bba	254	* nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
d8a5ad75 TM	255	* @desc: destination io descriptor
	256	* @req: request
	257	*
	258	* Returns true if the request 'req' was successfully coalesced into the
	259	* existing list of pages 'desc'.
	260	*/
bcb71bba TM	261	static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
bcb71bba TM	262	struct nfs_page *req)
d8a5ad75	263	{
d8a5ad75 TM	264	if (desc->pg_count != 0) {
	265	struct nfs_page *prev;
	266
d8a5ad75	267	prev = nfs_list_entry(desc->pg_list.prev);
94ad1c80	268	if (!nfs_can_coalesce_requests(prev, req, desc))
d8a5ad75	269	return 0;
5b36c7dc	270	} else {
d8007d4d TM	271	if (desc->pg_ops->pg_init)
d8007d4d TM	272	desc->pg_ops->pg_init(desc, req);
d8a5ad75	273	desc->pg_base = req->wb_pgbase;
5b36c7dc	274	}
d8a5ad75 TM	275	nfs_list_remove_request(req);
d8a5ad75 TM	276	nfs_list_add_request(req, &desc->pg_list);
5b36c7dc	277	desc->pg_count += req->wb_bytes;
d8a5ad75 TM	278	return 1;
	279	}
	280
bcb71bba TM	281	/*
	282	* Helper for nfs_pageio_add_request and nfs_pageio_complete
	283	*/
	284	static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
	285	{
	286	if (!list_empty(&desc->pg_list)) {
1751c363	287	int error = desc->pg_ops->pg_doio(desc);
bcb71bba TM	288	if (error < 0)
	289	desc->pg_error = error;
	290	else
	291	desc->pg_bytes_written += desc->pg_count;
	292	}
	293	if (list_empty(&desc->pg_list)) {
	294	desc->pg_count = 0;
	295	desc->pg_base = 0;
	296	}
	297	}
	298
	299	/**
	300	* nfs_pageio_add_request - Attempt to coalesce a request into a page list.
	301	* @desc: destination io descriptor
	302	* @req: request
	303	*
	304	* Returns true if the request 'req' was successfully coalesced into the
	305	* existing list of pages 'desc'.
	306	*/
d9156f9f	307	static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
8b09bee3	308	struct nfs_page *req)
bcb71bba TM	309	{
bcb71bba TM	310	while (!nfs_pageio_do_add_request(desc, req)) {
b31268ac	311	desc->pg_moreio = 1;
bcb71bba TM	312	nfs_pageio_doio(desc);
	313	if (desc->pg_error < 0)
	314	return 0;
b31268ac	315	desc->pg_moreio = 0;
d9156f9f TM	316	if (desc->pg_recoalesce)
d9156f9f TM	317	return 0;
bcb71bba TM	318	}
	319	return 1;
	320	}
	321
d9156f9f TM	322	static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
	323	{
	324	LIST_HEAD(head);
	325
	326	do {
	327	list_splice_init(&desc->pg_list, &head);
	328	desc->pg_bytes_written -= desc->pg_count;
	329	desc->pg_count = 0;
	330	desc->pg_base = 0;
	331	desc->pg_recoalesce = 0;
	332
	333	while (!list_empty(&head)) {
	334	struct nfs_page *req;
	335
	336	req = list_first_entry(&head, struct nfs_page, wb_list);
	337	nfs_list_remove_request(req);
	338	if (__nfs_pageio_add_request(desc, req))
	339	continue;
	340	if (desc->pg_error < 0)
	341	return 0;
	342	break;
	343	}
	344	} while (desc->pg_recoalesce);
	345	return 1;
	346	}
	347
	348	int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
	349	struct nfs_page *req)
	350	{
	351	int ret;
	352
	353	do {
	354	ret = __nfs_pageio_add_request(desc, req);
	355	if (ret)
	356	break;
	357	if (desc->pg_error < 0)
	358	break;
	359	ret = nfs_do_recoalesce(desc);
	360	} while (ret);
	361	return ret;
	362	}
	363
bcb71bba TM	364	/**
	365	* nfs_pageio_complete - Complete I/O on an nfs_pageio_descriptor
	366	* @desc: pointer to io descriptor
	367	*/
	368	void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
	369	{
d9156f9f TM	370	for (;;) {
	371	nfs_pageio_doio(desc);
	372	if (!desc->pg_recoalesce)
	373	break;
	374	if (!nfs_do_recoalesce(desc))
	375	break;
	376	}
bcb71bba TM	377	}
bcb71bba TM	378
7fe7f848 TM	379	/**
	380	* nfs_pageio_cond_complete - Conditional I/O completion
	381	* @desc: pointer to io descriptor
	382	* @index: page index
	383	*
	384	* It is important to ensure that processes don't try to take locks
	385	* on non-contiguous ranges of pages as that might deadlock. This
	386	* function should be called before attempting to wait on a locked
	387	* nfs_page. It will complete the I/O if the page index 'index'
	388	* is not contiguous with the existing list of pages in 'desc'.
	389	*/
	390	void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
	391	{
	392	if (!list_empty(&desc->pg_list)) {
	393	struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev);
	394	if (index != prev->wb_index + 1)
d9156f9f	395	nfs_pageio_complete(desc);
7fe7f848 TM	396	}
	397	}
	398
f7b422b1	399	int __init nfs_init_nfspagecache(void)
1da177e4 LT	400	{
	401	nfs_page_cachep = kmem_cache_create("nfs_page",
	402	sizeof(struct nfs_page),
	403	0, SLAB_HWCACHE_ALIGN,
20c2df83	404	NULL);
1da177e4 LT	405	if (nfs_page_cachep == NULL)
	406	return -ENOMEM;
	407
	408	return 0;
	409	}
	410
266bee88	411	void nfs_destroy_nfspagecache(void)
1da177e4	412	{
1a1d92c1	413	kmem_cache_destroy(nfs_page_cachep);
1da177e4 LT	414	}
1da177e4 LT	415