[linux-2.6-block.git] / mm / iov_iter.c

#include <linux/export.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *from;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (!fault_in_pages_writeable(buf, copy)) {
		kaddr = kmap_atomic(page);
		from = kaddr + offset;

		/* first chunk, usually the only one */
		left = __copy_to_user_inatomic(buf, from, copy);
		copy -= left;
		skip += copy;
		from += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = __copy_to_user_inatomic(buf, from, copy);
			copy -= left;
			skip = copy;
			from += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = from - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
	kaddr = kmap(page);
	from = kaddr + offset;
	left = __copy_to_user(buf, from, copy);
	copy -= left;
	skip += copy;
	from += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_to_user(buf, from, copy);
		copy -= left;
		skip = copy;
		from += copy;
		bytes -= copy;
	}
	kunmap(page);
done:
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}
EXPORT_SYMBOL(copy_page_to_iter);

size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *to;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (!fault_in_pages_readable(buf, copy)) {
		kaddr = kmap_atomic(page);
		to = kaddr + offset;

		/* first chunk, usually the only one */
		left = __copy_from_user_inatomic(to, buf, copy);
		copy -= left;
		skip += copy;
		to += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = __copy_from_user_inatomic(to, buf, copy);
			copy -= left;
			skip = copy;
			to += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = to - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */
	kaddr = kmap(page);
	to = kaddr + offset;
	left = __copy_from_user(to, buf, copy);
	copy -= left;
	skip += copy;
	to += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = __copy_from_user(to, buf, copy);
		copy -= left;
		skip = copy;
		to += copy;
		bytes -= copy;
	}
	kunmap(page);
done:
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}
EXPORT_SYMBOL(copy_page_from_iter);

static size_t __iovec_copy_from_user_inatomic(char *vaddr,
			const struct iovec *iov, size_t base, size_t bytes)
{
	size_t copied = 0, left = 0;

	while (bytes) {
		char __user *buf = iov->iov_base + base;
		int copy = min(bytes, iov->iov_len - base);

		base = 0;
		left = __copy_from_user_inatomic(vaddr, buf, copy);
		copied += copy;
		bytes -= copy;
		vaddr += copy;
		iov++;

		if (unlikely(left))
			break;
	}
	return copied - left;
}

/*
 * Copy as much as we can into the page and return the number of bytes which
 * were successfully copied.  If a fault is encountered then return the number of
 * bytes which were copied.
 */
size_t iov_iter_copy_from_user_atomic(struct page *page,
		struct iov_iter *i, unsigned long offset, size_t bytes)
{
	char *kaddr;
	size_t copied;

	kaddr = kmap_atomic(page);
	if (likely(i->nr_segs == 1)) {
		int left;
		char __user *buf = i->iov->iov_base + i->iov_offset;
		left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
		copied = bytes - left;
	} else {
		copied = __iovec_copy_from_user_inatomic(kaddr + offset,
						i->iov, i->iov_offset, bytes);
	}
	kunmap_atomic(kaddr);

	return copied;
}
EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);

void iov_iter_advance(struct iov_iter *i, size_t bytes)
{
	BUG_ON(i->count < bytes);

	if (likely(i->nr_segs == 1)) {
		i->iov_offset += bytes;
		i->count -= bytes;
	} else {
		const struct iovec *iov = i->iov;
		size_t base = i->iov_offset;
		unsigned long nr_segs = i->nr_segs;

		/*
		 * The !iov->iov_len check ensures we skip over unlikely
		 * zero-length segments (without overruning the iovec).
		 */
		while (bytes || unlikely(i->count && !iov->iov_len)) {
			int copy;

			copy = min(bytes, iov->iov_len - base);
			BUG_ON(!i->count || i->count < copy);
			i->count -= copy;
			bytes -= copy;
			base += copy;
			if (iov->iov_len == base) {
				iov++;
				nr_segs--;
				base = 0;
			}
		}
		i->iov = iov;
		i->iov_offset = base;
		i->nr_segs = nr_segs;
	}
}
EXPORT_SYMBOL(iov_iter_advance);

/*
 * Fault in the first iovec of the given iov_iter, to a maximum length
 * of bytes. Returns 0 on success, or non-zero if the memory could not be
 * accessed (ie. because it is an invalid address).
 *
 * writev-intensive code may want this to prefault several iovecs -- that
 * would be possible (callers must not rely on the fact that _only_ the
 * first iovec will be faulted with the current implementation).
 */
int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
{
	char __user *buf = i->iov->iov_base + i->iov_offset;
	bytes = min(bytes, i->iov->iov_len - i->iov_offset);
	return fault_in_pages_readable(buf, bytes);
}
EXPORT_SYMBOL(iov_iter_fault_in_readable);

/*
 * Return the count of just the current iov_iter segment.
 */
size_t iov_iter_single_seg_count(const struct iov_iter *i)
{
	const struct iovec *iov = i->iov;
	if (i->nr_segs == 1)
		return i->count;
	else
		return min(i->count, iov->iov_len - i->iov_offset);
}
EXPORT_SYMBOL(iov_iter_single_seg_count);

unsigned long iov_iter_alignment(const struct iov_iter *i)
{
	const struct iovec *iov = i->iov;
	unsigned long res;
	size_t size = i->count;
	size_t n;

	if (!size)
		return 0;

	res = (unsigned long)iov->iov_base + i->iov_offset;
	n = iov->iov_len - i->iov_offset;
	if (n >= size)
		return res | size;
	size -= n;
	res |= n;
	while (size > (++iov)->iov_len) {
		res |= (unsigned long)iov->iov_base | iov->iov_len;
		size -= iov->iov_len;
	}
	res |= (unsigned long)iov->iov_base | size;
	return res;
}
EXPORT_SYMBOL(iov_iter_alignment);

void iov_iter_init(struct iov_iter *i, int direction,
			const struct iovec *iov, unsigned long nr_segs,
			size_t count)
{
	/* It will get better.  Eventually... */
	if (segment_eq(get_fs(), KERNEL_DS))
		direction |= REQ_KERNEL;
	i->type = direction;
	i->iov = iov;
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
EXPORT_SYMBOL(iov_iter_init);

ssize_t iov_iter_get_pages(struct iov_iter *i,
		   struct page **pages, size_t maxsize,
		   size_t *start)
{
	size_t offset = i->iov_offset;
	const struct iovec *iov = i->iov;
	size_t len;
	unsigned long addr;
	int n;
	int res;

	len = iov->iov_len - offset;
	if (len > i->count)
		len = i->count;
	if (len > maxsize)
		len = maxsize;
	addr = (unsigned long)iov->iov_base + offset;
	len += *start = addr & (PAGE_SIZE - 1);
	addr &= ~(PAGE_SIZE - 1);
	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
	if (unlikely(res < 0))
		return res;
	return (res == n ? len : res * PAGE_SIZE) - *start;
}
EXPORT_SYMBOL(iov_iter_get_pages);

ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
		   struct page ***pages, size_t maxsize,
		   size_t *start)
{
	size_t offset = i->iov_offset;
	const struct iovec *iov = i->iov;
	size_t len;
	unsigned long addr;
	void *p;
	int n;
	int res;

	len = iov->iov_len - offset;
	if (len > i->count)
		len = i->count;
	if (len > maxsize)
		len = maxsize;
	addr = (unsigned long)iov->iov_base + offset;
	len += *start = addr & (PAGE_SIZE - 1);
	addr &= ~(PAGE_SIZE - 1);
	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	
	p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
	if (!p)
		p = vmalloc(n * sizeof(struct page *));
	if (!p)
		return -ENOMEM;

	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
	if (unlikely(res < 0)) {
		kvfree(p);
		return res;
	}
	*pages = p;
	return (res == n ? len : res * PAGE_SIZE) - *start;
}
EXPORT_SYMBOL(iov_iter_get_pages_alloc);

int iov_iter_npages(const struct iov_iter *i, int maxpages)
{
	size_t offset = i->iov_offset;
	size_t size = i->count;
	const struct iovec *iov = i->iov;
	int npages = 0;
	int n;

	for (n = 0; size && n < i->nr_segs; n++, iov++) {
		unsigned long addr = (unsigned long)iov->iov_base + offset;
		size_t len = iov->iov_len - offset;
		offset = 0;
		if (unlikely(!len))	/* empty segment */
			continue;
		if (len > size)
			len = size;
		npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
			  - addr / PAGE_SIZE;
		if (npages >= maxpages)	/* don't bother going further */
			return maxpages;
		size -= len;
		offset = 0;
	}
	return min(npages, maxpages);
}
EXPORT_SYMBOL(iov_iter_npages);
Commit	Line	Data
4f18cd31 AV	1	#include <linux/export.h>
	2	#include <linux/uio.h>
	3	#include <linux/pagemap.h>
91f79c43 AV	4	#include <linux/slab.h>
91f79c43 AV	5	#include <linux/vmalloc.h>
4f18cd31 AV	6
	7	size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
	8	struct iov_iter *i)
	9	{
	10	size_t skip, copy, left, wanted;
	11	const struct iovec *iov;
	12	char __user *buf;
	13	void kaddr, from;
	14
	15	if (unlikely(bytes > i->count))
	16	bytes = i->count;
	17
	18	if (unlikely(!bytes))
	19	return 0;
	20
	21	wanted = bytes;
	22	iov = i->iov;
	23	skip = i->iov_offset;
	24	buf = iov->iov_base + skip;
	25	copy = min(bytes, iov->iov_len - skip);
	26
	27	if (!fault_in_pages_writeable(buf, copy)) {
	28	kaddr = kmap_atomic(page);
	29	from = kaddr + offset;
	30
	31	/* first chunk, usually the only one */
	32	left = __copy_to_user_inatomic(buf, from, copy);
	33	copy -= left;
	34	skip += copy;
	35	from += copy;
	36	bytes -= copy;
	37
	38	while (unlikely(!left && bytes)) {
	39	iov++;
	40	buf = iov->iov_base;
	41	copy = min(bytes, iov->iov_len);
	42	left = __copy_to_user_inatomic(buf, from, copy);
	43	copy -= left;
	44	skip = copy;
	45	from += copy;
	46	bytes -= copy;
	47	}
	48	if (likely(!bytes)) {
	49	kunmap_atomic(kaddr);
	50	goto done;
	51	}
	52	offset = from - kaddr;
	53	buf += copy;
	54	kunmap_atomic(kaddr);
	55	copy = min(bytes, iov->iov_len - skip);
	56	}
	57	/* Too bad - revert to non-atomic kmap */
	58	kaddr = kmap(page);
	59	from = kaddr + offset;
	60	left = __copy_to_user(buf, from, copy);
	61	copy -= left;
	62	skip += copy;
	63	from += copy;
	64	bytes -= copy;
	65	while (unlikely(!left && bytes)) {
	66	iov++;
	67	buf = iov->iov_base;
	68	copy = min(bytes, iov->iov_len);
	69	left = __copy_to_user(buf, from, copy);
70	copy -= left;
71	skip = copy;
72	from += copy;
73	bytes -= copy;
74	}
75	kunmap(page);
76	done:
77	i->count -= wanted - bytes;
78	i->nr_segs -= iov - i->iov;
79	i->iov = iov;
80	i->iov_offset = skip;
81	return wanted - bytes;
82	}
83	EXPORT_SYMBOL(copy_page_to_iter);
84
f0d1bec9 AV	85	size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
	86	struct iov_iter *i)
	87	{
	88	size_t skip, copy, left, wanted;
	89	const struct iovec *iov;
	90	char __user *buf;
	91	void kaddr, to;
	92
	93	if (unlikely(bytes > i->count))
	94	bytes = i->count;
	95
	96	if (unlikely(!bytes))
	97	return 0;
	98
	99	wanted = bytes;
	100	iov = i->iov;
	101	skip = i->iov_offset;
	102	buf = iov->iov_base + skip;
	103	copy = min(bytes, iov->iov_len - skip);
	104
	105	if (!fault_in_pages_readable(buf, copy)) {
	106	kaddr = kmap_atomic(page);
	107	to = kaddr + offset;
	108
	109	/* first chunk, usually the only one */
	110	left = __copy_from_user_inatomic(to, buf, copy);
	111	copy -= left;
	112	skip += copy;
	113	to += copy;
	114	bytes -= copy;
	115
	116	while (unlikely(!left && bytes)) {
	117	iov++;
	118	buf = iov->iov_base;
	119	copy = min(bytes, iov->iov_len);
	120	left = __copy_from_user_inatomic(to, buf, copy);
	121	copy -= left;
	122	skip = copy;
	123	to += copy;
	124	bytes -= copy;
	125	}
	126	if (likely(!bytes)) {
	127	kunmap_atomic(kaddr);
	128	goto done;
	129	}
	130	offset = to - kaddr;
	131	buf += copy;
	132	kunmap_atomic(kaddr);
	133	copy = min(bytes, iov->iov_len - skip);
	134	}
	135	/* Too bad - revert to non-atomic kmap */
	136	kaddr = kmap(page);
	137	to = kaddr + offset;
	138	left = __copy_from_user(to, buf, copy);
	139	copy -= left;
	140	skip += copy;
	141	to += copy;
	142	bytes -= copy;
	143	while (unlikely(!left && bytes)) {
	144	iov++;
	145	buf = iov->iov_base;
	146	copy = min(bytes, iov->iov_len);
	147	left = __copy_from_user(to, buf, copy);
	148	copy -= left;
149	skip = copy;
150	to += copy;
151	bytes -= copy;
152	}
153	kunmap(page);
154	done:
155	i->count -= wanted - bytes;
156	i->nr_segs -= iov - i->iov;
157	i->iov = iov;
158	i->iov_offset = skip;
159	return wanted - bytes;
160	}
161	EXPORT_SYMBOL(copy_page_from_iter);
162
4f18cd31 AV	163	static size_t __iovec_copy_from_user_inatomic(char *vaddr,
	164	const struct iovec *iov, size_t base, size_t bytes)
	165	{
	166	size_t copied = 0, left = 0;
	167
	168	while (bytes) {
	169	char __user *buf = iov->iov_base + base;
	170	int copy = min(bytes, iov->iov_len - base);
	171
	172	base = 0;
	173	left = __copy_from_user_inatomic(vaddr, buf, copy);
	174	copied += copy;
	175	bytes -= copy;
	176	vaddr += copy;
	177	iov++;
	178
	179	if (unlikely(left))
	180	break;
	181	}
	182	return copied - left;
	183	}
	184
	185	/*
	186	* Copy as much as we can into the page and return the number of bytes which
	187	* were successfully copied. If a fault is encountered then return the number of
	188	* bytes which were copied.
	189	*/
	190	size_t iov_iter_copy_from_user_atomic(struct page *page,
	191	struct iov_iter *i, unsigned long offset, size_t bytes)
	192	{
	193	char *kaddr;
	194	size_t copied;
	195
	196	kaddr = kmap_atomic(page);
	197	if (likely(i->nr_segs == 1)) {
	198	int left;
	199	char __user *buf = i->iov->iov_base + i->iov_offset;
	200	left = __copy_from_user_inatomic(kaddr + offset, buf, bytes);
	201	copied = bytes - left;
	202	} else {
	203	copied = __iovec_copy_from_user_inatomic(kaddr + offset,
	204	i->iov, i->iov_offset, bytes);
	205	}
	206	kunmap_atomic(kaddr);
	207
	208	return copied;
	209	}
	210	EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
	211
4f18cd31 AV	212	void iov_iter_advance(struct iov_iter *i, size_t bytes)
	213	{
	214	BUG_ON(i->count < bytes);
	215
	216	if (likely(i->nr_segs == 1)) {
	217	i->iov_offset += bytes;
	218	i->count -= bytes;
	219	} else {
	220	const struct iovec *iov = i->iov;
	221	size_t base = i->iov_offset;
	222	unsigned long nr_segs = i->nr_segs;
	223
	224	/*
	225	* The !iov->iov_len check ensures we skip over unlikely
	226	* zero-length segments (without overruning the iovec).
	227	*/
	228	while (bytes \|\| unlikely(i->count && !iov->iov_len)) {
	229	int copy;
	230
	231	copy = min(bytes, iov->iov_len - base);
	232	BUG_ON(!i->count \|\| i->count < copy);
	233	i->count -= copy;
	234	bytes -= copy;
	235	base += copy;
	236	if (iov->iov_len == base) {
	237	iov++;
	238	nr_segs--;
	239	base = 0;
	240	}
	241	}
	242	i->iov = iov;
	243	i->iov_offset = base;
	244	i->nr_segs = nr_segs;
	245	}
	246	}
	247	EXPORT_SYMBOL(iov_iter_advance);
	248
	249	/*
	250	* Fault in the first iovec of the given iov_iter, to a maximum length
	251	* of bytes. Returns 0 on success, or non-zero if the memory could not be
	252	* accessed (ie. because it is an invalid address).
	253	*
	254	* writev-intensive code may want this to prefault several iovecs -- that
	255	* would be possible (callers must not rely on the fact that _only_ the
	256	* first iovec will be faulted with the current implementation).
	257	*/
	258	int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
	259	{
	260	char __user *buf = i->iov->iov_base + i->iov_offset;
	261	bytes = min(bytes, i->iov->iov_len - i->iov_offset);
	262	return fault_in_pages_readable(buf, bytes);
	263	}
	264	EXPORT_SYMBOL(iov_iter_fault_in_readable);
	265
	266	/*
	267	* Return the count of just the current iov_iter segment.
	268	*/
	269	size_t iov_iter_single_seg_count(const struct iov_iter *i)
	270	{
	271	const struct iovec *iov = i->iov;
	272	if (i->nr_segs == 1)
	273	return i->count;
	274	else
	275	return min(i->count, iov->iov_len - i->iov_offset);
276	}
277	EXPORT_SYMBOL(iov_iter_single_seg_count);
886a3911 AV	278
	279	unsigned long iov_iter_alignment(const struct iov_iter *i)
	280	{
	281	const struct iovec *iov = i->iov;
	282	unsigned long res;
	283	size_t size = i->count;
	284	size_t n;
	285
	286	if (!size)
	287	return 0;
	288
	289	res = (unsigned long)iov->iov_base + i->iov_offset;
	290	n = iov->iov_len - i->iov_offset;
	291	if (n >= size)
	292	return res \| size;
	293	size -= n;
	294	res \|= n;
	295	while (size > (++iov)->iov_len) {
	296	res \|= (unsigned long)iov->iov_base \| iov->iov_len;
	297	size -= iov->iov_len;
	298	}
	299	res \|= (unsigned long)iov->iov_base \| size;
	300	return res;
	301	}
	302	EXPORT_SYMBOL(iov_iter_alignment);
71d8e532 AV	303
	304	void iov_iter_init(struct iov_iter *i, int direction,
	305	const struct iovec *iov, unsigned long nr_segs,
	306	size_t count)
	307	{
	308	/* It will get better. Eventually... */
	309	if (segment_eq(get_fs(), KERNEL_DS))
	310	direction \|= REQ_KERNEL;
	311	i->type = direction;
	312	i->iov = iov;
	313	i->nr_segs = nr_segs;
	314	i->iov_offset = 0;
	315	i->count = count;
	316	}
	317	EXPORT_SYMBOL(iov_iter_init);
7b2c99d1 AV	318
	319	ssize_t iov_iter_get_pages(struct iov_iter *i,
	320	struct page **pages, size_t maxsize,
	321	size_t *start)
	322	{
	323	size_t offset = i->iov_offset;
	324	const struct iovec *iov = i->iov;
	325	size_t len;
	326	unsigned long addr;
	327	int n;
	328	int res;
	329
	330	len = iov->iov_len - offset;
	331	if (len > i->count)
	332	len = i->count;
	333	if (len > maxsize)
	334	len = maxsize;
	335	addr = (unsigned long)iov->iov_base + offset;
	336	len += *start = addr & (PAGE_SIZE - 1);
	337	addr &= ~(PAGE_SIZE - 1);
	338	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	339	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
	340	if (unlikely(res < 0))
	341	return res;
	342	return (res == n ? len : res * PAGE_SIZE) - *start;
	343	}
	344	EXPORT_SYMBOL(iov_iter_get_pages);
f67da30c	345
91f79c43 AV	346	ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
	347	struct page ***pages, size_t maxsize,
	348	size_t *start)
	349	{
	350	size_t offset = i->iov_offset;
	351	const struct iovec *iov = i->iov;
	352	size_t len;
	353	unsigned long addr;
	354	void *p;
	355	int n;
	356	int res;
	357
	358	len = iov->iov_len - offset;
	359	if (len > i->count)
	360	len = i->count;
	361	if (len > maxsize)
	362	len = maxsize;
	363	addr = (unsigned long)iov->iov_base + offset;
	364	len += *start = addr & (PAGE_SIZE - 1);
	365	addr &= ~(PAGE_SIZE - 1);
	366	n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
	367
	368	p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
	369	if (!p)
	370	p = vmalloc(n * sizeof(struct page *));
	371	if (!p)
	372	return -ENOMEM;
	373
	374	res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
	375	if (unlikely(res < 0)) {
	376	kvfree(p);
	377	return res;
	378	}
	379	*pages = p;
	380	return (res == n ? len : res * PAGE_SIZE) - *start;
	381	}
	382	EXPORT_SYMBOL(iov_iter_get_pages_alloc);
	383
f67da30c AV	384	int iov_iter_npages(const struct iov_iter *i, int maxpages)
	385	{
	386	size_t offset = i->iov_offset;
	387	size_t size = i->count;
	388	const struct iovec *iov = i->iov;
	389	int npages = 0;
	390	int n;
	391
	392	for (n = 0; size && n < i->nr_segs; n++, iov++) {
	393	unsigned long addr = (unsigned long)iov->iov_base + offset;
	394	size_t len = iov->iov_len - offset;
	395	offset = 0;
	396	if (unlikely(!len)) /* empty segment */
	397	continue;
	398	if (len > size)
	399	len = size;
	400	npages += (addr + len + PAGE_SIZE - 1) / PAGE_SIZE
	401	- addr / PAGE_SIZE;
	402	if (npages >= maxpages) /* don't bother going further */
	403	return maxpages;
	404	size -= len;
	405	offset = 0;
	406	}
	407	return min(npages, maxpages);
	408	}
	409	EXPORT_SYMBOL(iov_iter_npages);