while (n) { \
unsigned offset = p->bv_offset + skip; \
unsigned left; \
- __v.bv_offset = offset % PAGE_SIZE; \
- __v.bv_page = p->bv_page + offset / PAGE_SIZE; \
- __v.bv_len = min(min(n, p->bv_len - skip), \
+ void *kaddr = kmap_local_page(p->bv_page + \
+ offset / PAGE_SIZE); \
+ __v.iov_base = kaddr + offset % PAGE_SIZE; \
+ __v.iov_len = min(min(n, p->bv_len - skip), \
(size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \
left = (STEP); \
- __v.bv_len -= left; \
- skip += __v.bv_len; \
+ kunmap_local(kaddr); \
+ __v.iov_len -= left; \
+ skip += __v.iov_len; \
if (skip == p->bv_len) { \
skip = 0; \
p++; \
} \
- n -= __v.bv_len; \
+ n -= __v.iov_len; \
if (left) \
break; \
} \
break; \
for (j = (head->index < index) ? index - head->index : 0; \
j < thp_nr_pages(head); j++) { \
- __v.bv_page = head + j; \
- offset = (i->xarray_start + skip) & ~PAGE_MASK; \
+ void *kaddr = kmap_local_page(head + j); \
+ offset = (i->xarray_start + skip) % PAGE_SIZE; \
+ __v.iov_base = kaddr + offset; \
seg = PAGE_SIZE - offset; \
- __v.bv_offset = offset; \
- __v.bv_len = min(n, seg); \
+ __v.iov_len = min(n, seg); \
left = (STEP); \
- __v.bv_len -= left; \
- n -= __v.bv_len; \
- skip += __v.bv_len; \
+ kunmap_local(kaddr); \
+ __v.iov_len -= left; \
+ n -= __v.iov_len; \
+ skip += __v.iov_len; \
if (left || n == 0) \
goto __out; \
} \
n = wanted - n; \
}
-#define __iterate_and_advance(i, n, v, I, B, K, X) { \
+#define __iterate_and_advance(i, n, v, I, K) { \
if (unlikely(i->count < n)) \
n = i->count; \
if (likely(n)) { \
i->iov = iov; \
} else if (iov_iter_is_bvec(i)) { \
const struct bio_vec *bvec = i->bvec; \
- struct bio_vec v; \
- iterate_bvec(i, n, v, bvec, skip, (B)) \
+ struct kvec v; \
+ iterate_bvec(i, n, v, bvec, skip, (K)) \
i->nr_segs -= bvec - i->bvec; \
i->bvec = bvec; \
} else if (iov_iter_is_kvec(i)) { \
i->nr_segs -= kvec - i->kvec; \
i->kvec = kvec; \
} else if (iov_iter_is_xarray(i)) { \
- struct bio_vec v; \
- iterate_xarray(i, n, v, skip, (X)) \
+ struct kvec v; \
+ iterate_xarray(i, n, v, skip, (K)) \
} \
i->count -= n; \
i->iov_offset = skip; \
} \
}
-#define iterate_and_advance(i, n, v, I, B, K, X) \
- __iterate_and_advance(i, n, v, I, ((void)(B),0), \
- ((void)(K),0), ((void)(X),0))
+#define iterate_and_advance(i, n, v, I, K) \
+ __iterate_and_advance(i, n, v, I, ((void)(K),0))
static int copyout(void __user *to, const void *from, size_t n)
{
might_fault();
iterate_and_advance(i, bytes, v,
copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
- memcpy_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len),
- memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
- memcpy_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len)
+ memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
)
return bytes;
__iterate_and_advance(i, bytes, v,
copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
v.iov_len),
- copy_mc_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len),
copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
- - v.iov_len, v.iov_len),
- copy_mc_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len)
+ - v.iov_len, v.iov_len)
)
return bytes;
might_fault();
iterate_and_advance(i, bytes, v,
copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
- memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len),
- memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
- memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len)
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
)
return bytes;
iterate_and_advance(i, bytes, v,
__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
v.iov_base, v.iov_len),
- memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len),
- memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
- memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len)
+ memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
)
return bytes;
iterate_and_advance(i, bytes, v,
__copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
v.iov_base, v.iov_len),
- memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len),
memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
- v.iov_len),
- memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len)
+ v.iov_len)
)
return bytes;
return pipe_zero(bytes, i);
iterate_and_advance(i, bytes, v,
clear_user(v.iov_base, v.iov_len),
- memzero_page(v.bv_page, v.bv_offset, v.bv_len),
- memset(v.iov_base, 0, v.iov_len),
- memzero_page(v.bv_page, v.bv_offset, v.bv_len)
+ memset(v.iov_base, 0, v.iov_len)
)
return bytes;
}
iterate_and_advance(i, bytes, v,
copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
- memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len),
- memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
- memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len)
+ memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
)
kunmap_atomic(kaddr);
return bytes;
}
next ? 0 : v.iov_len;
}), ({
- char *p = kmap_atomic(v.bv_page);
- sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
- p + v.bv_offset, v.bv_len,
- sum, off);
- kunmap_atomic(p);
- off += v.bv_len;
- }),({
sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
v.iov_base, v.iov_len,
sum, off);
off += v.iov_len;
- }), ({
- char *p = kmap_atomic(v.bv_page);
- sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
- p + v.bv_offset, v.bv_len,
- sum, off);
- kunmap_atomic(p);
- off += v.bv_len;
})
)
*csum = sum;
}
next ? 0 : v.iov_len;
}), ({
- char *p = kmap_atomic(v.bv_page);
- sum = csum_and_memcpy(p + v.bv_offset,
- (from += v.bv_len) - v.bv_len,
- v.bv_len, sum, off);
- kunmap_atomic(p);
- off += v.bv_len;
- }),({
sum = csum_and_memcpy(v.iov_base,
(from += v.iov_len) - v.iov_len,
v.iov_len, sum, off);
off += v.iov_len;
- }), ({
- char *p = kmap_atomic(v.bv_page);
- sum = csum_and_memcpy(p + v.bv_offset,
- (from += v.bv_len) - v.bv_len,
- v.bv_len, sum, off);
- kunmap_atomic(p);
- off += v.bv_len;
})
)
csstate->csum = csum_shift(sum, csstate->off);