Commit | Line | Data |
---|---|---|
eff5f53b | 1 | /* |
eee2fa6a | 2 | * Copyright (c) 2007, 2017 Oracle and/or its affiliates. All rights reserved. |
eff5f53b AG |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/pagemap.h> | |
5a0e3ad6 | 34 | #include <linux/slab.h> |
eff5f53b AG |
35 | #include <linux/rbtree.h> |
36 | #include <linux/dma-mapping.h> /* for DMA_*_DEVICE */ | |
37 | ||
21f79afa | 38 | #include "rds.h" |
eff5f53b AG |
39 | |
40 | /* | |
41 | * XXX | |
42 | * - build with sparse | |
eff5f53b AG |
43 | * - should we detect duplicate keys on a socket? hmm. |
44 | * - an rdma is an mlock, apply rlimit? | |
45 | */ | |
46 | ||
47 | /* | |
48 | * get the number of pages by looking at the page indices that the start and | |
49 | * end addresses fall in. | |
50 | * | |
51 | * Returns 0 if the vec is invalid. It is invalid if the number of bytes | |
52 | * causes the address to wrap or overflows an unsigned int. This comes | |
53 | * from being stored in the 'length' member of 'struct scatterlist'. | |
54 | */ | |
55 | static unsigned int rds_pages_in_vec(struct rds_iovec *vec) | |
56 | { | |
57 | if ((vec->addr + vec->bytes <= vec->addr) || | |
58 | (vec->bytes > (u64)UINT_MAX)) | |
59 | return 0; | |
60 | ||
61 | return ((vec->addr + vec->bytes + PAGE_SIZE - 1) >> PAGE_SHIFT) - | |
62 | (vec->addr >> PAGE_SHIFT); | |
63 | } | |
64 | ||
65 | static struct rds_mr *rds_mr_tree_walk(struct rb_root *root, u64 key, | |
66 | struct rds_mr *insert) | |
67 | { | |
68 | struct rb_node **p = &root->rb_node; | |
69 | struct rb_node *parent = NULL; | |
70 | struct rds_mr *mr; | |
71 | ||
72 | while (*p) { | |
73 | parent = *p; | |
74 | mr = rb_entry(parent, struct rds_mr, r_rb_node); | |
75 | ||
76 | if (key < mr->r_key) | |
77 | p = &(*p)->rb_left; | |
78 | else if (key > mr->r_key) | |
79 | p = &(*p)->rb_right; | |
80 | else | |
81 | return mr; | |
82 | } | |
83 | ||
84 | if (insert) { | |
85 | rb_link_node(&insert->r_rb_node, parent, p); | |
86 | rb_insert_color(&insert->r_rb_node, root); | |
803ea850 | 87 | refcount_inc(&insert->r_refcount); |
eff5f53b AG |
88 | } |
89 | return NULL; | |
90 | } | |
91 | ||
92 | /* | |
93 | * Destroy the transport-specific part of a MR. | |
94 | */ | |
95 | static void rds_destroy_mr(struct rds_mr *mr) | |
96 | { | |
97 | struct rds_sock *rs = mr->r_sock; | |
98 | void *trans_private = NULL; | |
99 | unsigned long flags; | |
100 | ||
101 | rdsdebug("RDS: destroy mr key is %x refcnt %u\n", | |
803ea850 | 102 | mr->r_key, refcount_read(&mr->r_refcount)); |
eff5f53b AG |
103 | |
104 | if (test_and_set_bit(RDS_MR_DEAD, &mr->r_state)) | |
105 | return; | |
106 | ||
107 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); | |
108 | if (!RB_EMPTY_NODE(&mr->r_rb_node)) | |
109 | rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys); | |
110 | trans_private = mr->r_trans_private; | |
111 | mr->r_trans_private = NULL; | |
112 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); | |
113 | ||
114 | if (trans_private) | |
115 | mr->r_trans->free_mr(trans_private, mr->r_invalidate); | |
116 | } | |
117 | ||
118 | void __rds_put_mr_final(struct rds_mr *mr) | |
119 | { | |
120 | rds_destroy_mr(mr); | |
121 | kfree(mr); | |
122 | } | |
123 | ||
124 | /* | |
125 | * By the time this is called we can't have any more ioctls called on | |
126 | * the socket so we don't need to worry about racing with others. | |
127 | */ | |
128 | void rds_rdma_drop_keys(struct rds_sock *rs) | |
129 | { | |
130 | struct rds_mr *mr; | |
131 | struct rb_node *node; | |
35b52c70 | 132 | unsigned long flags; |
eff5f53b AG |
133 | |
134 | /* Release any MRs associated with this socket */ | |
35b52c70 | 135 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); |
eff5f53b | 136 | while ((node = rb_first(&rs->rs_rdma_keys))) { |
a763f78c | 137 | mr = rb_entry(node, struct rds_mr, r_rb_node); |
eff5f53b AG |
138 | if (mr->r_trans == rs->rs_transport) |
139 | mr->r_invalidate = 0; | |
35b52c70 TY |
140 | rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys); |
141 | RB_CLEAR_NODE(&mr->r_rb_node); | |
142 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); | |
143 | rds_destroy_mr(mr); | |
eff5f53b | 144 | rds_mr_put(mr); |
35b52c70 | 145 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); |
eff5f53b | 146 | } |
35b52c70 | 147 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); |
eff5f53b AG |
148 | |
149 | if (rs->rs_transport && rs->rs_transport->flush_mrs) | |
150 | rs->rs_transport->flush_mrs(); | |
151 | } | |
152 | ||
153 | /* | |
154 | * Helper function to pin user pages. | |
155 | */ | |
156 | static int rds_pin_pages(unsigned long user_addr, unsigned int nr_pages, | |
157 | struct page **pages, int write) | |
158 | { | |
c4c86abb | 159 | unsigned int gup_flags = FOLL_LONGTERM; |
eff5f53b AG |
160 | int ret; |
161 | ||
c4c86abb HWR |
162 | if (write) |
163 | gup_flags |= FOLL_WRITE; | |
eff5f53b | 164 | |
0d4597c8 | 165 | ret = pin_user_pages_fast(user_addr, nr_pages, gup_flags, pages); |
7acd4a79 | 166 | if (ret >= 0 && ret < nr_pages) { |
0d4597c8 | 167 | unpin_user_pages(pages, ret); |
eff5f53b AG |
168 | ret = -EFAULT; |
169 | } | |
170 | ||
171 | return ret; | |
172 | } | |
173 | ||
174 | static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args, | |
9e630bcb AR |
175 | u64 *cookie_ret, struct rds_mr **mr_ret, |
176 | struct rds_conn_path *cp) | |
eff5f53b AG |
177 | { |
178 | struct rds_mr *mr = NULL, *found; | |
2eafa174 | 179 | struct scatterlist *sg = NULL; |
eff5f53b AG |
180 | unsigned int nr_pages; |
181 | struct page **pages = NULL; | |
eff5f53b AG |
182 | void *trans_private; |
183 | unsigned long flags; | |
184 | rds_rdma_cookie_t cookie; | |
2eafa174 HWR |
185 | unsigned int nents = 0; |
186 | int need_odp = 0; | |
eff5f53b AG |
187 | long i; |
188 | int ret; | |
189 | ||
eee2fa6a | 190 | if (ipv6_addr_any(&rs->rs_bound_addr) || !rs->rs_transport) { |
eff5f53b AG |
191 | ret = -ENOTCONN; /* XXX not a great errno */ |
192 | goto out; | |
193 | } | |
194 | ||
8690bfa1 | 195 | if (!rs->rs_transport->get_mr) { |
eff5f53b AG |
196 | ret = -EOPNOTSUPP; |
197 | goto out; | |
198 | } | |
199 | ||
2eafa174 HWR |
200 | /* If the combination of the addr and size requested for this memory |
201 | * region causes an integer overflow, return error. | |
202 | */ | |
203 | if (((args->vec.addr + args->vec.bytes) < args->vec.addr) || | |
204 | PAGE_ALIGN(args->vec.addr + args->vec.bytes) < | |
205 | (args->vec.addr + args->vec.bytes)) { | |
206 | ret = -EINVAL; | |
207 | goto out; | |
208 | } | |
209 | ||
210 | if (!can_do_mlock()) { | |
211 | ret = -EPERM; | |
212 | goto out; | |
213 | } | |
214 | ||
eff5f53b AG |
215 | nr_pages = rds_pages_in_vec(&args->vec); |
216 | if (nr_pages == 0) { | |
217 | ret = -EINVAL; | |
218 | goto out; | |
219 | } | |
220 | ||
f9fb69ad AR |
221 | /* Restrict the size of mr irrespective of underlying transport |
222 | * To account for unaligned mr regions, subtract one from nr_pages | |
223 | */ | |
224 | if ((nr_pages - 1) > (RDS_MAX_MSG_SIZE >> PAGE_SHIFT)) { | |
225 | ret = -EMSGSIZE; | |
226 | goto out; | |
227 | } | |
228 | ||
eff5f53b AG |
229 | rdsdebug("RDS: get_mr addr %llx len %llu nr_pages %u\n", |
230 | args->vec.addr, args->vec.bytes, nr_pages); | |
231 | ||
232 | /* XXX clamp nr_pages to limit the size of this alloc? */ | |
233 | pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); | |
8690bfa1 | 234 | if (!pages) { |
eff5f53b AG |
235 | ret = -ENOMEM; |
236 | goto out; | |
237 | } | |
238 | ||
239 | mr = kzalloc(sizeof(struct rds_mr), GFP_KERNEL); | |
8690bfa1 | 240 | if (!mr) { |
eff5f53b AG |
241 | ret = -ENOMEM; |
242 | goto out; | |
243 | } | |
244 | ||
803ea850 | 245 | refcount_set(&mr->r_refcount, 1); |
eff5f53b AG |
246 | RB_CLEAR_NODE(&mr->r_rb_node); |
247 | mr->r_trans = rs->rs_transport; | |
248 | mr->r_sock = rs; | |
249 | ||
250 | if (args->flags & RDS_RDMA_USE_ONCE) | |
251 | mr->r_use_once = 1; | |
252 | if (args->flags & RDS_RDMA_INVALIDATE) | |
253 | mr->r_invalidate = 1; | |
254 | if (args->flags & RDS_RDMA_READWRITE) | |
255 | mr->r_write = 1; | |
256 | ||
257 | /* | |
258 | * Pin the pages that make up the user buffer and transfer the page | |
259 | * pointers to the mr's sg array. We check to see if we've mapped | |
260 | * the whole region after transferring the partial page references | |
261 | * to the sg array so that we can have one page ref cleanup path. | |
262 | * | |
263 | * For now we have no flag that tells us whether the mapping is | |
264 | * r/o or r/w. We need to assume r/w, or we'll do a lot of RDMA to | |
265 | * the zero page. | |
266 | */ | |
d22faec2 | 267 | ret = rds_pin_pages(args->vec.addr, nr_pages, pages, 1); |
2eafa174 HWR |
268 | if (ret == -EOPNOTSUPP) { |
269 | need_odp = 1; | |
270 | } else if (ret <= 0) { | |
eff5f53b | 271 | goto out; |
2eafa174 HWR |
272 | } else { |
273 | nents = ret; | |
274 | sg = kcalloc(nents, sizeof(*sg), GFP_KERNEL); | |
275 | if (!sg) { | |
276 | ret = -ENOMEM; | |
277 | goto out; | |
278 | } | |
279 | WARN_ON(!nents); | |
280 | sg_init_table(sg, nents); | |
eff5f53b | 281 | |
2eafa174 HWR |
282 | /* Stick all pages into the scatterlist */ |
283 | for (i = 0 ; i < nents; i++) | |
284 | sg_set_page(&sg[i], pages[i], PAGE_SIZE, 0); | |
eff5f53b | 285 | |
2eafa174 HWR |
286 | rdsdebug("RDS: trans_private nents is %u\n", nents); |
287 | } | |
eff5f53b AG |
288 | /* Obtain a transport specific MR. If this succeeds, the |
289 | * s/g list is now owned by the MR. | |
290 | * Note that dma_map() implies that pending writes are | |
291 | * flushed to RAM, so no dma_sync is needed here. */ | |
2eafa174 HWR |
292 | trans_private = rs->rs_transport->get_mr( |
293 | sg, nents, rs, &mr->r_key, cp ? cp->cp_conn : NULL, | |
294 | args->vec.addr, args->vec.bytes, | |
295 | need_odp ? ODP_ZEROBASED : ODP_NOT_NEEDED); | |
eff5f53b AG |
296 | |
297 | if (IS_ERR(trans_private)) { | |
2eafa174 HWR |
298 | /* In ODP case, we don't GUP pages, so don't need |
299 | * to release anything. | |
300 | */ | |
301 | if (!need_odp) { | |
0d4597c8 | 302 | unpin_user_pages(pages, nr_pages); |
2eafa174 HWR |
303 | kfree(sg); |
304 | } | |
eff5f53b AG |
305 | ret = PTR_ERR(trans_private); |
306 | goto out; | |
307 | } | |
308 | ||
309 | mr->r_trans_private = trans_private; | |
310 | ||
311 | rdsdebug("RDS: get_mr put_user key is %x cookie_addr %p\n", | |
312 | mr->r_key, (void *)(unsigned long) args->cookie_addr); | |
313 | ||
314 | /* The user may pass us an unaligned address, but we can only | |
315 | * map page aligned regions. So we keep the offset, and build | |
316 | * a 64bit cookie containing <R_Key, offset> and pass that | |
317 | * around. */ | |
2eafa174 HWR |
318 | if (need_odp) |
319 | cookie = rds_rdma_make_cookie(mr->r_key, 0); | |
320 | else | |
321 | cookie = rds_rdma_make_cookie(mr->r_key, | |
322 | args->vec.addr & ~PAGE_MASK); | |
eff5f53b AG |
323 | if (cookie_ret) |
324 | *cookie_ret = cookie; | |
325 | ||
0d4597c8 LR |
326 | if (args->cookie_addr && |
327 | put_user(cookie, (u64 __user *)(unsigned long)args->cookie_addr)) { | |
328 | if (!need_odp) { | |
329 | unpin_user_pages(pages, nr_pages); | |
330 | kfree(sg); | |
331 | } | |
eff5f53b AG |
332 | ret = -EFAULT; |
333 | goto out; | |
334 | } | |
335 | ||
336 | /* Inserting the new MR into the rbtree bumps its | |
337 | * reference count. */ | |
338 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); | |
339 | found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr); | |
340 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); | |
341 | ||
342 | BUG_ON(found && found != mr); | |
343 | ||
344 | rdsdebug("RDS: get_mr key is %x\n", mr->r_key); | |
345 | if (mr_ret) { | |
803ea850 | 346 | refcount_inc(&mr->r_refcount); |
eff5f53b AG |
347 | *mr_ret = mr; |
348 | } | |
349 | ||
350 | ret = 0; | |
351 | out: | |
352 | kfree(pages); | |
353 | if (mr) | |
354 | rds_mr_put(mr); | |
355 | return ret; | |
356 | } | |
357 | ||
358 | int rds_get_mr(struct rds_sock *rs, char __user *optval, int optlen) | |
359 | { | |
360 | struct rds_get_mr_args args; | |
361 | ||
362 | if (optlen != sizeof(struct rds_get_mr_args)) | |
363 | return -EINVAL; | |
364 | ||
365 | if (copy_from_user(&args, (struct rds_get_mr_args __user *)optval, | |
366 | sizeof(struct rds_get_mr_args))) | |
367 | return -EFAULT; | |
368 | ||
9e630bcb | 369 | return __rds_rdma_map(rs, &args, NULL, NULL, NULL); |
eff5f53b AG |
370 | } |
371 | ||
244546f0 AG |
372 | int rds_get_mr_for_dest(struct rds_sock *rs, char __user *optval, int optlen) |
373 | { | |
374 | struct rds_get_mr_for_dest_args args; | |
375 | struct rds_get_mr_args new_args; | |
376 | ||
377 | if (optlen != sizeof(struct rds_get_mr_for_dest_args)) | |
378 | return -EINVAL; | |
379 | ||
380 | if (copy_from_user(&args, (struct rds_get_mr_for_dest_args __user *)optval, | |
381 | sizeof(struct rds_get_mr_for_dest_args))) | |
382 | return -EFAULT; | |
383 | ||
384 | /* | |
385 | * Initially, just behave like get_mr(). | |
386 | * TODO: Implement get_mr as wrapper around this | |
387 | * and deprecate it. | |
388 | */ | |
389 | new_args.vec = args.vec; | |
390 | new_args.cookie_addr = args.cookie_addr; | |
391 | new_args.flags = args.flags; | |
392 | ||
9e630bcb | 393 | return __rds_rdma_map(rs, &new_args, NULL, NULL, NULL); |
244546f0 AG |
394 | } |
395 | ||
eff5f53b AG |
396 | /* |
397 | * Free the MR indicated by the given R_Key | |
398 | */ | |
399 | int rds_free_mr(struct rds_sock *rs, char __user *optval, int optlen) | |
400 | { | |
401 | struct rds_free_mr_args args; | |
402 | struct rds_mr *mr; | |
403 | unsigned long flags; | |
404 | ||
405 | if (optlen != sizeof(struct rds_free_mr_args)) | |
406 | return -EINVAL; | |
407 | ||
408 | if (copy_from_user(&args, (struct rds_free_mr_args __user *)optval, | |
409 | sizeof(struct rds_free_mr_args))) | |
410 | return -EFAULT; | |
411 | ||
412 | /* Special case - a null cookie means flush all unused MRs */ | |
413 | if (args.cookie == 0) { | |
414 | if (!rs->rs_transport || !rs->rs_transport->flush_mrs) | |
415 | return -EINVAL; | |
416 | rs->rs_transport->flush_mrs(); | |
417 | return 0; | |
418 | } | |
419 | ||
420 | /* Look up the MR given its R_key and remove it from the rbtree | |
421 | * so nobody else finds it. | |
422 | * This should also prevent races with rds_rdma_unuse. | |
423 | */ | |
424 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); | |
425 | mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL); | |
426 | if (mr) { | |
427 | rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys); | |
428 | RB_CLEAR_NODE(&mr->r_rb_node); | |
429 | if (args.flags & RDS_RDMA_INVALIDATE) | |
430 | mr->r_invalidate = 1; | |
431 | } | |
432 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); | |
433 | ||
434 | if (!mr) | |
435 | return -EINVAL; | |
436 | ||
437 | /* | |
438 | * call rds_destroy_mr() ourselves so that we're sure it's done by the time | |
439 | * we return. If we let rds_mr_put() do it it might not happen until | |
440 | * someone else drops their ref. | |
441 | */ | |
442 | rds_destroy_mr(mr); | |
443 | rds_mr_put(mr); | |
444 | return 0; | |
445 | } | |
446 | ||
447 | /* | |
448 | * This is called when we receive an extension header that | |
449 | * tells us this MR was used. It allows us to implement | |
450 | * use_once semantics | |
451 | */ | |
452 | void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force) | |
453 | { | |
454 | struct rds_mr *mr; | |
455 | unsigned long flags; | |
456 | int zot_me = 0; | |
457 | ||
458 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); | |
459 | mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL); | |
3ef13f3c | 460 | if (!mr) { |
c536a068 SS |
461 | pr_debug("rds: trying to unuse MR with unknown r_key %u!\n", |
462 | r_key); | |
3ef13f3c AG |
463 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); |
464 | return; | |
465 | } | |
466 | ||
467 | if (mr->r_use_once || force) { | |
eff5f53b AG |
468 | rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys); |
469 | RB_CLEAR_NODE(&mr->r_rb_node); | |
470 | zot_me = 1; | |
3ef13f3c | 471 | } |
eff5f53b AG |
472 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); |
473 | ||
474 | /* May have to issue a dma_sync on this memory region. | |
475 | * Note we could avoid this if the operation was a RDMA READ, | |
476 | * but at this point we can't tell. */ | |
3ef13f3c AG |
477 | if (mr->r_trans->sync_mr) |
478 | mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE); | |
eff5f53b | 479 | |
3ef13f3c AG |
480 | /* If the MR was marked as invalidate, this will |
481 | * trigger an async flush. */ | |
3f6b3143 | 482 | if (zot_me) { |
3ef13f3c | 483 | rds_destroy_mr(mr); |
3f6b3143 | 484 | rds_mr_put(mr); |
485 | } | |
eff5f53b AG |
486 | } |
487 | ||
f8b3aaf2 | 488 | void rds_rdma_free_op(struct rm_rdma_op *ro) |
eff5f53b AG |
489 | { |
490 | unsigned int i; | |
491 | ||
2eafa174 HWR |
492 | if (ro->op_odp_mr) { |
493 | rds_mr_put(ro->op_odp_mr); | |
494 | } else { | |
495 | for (i = 0; i < ro->op_nents; i++) { | |
496 | struct page *page = sg_page(&ro->op_sg[i]); | |
497 | ||
498 | /* Mark page dirty if it was possibly modified, which | |
499 | * is the case for a RDMA_READ which copies from remote | |
500 | * to local memory | |
501 | */ | |
0d4597c8 | 502 | unpin_user_pages_dirty_lock(&page, 1, !ro->op_write); |
561c7df6 | 503 | } |
eff5f53b AG |
504 | } |
505 | ||
f8b3aaf2 AG |
506 | kfree(ro->op_notifier); |
507 | ro->op_notifier = NULL; | |
508 | ro->op_active = 0; | |
2eafa174 | 509 | ro->op_odp_mr = NULL; |
ff87e97a AG |
510 | } |
511 | ||
d0ab25a8 AG |
512 | void rds_atomic_free_op(struct rm_atomic_op *ao) |
513 | { | |
514 | struct page *page = sg_page(ao->op_sg); | |
515 | ||
516 | /* Mark page dirty if it was possibly modified, which | |
517 | * is the case for a RDMA_READ which copies from remote | |
518 | * to local memory */ | |
0d4597c8 | 519 | unpin_user_pages_dirty_lock(&page, 1, true); |
d0ab25a8 AG |
520 | |
521 | kfree(ao->op_notifier); | |
522 | ao->op_notifier = NULL; | |
523 | ao->op_active = 0; | |
524 | } | |
525 | ||
526 | ||
ff87e97a | 527 | /* |
fc8162e3 | 528 | * Count the number of pages needed to describe an incoming iovec array. |
ff87e97a | 529 | */ |
fc8162e3 AG |
530 | static int rds_rdma_pages(struct rds_iovec iov[], int nr_iovecs) |
531 | { | |
532 | int tot_pages = 0; | |
533 | unsigned int nr_pages; | |
534 | unsigned int i; | |
535 | ||
536 | /* figure out the number of pages in the vector */ | |
537 | for (i = 0; i < nr_iovecs; i++) { | |
538 | nr_pages = rds_pages_in_vec(&iov[i]); | |
539 | if (nr_pages == 0) | |
540 | return -EINVAL; | |
541 | ||
542 | tot_pages += nr_pages; | |
543 | ||
544 | /* | |
545 | * nr_pages for one entry is limited to (UINT_MAX>>PAGE_SHIFT)+1, | |
546 | * so tot_pages cannot overflow without first going negative. | |
547 | */ | |
548 | if (tot_pages < 0) | |
549 | return -EINVAL; | |
550 | } | |
551 | ||
552 | return tot_pages; | |
553 | } | |
554 | ||
ea010070 | 555 | int rds_rdma_extra_size(struct rds_rdma_args *args, |
556 | struct rds_iov_vector *iov) | |
ff87e97a | 557 | { |
ea010070 | 558 | struct rds_iovec *vec; |
ff87e97a | 559 | struct rds_iovec __user *local_vec; |
fc8162e3 | 560 | int tot_pages = 0; |
ff87e97a AG |
561 | unsigned int nr_pages; |
562 | unsigned int i; | |
563 | ||
564 | local_vec = (struct rds_iovec __user *)(unsigned long) args->local_vec_addr; | |
565 | ||
c0955087 MG |
566 | if (args->nr_local == 0) |
567 | return -EINVAL; | |
568 | ||
ea010070 | 569 | iov->iov = kcalloc(args->nr_local, |
570 | sizeof(struct rds_iovec), | |
571 | GFP_KERNEL); | |
572 | if (!iov->iov) | |
573 | return -ENOMEM; | |
574 | ||
575 | vec = &iov->iov[0]; | |
576 | ||
577 | if (copy_from_user(vec, local_vec, args->nr_local * | |
578 | sizeof(struct rds_iovec))) | |
579 | return -EFAULT; | |
580 | iov->len = args->nr_local; | |
581 | ||
ff87e97a | 582 | /* figure out the number of pages in the vector */ |
ea010070 | 583 | for (i = 0; i < args->nr_local; i++, vec++) { |
ff87e97a | 584 | |
ea010070 | 585 | nr_pages = rds_pages_in_vec(vec); |
ff87e97a AG |
586 | if (nr_pages == 0) |
587 | return -EINVAL; | |
588 | ||
589 | tot_pages += nr_pages; | |
1b1f693d LT |
590 | |
591 | /* | |
592 | * nr_pages for one entry is limited to (UINT_MAX>>PAGE_SHIFT)+1, | |
593 | * so tot_pages cannot overflow without first going negative. | |
594 | */ | |
fc8162e3 | 595 | if (tot_pages < 0) |
1b1f693d | 596 | return -EINVAL; |
ff87e97a AG |
597 | } |
598 | ||
fc8162e3 | 599 | return tot_pages * sizeof(struct scatterlist); |
eff5f53b AG |
600 | } |
601 | ||
602 | /* | |
4324879d AG |
603 | * The application asks for a RDMA transfer. |
604 | * Extract all arguments and set up the rdma_op | |
eff5f53b | 605 | */ |
4324879d | 606 | int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm, |
ea010070 | 607 | struct cmsghdr *cmsg, |
608 | struct rds_iov_vector *vec) | |
eff5f53b | 609 | { |
4324879d | 610 | struct rds_rdma_args *args; |
f8b3aaf2 | 611 | struct rm_rdma_op *op = &rm->rdma; |
9b9d2e00 | 612 | int nr_pages; |
eff5f53b AG |
613 | unsigned int nr_bytes; |
614 | struct page **pages = NULL; | |
ea010070 | 615 | struct rds_iovec *iovs; |
eff5f53b | 616 | unsigned int i, j; |
ff87e97a | 617 | int ret = 0; |
2eafa174 | 618 | bool odp_supported = true; |
eff5f53b | 619 | |
4324879d | 620 | if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_rdma_args)) |
f8b3aaf2 | 621 | || rm->rdma.op_active) |
4324879d AG |
622 | return -EINVAL; |
623 | ||
624 | args = CMSG_DATA(cmsg); | |
eff5f53b | 625 | |
eee2fa6a | 626 | if (ipv6_addr_any(&rs->rs_bound_addr)) { |
eff5f53b | 627 | ret = -ENOTCONN; /* XXX not a great errno */ |
dee49f20 | 628 | goto out_ret; |
eff5f53b AG |
629 | } |
630 | ||
218854af | 631 | if (args->nr_local > UIO_MAXIOV) { |
eff5f53b | 632 | ret = -EMSGSIZE; |
dee49f20 | 633 | goto out_ret; |
eff5f53b AG |
634 | } |
635 | ||
ea010070 | 636 | if (vec->len != args->nr_local) { |
637 | ret = -EINVAL; | |
638 | goto out_ret; | |
fc8162e3 | 639 | } |
2eafa174 HWR |
640 | /* odp-mr is not supported for multiple requests within one message */ |
641 | if (args->nr_local != 1) | |
642 | odp_supported = false; | |
fc8162e3 | 643 | |
ea010070 | 644 | iovs = vec->iov; |
fc8162e3 AG |
645 | |
646 | nr_pages = rds_rdma_pages(iovs, args->nr_local); | |
a09f69c4 AG |
647 | if (nr_pages < 0) { |
648 | ret = -EINVAL; | |
ea010070 | 649 | goto out_ret; |
a09f69c4 | 650 | } |
eff5f53b | 651 | |
ff87e97a AG |
652 | pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); |
653 | if (!pages) { | |
eff5f53b | 654 | ret = -ENOMEM; |
ea010070 | 655 | goto out_ret; |
eff5f53b AG |
656 | } |
657 | ||
f8b3aaf2 AG |
658 | op->op_write = !!(args->flags & RDS_RDMA_READWRITE); |
659 | op->op_fence = !!(args->flags & RDS_RDMA_FENCE); | |
660 | op->op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME); | |
2c3a5f9a | 661 | op->op_silent = !!(args->flags & RDS_RDMA_SILENT); |
f8b3aaf2 AG |
662 | op->op_active = 1; |
663 | op->op_recverr = rs->rs_recverr; | |
2eafa174 HWR |
664 | op->op_odp_mr = NULL; |
665 | ||
eff5f53b | 666 | WARN_ON(!nr_pages); |
c75ab8a5 | 667 | op->op_sg = rds_message_alloc_sgs(rm, nr_pages, &ret); |
668 | if (!op->op_sg) | |
ea010070 | 669 | goto out_pages; |
eff5f53b | 670 | |
f8b3aaf2 | 671 | if (op->op_notify || op->op_recverr) { |
eff5f53b AG |
672 | /* We allocate an uninitialized notifier here, because |
673 | * we don't want to do that in the completion handler. We | |
674 | * would have to use GFP_ATOMIC there, and don't want to deal | |
675 | * with failed allocations. | |
676 | */ | |
f8b3aaf2 AG |
677 | op->op_notifier = kmalloc(sizeof(struct rds_notifier), GFP_KERNEL); |
678 | if (!op->op_notifier) { | |
eff5f53b | 679 | ret = -ENOMEM; |
ea010070 | 680 | goto out_pages; |
eff5f53b | 681 | } |
f8b3aaf2 AG |
682 | op->op_notifier->n_user_token = args->user_token; |
683 | op->op_notifier->n_status = RDS_RDMA_SUCCESS; | |
eff5f53b AG |
684 | } |
685 | ||
686 | /* The cookie contains the R_Key of the remote memory region, and | |
687 | * optionally an offset into it. This is how we implement RDMA into | |
688 | * unaligned memory. | |
689 | * When setting up the RDMA, we need to add that offset to the | |
690 | * destination address (which is really an offset into the MR) | |
691 | * FIXME: We may want to move this into ib_rdma.c | |
692 | */ | |
f8b3aaf2 AG |
693 | op->op_rkey = rds_rdma_cookie_key(args->cookie); |
694 | op->op_remote_addr = args->remote_vec.addr + rds_rdma_cookie_offset(args->cookie); | |
eff5f53b AG |
695 | |
696 | nr_bytes = 0; | |
697 | ||
698 | rdsdebug("RDS: rdma prepare nr_local %llu rva %llx rkey %x\n", | |
699 | (unsigned long long)args->nr_local, | |
700 | (unsigned long long)args->remote_vec.addr, | |
f8b3aaf2 | 701 | op->op_rkey); |
eff5f53b AG |
702 | |
703 | for (i = 0; i < args->nr_local; i++) { | |
fc8162e3 AG |
704 | struct rds_iovec *iov = &iovs[i]; |
705 | /* don't need to check, rds_rdma_pages() verified nr will be +nonzero */ | |
706 | unsigned int nr = rds_pages_in_vec(iov); | |
eff5f53b | 707 | |
fc8162e3 AG |
708 | rs->rs_user_addr = iov->addr; |
709 | rs->rs_user_bytes = iov->bytes; | |
eff5f53b | 710 | |
eff5f53b AG |
711 | /* If it's a WRITE operation, we want to pin the pages for reading. |
712 | * If it's a READ operation, we need to pin the pages for writing. | |
713 | */ | |
fc8162e3 | 714 | ret = rds_pin_pages(iov->addr, nr, pages, !op->op_write); |
2eafa174 HWR |
715 | if ((!odp_supported && ret <= 0) || |
716 | (odp_supported && ret <= 0 && ret != -EOPNOTSUPP)) | |
ea010070 | 717 | goto out_pages; |
2eafa174 HWR |
718 | |
719 | if (ret == -EOPNOTSUPP) { | |
720 | struct rds_mr *local_odp_mr; | |
721 | ||
722 | if (!rs->rs_transport->get_mr) { | |
723 | ret = -EOPNOTSUPP; | |
724 | goto out_pages; | |
725 | } | |
726 | local_odp_mr = | |
727 | kzalloc(sizeof(*local_odp_mr), GFP_KERNEL); | |
728 | if (!local_odp_mr) { | |
729 | ret = -ENOMEM; | |
730 | goto out_pages; | |
731 | } | |
732 | RB_CLEAR_NODE(&local_odp_mr->r_rb_node); | |
733 | refcount_set(&local_odp_mr->r_refcount, 1); | |
734 | local_odp_mr->r_trans = rs->rs_transport; | |
735 | local_odp_mr->r_sock = rs; | |
736 | local_odp_mr->r_trans_private = | |
737 | rs->rs_transport->get_mr( | |
738 | NULL, 0, rs, &local_odp_mr->r_key, NULL, | |
739 | iov->addr, iov->bytes, ODP_VIRTUAL); | |
740 | if (IS_ERR(local_odp_mr->r_trans_private)) { | |
741 | ret = IS_ERR(local_odp_mr->r_trans_private); | |
742 | rdsdebug("get_mr ret %d %p\"", ret, | |
743 | local_odp_mr->r_trans_private); | |
744 | kfree(local_odp_mr); | |
745 | ret = -EOPNOTSUPP; | |
746 | goto out_pages; | |
747 | } | |
748 | rdsdebug("Need odp; local_odp_mr %p trans_private %p\n", | |
749 | local_odp_mr, local_odp_mr->r_trans_private); | |
750 | op->op_odp_mr = local_odp_mr; | |
751 | op->op_odp_addr = iov->addr; | |
752 | } | |
eff5f53b | 753 | |
fc8162e3 AG |
754 | rdsdebug("RDS: nr_bytes %u nr %u iov->bytes %llu iov->addr %llx\n", |
755 | nr_bytes, nr, iov->bytes, iov->addr); | |
eff5f53b | 756 | |
fc8162e3 | 757 | nr_bytes += iov->bytes; |
eff5f53b AG |
758 | |
759 | for (j = 0; j < nr; j++) { | |
fc8162e3 | 760 | unsigned int offset = iov->addr & ~PAGE_MASK; |
ff87e97a | 761 | struct scatterlist *sg; |
eff5f53b | 762 | |
f8b3aaf2 | 763 | sg = &op->op_sg[op->op_nents + j]; |
eff5f53b | 764 | sg_set_page(sg, pages[j], |
fc8162e3 | 765 | min_t(unsigned int, iov->bytes, PAGE_SIZE - offset), |
eff5f53b AG |
766 | offset); |
767 | ||
2eafa174 | 768 | sg_dma_len(sg) = sg->length; |
fc8162e3 AG |
769 | rdsdebug("RDS: sg->offset %x sg->len %x iov->addr %llx iov->bytes %llu\n", |
770 | sg->offset, sg->length, iov->addr, iov->bytes); | |
eff5f53b | 771 | |
fc8162e3 AG |
772 | iov->addr += sg->length; |
773 | iov->bytes -= sg->length; | |
eff5f53b AG |
774 | } |
775 | ||
f8b3aaf2 | 776 | op->op_nents += nr; |
eff5f53b AG |
777 | } |
778 | ||
eff5f53b AG |
779 | if (nr_bytes > args->remote_vec.bytes) { |
780 | rdsdebug("RDS nr_bytes %u remote_bytes %u do not match\n", | |
781 | nr_bytes, | |
782 | (unsigned int) args->remote_vec.bytes); | |
783 | ret = -EINVAL; | |
ea010070 | 784 | goto out_pages; |
eff5f53b | 785 | } |
f8b3aaf2 | 786 | op->op_bytes = nr_bytes; |
2eafa174 | 787 | ret = 0; |
eff5f53b | 788 | |
ea010070 | 789 | out_pages: |
eff5f53b | 790 | kfree(pages); |
dee49f20 | 791 | out_ret: |
ff87e97a AG |
792 | if (ret) |
793 | rds_rdma_free_op(op); | |
f4a3fc03 AG |
794 | else |
795 | rds_stats_inc(s_send_rdma); | |
4324879d AG |
796 | |
797 | return ret; | |
eff5f53b AG |
798 | } |
799 | ||
800 | /* | |
801 | * The application wants us to pass an RDMA destination (aka MR) | |
802 | * to the remote | |
803 | */ | |
804 | int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm, | |
805 | struct cmsghdr *cmsg) | |
806 | { | |
807 | unsigned long flags; | |
808 | struct rds_mr *mr; | |
809 | u32 r_key; | |
810 | int err = 0; | |
811 | ||
f64f9e71 JP |
812 | if (cmsg->cmsg_len < CMSG_LEN(sizeof(rds_rdma_cookie_t)) || |
813 | rm->m_rdma_cookie != 0) | |
eff5f53b AG |
814 | return -EINVAL; |
815 | ||
816 | memcpy(&rm->m_rdma_cookie, CMSG_DATA(cmsg), sizeof(rm->m_rdma_cookie)); | |
817 | ||
818 | /* We are reusing a previously mapped MR here. Most likely, the | |
819 | * application has written to the buffer, so we need to explicitly | |
820 | * flush those writes to RAM. Otherwise the HCA may not see them | |
821 | * when doing a DMA from that buffer. | |
822 | */ | |
823 | r_key = rds_rdma_cookie_key(rm->m_rdma_cookie); | |
824 | ||
825 | spin_lock_irqsave(&rs->rs_rdma_lock, flags); | |
826 | mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL); | |
8690bfa1 | 827 | if (!mr) |
eff5f53b AG |
828 | err = -EINVAL; /* invalid r_key */ |
829 | else | |
803ea850 | 830 | refcount_inc(&mr->r_refcount); |
eff5f53b AG |
831 | spin_unlock_irqrestore(&rs->rs_rdma_lock, flags); |
832 | ||
833 | if (mr) { | |
2eafa174 HWR |
834 | mr->r_trans->sync_mr(mr->r_trans_private, |
835 | DMA_TO_DEVICE); | |
f8b3aaf2 | 836 | rm->rdma.op_rdma_mr = mr; |
eff5f53b AG |
837 | } |
838 | return err; | |
839 | } | |
840 | ||
841 | /* | |
842 | * The application passes us an address range it wants to enable RDMA | |
843 | * to/from. We map the area, and save the <R_Key,offset> pair | |
844 | * in rm->m_rdma_cookie. This causes it to be sent along to the peer | |
845 | * in an extension header. | |
846 | */ | |
847 | int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm, | |
848 | struct cmsghdr *cmsg) | |
849 | { | |
f64f9e71 JP |
850 | if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_get_mr_args)) || |
851 | rm->m_rdma_cookie != 0) | |
eff5f53b AG |
852 | return -EINVAL; |
853 | ||
9e630bcb AR |
854 | return __rds_rdma_map(rs, CMSG_DATA(cmsg), &rm->m_rdma_cookie, |
855 | &rm->rdma.op_rdma_mr, rm->m_conn_path); | |
eff5f53b | 856 | } |
15133f6e AG |
857 | |
858 | /* | |
859 | * Fill in rds_message for an atomic request. | |
860 | */ | |
861 | int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm, | |
862 | struct cmsghdr *cmsg) | |
863 | { | |
864 | struct page *page = NULL; | |
865 | struct rds_atomic_args *args; | |
866 | int ret = 0; | |
867 | ||
868 | if (cmsg->cmsg_len < CMSG_LEN(sizeof(struct rds_atomic_args)) | |
869 | || rm->atomic.op_active) | |
870 | return -EINVAL; | |
871 | ||
872 | args = CMSG_DATA(cmsg); | |
873 | ||
20c72bd5 AG |
874 | /* Nonmasked & masked cmsg ops converted to masked hw ops */ |
875 | switch (cmsg->cmsg_type) { | |
876 | case RDS_CMSG_ATOMIC_FADD: | |
877 | rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD; | |
878 | rm->atomic.op_m_fadd.add = args->fadd.add; | |
879 | rm->atomic.op_m_fadd.nocarry_mask = 0; | |
880 | break; | |
881 | case RDS_CMSG_MASKED_ATOMIC_FADD: | |
15133f6e | 882 | rm->atomic.op_type = RDS_ATOMIC_TYPE_FADD; |
20c72bd5 AG |
883 | rm->atomic.op_m_fadd.add = args->m_fadd.add; |
884 | rm->atomic.op_m_fadd.nocarry_mask = args->m_fadd.nocarry_mask; | |
885 | break; | |
886 | case RDS_CMSG_ATOMIC_CSWP: | |
887 | rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP; | |
888 | rm->atomic.op_m_cswp.compare = args->cswp.compare; | |
889 | rm->atomic.op_m_cswp.swap = args->cswp.swap; | |
890 | rm->atomic.op_m_cswp.compare_mask = ~0; | |
891 | rm->atomic.op_m_cswp.swap_mask = ~0; | |
892 | break; | |
893 | case RDS_CMSG_MASKED_ATOMIC_CSWP: | |
894 | rm->atomic.op_type = RDS_ATOMIC_TYPE_CSWP; | |
895 | rm->atomic.op_m_cswp.compare = args->m_cswp.compare; | |
896 | rm->atomic.op_m_cswp.swap = args->m_cswp.swap; | |
897 | rm->atomic.op_m_cswp.compare_mask = args->m_cswp.compare_mask; | |
898 | rm->atomic.op_m_cswp.swap_mask = args->m_cswp.swap_mask; | |
899 | break; | |
900 | default: | |
901 | BUG(); /* should never happen */ | |
15133f6e AG |
902 | } |
903 | ||
15133f6e | 904 | rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME); |
2c3a5f9a | 905 | rm->atomic.op_silent = !!(args->flags & RDS_RDMA_SILENT); |
7e3bd65e | 906 | rm->atomic.op_active = 1; |
15133f6e | 907 | rm->atomic.op_recverr = rs->rs_recverr; |
c75ab8a5 | 908 | rm->atomic.op_sg = rds_message_alloc_sgs(rm, 1, &ret); |
909 | if (!rm->atomic.op_sg) | |
d139ff09 | 910 | goto err; |
15133f6e AG |
911 | |
912 | /* verify 8 byte-aligned */ | |
913 | if (args->local_addr & 0x7) { | |
914 | ret = -EFAULT; | |
915 | goto err; | |
916 | } | |
917 | ||
918 | ret = rds_pin_pages(args->local_addr, 1, &page, 1); | |
919 | if (ret != 1) | |
920 | goto err; | |
921 | ret = 0; | |
922 | ||
923 | sg_set_page(rm->atomic.op_sg, page, 8, offset_in_page(args->local_addr)); | |
924 | ||
925 | if (rm->atomic.op_notify || rm->atomic.op_recverr) { | |
926 | /* We allocate an uninitialized notifier here, because | |
927 | * we don't want to do that in the completion handler. We | |
928 | * would have to use GFP_ATOMIC there, and don't want to deal | |
929 | * with failed allocations. | |
930 | */ | |
931 | rm->atomic.op_notifier = kmalloc(sizeof(*rm->atomic.op_notifier), GFP_KERNEL); | |
932 | if (!rm->atomic.op_notifier) { | |
933 | ret = -ENOMEM; | |
934 | goto err; | |
935 | } | |
936 | ||
937 | rm->atomic.op_notifier->n_user_token = args->user_token; | |
938 | rm->atomic.op_notifier->n_status = RDS_RDMA_SUCCESS; | |
939 | } | |
940 | ||
40589e74 | 941 | rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie); |
15133f6e AG |
942 | rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie); |
943 | ||
15133f6e AG |
944 | return ret; |
945 | err: | |
946 | if (page) | |
0d4597c8 | 947 | unpin_user_page(page); |
7d11f77f | 948 | rm->atomic.op_active = 0; |
15133f6e AG |
949 | kfree(rm->atomic.op_notifier); |
950 | ||
951 | return ret; | |
952 | } |