Commit | Line | Data |
---|---|---|
a2268cfb | 1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
f58851e6 | 2 | /* |
62b56a67 | 3 | * Copyright (c) 2014-2017 Oracle. All rights reserved. |
c56c65fb TT |
4 | * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. |
5 | * | |
6 | * This software is available to you under a choice of one of two | |
7 | * licenses. You may choose to be licensed under the terms of the GNU | |
8 | * General Public License (GPL) Version 2, available from the file | |
9 | * COPYING in the main directory of this source tree, or the BSD-type | |
10 | * license below: | |
11 | * | |
12 | * Redistribution and use in source and binary forms, with or without | |
13 | * modification, are permitted provided that the following conditions | |
14 | * are met: | |
15 | * | |
16 | * Redistributions of source code must retain the above copyright | |
17 | * notice, this list of conditions and the following disclaimer. | |
18 | * | |
19 | * Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials provided | |
22 | * with the distribution. | |
23 | * | |
24 | * Neither the name of the Network Appliance, Inc. nor the names of | |
25 | * its contributors may be used to endorse or promote products | |
26 | * derived from this software without specific prior written | |
27 | * permission. | |
28 | * | |
29 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
30 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
31 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
32 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
33 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
34 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
35 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
36 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
37 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
38 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
39 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
f58851e6 TT |
40 | */ |
41 | ||
c56c65fb TT |
42 | /* |
43 | * verbs.c | |
44 | * | |
45 | * Encapsulates the major functions managing: | |
46 | * o adapters | |
47 | * o endpoints | |
48 | * o connections | |
49 | * o buffer memory | |
50 | */ | |
51 | ||
a6b7a407 | 52 | #include <linux/interrupt.h> |
5a0e3ad6 | 53 | #include <linux/slab.h> |
0dd39cae | 54 | #include <linux/sunrpc/addr.h> |
05c97466 | 55 | #include <linux/sunrpc/svc_rdma.h> |
f3c66a2f | 56 | #include <linux/log2.h> |
ae72950a CL |
57 | |
58 | #include <asm-generic/barrier.h> | |
65866f82 | 59 | #include <asm/bitops.h> |
56a6bd15 | 60 | |
0a90487b | 61 | #include <rdma/ib_cm.h> |
c56c65fb | 62 | |
f58851e6 | 63 | #include "xprt_rdma.h" |
b6e717cb | 64 | #include <trace/events/rpcrdma.h> |
f58851e6 | 65 | |
cb586dec CL |
66 | static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt); |
67 | static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt); | |
f995879e CL |
68 | static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, |
69 | struct rpcrdma_sendctx *sc); | |
b78de1dc | 70 | static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt); |
a31b2f93 | 71 | static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt); |
85810388 | 72 | static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep); |
671c450b | 73 | static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt); |
96ceddea | 74 | static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt); |
9d2da4ff | 75 | static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); |
2acc5cae CL |
76 | static void rpcrdma_ep_get(struct rpcrdma_ep *ep); |
77 | static int rpcrdma_ep_put(struct rpcrdma_ep *ep); | |
d2832af3 | 78 | static struct rpcrdma_regbuf * |
7ac18798 | 79 | rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction); |
d2832af3 CL |
80 | static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb); |
81 | static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); | |
c56c65fb | 82 | |
b8fe677f CL |
83 | /* Wait for outstanding transport work to finish. ib_drain_qp |
84 | * handles the drains in the wrong order for us, so open code | |
85 | * them here. | |
6d2d0ee2 CL |
86 | */ |
87 | static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt) | |
c56c65fb | 88 | { |
2acc5cae CL |
89 | struct rpcrdma_ep *ep = r_xprt->rx_ep; |
90 | struct rdma_cm_id *id = ep->re_id; | |
c56c65fb | 91 | |
15788d1d CL |
92 | /* Wait for rpcrdma_post_recvs() to leave its critical |
93 | * section. | |
94 | */ | |
95 | if (atomic_inc_return(&ep->re_receiving) > 1) | |
96 | wait_for_completion(&ep->re_done); | |
97 | ||
6d2d0ee2 CL |
98 | /* Flush Receives, then wait for deferred Reply work |
99 | * to complete. | |
100 | */ | |
93aa8e0a | 101 | ib_drain_rq(id->qp); |
f1a03b76 | 102 | |
6d2d0ee2 CL |
103 | /* Deferred Reply processing might have scheduled |
104 | * local invalidations. | |
105 | */ | |
93aa8e0a | 106 | ib_drain_sq(id->qp); |
2acc5cae CL |
107 | |
108 | rpcrdma_ep_put(ep); | |
f1a03b76 CL |
109 | } |
110 | ||
c487eb7d CL |
111 | /* Ensure xprt_force_disconnect() is invoked exactly once when a |
112 | * connection is closed or lost. (The important thing is it needs | |
113 | * to be invoked "at least" once). | |
114 | */ | |
1143129e | 115 | void rpcrdma_force_disconnect(struct rpcrdma_ep *ep) |
c487eb7d CL |
116 | { |
117 | if (atomic_add_unless(&ep->re_force_disconnect, 1, 1)) | |
118 | xprt_force_disconnect(ep->re_xprt); | |
119 | } | |
120 | ||
d6ccebf9 CL |
121 | /** |
122 | * rpcrdma_flush_disconnect - Disconnect on flushed completion | |
f423f755 | 123 | * @r_xprt: transport to disconnect |
d6ccebf9 CL |
124 | * @wc: work completion entry |
125 | * | |
126 | * Must be called in process context. | |
127 | */ | |
f423f755 | 128 | void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc) |
d6ccebf9 | 129 | { |
c487eb7d CL |
130 | if (wc->status != IB_WC_SUCCESS) |
131 | rpcrdma_force_disconnect(r_xprt->rx_ep); | |
d6ccebf9 CL |
132 | } |
133 | ||
2fa8f88d CL |
134 | /** |
135 | * rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC | |
f995879e | 136 | * @cq: completion queue |
d6ccebf9 | 137 | * @wc: WCE for a completed Send WR |
2fa8f88d | 138 | * |
fc664485 | 139 | */ |
d6ccebf9 | 140 | static void rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) |
fc664485 | 141 | { |
ae72950a CL |
142 | struct ib_cqe *cqe = wc->wr_cqe; |
143 | struct rpcrdma_sendctx *sc = | |
144 | container_of(cqe, struct rpcrdma_sendctx, sc_cqe); | |
f423f755 | 145 | struct rpcrdma_xprt *r_xprt = cq->cq_context; |
ae72950a | 146 | |
2fa8f88d | 147 | /* WARNING: Only wr_cqe and status are reliable at this point */ |
b2e7467f | 148 | trace_xprtrdma_wc_send(wc, &sc->sc_cid); |
f423f755 CL |
149 | rpcrdma_sendctx_put_locked(r_xprt, sc); |
150 | rpcrdma_flush_disconnect(r_xprt, wc); | |
fc664485 | 151 | } |
c56c65fb | 152 | |
552bf225 | 153 | /** |
1519e969 | 154 | * rpcrdma_wc_receive - Invoked by RDMA provider for each polled Receive WC |
d6ccebf9 CL |
155 | * @cq: completion queue |
156 | * @wc: WCE for a completed Receive WR | |
552bf225 CL |
157 | * |
158 | */ | |
d6ccebf9 | 159 | static void rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) |
fc664485 | 160 | { |
552bf225 CL |
161 | struct ib_cqe *cqe = wc->wr_cqe; |
162 | struct rpcrdma_rep *rep = container_of(cqe, struct rpcrdma_rep, | |
163 | rr_cqe); | |
d6ccebf9 | 164 | struct rpcrdma_xprt *r_xprt = cq->cq_context; |
fc664485 | 165 | |
6ceea368 | 166 | /* WARNING: Only wr_cqe and status are reliable at this point */ |
af5865d2 | 167 | trace_xprtrdma_wc_receive(wc, &rep->rr_cid); |
e28ce900 | 168 | --r_xprt->rx_ep->re_receive_count; |
8502427c | 169 | if (wc->status != IB_WC_SUCCESS) |
6ceea368 | 170 | goto out_flushed; |
fc664485 | 171 | |
8502427c | 172 | /* status == SUCCESS means all fields in wc are trustworthy */ |
96f8778f | 173 | rpcrdma_set_xdrlen(&rep->rr_hdrbuf, wc->byte_len); |
c8b920bb CL |
174 | rep->rr_wc_flags = wc->wc_flags; |
175 | rep->rr_inv_rkey = wc->ex.invalidate_rkey; | |
176 | ||
91a10c52 | 177 | ib_dma_sync_single_for_cpu(rdmab_device(rep->rr_rdmabuf), |
6b1184cd | 178 | rdmab_addr(rep->rr_rdmabuf), |
e2a67190 | 179 | wc->byte_len, DMA_FROM_DEVICE); |
23826c7a | 180 | |
d8f532d2 | 181 | rpcrdma_reply_handler(rep); |
8502427c | 182 | return; |
fe97b47c | 183 | |
6ceea368 | 184 | out_flushed: |
f423f755 | 185 | rpcrdma_flush_disconnect(r_xprt, wc); |
5030c9a9 | 186 | rpcrdma_rep_put(&r_xprt->rx_buf, rep); |
fc664485 CL |
187 | } |
188 | ||
745b734c | 189 | static void rpcrdma_update_cm_private(struct rpcrdma_ep *ep, |
f54c870d | 190 | struct rdma_conn_param *param) |
87cfb9a0 | 191 | { |
87cfb9a0 CL |
192 | const struct rpcrdma_connect_private *pmsg = param->private_data; |
193 | unsigned int rsize, wsize; | |
194 | ||
c8b920bb | 195 | /* Default settings for RPC-over-RDMA Version One */ |
87cfb9a0 CL |
196 | rsize = RPCRDMA_V1_DEF_INLINE_SIZE; |
197 | wsize = RPCRDMA_V1_DEF_INLINE_SIZE; | |
198 | ||
199 | if (pmsg && | |
200 | pmsg->cp_magic == rpcrdma_cmp_magic && | |
201 | pmsg->cp_version == RPCRDMA_CMP_VERSION) { | |
202 | rsize = rpcrdma_decode_buffer_size(pmsg->cp_send_size); | |
203 | wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size); | |
204 | } | |
205 | ||
93aa8e0a CL |
206 | if (rsize < ep->re_inline_recv) |
207 | ep->re_inline_recv = rsize; | |
208 | if (wsize < ep->re_inline_send) | |
209 | ep->re_inline_send = wsize; | |
f54c870d | 210 | |
93aa8e0a | 211 | rpcrdma_set_max_header_sizes(ep); |
87cfb9a0 CL |
212 | } |
213 | ||
ae38288e CL |
214 | /** |
215 | * rpcrdma_cm_event_handler - Handle RDMA CM events | |
216 | * @id: rdma_cm_id on which an event has occurred | |
217 | * @event: details of the event | |
218 | * | |
219 | * Called with @id's mutex held. Returns 1 if caller should | |
220 | * destroy @id, otherwise 0. | |
221 | */ | |
c56c65fb | 222 | static int |
ae38288e | 223 | rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) |
c56c65fb | 224 | { |
745b734c | 225 | struct sockaddr *sap = (struct sockaddr *)&id->route.addr.dst_addr; |
e28ce900 | 226 | struct rpcrdma_ep *ep = id->context; |
c56c65fb | 227 | |
ae38288e CL |
228 | might_sleep(); |
229 | ||
c56c65fb TT |
230 | switch (event->event) { |
231 | case RDMA_CM_EVENT_ADDR_RESOLVED: | |
232 | case RDMA_CM_EVENT_ROUTE_RESOLVED: | |
93aa8e0a CL |
233 | ep->re_async_rc = 0; |
234 | complete(&ep->re_done); | |
316a616e | 235 | return 0; |
c56c65fb | 236 | case RDMA_CM_EVENT_ADDR_ERROR: |
93aa8e0a CL |
237 | ep->re_async_rc = -EPROTO; |
238 | complete(&ep->re_done); | |
316a616e | 239 | return 0; |
c56c65fb | 240 | case RDMA_CM_EVENT_ROUTE_ERROR: |
93aa8e0a CL |
241 | ep->re_async_rc = -ENETUNREACH; |
242 | complete(&ep->re_done); | |
316a616e | 243 | return 0; |
bebd0318 | 244 | case RDMA_CM_EVENT_DEVICE_REMOVAL: |
745b734c CL |
245 | pr_info("rpcrdma: removing device %s for %pISpc\n", |
246 | ep->re_id->device->name, sap); | |
df561f66 | 247 | fallthrough; |
e28ce900 | 248 | case RDMA_CM_EVENT_ADDR_CHANGE: |
93aa8e0a | 249 | ep->re_connect_status = -ENODEV; |
e28ce900 | 250 | goto disconnected; |
c56c65fb | 251 | case RDMA_CM_EVENT_ESTABLISHED: |
2acc5cae | 252 | rpcrdma_ep_get(ep); |
93aa8e0a | 253 | ep->re_connect_status = 1; |
745b734c CL |
254 | rpcrdma_update_cm_private(ep, &event->param.conn); |
255 | trace_xprtrdma_inline_thresh(ep); | |
93aa8e0a | 256 | wake_up_all(&ep->re_connect_wait); |
31e62d25 | 257 | break; |
c56c65fb | 258 | case RDMA_CM_EVENT_CONNECT_ERROR: |
93aa8e0a | 259 | ep->re_connect_status = -ENOTCONN; |
af667527 | 260 | goto wake_connect_worker; |
c56c65fb | 261 | case RDMA_CM_EVENT_UNREACHABLE: |
93aa8e0a | 262 | ep->re_connect_status = -ENETUNREACH; |
af667527 | 263 | goto wake_connect_worker; |
c56c65fb | 264 | case RDMA_CM_EVENT_REJECTED: |
93aa8e0a | 265 | ep->re_connect_status = -ECONNREFUSED; |
0a90487b | 266 | if (event->status == IB_CM_REJ_STALE_CONN) |
4cf44be6 | 267 | ep->re_connect_status = -ENOTCONN; |
af667527 CL |
268 | wake_connect_worker: |
269 | wake_up_all(&ep->re_connect_wait); | |
270 | return 0; | |
c56c65fb | 271 | case RDMA_CM_EVENT_DISCONNECTED: |
93aa8e0a | 272 | ep->re_connect_status = -ECONNABORTED; |
31e62d25 | 273 | disconnected: |
c487eb7d | 274 | rpcrdma_force_disconnect(ep); |
2acc5cae | 275 | return rpcrdma_ep_put(ep); |
c56c65fb | 276 | default: |
c56c65fb TT |
277 | break; |
278 | } | |
279 | ||
280 | return 0; | |
281 | } | |
282 | ||
93aa8e0a CL |
283 | static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt, |
284 | struct rpcrdma_ep *ep) | |
c56c65fb | 285 | { |
109b88ab | 286 | unsigned long wtimeout = msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1; |
93aa8e0a | 287 | struct rpc_xprt *xprt = &r_xprt->rx_xprt; |
c56c65fb TT |
288 | struct rdma_cm_id *id; |
289 | int rc; | |
290 | ||
93aa8e0a | 291 | init_completion(&ep->re_done); |
1a954051 | 292 | |
e28ce900 | 293 | id = rdma_create_id(xprt->xprt_net, rpcrdma_cm_event_handler, ep, |
93aa8e0a | 294 | RDMA_PS_TCP, IB_QPT_RC); |
ddbb347f | 295 | if (IS_ERR(id)) |
c56c65fb | 296 | return id; |
c56c65fb | 297 | |
93aa8e0a CL |
298 | ep->re_async_rc = -ETIMEDOUT; |
299 | rc = rdma_resolve_addr(id, NULL, (struct sockaddr *)&xprt->addr, | |
dd229cee | 300 | RDMA_RESOLVE_TIMEOUT); |
ddbb347f | 301 | if (rc) |
c56c65fb | 302 | goto out; |
93aa8e0a | 303 | rc = wait_for_completion_interruptible_timeout(&ep->re_done, wtimeout); |
7b020f17 | 304 | if (rc < 0) |
109b88ab | 305 | goto out; |
d0f36c46 | 306 | |
93aa8e0a | 307 | rc = ep->re_async_rc; |
c56c65fb TT |
308 | if (rc) |
309 | goto out; | |
310 | ||
93aa8e0a | 311 | ep->re_async_rc = -ETIMEDOUT; |
c56c65fb | 312 | rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); |
ddbb347f | 313 | if (rc) |
56a6bd15 | 314 | goto out; |
93aa8e0a | 315 | rc = wait_for_completion_interruptible_timeout(&ep->re_done, wtimeout); |
7b020f17 | 316 | if (rc < 0) |
56a6bd15 | 317 | goto out; |
93aa8e0a | 318 | rc = ep->re_async_rc; |
c56c65fb | 319 | if (rc) |
56a6bd15 | 320 | goto out; |
c56c65fb TT |
321 | |
322 | return id; | |
56a6bd15 | 323 | |
c56c65fb TT |
324 | out: |
325 | rdma_destroy_id(id); | |
326 | return ERR_PTR(rc); | |
327 | } | |
328 | ||
2acc5cae | 329 | static void rpcrdma_ep_destroy(struct kref *kref) |
e28ce900 CL |
330 | { |
331 | struct rpcrdma_ep *ep = container_of(kref, struct rpcrdma_ep, re_kref); | |
332 | ||
333 | if (ep->re_id->qp) { | |
334 | rdma_destroy_qp(ep->re_id); | |
335 | ep->re_id->qp = NULL; | |
336 | } | |
337 | ||
338 | if (ep->re_attr.recv_cq) | |
339 | ib_free_cq(ep->re_attr.recv_cq); | |
340 | ep->re_attr.recv_cq = NULL; | |
341 | if (ep->re_attr.send_cq) | |
342 | ib_free_cq(ep->re_attr.send_cq); | |
343 | ep->re_attr.send_cq = NULL; | |
344 | ||
345 | if (ep->re_pd) | |
346 | ib_dealloc_pd(ep->re_pd); | |
347 | ep->re_pd = NULL; | |
348 | ||
349 | kfree(ep); | |
350 | module_put(THIS_MODULE); | |
351 | } | |
352 | ||
2acc5cae CL |
353 | static noinline void rpcrdma_ep_get(struct rpcrdma_ep *ep) |
354 | { | |
355 | kref_get(&ep->re_kref); | |
356 | } | |
357 | ||
e28ce900 CL |
358 | /* Returns: |
359 | * %0 if @ep still has a positive kref count, or | |
360 | * %1 if @ep was destroyed successfully. | |
c56c65fb | 361 | */ |
2acc5cae | 362 | static noinline int rpcrdma_ep_put(struct rpcrdma_ep *ep) |
e28ce900 | 363 | { |
2acc5cae | 364 | return kref_put(&ep->re_kref, rpcrdma_ep_destroy); |
e28ce900 | 365 | } |
c56c65fb | 366 | |
81fe0c57 | 367 | static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) |
c56c65fb | 368 | { |
e28ce900 CL |
369 | struct rpcrdma_connect_private *pmsg; |
370 | struct ib_device *device; | |
81fe0c57 | 371 | struct rdma_cm_id *id; |
e28ce900 | 372 | struct rpcrdma_ep *ep; |
2fa8f88d | 373 | int rc; |
c56c65fb | 374 | |
9c8f332f | 375 | ep = kzalloc(sizeof(*ep), XPRTRDMA_GFP_FLAGS); |
e28ce900 | 376 | if (!ep) |
dda9a951 | 377 | return -ENOTCONN; |
e28ce900 CL |
378 | ep->re_xprt = &r_xprt->rx_xprt; |
379 | kref_init(&ep->re_kref); | |
380 | ||
93aa8e0a | 381 | id = rpcrdma_create_id(r_xprt, ep); |
e28ce900 | 382 | if (IS_ERR(id)) { |
85bfd71b CL |
383 | kfree(ep); |
384 | return PTR_ERR(id); | |
e28ce900 CL |
385 | } |
386 | __module_get(THIS_MODULE); | |
387 | device = id->device; | |
388 | ep->re_id = id; | |
15788d1d | 389 | reinit_completion(&ep->re_done); |
81fe0c57 | 390 | |
93aa8e0a CL |
391 | ep->re_max_requests = r_xprt->rx_xprt.max_reqs; |
392 | ep->re_inline_send = xprt_rdma_max_inline_write; | |
393 | ep->re_inline_recv = xprt_rdma_max_inline_read; | |
e28ce900 | 394 | rc = frwr_query_device(ep, device); |
914fcad9 | 395 | if (rc) |
81fe0c57 CL |
396 | goto out_destroy; |
397 | ||
93aa8e0a | 398 | r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests); |
c56c65fb | 399 | |
93aa8e0a CL |
400 | ep->re_attr.srq = NULL; |
401 | ep->re_attr.cap.max_inline_data = 0; | |
402 | ep->re_attr.sq_sig_type = IB_SIGNAL_REQ_WR; | |
403 | ep->re_attr.qp_type = IB_QPT_RC; | |
404 | ep->re_attr.port_num = ~0; | |
c56c65fb | 405 | |
93aa8e0a CL |
406 | ep->re_send_batch = ep->re_max_requests >> 3; |
407 | ep->re_send_count = ep->re_send_batch; | |
408 | init_waitqueue_head(&ep->re_connect_wait); | |
409 | ||
e28ce900 | 410 | ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt, |
93aa8e0a CL |
411 | ep->re_attr.cap.max_send_wr, |
412 | IB_POLL_WORKQUEUE); | |
413 | if (IS_ERR(ep->re_attr.send_cq)) { | |
414 | rc = PTR_ERR(ep->re_attr.send_cq); | |
a9c10b5b | 415 | ep->re_attr.send_cq = NULL; |
85cd8e2b | 416 | goto out_destroy; |
c56c65fb TT |
417 | } |
418 | ||
e28ce900 | 419 | ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt, |
93aa8e0a CL |
420 | ep->re_attr.cap.max_recv_wr, |
421 | IB_POLL_WORKQUEUE); | |
422 | if (IS_ERR(ep->re_attr.recv_cq)) { | |
423 | rc = PTR_ERR(ep->re_attr.recv_cq); | |
a9c10b5b | 424 | ep->re_attr.recv_cq = NULL; |
85cd8e2b | 425 | goto out_destroy; |
fc664485 | 426 | } |
93aa8e0a | 427 | ep->re_receive_count = 0; |
fc664485 | 428 | |
c56c65fb | 429 | /* Initialize cma parameters */ |
93aa8e0a | 430 | memset(&ep->re_remote_cma, 0, sizeof(ep->re_remote_cma)); |
c56c65fb | 431 | |
87cfb9a0 | 432 | /* Prepare RDMA-CM private message */ |
e28ce900 | 433 | pmsg = &ep->re_cm_private; |
87cfb9a0 CL |
434 | pmsg->cp_magic = rpcrdma_cmp_magic; |
435 | pmsg->cp_version = RPCRDMA_CMP_VERSION; | |
5f62412b | 436 | pmsg->cp_flags |= RPCRDMA_CMP_F_SND_W_INV_OK; |
93aa8e0a CL |
437 | pmsg->cp_send_size = rpcrdma_encode_buffer_size(ep->re_inline_send); |
438 | pmsg->cp_recv_size = rpcrdma_encode_buffer_size(ep->re_inline_recv); | |
439 | ep->re_remote_cma.private_data = pmsg; | |
440 | ep->re_remote_cma.private_data_len = sizeof(*pmsg); | |
c56c65fb TT |
441 | |
442 | /* Client offers RDMA Read but does not initiate */ | |
93aa8e0a CL |
443 | ep->re_remote_cma.initiator_depth = 0; |
444 | ep->re_remote_cma.responder_resources = | |
e28ce900 | 445 | min_t(int, U8_MAX, device->attrs.max_qp_rd_atom); |
c56c65fb | 446 | |
b2dde94b CL |
447 | /* Limit transport retries so client can detect server |
448 | * GID changes quickly. RPC layer handles re-establishing | |
449 | * transport connection and retransmission. | |
450 | */ | |
93aa8e0a | 451 | ep->re_remote_cma.retry_count = 6; |
b2dde94b CL |
452 | |
453 | /* RPC-over-RDMA handles its own flow control. In addition, | |
454 | * make all RNR NAKs visible so we know that RPC-over-RDMA | |
455 | * flow control is working correctly (no NAKs should be seen). | |
456 | */ | |
93aa8e0a CL |
457 | ep->re_remote_cma.flow_control = 0; |
458 | ep->re_remote_cma.rnr_retry_count = 0; | |
c56c65fb | 459 | |
e28ce900 | 460 | ep->re_pd = ib_alloc_pd(device, 0); |
93aa8e0a CL |
461 | if (IS_ERR(ep->re_pd)) { |
462 | rc = PTR_ERR(ep->re_pd); | |
a9c10b5b | 463 | ep->re_pd = NULL; |
9ba373ee CL |
464 | goto out_destroy; |
465 | } | |
466 | ||
93aa8e0a | 467 | rc = rdma_create_qp(id, ep->re_pd, &ep->re_attr); |
85cd8e2b CL |
468 | if (rc) |
469 | goto out_destroy; | |
93aa8e0a | 470 | |
e28ce900 | 471 | r_xprt->rx_ep = ep; |
c56c65fb TT |
472 | return 0; |
473 | ||
85cd8e2b | 474 | out_destroy: |
2acc5cae | 475 | rpcrdma_ep_put(ep); |
81fe0c57 | 476 | rdma_destroy_id(id); |
c56c65fb TT |
477 | return rc; |
478 | } | |
479 | ||
e28ce900 CL |
480 | /** |
481 | * rpcrdma_xprt_connect - Connect an unconnected transport | |
482 | * @r_xprt: controlling transport instance | |
483 | * | |
484 | * Returns 0 on success or a negative errno. | |
c56c65fb | 485 | */ |
9144a803 | 486 | int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt) |
c56c65fb | 487 | { |
31e62d25 | 488 | struct rpc_xprt *xprt = &r_xprt->rx_xprt; |
e28ce900 | 489 | struct rpcrdma_ep *ep; |
1890896b | 490 | int rc; |
c56c65fb | 491 | |
81fe0c57 CL |
492 | rc = rpcrdma_ep_create(r_xprt); |
493 | if (rc) | |
e28ce900 CL |
494 | return rc; |
495 | ep = r_xprt->rx_ep; | |
c56c65fb | 496 | |
31e62d25 | 497 | xprt_clear_connected(xprt); |
eea63ca7 | 498 | rpcrdma_reset_cwnd(r_xprt); |
2acc5cae CL |
499 | |
500 | /* Bump the ep's reference count while there are | |
501 | * outstanding Receives. | |
502 | */ | |
503 | rpcrdma_ep_get(ep); | |
35d8b10a | 504 | rpcrdma_post_recvs(r_xprt, 1, true); |
c56c65fb | 505 | |
93aa8e0a | 506 | rc = rdma_connect(ep->re_id, &ep->re_remote_cma); |
ddbb347f | 507 | if (rc) |
c56c65fb | 508 | goto out; |
c56c65fb | 509 | |
f9e1afe0 CL |
510 | if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO) |
511 | xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; | |
93aa8e0a CL |
512 | wait_event_interruptible(ep->re_connect_wait, |
513 | ep->re_connect_status != 0); | |
514 | if (ep->re_connect_status <= 0) { | |
93aa8e0a | 515 | rc = ep->re_connect_status; |
0a90487b | 516 | goto out; |
c56c65fb TT |
517 | } |
518 | ||
dda9a951 CL |
519 | rc = rpcrdma_sendctxs_create(r_xprt); |
520 | if (rc) { | |
521 | rc = -ENOTCONN; | |
522 | goto out; | |
523 | } | |
524 | ||
b78de1dc | 525 | rc = rpcrdma_reqs_setup(r_xprt); |
dda9a951 CL |
526 | if (rc) { |
527 | rc = -ENOTCONN; | |
b78de1dc | 528 | goto out; |
dda9a951 | 529 | } |
9d2da4ff | 530 | rpcrdma_mrs_create(r_xprt); |
21037b8c | 531 | frwr_wp_create(r_xprt); |
7c8d9e7c | 532 | |
c56c65fb | 533 | out: |
7b020f17 | 534 | trace_xprtrdma_connect(r_xprt, rc); |
c56c65fb TT |
535 | return rc; |
536 | } | |
537 | ||
6d2d0ee2 | 538 | /** |
9144a803 CL |
539 | * rpcrdma_xprt_disconnect - Disconnect underlying transport |
540 | * @r_xprt: controlling transport instance | |
c56c65fb | 541 | * |
9d2da4ff CL |
542 | * Caller serializes. Either the transport send lock is held, |
543 | * or we're being called to destroy the transport. | |
e28ce900 CL |
544 | * |
545 | * On return, @r_xprt is completely divested of all hardware | |
546 | * resources and prepared for the next ->connect operation. | |
c56c65fb | 547 | */ |
9144a803 | 548 | void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) |
c56c65fb | 549 | { |
e28ce900 CL |
550 | struct rpcrdma_ep *ep = r_xprt->rx_ep; |
551 | struct rdma_cm_id *id; | |
552 | int rc; | |
c56c65fb | 553 | |
e28ce900 | 554 | if (!ep) |
897b7be9 | 555 | return; |
85cd8e2b | 556 | |
e28ce900 | 557 | id = ep->re_id; |
85cd8e2b | 558 | rc = rdma_disconnect(id); |
6d2d0ee2 | 559 | trace_xprtrdma_disconnect(r_xprt, rc); |
550d7502 | 560 | |
e28ce900 | 561 | rpcrdma_xprt_drain(r_xprt); |
897b7be9 | 562 | rpcrdma_reps_unmap(r_xprt); |
a31b2f93 | 563 | rpcrdma_reqs_reset(r_xprt); |
9d2da4ff | 564 | rpcrdma_mrs_destroy(r_xprt); |
cb586dec | 565 | rpcrdma_sendctxs_destroy(r_xprt); |
85cd8e2b | 566 | |
2acc5cae | 567 | if (rpcrdma_ep_put(ep)) |
897b7be9 | 568 | rdma_destroy_id(id); |
e28ce900 CL |
569 | |
570 | r_xprt->rx_ep = NULL; | |
c56c65fb TT |
571 | } |
572 | ||
ae72950a CL |
573 | /* Fixed-size circular FIFO queue. This implementation is wait-free and |
574 | * lock-free. | |
575 | * | |
576 | * Consumer is the code path that posts Sends. This path dequeues a | |
577 | * sendctx for use by a Send operation. Multiple consumer threads | |
578 | * are serialized by the RPC transport lock, which allows only one | |
579 | * ->send_request call at a time. | |
580 | * | |
581 | * Producer is the code path that handles Send completions. This path | |
582 | * enqueues a sendctx that has been completed. Multiple producer | |
583 | * threads are serialized by the ib_poll_cq() function. | |
584 | */ | |
585 | ||
586 | /* rpcrdma_sendctxs_destroy() assumes caller has already quiesced | |
b8fe677f CL |
587 | * queue activity, and rpcrdma_xprt_drain has flushed all remaining |
588 | * Send requests. | |
ae72950a | 589 | */ |
cb586dec | 590 | static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt) |
ae72950a | 591 | { |
cb586dec | 592 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
ae72950a CL |
593 | unsigned long i; |
594 | ||
cb586dec CL |
595 | if (!buf->rb_sc_ctxs) |
596 | return; | |
ae72950a CL |
597 | for (i = 0; i <= buf->rb_sc_last; i++) |
598 | kfree(buf->rb_sc_ctxs[i]); | |
599 | kfree(buf->rb_sc_ctxs); | |
cb586dec | 600 | buf->rb_sc_ctxs = NULL; |
ae72950a CL |
601 | } |
602 | ||
2e870368 | 603 | static struct rpcrdma_sendctx *rpcrdma_sendctx_create(struct rpcrdma_ep *ep) |
ae72950a CL |
604 | { |
605 | struct rpcrdma_sendctx *sc; | |
606 | ||
93aa8e0a | 607 | sc = kzalloc(struct_size(sc, sc_sges, ep->re_attr.cap.max_send_sge), |
9c8f332f | 608 | XPRTRDMA_GFP_FLAGS); |
ae72950a CL |
609 | if (!sc) |
610 | return NULL; | |
611 | ||
ae72950a | 612 | sc->sc_cqe.done = rpcrdma_wc_send; |
b2e7467f CL |
613 | sc->sc_cid.ci_queue_id = ep->re_attr.send_cq->res.id; |
614 | sc->sc_cid.ci_completion_id = | |
615 | atomic_inc_return(&ep->re_completion_ids); | |
ae72950a CL |
616 | return sc; |
617 | } | |
618 | ||
619 | static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) | |
620 | { | |
621 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; | |
622 | struct rpcrdma_sendctx *sc; | |
623 | unsigned long i; | |
624 | ||
625 | /* Maximum number of concurrent outstanding Send WRs. Capping | |
626 | * the circular queue size stops Send Queue overflow by causing | |
627 | * the ->send_request call to fail temporarily before too many | |
628 | * Sends are posted. | |
629 | */ | |
e28ce900 | 630 | i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS; |
9c8f332f | 631 | buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), XPRTRDMA_GFP_FLAGS); |
ae72950a CL |
632 | if (!buf->rb_sc_ctxs) |
633 | return -ENOMEM; | |
634 | ||
635 | buf->rb_sc_last = i - 1; | |
636 | for (i = 0; i <= buf->rb_sc_last; i++) { | |
e28ce900 | 637 | sc = rpcrdma_sendctx_create(r_xprt->rx_ep); |
ae72950a | 638 | if (!sc) |
6e17f58c | 639 | return -ENOMEM; |
ae72950a | 640 | |
ae72950a CL |
641 | buf->rb_sc_ctxs[i] = sc; |
642 | } | |
643 | ||
cb586dec CL |
644 | buf->rb_sc_head = 0; |
645 | buf->rb_sc_tail = 0; | |
ae72950a | 646 | return 0; |
ae72950a CL |
647 | } |
648 | ||
649 | /* The sendctx queue is not guaranteed to have a size that is a | |
650 | * power of two, thus the helpers in circ_buf.h cannot be used. | |
651 | * The other option is to use modulus (%), which can be expensive. | |
652 | */ | |
653 | static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf, | |
654 | unsigned long item) | |
655 | { | |
656 | return likely(item < buf->rb_sc_last) ? item + 1 : 0; | |
657 | } | |
658 | ||
659 | /** | |
660 | * rpcrdma_sendctx_get_locked - Acquire a send context | |
dbcc53a5 | 661 | * @r_xprt: controlling transport instance |
ae72950a CL |
662 | * |
663 | * Returns pointer to a free send completion context; or NULL if | |
664 | * the queue is empty. | |
665 | * | |
666 | * Usage: Called to acquire an SGE array before preparing a Send WR. | |
667 | * | |
dbcc53a5 CL |
668 | * The caller serializes calls to this function (per transport), and |
669 | * provides an effective memory barrier that flushes the new value | |
ae72950a CL |
670 | * of rb_sc_head. |
671 | */ | |
dbcc53a5 | 672 | struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt) |
ae72950a | 673 | { |
dbcc53a5 | 674 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
ae72950a CL |
675 | struct rpcrdma_sendctx *sc; |
676 | unsigned long next_head; | |
677 | ||
678 | next_head = rpcrdma_sendctx_next(buf, buf->rb_sc_head); | |
679 | ||
680 | if (next_head == READ_ONCE(buf->rb_sc_tail)) | |
681 | goto out_emptyq; | |
682 | ||
683 | /* ORDER: item must be accessed _before_ head is updated */ | |
684 | sc = buf->rb_sc_ctxs[next_head]; | |
685 | ||
686 | /* Releasing the lock in the caller acts as a memory | |
687 | * barrier that flushes rb_sc_head. | |
688 | */ | |
689 | buf->rb_sc_head = next_head; | |
690 | ||
691 | return sc; | |
692 | ||
693 | out_emptyq: | |
694 | /* The queue is "empty" if there have not been enough Send | |
695 | * completions recently. This is a sign the Send Queue is | |
696 | * backing up. Cause the caller to pause and try again. | |
697 | */ | |
05eb06d8 | 698 | xprt_wait_for_buffer_space(&r_xprt->rx_xprt); |
ae72950a CL |
699 | r_xprt->rx_stats.empty_sendctx_q++; |
700 | return NULL; | |
701 | } | |
702 | ||
703 | /** | |
704 | * rpcrdma_sendctx_put_locked - Release a send context | |
f995879e | 705 | * @r_xprt: controlling transport instance |
ae72950a CL |
706 | * @sc: send context to release |
707 | * | |
708 | * Usage: Called from Send completion to return a sendctxt | |
709 | * to the queue. | |
710 | * | |
dbcc53a5 | 711 | * The caller serializes calls to this function (per transport). |
ae72950a | 712 | */ |
f995879e CL |
713 | static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt, |
714 | struct rpcrdma_sendctx *sc) | |
ae72950a | 715 | { |
f995879e | 716 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
ae72950a CL |
717 | unsigned long next_tail; |
718 | ||
dbcc53a5 | 719 | /* Unmap SGEs of previously completed but unsignaled |
ae72950a CL |
720 | * Sends by walking up the queue until @sc is found. |
721 | */ | |
722 | next_tail = buf->rb_sc_tail; | |
723 | do { | |
724 | next_tail = rpcrdma_sendctx_next(buf, next_tail); | |
725 | ||
726 | /* ORDER: item must be accessed _before_ tail is updated */ | |
dbcc53a5 | 727 | rpcrdma_sendctx_unmap(buf->rb_sc_ctxs[next_tail]); |
ae72950a CL |
728 | |
729 | } while (buf->rb_sc_ctxs[next_tail] != sc); | |
730 | ||
731 | /* Paired with READ_ONCE */ | |
732 | smp_store_release(&buf->rb_sc_tail, next_tail); | |
2fad6592 | 733 | |
f995879e | 734 | xprt_write_space(&r_xprt->rx_xprt); |
ae72950a CL |
735 | } |
736 | ||
e2ac236c | 737 | static void |
96ceddea | 738 | rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) |
e2ac236c CL |
739 | { |
740 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; | |
e28ce900 | 741 | struct rpcrdma_ep *ep = r_xprt->rx_ep; |
2d77058c | 742 | struct ib_device *device = ep->re_id->device; |
e2ac236c | 743 | unsigned int count; |
e2ac236c | 744 | |
2d77058c | 745 | /* Try to allocate enough to perform one full-sized I/O */ |
93aa8e0a | 746 | for (count = 0; count < ep->re_max_rdma_segs; count++) { |
96ceddea | 747 | struct rpcrdma_mr *mr; |
e2ac236c CL |
748 | int rc; |
749 | ||
2d77058c CL |
750 | mr = kzalloc_node(sizeof(*mr), XPRTRDMA_GFP_FLAGS, |
751 | ibdev_to_node(device)); | |
96ceddea | 752 | if (!mr) |
e2ac236c CL |
753 | break; |
754 | ||
253a5162 | 755 | rc = frwr_mr_init(r_xprt, mr); |
e2ac236c | 756 | if (rc) { |
96ceddea | 757 | kfree(mr); |
e2ac236c CL |
758 | break; |
759 | } | |
760 | ||
4d6b8890 | 761 | spin_lock(&buf->rb_lock); |
c3700780 | 762 | rpcrdma_mr_push(mr, &buf->rb_mrs); |
eed48a9c | 763 | list_add(&mr->mr_all, &buf->rb_all_mrs); |
4d6b8890 | 764 | spin_unlock(&buf->rb_lock); |
e2ac236c CL |
765 | } |
766 | ||
e2ac236c | 767 | r_xprt->rx_stats.mrs_allocated += count; |
1c443eff | 768 | trace_xprtrdma_createmrs(r_xprt, count); |
e2ac236c CL |
769 | } |
770 | ||
771 | static void | |
772 | rpcrdma_mr_refresh_worker(struct work_struct *work) | |
773 | { | |
774 | struct rpcrdma_buffer *buf = container_of(work, struct rpcrdma_buffer, | |
3b39f52a | 775 | rb_refresh_worker); |
e2ac236c CL |
776 | struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt, |
777 | rx_buf); | |
778 | ||
96ceddea | 779 | rpcrdma_mrs_create(r_xprt); |
05eb06d8 | 780 | xprt_write_space(&r_xprt->rx_xprt); |
e2ac236c CL |
781 | } |
782 | ||
9d2da4ff CL |
783 | /** |
784 | * rpcrdma_mrs_refresh - Wake the MR refresh worker | |
785 | * @r_xprt: controlling transport instance | |
786 | * | |
787 | */ | |
788 | void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt) | |
789 | { | |
790 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; | |
e28ce900 | 791 | struct rpcrdma_ep *ep = r_xprt->rx_ep; |
9d2da4ff | 792 | |
897b7be9 CL |
793 | /* If there is no underlying connection, it's no use |
794 | * to wake the refresh worker. | |
9d2da4ff | 795 | */ |
6b1eb3b2 CL |
796 | if (ep->re_connect_status != 1) |
797 | return; | |
798 | queue_work(system_highpri_wq, &buf->rb_refresh_worker); | |
9d2da4ff CL |
799 | } |
800 | ||
1769e6a8 CL |
801 | /** |
802 | * rpcrdma_req_create - Allocate an rpcrdma_req object | |
803 | * @r_xprt: controlling r_xprt | |
bb93a1ae | 804 | * @size: initial size, in bytes, of send and receive buffers |
1769e6a8 CL |
805 | * |
806 | * Returns an allocated and fully initialized rpcrdma_req or NULL. | |
807 | */ | |
3b50cc1c CL |
808 | struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, |
809 | size_t size) | |
1392402c | 810 | { |
f531a5db | 811 | struct rpcrdma_buffer *buffer = &r_xprt->rx_buf; |
1392402c | 812 | struct rpcrdma_req *req; |
1392402c | 813 | |
f20f18c9 | 814 | req = kzalloc(sizeof(*req), XPRTRDMA_GFP_FLAGS); |
1392402c | 815 | if (req == NULL) |
bb93a1ae | 816 | goto out1; |
1392402c | 817 | |
7ac18798 | 818 | req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE); |
bb93a1ae | 819 | if (!req->rl_sendbuf) |
b78de1dc | 820 | goto out2; |
bb93a1ae | 821 | |
7ac18798 | 822 | req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE); |
bb93a1ae | 823 | if (!req->rl_recvbuf) |
b78de1dc | 824 | goto out3; |
bb93a1ae | 825 | |
6dc6ec9e | 826 | INIT_LIST_HEAD(&req->rl_free_mrs); |
2dd4a012 | 827 | INIT_LIST_HEAD(&req->rl_registered); |
92f4433e | 828 | spin_lock(&buffer->rb_lock); |
f531a5db | 829 | list_add(&req->rl_all, &buffer->rb_allreqs); |
92f4433e | 830 | spin_unlock(&buffer->rb_lock); |
1392402c | 831 | return req; |
bb93a1ae | 832 | |
bb93a1ae | 833 | out3: |
9181f40f | 834 | rpcrdma_regbuf_free(req->rl_sendbuf); |
bb93a1ae CL |
835 | out2: |
836 | kfree(req); | |
837 | out1: | |
838 | return NULL; | |
1392402c CL |
839 | } |
840 | ||
a31b2f93 | 841 | /** |
b78de1dc | 842 | * rpcrdma_req_setup - Per-connection instance setup of an rpcrdma_req object |
a31b2f93 | 843 | * @r_xprt: controlling transport instance |
b78de1dc | 844 | * @req: rpcrdma_req object to set up |
a31b2f93 | 845 | * |
b78de1dc CL |
846 | * Returns zero on success, and a negative errno on failure. |
847 | */ | |
848 | int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) | |
849 | { | |
850 | struct rpcrdma_regbuf *rb; | |
851 | size_t maxhdrsize; | |
852 | ||
853 | /* Compute maximum header buffer size in bytes */ | |
854 | maxhdrsize = rpcrdma_fixed_maxsz + 3 + | |
e28ce900 | 855 | r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz; |
b78de1dc CL |
856 | maxhdrsize *= sizeof(__be32); |
857 | rb = rpcrdma_regbuf_alloc(__roundup_pow_of_two(maxhdrsize), | |
7ac18798 | 858 | DMA_TO_DEVICE); |
b78de1dc CL |
859 | if (!rb) |
860 | goto out; | |
861 | ||
862 | if (!__rpcrdma_regbuf_dma_map(r_xprt, rb)) | |
863 | goto out_free; | |
864 | ||
865 | req->rl_rdmabuf = rb; | |
866 | xdr_buf_init(&req->rl_hdrbuf, rdmab_data(rb), rdmab_length(rb)); | |
867 | return 0; | |
868 | ||
869 | out_free: | |
870 | rpcrdma_regbuf_free(rb); | |
871 | out: | |
872 | return -ENOMEM; | |
873 | } | |
874 | ||
875 | /* ASSUMPTION: the rb_allreqs list is stable for the duration, | |
a31b2f93 CL |
876 | * and thus can be walked without holding rb_lock. Eg. the |
877 | * caller is holding the transport send lock to exclude | |
878 | * device removal or disconnection. | |
879 | */ | |
b78de1dc | 880 | static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt) |
a31b2f93 CL |
881 | { |
882 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; | |
883 | struct rpcrdma_req *req; | |
b78de1dc | 884 | int rc; |
a31b2f93 CL |
885 | |
886 | list_for_each_entry(req, &buf->rb_allreqs, rl_all) { | |
b78de1dc CL |
887 | rc = rpcrdma_req_setup(r_xprt, req); |
888 | if (rc) | |
889 | return rc; | |
a31b2f93 | 890 | } |
b78de1dc CL |
891 | return 0; |
892 | } | |
893 | ||
894 | static void rpcrdma_req_reset(struct rpcrdma_req *req) | |
895 | { | |
896 | /* Credits are valid for only one connection */ | |
897 | req->rl_slot.rq_cong = 0; | |
898 | ||
899 | rpcrdma_regbuf_free(req->rl_rdmabuf); | |
900 | req->rl_rdmabuf = NULL; | |
901 | ||
902 | rpcrdma_regbuf_dma_unmap(req->rl_sendbuf); | |
903 | rpcrdma_regbuf_dma_unmap(req->rl_recvbuf); | |
5de55ce9 CL |
904 | |
905 | frwr_reset(req); | |
b78de1dc CL |
906 | } |
907 | ||
908 | /* ASSUMPTION: the rb_allreqs list is stable for the duration, | |
909 | * and thus can be walked without holding rb_lock. Eg. the | |
910 | * caller is holding the transport send lock to exclude | |
911 | * device removal or disconnection. | |
912 | */ | |
913 | static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt) | |
914 | { | |
915 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; | |
916 | struct rpcrdma_req *req; | |
917 | ||
918 | list_for_each_entry(req, &buf->rb_allreqs, rl_all) | |
919 | rpcrdma_req_reset(req); | |
a31b2f93 CL |
920 | } |
921 | ||
e515dd9d CL |
922 | static noinline |
923 | struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, | |
924 | bool temp) | |
1392402c | 925 | { |
9e3ca33b | 926 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
1392402c | 927 | struct rpcrdma_rep *rep; |
1392402c | 928 | |
f20f18c9 | 929 | rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS); |
1392402c CL |
930 | if (rep == NULL) |
931 | goto out; | |
1392402c | 932 | |
e28ce900 | 933 | rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv, |
7ac18798 | 934 | DMA_FROM_DEVICE); |
8cec3dba | 935 | if (!rep->rr_rdmabuf) |
1392402c | 936 | goto out_free; |
379d1bc5 | 937 | |
e515dd9d CL |
938 | if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) |
939 | goto out_free_regbuf; | |
940 | ||
af5865d2 CL |
941 | rep->rr_cid.ci_completion_id = |
942 | atomic_inc_return(&r_xprt->rx_ep->re_completion_ids); | |
943 | ||
8cec3dba | 944 | xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf), |
96f8778f | 945 | rdmab_length(rep->rr_rdmabuf)); |
1519e969 | 946 | rep->rr_cqe.done = rpcrdma_wc_receive; |
fed171b3 | 947 | rep->rr_rxprt = r_xprt; |
6ea8e711 CL |
948 | rep->rr_recv_wr.next = NULL; |
949 | rep->rr_recv_wr.wr_cqe = &rep->rr_cqe; | |
950 | rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov; | |
951 | rep->rr_recv_wr.num_sge = 1; | |
7c8d9e7c | 952 | rep->rr_temp = temp; |
9e3ca33b CL |
953 | |
954 | spin_lock(&buf->rb_lock); | |
955 | list_add(&rep->rr_all, &buf->rb_all_reps); | |
956 | spin_unlock(&buf->rb_lock); | |
379d1bc5 | 957 | return rep; |
1392402c | 958 | |
e515dd9d CL |
959 | out_free_regbuf: |
960 | rpcrdma_regbuf_free(rep->rr_rdmabuf); | |
1392402c CL |
961 | out_free: |
962 | kfree(rep); | |
963 | out: | |
379d1bc5 | 964 | return NULL; |
1392402c CL |
965 | } |
966 | ||
eaf86e8c | 967 | static void rpcrdma_rep_free(struct rpcrdma_rep *rep) |
b0b227f0 CL |
968 | { |
969 | rpcrdma_regbuf_free(rep->rr_rdmabuf); | |
970 | kfree(rep); | |
971 | } | |
972 | ||
eaf86e8c CL |
973 | static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep) |
974 | { | |
975 | struct rpcrdma_buffer *buf = &rep->rr_rxprt->rx_buf; | |
976 | ||
977 | spin_lock(&buf->rb_lock); | |
978 | list_del(&rep->rr_all); | |
979 | spin_unlock(&buf->rb_lock); | |
980 | ||
981 | rpcrdma_rep_free(rep); | |
982 | } | |
983 | ||
b0b227f0 CL |
984 | static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf) |
985 | { | |
986 | struct llist_node *node; | |
987 | ||
988 | /* Calls to llist_del_first are required to be serialized */ | |
989 | node = llist_del_first(&buf->rb_free_reps); | |
990 | if (!node) | |
991 | return NULL; | |
992 | return llist_entry(node, struct rpcrdma_rep, rr_node); | |
993 | } | |
994 | ||
c35ca60d CL |
995 | /** |
996 | * rpcrdma_rep_put - Release rpcrdma_rep back to free list | |
997 | * @buf: buffer pool | |
998 | * @rep: rep to release | |
999 | * | |
1000 | */ | |
1001 | void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep) | |
b0b227f0 | 1002 | { |
671c450b CL |
1003 | llist_add(&rep->rr_node, &buf->rb_free_reps); |
1004 | } | |
1005 | ||
8b5292be CL |
1006 | /* Caller must ensure the QP is quiescent (RQ is drained) before |
1007 | * invoking this function, to guarantee rb_all_reps is not | |
1008 | * changing. | |
1009 | */ | |
671c450b CL |
1010 | static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt) |
1011 | { | |
1012 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; | |
1013 | struct rpcrdma_rep *rep; | |
1014 | ||
b7ff0185 | 1015 | list_for_each_entry(rep, &buf->rb_all_reps, rr_all) { |
671c450b | 1016 | rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf); |
8b5292be | 1017 | rep->rr_temp = true; /* Mark this rep for destruction */ |
b7ff0185 | 1018 | } |
b0b227f0 CL |
1019 | } |
1020 | ||
1021 | static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) | |
1022 | { | |
1023 | struct rpcrdma_rep *rep; | |
1024 | ||
eaf86e8c CL |
1025 | spin_lock(&buf->rb_lock); |
1026 | while ((rep = list_first_entry_or_null(&buf->rb_all_reps, | |
1027 | struct rpcrdma_rep, | |
1028 | rr_all)) != NULL) { | |
1029 | list_del(&rep->rr_all); | |
1030 | spin_unlock(&buf->rb_lock); | |
1031 | ||
1032 | rpcrdma_rep_free(rep); | |
1033 | ||
1034 | spin_lock(&buf->rb_lock); | |
1035 | } | |
1036 | spin_unlock(&buf->rb_lock); | |
b0b227f0 CL |
1037 | } |
1038 | ||
86c4ccd9 CL |
1039 | /** |
1040 | * rpcrdma_buffer_create - Create initial set of req/rep objects | |
1041 | * @r_xprt: transport instance to (re)initialize | |
1042 | * | |
1043 | * Returns zero on success, otherwise a negative errno. | |
1044 | */ | |
1045 | int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) | |
c56c65fb | 1046 | { |
ac920d04 | 1047 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
c56c65fb TT |
1048 | int i, rc; |
1049 | ||
f531a5db | 1050 | buf->rb_bc_srv_max_requests = 0; |
505bbe64 | 1051 | spin_lock_init(&buf->rb_lock); |
96ceddea | 1052 | INIT_LIST_HEAD(&buf->rb_mrs); |
eed48a9c | 1053 | INIT_LIST_HEAD(&buf->rb_all_mrs); |
3b39f52a | 1054 | INIT_WORK(&buf->rb_refresh_worker, rpcrdma_mr_refresh_worker); |
c56c65fb | 1055 | |
1e465fd4 | 1056 | INIT_LIST_HEAD(&buf->rb_send_bufs); |
f531a5db | 1057 | INIT_LIST_HEAD(&buf->rb_allreqs); |
671c450b | 1058 | INIT_LIST_HEAD(&buf->rb_all_reps); |
1769e6a8 CL |
1059 | |
1060 | rc = -ENOMEM; | |
7581d901 | 1061 | for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) { |
c56c65fb | 1062 | struct rpcrdma_req *req; |
c56c65fb | 1063 | |
3b50cc1c CL |
1064 | req = rpcrdma_req_create(r_xprt, |
1065 | RPCRDMA_V1_DEF_INLINE_SIZE * 2); | |
1769e6a8 | 1066 | if (!req) |
c56c65fb | 1067 | goto out; |
a80d66c9 | 1068 | list_add(&req->rl_list, &buf->rb_send_bufs); |
1e465fd4 CL |
1069 | } |
1070 | ||
b0b227f0 | 1071 | init_llist_head(&buf->rb_free_reps); |
1392402c | 1072 | |
c56c65fb TT |
1073 | return 0; |
1074 | out: | |
1075 | rpcrdma_buffer_destroy(buf); | |
1076 | return rc; | |
1077 | } | |
1078 | ||
92f4433e CL |
1079 | /** |
1080 | * rpcrdma_req_destroy - Destroy an rpcrdma_req object | |
1081 | * @req: unused object to be destroyed | |
1082 | * | |
9d2da4ff CL |
1083 | * Relies on caller holding the transport send lock to protect |
1084 | * removing req->rl_all from buf->rb_all_reqs safely. | |
92f4433e | 1085 | */ |
6dc6ec9e | 1086 | void rpcrdma_req_destroy(struct rpcrdma_req *req) |
1392402c | 1087 | { |
c3700780 CL |
1088 | struct rpcrdma_mr *mr; |
1089 | ||
92f4433e CL |
1090 | list_del(&req->rl_all); |
1091 | ||
c3700780 CL |
1092 | while ((mr = rpcrdma_mr_pop(&req->rl_free_mrs))) { |
1093 | struct rpcrdma_buffer *buf = &mr->mr_xprt->rx_buf; | |
1094 | ||
1095 | spin_lock(&buf->rb_lock); | |
1096 | list_del(&mr->mr_all); | |
1097 | spin_unlock(&buf->rb_lock); | |
1098 | ||
f912af77 | 1099 | frwr_mr_release(mr); |
c3700780 | 1100 | } |
6dc6ec9e | 1101 | |
d2832af3 CL |
1102 | rpcrdma_regbuf_free(req->rl_recvbuf); |
1103 | rpcrdma_regbuf_free(req->rl_sendbuf); | |
1104 | rpcrdma_regbuf_free(req->rl_rdmabuf); | |
1392402c CL |
1105 | kfree(req); |
1106 | } | |
1107 | ||
c3700780 CL |
1108 | /** |
1109 | * rpcrdma_mrs_destroy - Release all of a transport's MRs | |
9d2da4ff | 1110 | * @r_xprt: controlling transport instance |
c3700780 CL |
1111 | * |
1112 | * Relies on caller holding the transport send lock to protect | |
1113 | * removing mr->mr_list from req->rl_free_mrs safely. | |
1114 | */ | |
9d2da4ff | 1115 | static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt) |
e2ac236c | 1116 | { |
9d2da4ff | 1117 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
96ceddea | 1118 | struct rpcrdma_mr *mr; |
e2ac236c | 1119 | |
9d2da4ff CL |
1120 | cancel_work_sync(&buf->rb_refresh_worker); |
1121 | ||
4d6b8890 | 1122 | spin_lock(&buf->rb_lock); |
eed48a9c CL |
1123 | while ((mr = list_first_entry_or_null(&buf->rb_all_mrs, |
1124 | struct rpcrdma_mr, | |
1125 | mr_all)) != NULL) { | |
c3700780 | 1126 | list_del(&mr->mr_list); |
96ceddea | 1127 | list_del(&mr->mr_all); |
4d6b8890 | 1128 | spin_unlock(&buf->rb_lock); |
054f1557 | 1129 | |
f912af77 | 1130 | frwr_mr_release(mr); |
9d2da4ff | 1131 | |
4d6b8890 | 1132 | spin_lock(&buf->rb_lock); |
e2ac236c | 1133 | } |
4d6b8890 | 1134 | spin_unlock(&buf->rb_lock); |
e2ac236c CL |
1135 | } |
1136 | ||
af65ed40 CL |
1137 | /** |
1138 | * rpcrdma_buffer_destroy - Release all hw resources | |
1139 | * @buf: root control block for resources | |
1140 | * | |
b8fe677f | 1141 | * ORDERING: relies on a prior rpcrdma_xprt_drain : |
af65ed40 CL |
1142 | * - No more Send or Receive completions can occur |
1143 | * - All MRs, reps, and reqs are returned to their free lists | |
1144 | */ | |
c56c65fb TT |
1145 | void |
1146 | rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) | |
1147 | { | |
b0b227f0 | 1148 | rpcrdma_reps_destroy(buf); |
c56c65fb | 1149 | |
92f4433e | 1150 | while (!list_empty(&buf->rb_send_bufs)) { |
1e465fd4 | 1151 | struct rpcrdma_req *req; |
4034ba04 | 1152 | |
92f4433e CL |
1153 | req = list_first_entry(&buf->rb_send_bufs, |
1154 | struct rpcrdma_req, rl_list); | |
1155 | list_del(&req->rl_list); | |
1156 | rpcrdma_req_destroy(req); | |
1e465fd4 | 1157 | } |
c56c65fb TT |
1158 | } |
1159 | ||
96ceddea CL |
1160 | /** |
1161 | * rpcrdma_mr_get - Allocate an rpcrdma_mr object | |
1162 | * @r_xprt: controlling transport | |
1163 | * | |
1164 | * Returns an initialized rpcrdma_mr or NULL if no free | |
1165 | * rpcrdma_mr objects are available. | |
1166 | */ | |
1167 | struct rpcrdma_mr * | |
1168 | rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt) | |
c2922c02 | 1169 | { |
346aa66b | 1170 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
265a38d4 | 1171 | struct rpcrdma_mr *mr; |
346aa66b | 1172 | |
4d6b8890 | 1173 | spin_lock(&buf->rb_lock); |
265a38d4 | 1174 | mr = rpcrdma_mr_pop(&buf->rb_mrs); |
4d6b8890 | 1175 | spin_unlock(&buf->rb_lock); |
96ceddea | 1176 | return mr; |
ec12e479 CL |
1177 | } |
1178 | ||
e86be3a0 TM |
1179 | /** |
1180 | * rpcrdma_reply_put - Put reply buffers back into pool | |
1181 | * @buffers: buffer pool | |
1182 | * @req: object to return | |
1183 | * | |
1184 | */ | |
1185 | void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) | |
1186 | { | |
1187 | if (req->rl_reply) { | |
1188 | rpcrdma_rep_put(buffers, req->rl_reply); | |
1189 | req->rl_reply = NULL; | |
1190 | } | |
1191 | } | |
1192 | ||
7c8d9e7c CL |
1193 | /** |
1194 | * rpcrdma_buffer_get - Get a request buffer | |
1195 | * @buffers: Buffer pool from which to obtain a buffer | |
78d506e1 | 1196 | * |
7c8d9e7c | 1197 | * Returns a fresh rpcrdma_req, or NULL if none are available. |
c56c65fb TT |
1198 | */ |
1199 | struct rpcrdma_req * | |
1200 | rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) | |
1201 | { | |
1202 | struct rpcrdma_req *req; | |
c14d86e5 | 1203 | |
a5b027e1 | 1204 | spin_lock(&buffers->rb_lock); |
e68699cc CL |
1205 | req = list_first_entry_or_null(&buffers->rb_send_bufs, |
1206 | struct rpcrdma_req, rl_list); | |
1207 | if (req) | |
1208 | list_del_init(&req->rl_list); | |
a5b027e1 | 1209 | spin_unlock(&buffers->rb_lock); |
1e465fd4 | 1210 | return req; |
c56c65fb TT |
1211 | } |
1212 | ||
7c8d9e7c CL |
1213 | /** |
1214 | * rpcrdma_buffer_put - Put request/reply buffers back into pool | |
5828ceba | 1215 | * @buffers: buffer pool |
7c8d9e7c CL |
1216 | * @req: object to return |
1217 | * | |
c56c65fb | 1218 | */ |
5828ceba | 1219 | void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) |
c56c65fb | 1220 | { |
e86be3a0 | 1221 | rpcrdma_reply_put(buffers, req); |
1e465fd4 | 1222 | |
a5b027e1 | 1223 | spin_lock(&buffers->rb_lock); |
7c8d9e7c | 1224 | list_add(&req->rl_list, &buffers->rb_send_bufs); |
a5b027e1 | 1225 | spin_unlock(&buffers->rb_lock); |
c56c65fb TT |
1226 | } |
1227 | ||
d2832af3 | 1228 | /* Returns a pointer to a rpcrdma_regbuf object, or NULL. |
9128c3e7 CL |
1229 | * |
1230 | * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for | |
99ef4db3 | 1231 | * receiving the payload of RDMA RECV operations. During Long Calls |
5f62412b | 1232 | * or Replies they may be registered externally via frwr_map. |
9128c3e7 | 1233 | */ |
d2832af3 | 1234 | static struct rpcrdma_regbuf * |
7ac18798 | 1235 | rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) |
9128c3e7 CL |
1236 | { |
1237 | struct rpcrdma_regbuf *rb; | |
9128c3e7 | 1238 | |
f20f18c9 | 1239 | rb = kmalloc(sizeof(*rb), XPRTRDMA_GFP_FLAGS); |
8cec3dba CL |
1240 | if (!rb) |
1241 | return NULL; | |
f20f18c9 | 1242 | rb->rg_data = kmalloc(size, XPRTRDMA_GFP_FLAGS); |
8cec3dba CL |
1243 | if (!rb->rg_data) { |
1244 | kfree(rb); | |
1245 | return NULL; | |
1246 | } | |
9128c3e7 | 1247 | |
54cbd6b0 | 1248 | rb->rg_device = NULL; |
99ef4db3 | 1249 | rb->rg_direction = direction; |
54cbd6b0 | 1250 | rb->rg_iov.length = size; |
9128c3e7 | 1251 | return rb; |
54cbd6b0 | 1252 | } |
9128c3e7 | 1253 | |
0f665ceb CL |
1254 | /** |
1255 | * rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer | |
1256 | * @rb: regbuf to reallocate | |
1257 | * @size: size of buffer to be allocated, in bytes | |
1258 | * @flags: GFP flags | |
1259 | * | |
1260 | * Returns true if reallocation was successful. If false is | |
1261 | * returned, @rb is left untouched. | |
1262 | */ | |
1263 | bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags) | |
1264 | { | |
1265 | void *buf; | |
1266 | ||
1267 | buf = kmalloc(size, flags); | |
1268 | if (!buf) | |
1269 | return false; | |
1270 | ||
d2832af3 | 1271 | rpcrdma_regbuf_dma_unmap(rb); |
0f665ceb CL |
1272 | kfree(rb->rg_data); |
1273 | ||
1274 | rb->rg_data = buf; | |
1275 | rb->rg_iov.length = size; | |
1276 | return true; | |
1277 | } | |
1278 | ||
54cbd6b0 | 1279 | /** |
d2832af3 CL |
1280 | * __rpcrdma_regbuf_dma_map - DMA-map a regbuf |
1281 | * @r_xprt: controlling transport instance | |
54cbd6b0 | 1282 | * @rb: regbuf to be mapped |
d2832af3 CL |
1283 | * |
1284 | * Returns true if the buffer is now DMA mapped to @r_xprt's device | |
54cbd6b0 | 1285 | */ |
d2832af3 CL |
1286 | bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt, |
1287 | struct rpcrdma_regbuf *rb) | |
54cbd6b0 | 1288 | { |
e28ce900 | 1289 | struct ib_device *device = r_xprt->rx_ep->re_id->device; |
91a10c52 | 1290 | |
54cbd6b0 CL |
1291 | if (rb->rg_direction == DMA_NONE) |
1292 | return false; | |
1293 | ||
d2832af3 CL |
1294 | rb->rg_iov.addr = ib_dma_map_single(device, rdmab_data(rb), |
1295 | rdmab_length(rb), rb->rg_direction); | |
53b2c1cb CL |
1296 | if (ib_dma_mapping_error(device, rdmab_addr(rb))) { |
1297 | trace_xprtrdma_dma_maperr(rdmab_addr(rb)); | |
54cbd6b0 | 1298 | return false; |
53b2c1cb | 1299 | } |
54cbd6b0 | 1300 | |
91a10c52 | 1301 | rb->rg_device = device; |
e28ce900 | 1302 | rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey; |
54cbd6b0 CL |
1303 | return true; |
1304 | } | |
1305 | ||
d2832af3 | 1306 | static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb) |
54cbd6b0 | 1307 | { |
e89e8d8f CL |
1308 | if (!rb) |
1309 | return; | |
1310 | ||
54cbd6b0 CL |
1311 | if (!rpcrdma_regbuf_is_mapped(rb)) |
1312 | return; | |
1313 | ||
d2832af3 CL |
1314 | ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb), rdmab_length(rb), |
1315 | rb->rg_direction); | |
54cbd6b0 | 1316 | rb->rg_device = NULL; |
9128c3e7 CL |
1317 | } |
1318 | ||
d2832af3 | 1319 | static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb) |
9128c3e7 | 1320 | { |
d2832af3 | 1321 | rpcrdma_regbuf_dma_unmap(rb); |
8cec3dba CL |
1322 | if (rb) |
1323 | kfree(rb->rg_data); | |
e531dcab | 1324 | kfree(rb); |
9128c3e7 CL |
1325 | } |
1326 | ||
2ae50ad6 CL |
1327 | /** |
1328 | * rpcrdma_post_recvs - Refill the Receive Queue | |
1329 | * @r_xprt: controlling transport instance | |
35d8b10a CL |
1330 | * @needed: current credit grant |
1331 | * @temp: mark Receive buffers to be deleted after one use | |
2ae50ad6 CL |
1332 | * |
1333 | */ | |
35d8b10a | 1334 | void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp) |
f531a5db | 1335 | { |
7c8d9e7c | 1336 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
e28ce900 | 1337 | struct rpcrdma_ep *ep = r_xprt->rx_ep; |
e515dd9d | 1338 | struct ib_recv_wr *wr, *bad_wr; |
9ef33ef5 | 1339 | struct rpcrdma_rep *rep; |
35d8b10a | 1340 | int count, rc; |
f531a5db | 1341 | |
61c208a5 CL |
1342 | rc = 0; |
1343 | count = 0; | |
9ef33ef5 | 1344 | |
93aa8e0a | 1345 | if (likely(ep->re_receive_count > needed)) |
61c208a5 | 1346 | goto out; |
93aa8e0a | 1347 | needed -= ep->re_receive_count; |
e340c2d6 CL |
1348 | if (!temp) |
1349 | needed += RPCRDMA_MAX_RECV_BATCH; | |
f531a5db | 1350 | |
15788d1d CL |
1351 | if (atomic_inc_return(&ep->re_receiving) > 1) |
1352 | goto out; | |
1353 | ||
9ef33ef5 | 1354 | /* fast path: all needed reps can be found on the free list */ |
7c8d9e7c CL |
1355 | wr = NULL; |
1356 | while (needed) { | |
b0b227f0 | 1357 | rep = rpcrdma_rep_get_locked(buf); |
671c450b CL |
1358 | if (rep && rep->rr_temp) { |
1359 | rpcrdma_rep_destroy(rep); | |
1360 | continue; | |
1361 | } | |
9ef33ef5 | 1362 | if (!rep) |
b0b227f0 | 1363 | rep = rpcrdma_rep_create(r_xprt, temp); |
9ef33ef5 | 1364 | if (!rep) |
d2832af3 | 1365 | break; |
f531a5db | 1366 | |
af5865d2 | 1367 | rep->rr_cid.ci_queue_id = ep->re_attr.recv_cq->res.id; |
e515dd9d | 1368 | trace_xprtrdma_post_recv(rep); |
7c8d9e7c CL |
1369 | rep->rr_recv_wr.next = wr; |
1370 | wr = &rep->rr_recv_wr; | |
7c8d9e7c | 1371 | --needed; |
e515dd9d | 1372 | ++count; |
7c8d9e7c | 1373 | } |
9ef33ef5 | 1374 | if (!wr) |
61c208a5 | 1375 | goto out; |
7c8d9e7c | 1376 | |
e28ce900 | 1377 | rc = ib_post_recv(ep->re_id->qp, wr, |
d34ac5cd | 1378 | (const struct ib_recv_wr **)&bad_wr); |
7c8d9e7c | 1379 | if (rc) { |
683f31c3 | 1380 | trace_xprtrdma_post_recvs_err(r_xprt, rc); |
2d0abe36 | 1381 | for (wr = bad_wr; wr;) { |
7c8d9e7c CL |
1382 | struct rpcrdma_rep *rep; |
1383 | ||
1384 | rep = container_of(wr, struct rpcrdma_rep, rr_recv_wr); | |
2d0abe36 | 1385 | wr = wr->next; |
c35ca60d | 1386 | rpcrdma_rep_put(buf, rep); |
7c8d9e7c CL |
1387 | --count; |
1388 | } | |
1389 | } | |
97480cae CL |
1390 | if (atomic_dec_return(&ep->re_receiving) > 0) |
1391 | complete(&ep->re_done); | |
1392 | ||
1393 | out: | |
683f31c3 | 1394 | trace_xprtrdma_post_recvs(r_xprt, count); |
93aa8e0a | 1395 | ep->re_receive_count += count; |
9ef33ef5 | 1396 | return; |
f531a5db | 1397 | } |