Commit | Line | Data |
---|---|---|
f931551b RC |
1 | /* |
2 | * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation. | |
3 | * All rights reserved. | |
4 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. | |
5 | * | |
6 | * This software is available to you under a choice of one of two | |
7 | * licenses. You may choose to be licensed under the terms of the GNU | |
8 | * General Public License (GPL) Version 2, available from the file | |
9 | * COPYING in the main directory of this source tree, or the | |
10 | * OpenIB.org BSD license below: | |
11 | * | |
12 | * Redistribution and use in source and binary forms, with or | |
13 | * without modification, are permitted provided that the following | |
14 | * conditions are met: | |
15 | * | |
16 | * - Redistributions of source code must retain the above | |
17 | * copyright notice, this list of conditions and the following | |
18 | * disclaimer. | |
19 | * | |
20 | * - Redistributions in binary form must reproduce the above | |
21 | * copyright notice, this list of conditions and the following | |
22 | * disclaimer in the documentation and/or other materials | |
23 | * provided with the distribution. | |
24 | * | |
25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
32 | * SOFTWARE. | |
33 | */ | |
34 | ||
35 | #include "qib.h" | |
36 | ||
37 | /* cut down ridiculously long IB macro names */ | |
38 | #define OP(x) IB_OPCODE_UC_##x | |
39 | ||
40 | /** | |
41 | * qib_make_uc_req - construct a request packet (SEND, RDMA write) | |
42 | * @qp: a pointer to the QP | |
f8005fa8 | 43 | * @flags: unused |
f931551b | 44 | * |
46a80d62 MM |
45 | * Assumes the s_lock is held. |
46 | * | |
f931551b RC |
47 | * Return 1 if constructed; otherwise, return 0. |
48 | */ | |
747f4d7a | 49 | int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags) |
f931551b | 50 | { |
ffc26907 | 51 | struct qib_qp_priv *priv = qp->priv; |
261a4351 | 52 | struct ib_other_headers *ohdr; |
7c2e11fe | 53 | struct rvt_swqe *wqe; |
f931551b RC |
54 | u32 hwords; |
55 | u32 bth0; | |
56 | u32 len; | |
cc6ea138 | 57 | u32 pmtu = qp->pmtu; |
f931551b RC |
58 | int ret = 0; |
59 | ||
db3ef0eb HC |
60 | if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { |
61 | if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) | |
f931551b RC |
62 | goto bail; |
63 | /* We are in the error state, flush the work request. */ | |
6aa7de05 | 64 | if (qp->s_last == READ_ONCE(qp->s_head)) |
f931551b RC |
65 | goto bail; |
66 | /* If DMAs are in progress, we can't flush immediately. */ | |
ffc26907 | 67 | if (atomic_read(&priv->s_dma_busy)) { |
01ba79d4 | 68 | qp->s_flags |= RVT_S_WAIT_DMA; |
f931551b RC |
69 | goto bail; |
70 | } | |
db3ef0eb | 71 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
116aa033 | 72 | rvt_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
f931551b RC |
73 | goto done; |
74 | } | |
75 | ||
ffc26907 | 76 | ohdr = &priv->s_hdr->u.oth; |
d8966fcd | 77 | if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) |
ffc26907 | 78 | ohdr = &priv->s_hdr->u.l.oth; |
f931551b RC |
79 | |
80 | /* header size in 32-bit words LRH+BTH = (8+12)/4. */ | |
81 | hwords = 5; | |
82 | bth0 = 0; | |
83 | ||
84 | /* Get the next send request. */ | |
db3ef0eb | 85 | wqe = rvt_get_swqe_ptr(qp, qp->s_cur); |
f931551b RC |
86 | qp->s_wqe = NULL; |
87 | switch (qp->s_state) { | |
88 | default: | |
db3ef0eb HC |
89 | if (!(ib_rvt_state_ops[qp->state] & |
90 | RVT_PROCESS_NEXT_SEND_OK)) | |
f931551b RC |
91 | goto bail; |
92 | /* Check if send work queue is empty. */ | |
6aa7de05 | 93 | if (qp->s_cur == READ_ONCE(qp->s_head)) |
f931551b RC |
94 | goto bail; |
95 | /* | |
96 | * Start a new request. | |
97 | */ | |
46a80d62 | 98 | qp->s_psn = wqe->psn; |
f931551b RC |
99 | qp->s_sge.sge = wqe->sg_list[0]; |
100 | qp->s_sge.sg_list = wqe->sg_list + 1; | |
101 | qp->s_sge.num_sge = wqe->wr.num_sge; | |
102 | qp->s_sge.total_len = wqe->length; | |
103 | len = wqe->length; | |
104 | qp->s_len = len; | |
105 | switch (wqe->wr.opcode) { | |
106 | case IB_WR_SEND: | |
107 | case IB_WR_SEND_WITH_IMM: | |
108 | if (len > pmtu) { | |
109 | qp->s_state = OP(SEND_FIRST); | |
110 | len = pmtu; | |
111 | break; | |
112 | } | |
113 | if (wqe->wr.opcode == IB_WR_SEND) | |
114 | qp->s_state = OP(SEND_ONLY); | |
115 | else { | |
116 | qp->s_state = | |
117 | OP(SEND_ONLY_WITH_IMMEDIATE); | |
118 | /* Immediate data comes after the BTH */ | |
119 | ohdr->u.imm_data = wqe->wr.ex.imm_data; | |
120 | hwords += 1; | |
121 | } | |
122 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
123 | bth0 |= IB_BTH_SOLICITED; | |
124 | qp->s_wqe = wqe; | |
125 | if (++qp->s_cur >= qp->s_size) | |
126 | qp->s_cur = 0; | |
127 | break; | |
128 | ||
129 | case IB_WR_RDMA_WRITE: | |
130 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
131 | ohdr->u.rc.reth.vaddr = | |
e622f2f4 | 132 | cpu_to_be64(wqe->rdma_wr.remote_addr); |
f931551b | 133 | ohdr->u.rc.reth.rkey = |
e622f2f4 | 134 | cpu_to_be32(wqe->rdma_wr.rkey); |
f931551b RC |
135 | ohdr->u.rc.reth.length = cpu_to_be32(len); |
136 | hwords += sizeof(struct ib_reth) / 4; | |
137 | if (len > pmtu) { | |
138 | qp->s_state = OP(RDMA_WRITE_FIRST); | |
139 | len = pmtu; | |
140 | break; | |
141 | } | |
142 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE) | |
143 | qp->s_state = OP(RDMA_WRITE_ONLY); | |
144 | else { | |
145 | qp->s_state = | |
146 | OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); | |
147 | /* Immediate data comes after the RETH */ | |
148 | ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; | |
149 | hwords += 1; | |
150 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
151 | bth0 |= IB_BTH_SOLICITED; | |
152 | } | |
153 | qp->s_wqe = wqe; | |
154 | if (++qp->s_cur >= qp->s_size) | |
155 | qp->s_cur = 0; | |
156 | break; | |
157 | ||
158 | default: | |
159 | goto bail; | |
160 | } | |
161 | break; | |
162 | ||
163 | case OP(SEND_FIRST): | |
164 | qp->s_state = OP(SEND_MIDDLE); | |
df561f66 | 165 | fallthrough; |
f931551b RC |
166 | case OP(SEND_MIDDLE): |
167 | len = qp->s_len; | |
168 | if (len > pmtu) { | |
169 | len = pmtu; | |
170 | break; | |
171 | } | |
172 | if (wqe->wr.opcode == IB_WR_SEND) | |
173 | qp->s_state = OP(SEND_LAST); | |
174 | else { | |
175 | qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); | |
176 | /* Immediate data comes after the BTH */ | |
177 | ohdr->u.imm_data = wqe->wr.ex.imm_data; | |
178 | hwords += 1; | |
179 | } | |
180 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
181 | bth0 |= IB_BTH_SOLICITED; | |
182 | qp->s_wqe = wqe; | |
183 | if (++qp->s_cur >= qp->s_size) | |
184 | qp->s_cur = 0; | |
185 | break; | |
186 | ||
187 | case OP(RDMA_WRITE_FIRST): | |
188 | qp->s_state = OP(RDMA_WRITE_MIDDLE); | |
df561f66 | 189 | fallthrough; |
f931551b RC |
190 | case OP(RDMA_WRITE_MIDDLE): |
191 | len = qp->s_len; | |
192 | if (len > pmtu) { | |
193 | len = pmtu; | |
194 | break; | |
195 | } | |
196 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE) | |
197 | qp->s_state = OP(RDMA_WRITE_LAST); | |
198 | else { | |
199 | qp->s_state = | |
200 | OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); | |
201 | /* Immediate data comes after the BTH */ | |
202 | ohdr->u.imm_data = wqe->wr.ex.imm_data; | |
203 | hwords += 1; | |
204 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
205 | bth0 |= IB_BTH_SOLICITED; | |
206 | } | |
207 | qp->s_wqe = wqe; | |
208 | if (++qp->s_cur >= qp->s_size) | |
209 | qp->s_cur = 0; | |
210 | break; | |
211 | } | |
212 | qp->s_len -= len; | |
213 | qp->s_hdrwords = hwords; | |
214 | qp->s_cur_sge = &qp->s_sge; | |
215 | qp->s_cur_size = len; | |
216 | qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), | |
46a80d62 | 217 | qp->s_psn++ & QIB_PSN_MASK); |
f931551b | 218 | done: |
46a80d62 | 219 | return 1; |
f931551b | 220 | bail: |
01ba79d4 | 221 | qp->s_flags &= ~RVT_S_BUSY; |
f931551b RC |
222 | return ret; |
223 | } | |
224 | ||
225 | /** | |
226 | * qib_uc_rcv - handle an incoming UC packet | |
227 | * @ibp: the port the packet came in on | |
228 | * @hdr: the header of the packet | |
229 | * @has_grh: true if the packet has a GRH | |
230 | * @data: the packet data | |
231 | * @tlen: the length of the packet | |
232 | * @qp: the QP for this packet. | |
233 | * | |
234 | * This is called from qib_qp_rcv() to process an incoming UC packet | |
235 | * for the given QP. | |
236 | * Called at interrupt level. | |
237 | */ | |
261a4351 | 238 | void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr, |
7c2e11fe | 239 | int has_grh, void *data, u32 tlen, struct rvt_qp *qp) |
f931551b | 240 | { |
261a4351 | 241 | struct ib_other_headers *ohdr; |
f931551b RC |
242 | u32 opcode; |
243 | u32 hdrsize; | |
244 | u32 psn; | |
245 | u32 pad; | |
246 | struct ib_wc wc; | |
cc6ea138 | 247 | u32 pmtu = qp->pmtu; |
f931551b RC |
248 | struct ib_reth *reth; |
249 | int ret; | |
250 | ||
251 | /* Check for GRH */ | |
252 | if (!has_grh) { | |
253 | ohdr = &hdr->u.oth; | |
254 | hdrsize = 8 + 12; /* LRH + BTH */ | |
255 | } else { | |
256 | ohdr = &hdr->u.l.oth; | |
257 | hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ | |
258 | } | |
259 | ||
260 | opcode = be32_to_cpu(ohdr->bth[0]); | |
f931551b | 261 | if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) |
9fd5473d | 262 | return; |
f931551b RC |
263 | |
264 | psn = be32_to_cpu(ohdr->bth[2]); | |
265 | opcode >>= 24; | |
f931551b | 266 | |
f931551b RC |
267 | /* Compare the PSN verses the expected PSN. */ |
268 | if (unlikely(qib_cmp24(psn, qp->r_psn) != 0)) { | |
269 | /* | |
270 | * Handle a sequence error. | |
271 | * Silently drop any current message. | |
272 | */ | |
273 | qp->r_psn = psn; | |
274 | inv: | |
275 | if (qp->r_state == OP(SEND_FIRST) || | |
276 | qp->r_state == OP(SEND_MIDDLE)) { | |
01ba79d4 | 277 | set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); |
f931551b RC |
278 | qp->r_sge.num_sge = 0; |
279 | } else | |
70696ea7 | 280 | rvt_put_ss(&qp->r_sge); |
f931551b RC |
281 | qp->r_state = OP(SEND_LAST); |
282 | switch (opcode) { | |
283 | case OP(SEND_FIRST): | |
284 | case OP(SEND_ONLY): | |
285 | case OP(SEND_ONLY_WITH_IMMEDIATE): | |
286 | goto send_first; | |
287 | ||
288 | case OP(RDMA_WRITE_FIRST): | |
289 | case OP(RDMA_WRITE_ONLY): | |
290 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): | |
291 | goto rdma_first; | |
292 | ||
293 | default: | |
294 | goto drop; | |
295 | } | |
296 | } | |
297 | ||
298 | /* Check for opcode sequence errors. */ | |
299 | switch (qp->r_state) { | |
300 | case OP(SEND_FIRST): | |
301 | case OP(SEND_MIDDLE): | |
302 | if (opcode == OP(SEND_MIDDLE) || | |
303 | opcode == OP(SEND_LAST) || | |
304 | opcode == OP(SEND_LAST_WITH_IMMEDIATE)) | |
305 | break; | |
306 | goto inv; | |
307 | ||
308 | case OP(RDMA_WRITE_FIRST): | |
309 | case OP(RDMA_WRITE_MIDDLE): | |
310 | if (opcode == OP(RDMA_WRITE_MIDDLE) || | |
311 | opcode == OP(RDMA_WRITE_LAST) || | |
312 | opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) | |
313 | break; | |
314 | goto inv; | |
315 | ||
316 | default: | |
317 | if (opcode == OP(SEND_FIRST) || | |
318 | opcode == OP(SEND_ONLY) || | |
319 | opcode == OP(SEND_ONLY_WITH_IMMEDIATE) || | |
320 | opcode == OP(RDMA_WRITE_FIRST) || | |
321 | opcode == OP(RDMA_WRITE_ONLY) || | |
322 | opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) | |
323 | break; | |
324 | goto inv; | |
325 | } | |
326 | ||
beb5a042 BW |
327 | if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) |
328 | rvt_comm_est(qp); | |
f931551b RC |
329 | |
330 | /* OK, process the packet. */ | |
331 | switch (opcode) { | |
332 | case OP(SEND_FIRST): | |
333 | case OP(SEND_ONLY): | |
334 | case OP(SEND_ONLY_WITH_IMMEDIATE): | |
335 | send_first: | |
01ba79d4 | 336 | if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) |
f931551b RC |
337 | qp->r_sge = qp->s_rdma_read_sge; |
338 | else { | |
832369fa | 339 | ret = rvt_get_rwqe(qp, false); |
f931551b RC |
340 | if (ret < 0) |
341 | goto op_err; | |
342 | if (!ret) | |
343 | goto drop; | |
344 | /* | |
345 | * qp->s_rdma_read_sge will be the owner | |
346 | * of the mr references. | |
347 | */ | |
348 | qp->s_rdma_read_sge = qp->r_sge; | |
349 | } | |
350 | qp->r_rcv_len = 0; | |
351 | if (opcode == OP(SEND_ONLY)) | |
2fc109c8 | 352 | goto no_immediate_data; |
f931551b RC |
353 | else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) |
354 | goto send_last_imm; | |
df561f66 | 355 | fallthrough; |
f931551b RC |
356 | case OP(SEND_MIDDLE): |
357 | /* Check for invalid length PMTU or posted rwqe len. */ | |
358 | if (unlikely(tlen != (hdrsize + pmtu + 4))) | |
359 | goto rewind; | |
360 | qp->r_rcv_len += pmtu; | |
361 | if (unlikely(qp->r_rcv_len > qp->r_len)) | |
362 | goto rewind; | |
019f118b | 363 | rvt_copy_sge(qp, &qp->r_sge, data, pmtu, false, false); |
f931551b RC |
364 | break; |
365 | ||
366 | case OP(SEND_LAST_WITH_IMMEDIATE): | |
367 | send_last_imm: | |
368 | wc.ex.imm_data = ohdr->u.imm_data; | |
369 | hdrsize += 4; | |
370 | wc.wc_flags = IB_WC_WITH_IMM; | |
2fc109c8 | 371 | goto send_last; |
f931551b | 372 | case OP(SEND_LAST): |
2fc109c8 MM |
373 | no_immediate_data: |
374 | wc.ex.imm_data = 0; | |
375 | wc.wc_flags = 0; | |
f931551b RC |
376 | send_last: |
377 | /* Get the number of bytes the message was padded by. */ | |
378 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
379 | /* Check for invalid length. */ | |
380 | /* XXX LAST len should be >= 1 */ | |
381 | if (unlikely(tlen < (hdrsize + pad + 4))) | |
382 | goto rewind; | |
383 | /* Don't count the CRC. */ | |
384 | tlen -= (hdrsize + pad + 4); | |
385 | wc.byte_len = tlen + qp->r_rcv_len; | |
386 | if (unlikely(wc.byte_len > qp->r_len)) | |
387 | goto rewind; | |
388 | wc.opcode = IB_WC_RECV; | |
019f118b | 389 | rvt_copy_sge(qp, &qp->r_sge, data, tlen, false, false); |
70696ea7 | 390 | rvt_put_ss(&qp->s_rdma_read_sge); |
354dff1b | 391 | last_imm: |
f931551b RC |
392 | wc.wr_id = qp->r_wr_id; |
393 | wc.status = IB_WC_SUCCESS; | |
394 | wc.qp = &qp->ibqp; | |
395 | wc.src_qp = qp->remote_qpn; | |
d8966fcd DC |
396 | wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); |
397 | wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); | |
2fc109c8 MM |
398 | /* zero fields that are N/A */ |
399 | wc.vendor_err = 0; | |
400 | wc.pkey_index = 0; | |
401 | wc.dlid_path_bits = 0; | |
402 | wc.port_num = 0; | |
f931551b | 403 | /* Signal completion event if the solicited bit is set. */ |
5136bfea | 404 | rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); |
f931551b RC |
405 | break; |
406 | ||
407 | case OP(RDMA_WRITE_FIRST): | |
408 | case OP(RDMA_WRITE_ONLY): | |
409 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */ | |
410 | rdma_first: | |
411 | if (unlikely(!(qp->qp_access_flags & | |
412 | IB_ACCESS_REMOTE_WRITE))) { | |
413 | goto drop; | |
414 | } | |
415 | reth = &ohdr->u.rc.reth; | |
416 | hdrsize += sizeof(*reth); | |
417 | qp->r_len = be32_to_cpu(reth->length); | |
418 | qp->r_rcv_len = 0; | |
419 | qp->r_sge.sg_list = NULL; | |
420 | if (qp->r_len != 0) { | |
421 | u32 rkey = be32_to_cpu(reth->rkey); | |
422 | u64 vaddr = be64_to_cpu(reth->vaddr); | |
423 | int ok; | |
424 | ||
425 | /* Check rkey */ | |
7c2e11fe | 426 | ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, |
f931551b RC |
427 | vaddr, rkey, IB_ACCESS_REMOTE_WRITE); |
428 | if (unlikely(!ok)) | |
429 | goto drop; | |
430 | qp->r_sge.num_sge = 1; | |
431 | } else { | |
432 | qp->r_sge.num_sge = 0; | |
433 | qp->r_sge.sge.mr = NULL; | |
434 | qp->r_sge.sge.vaddr = NULL; | |
435 | qp->r_sge.sge.length = 0; | |
436 | qp->r_sge.sge.sge_length = 0; | |
437 | } | |
438 | if (opcode == OP(RDMA_WRITE_ONLY)) | |
439 | goto rdma_last; | |
5715f5d4 JG |
440 | else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) { |
441 | wc.ex.imm_data = ohdr->u.rc.imm_data; | |
f931551b | 442 | goto rdma_last_imm; |
5715f5d4 | 443 | } |
df561f66 | 444 | fallthrough; |
f931551b RC |
445 | case OP(RDMA_WRITE_MIDDLE): |
446 | /* Check for invalid length PMTU or posted rwqe len. */ | |
447 | if (unlikely(tlen != (hdrsize + pmtu + 4))) | |
448 | goto drop; | |
449 | qp->r_rcv_len += pmtu; | |
450 | if (unlikely(qp->r_rcv_len > qp->r_len)) | |
451 | goto drop; | |
019f118b | 452 | rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); |
f931551b RC |
453 | break; |
454 | ||
455 | case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): | |
f931551b | 456 | wc.ex.imm_data = ohdr->u.imm_data; |
5715f5d4 | 457 | rdma_last_imm: |
f931551b RC |
458 | hdrsize += 4; |
459 | wc.wc_flags = IB_WC_WITH_IMM; | |
460 | ||
461 | /* Get the number of bytes the message was padded by. */ | |
462 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
463 | /* Check for invalid length. */ | |
464 | /* XXX LAST len should be >= 1 */ | |
465 | if (unlikely(tlen < (hdrsize + pad + 4))) | |
466 | goto drop; | |
467 | /* Don't count the CRC. */ | |
468 | tlen -= (hdrsize + pad + 4); | |
469 | if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) | |
470 | goto drop; | |
01ba79d4 | 471 | if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) |
70696ea7 | 472 | rvt_put_ss(&qp->s_rdma_read_sge); |
f931551b | 473 | else { |
832369fa | 474 | ret = rvt_get_rwqe(qp, true); |
f931551b RC |
475 | if (ret < 0) |
476 | goto op_err; | |
477 | if (!ret) | |
478 | goto drop; | |
479 | } | |
480 | wc.byte_len = qp->r_len; | |
481 | wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; | |
019f118b | 482 | rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); |
70696ea7 | 483 | rvt_put_ss(&qp->r_sge); |
f931551b RC |
484 | goto last_imm; |
485 | ||
486 | case OP(RDMA_WRITE_LAST): | |
487 | rdma_last: | |
488 | /* Get the number of bytes the message was padded by. */ | |
489 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
490 | /* Check for invalid length. */ | |
491 | /* XXX LAST len should be >= 1 */ | |
492 | if (unlikely(tlen < (hdrsize + pad + 4))) | |
493 | goto drop; | |
494 | /* Don't count the CRC. */ | |
495 | tlen -= (hdrsize + pad + 4); | |
496 | if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) | |
497 | goto drop; | |
019f118b | 498 | rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); |
70696ea7 | 499 | rvt_put_ss(&qp->r_sge); |
f931551b RC |
500 | break; |
501 | ||
502 | default: | |
503 | /* Drop packet for unknown opcodes. */ | |
504 | goto drop; | |
505 | } | |
506 | qp->r_psn++; | |
507 | qp->r_state = opcode; | |
f931551b RC |
508 | return; |
509 | ||
510 | rewind: | |
01ba79d4 | 511 | set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); |
f931551b RC |
512 | qp->r_sge.num_sge = 0; |
513 | drop: | |
f24a6d48 | 514 | ibp->rvp.n_pkt_drops++; |
f931551b RC |
515 | return; |
516 | ||
517 | op_err: | |
beb5a042 | 518 | rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); |
f931551b RC |
519 | return; |
520 | ||
f931551b | 521 | } |