Commit | Line | Data |
---|---|---|
77241056 | 1 | /* |
05d6ac1d | 2 | * Copyright(c) 2015, 2016 Intel Corporation. |
77241056 MM |
3 | * |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
77241056 MM |
9 | * This program is free software; you can redistribute it and/or modify |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
77241056 MM |
20 | * Redistribution and use in source and binary forms, with or without |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | ||
48 | #include "hfi.h" | |
bb5df5f9 | 49 | #include "verbs_txreq.h" |
77241056 MM |
50 | #include "qp.h" |
51 | ||
52 | /* cut down ridiculously long IB macro names */ | |
b374e060 | 53 | #define OP(x) UC_OP(x) |
14553ca1 | 54 | |
77241056 MM |
55 | /** |
56 | * hfi1_make_uc_req - construct a request packet (SEND, RDMA write) | |
57 | * @qp: a pointer to the QP | |
58 | * | |
46a80d62 MM |
59 | * Assume s_lock is held. |
60 | * | |
77241056 MM |
61 | * Return 1 if constructed; otherwise, return 0. |
62 | */ | |
bb5df5f9 | 63 | int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) |
77241056 | 64 | { |
4c6829c5 | 65 | struct hfi1_qp_priv *priv = qp->priv; |
261a4351 | 66 | struct ib_other_headers *ohdr; |
895420dd | 67 | struct rvt_swqe *wqe; |
77241056 MM |
68 | u32 hwords = 5; |
69 | u32 bth0 = 0; | |
70 | u32 len; | |
71 | u32 pmtu = qp->pmtu; | |
77241056 MM |
72 | int middle = 0; |
73 | ||
bb5df5f9 DD |
74 | ps->s_txreq = get_txreq(ps->dev, qp); |
75 | if (IS_ERR(ps->s_txreq)) | |
76 | goto bail_no_tx; | |
77 | ||
83693bd1 DD |
78 | if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { |
79 | if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) | |
77241056 MM |
80 | goto bail; |
81 | /* We are in the error state, flush the work request. */ | |
46a80d62 MM |
82 | smp_read_barrier_depends(); /* see post_one_send() */ |
83 | if (qp->s_last == ACCESS_ONCE(qp->s_head)) | |
77241056 MM |
84 | goto bail; |
85 | /* If DMAs are in progress, we can't flush immediately. */ | |
14553ca1 | 86 | if (iowait_sdma_pending(&priv->s_iowait)) { |
54d10c1e | 87 | qp->s_flags |= RVT_S_WAIT_DMA; |
77241056 MM |
88 | goto bail; |
89 | } | |
90 | clear_ahg(qp); | |
83693bd1 | 91 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
77241056 | 92 | hfi1_send_complete(qp, wqe, IB_WC_WR_FLUSH_ERR); |
bb5df5f9 | 93 | goto done_free_tx; |
77241056 MM |
94 | } |
95 | ||
bb5df5f9 | 96 | ohdr = &ps->s_txreq->phdr.hdr.u.oth; |
77241056 | 97 | if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) |
bb5df5f9 | 98 | ohdr = &ps->s_txreq->phdr.hdr.u.l.oth; |
77241056 MM |
99 | |
100 | /* Get the next send request. */ | |
83693bd1 | 101 | wqe = rvt_get_swqe_ptr(qp, qp->s_cur); |
77241056 MM |
102 | qp->s_wqe = NULL; |
103 | switch (qp->s_state) { | |
104 | default: | |
83693bd1 DD |
105 | if (!(ib_rvt_state_ops[qp->state] & |
106 | RVT_PROCESS_NEXT_SEND_OK)) | |
77241056 MM |
107 | goto bail; |
108 | /* Check if send work queue is empty. */ | |
46a80d62 MM |
109 | smp_read_barrier_depends(); /* see post_one_send() */ |
110 | if (qp->s_cur == ACCESS_ONCE(qp->s_head)) { | |
77241056 MM |
111 | clear_ahg(qp); |
112 | goto bail; | |
113 | } | |
0db3dfa0 JX |
114 | /* |
115 | * Local operations are processed immediately | |
116 | * after all prior requests have completed. | |
117 | */ | |
118 | if (wqe->wr.opcode == IB_WR_REG_MR || | |
119 | wqe->wr.opcode == IB_WR_LOCAL_INV) { | |
d9b13c20 JX |
120 | int local_ops = 0; |
121 | int err = 0; | |
122 | ||
0db3dfa0 JX |
123 | if (qp->s_last != qp->s_cur) |
124 | goto bail; | |
125 | if (++qp->s_cur == qp->s_size) | |
126 | qp->s_cur = 0; | |
d9b13c20 | 127 | if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { |
0db3dfa0 JX |
128 | err = rvt_invalidate_rkey( |
129 | qp, wqe->wr.ex.invalidate_rkey); | |
d9b13c20 JX |
130 | local_ops = 1; |
131 | } | |
0db3dfa0 JX |
132 | hfi1_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR |
133 | : IB_WC_SUCCESS); | |
d9b13c20 JX |
134 | if (local_ops) |
135 | atomic_dec(&qp->local_ops_pending); | |
0db3dfa0 JX |
136 | qp->s_hdrwords = 0; |
137 | goto done_free_tx; | |
138 | } | |
77241056 MM |
139 | /* |
140 | * Start a new request. | |
141 | */ | |
46a80d62 | 142 | qp->s_psn = wqe->psn; |
77241056 MM |
143 | qp->s_sge.sge = wqe->sg_list[0]; |
144 | qp->s_sge.sg_list = wqe->sg_list + 1; | |
145 | qp->s_sge.num_sge = wqe->wr.num_sge; | |
146 | qp->s_sge.total_len = wqe->length; | |
147 | len = wqe->length; | |
148 | qp->s_len = len; | |
149 | switch (wqe->wr.opcode) { | |
150 | case IB_WR_SEND: | |
151 | case IB_WR_SEND_WITH_IMM: | |
152 | if (len > pmtu) { | |
153 | qp->s_state = OP(SEND_FIRST); | |
154 | len = pmtu; | |
155 | break; | |
156 | } | |
e490974e | 157 | if (wqe->wr.opcode == IB_WR_SEND) { |
77241056 | 158 | qp->s_state = OP(SEND_ONLY); |
e490974e | 159 | } else { |
77241056 MM |
160 | qp->s_state = |
161 | OP(SEND_ONLY_WITH_IMMEDIATE); | |
162 | /* Immediate data comes after the BTH */ | |
163 | ohdr->u.imm_data = wqe->wr.ex.imm_data; | |
164 | hwords += 1; | |
165 | } | |
166 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
167 | bth0 |= IB_BTH_SOLICITED; | |
168 | qp->s_wqe = wqe; | |
169 | if (++qp->s_cur >= qp->s_size) | |
170 | qp->s_cur = 0; | |
171 | break; | |
172 | ||
173 | case IB_WR_RDMA_WRITE: | |
174 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
175 | ohdr->u.rc.reth.vaddr = | |
e622f2f4 | 176 | cpu_to_be64(wqe->rdma_wr.remote_addr); |
77241056 | 177 | ohdr->u.rc.reth.rkey = |
e622f2f4 | 178 | cpu_to_be32(wqe->rdma_wr.rkey); |
77241056 MM |
179 | ohdr->u.rc.reth.length = cpu_to_be32(len); |
180 | hwords += sizeof(struct ib_reth) / 4; | |
181 | if (len > pmtu) { | |
182 | qp->s_state = OP(RDMA_WRITE_FIRST); | |
183 | len = pmtu; | |
184 | break; | |
185 | } | |
e490974e | 186 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { |
77241056 | 187 | qp->s_state = OP(RDMA_WRITE_ONLY); |
e490974e | 188 | } else { |
77241056 MM |
189 | qp->s_state = |
190 | OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); | |
191 | /* Immediate data comes after the RETH */ | |
192 | ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; | |
193 | hwords += 1; | |
194 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
195 | bth0 |= IB_BTH_SOLICITED; | |
196 | } | |
197 | qp->s_wqe = wqe; | |
198 | if (++qp->s_cur >= qp->s_size) | |
199 | qp->s_cur = 0; | |
200 | break; | |
201 | ||
202 | default: | |
203 | goto bail; | |
204 | } | |
205 | break; | |
206 | ||
207 | case OP(SEND_FIRST): | |
208 | qp->s_state = OP(SEND_MIDDLE); | |
209 | /* FALLTHROUGH */ | |
210 | case OP(SEND_MIDDLE): | |
211 | len = qp->s_len; | |
212 | if (len > pmtu) { | |
213 | len = pmtu; | |
214 | middle = HFI1_CAP_IS_KSET(SDMA_AHG); | |
215 | break; | |
216 | } | |
e490974e | 217 | if (wqe->wr.opcode == IB_WR_SEND) { |
77241056 | 218 | qp->s_state = OP(SEND_LAST); |
e490974e | 219 | } else { |
77241056 MM |
220 | qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); |
221 | /* Immediate data comes after the BTH */ | |
222 | ohdr->u.imm_data = wqe->wr.ex.imm_data; | |
223 | hwords += 1; | |
224 | } | |
225 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
226 | bth0 |= IB_BTH_SOLICITED; | |
227 | qp->s_wqe = wqe; | |
228 | if (++qp->s_cur >= qp->s_size) | |
229 | qp->s_cur = 0; | |
230 | break; | |
231 | ||
232 | case OP(RDMA_WRITE_FIRST): | |
233 | qp->s_state = OP(RDMA_WRITE_MIDDLE); | |
234 | /* FALLTHROUGH */ | |
235 | case OP(RDMA_WRITE_MIDDLE): | |
236 | len = qp->s_len; | |
237 | if (len > pmtu) { | |
238 | len = pmtu; | |
239 | middle = HFI1_CAP_IS_KSET(SDMA_AHG); | |
240 | break; | |
241 | } | |
e490974e | 242 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { |
77241056 | 243 | qp->s_state = OP(RDMA_WRITE_LAST); |
e490974e | 244 | } else { |
77241056 MM |
245 | qp->s_state = |
246 | OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); | |
247 | /* Immediate data comes after the BTH */ | |
248 | ohdr->u.imm_data = wqe->wr.ex.imm_data; | |
249 | hwords += 1; | |
250 | if (wqe->wr.send_flags & IB_SEND_SOLICITED) | |
251 | bth0 |= IB_BTH_SOLICITED; | |
252 | } | |
253 | qp->s_wqe = wqe; | |
254 | if (++qp->s_cur >= qp->s_size) | |
255 | qp->s_cur = 0; | |
256 | break; | |
257 | } | |
258 | qp->s_len -= len; | |
259 | qp->s_hdrwords = hwords; | |
14553ca1 | 260 | ps->s_txreq->sde = priv->s_sde; |
77241056 | 261 | qp->s_cur_sge = &qp->s_sge; |
e922ae06 | 262 | ps->s_txreq->s_cur_size = len; |
77241056 | 263 | hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), |
bb5df5f9 | 264 | mask_psn(qp->s_psn++), middle, ps); |
aa0ad411 JX |
265 | /* pbc */ |
266 | ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2; | |
bb5df5f9 DD |
267 | return 1; |
268 | ||
269 | done_free_tx: | |
270 | hfi1_put_txreq(ps->s_txreq); | |
271 | ps->s_txreq = NULL; | |
46a80d62 | 272 | return 1; |
77241056 MM |
273 | |
274 | bail: | |
bb5df5f9 DD |
275 | hfi1_put_txreq(ps->s_txreq); |
276 | ||
277 | bail_no_tx: | |
278 | ps->s_txreq = NULL; | |
54d10c1e | 279 | qp->s_flags &= ~RVT_S_BUSY; |
bb5df5f9 DD |
280 | qp->s_hdrwords = 0; |
281 | return 0; | |
77241056 MM |
282 | } |
283 | ||
284 | /** | |
285 | * hfi1_uc_rcv - handle an incoming UC packet | |
286 | * @ibp: the port the packet came in on | |
287 | * @hdr: the header of the packet | |
288 | * @rcv_flags: flags relevant to rcv processing | |
289 | * @data: the packet data | |
290 | * @tlen: the length of the packet | |
291 | * @qp: the QP for this packet. | |
292 | * | |
293 | * This is called from qp_rcv() to process an incoming UC packet | |
294 | * for the given QP. | |
295 | * Called at interrupt level. | |
296 | */ | |
297 | void hfi1_uc_rcv(struct hfi1_packet *packet) | |
298 | { | |
299 | struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data; | |
261a4351 | 300 | struct ib_header *hdr = packet->hdr; |
77241056 MM |
301 | u32 rcv_flags = packet->rcv_flags; |
302 | void *data = packet->ebuf; | |
303 | u32 tlen = packet->tlen; | |
895420dd | 304 | struct rvt_qp *qp = packet->qp; |
261a4351 | 305 | struct ib_other_headers *ohdr = packet->ohdr; |
977940b8 | 306 | u32 bth0, opcode; |
77241056 MM |
307 | u32 hdrsize = packet->hlen; |
308 | u32 psn; | |
309 | u32 pad; | |
310 | struct ib_wc wc; | |
311 | u32 pmtu = qp->pmtu; | |
312 | struct ib_reth *reth; | |
313 | int has_grh = rcv_flags & HFI1_HAS_GRH; | |
314 | int ret; | |
77241056 | 315 | |
977940b8 AK |
316 | bth0 = be32_to_cpu(ohdr->bth[0]); |
317 | if (hfi1_ruc_check_hdr(ibp, hdr, has_grh, qp, bth0)) | |
77241056 MM |
318 | return; |
319 | ||
5fd2b562 | 320 | process_ecn(qp, packet, true); |
77241056 MM |
321 | |
322 | psn = be32_to_cpu(ohdr->bth[2]); | |
977940b8 | 323 | opcode = (bth0 >> 24) & 0xff; |
77241056 MM |
324 | |
325 | /* Compare the PSN verses the expected PSN. */ | |
326 | if (unlikely(cmp_psn(psn, qp->r_psn) != 0)) { | |
327 | /* | |
328 | * Handle a sequence error. | |
329 | * Silently drop any current message. | |
330 | */ | |
331 | qp->r_psn = psn; | |
332 | inv: | |
333 | if (qp->r_state == OP(SEND_FIRST) || | |
334 | qp->r_state == OP(SEND_MIDDLE)) { | |
54d10c1e | 335 | set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); |
77241056 | 336 | qp->r_sge.num_sge = 0; |
e490974e | 337 | } else { |
ec4274f1 | 338 | rvt_put_ss(&qp->r_sge); |
e490974e | 339 | } |
77241056 MM |
340 | qp->r_state = OP(SEND_LAST); |
341 | switch (opcode) { | |
342 | case OP(SEND_FIRST): | |
343 | case OP(SEND_ONLY): | |
344 | case OP(SEND_ONLY_WITH_IMMEDIATE): | |
345 | goto send_first; | |
346 | ||
347 | case OP(RDMA_WRITE_FIRST): | |
348 | case OP(RDMA_WRITE_ONLY): | |
349 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): | |
350 | goto rdma_first; | |
351 | ||
352 | default: | |
353 | goto drop; | |
354 | } | |
355 | } | |
356 | ||
357 | /* Check for opcode sequence errors. */ | |
358 | switch (qp->r_state) { | |
359 | case OP(SEND_FIRST): | |
360 | case OP(SEND_MIDDLE): | |
361 | if (opcode == OP(SEND_MIDDLE) || | |
362 | opcode == OP(SEND_LAST) || | |
363 | opcode == OP(SEND_LAST_WITH_IMMEDIATE)) | |
364 | break; | |
365 | goto inv; | |
366 | ||
367 | case OP(RDMA_WRITE_FIRST): | |
368 | case OP(RDMA_WRITE_MIDDLE): | |
369 | if (opcode == OP(RDMA_WRITE_MIDDLE) || | |
370 | opcode == OP(RDMA_WRITE_LAST) || | |
371 | opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) | |
372 | break; | |
373 | goto inv; | |
374 | ||
375 | default: | |
376 | if (opcode == OP(SEND_FIRST) || | |
377 | opcode == OP(SEND_ONLY) || | |
378 | opcode == OP(SEND_ONLY_WITH_IMMEDIATE) || | |
379 | opcode == OP(RDMA_WRITE_FIRST) || | |
380 | opcode == OP(RDMA_WRITE_ONLY) || | |
381 | opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) | |
382 | break; | |
383 | goto inv; | |
384 | } | |
385 | ||
54d10c1e | 386 | if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) |
77241056 MM |
387 | qp_comm_est(qp); |
388 | ||
389 | /* OK, process the packet. */ | |
390 | switch (opcode) { | |
391 | case OP(SEND_FIRST): | |
392 | case OP(SEND_ONLY): | |
393 | case OP(SEND_ONLY_WITH_IMMEDIATE): | |
394 | send_first: | |
e490974e | 395 | if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) { |
77241056 | 396 | qp->r_sge = qp->s_rdma_read_sge; |
e490974e | 397 | } else { |
ec4274f1 | 398 | ret = hfi1_rvt_get_rwqe(qp, 0); |
77241056 MM |
399 | if (ret < 0) |
400 | goto op_err; | |
401 | if (!ret) | |
402 | goto drop; | |
403 | /* | |
404 | * qp->s_rdma_read_sge will be the owner | |
405 | * of the mr references. | |
406 | */ | |
407 | qp->s_rdma_read_sge = qp->r_sge; | |
408 | } | |
409 | qp->r_rcv_len = 0; | |
410 | if (opcode == OP(SEND_ONLY)) | |
411 | goto no_immediate_data; | |
412 | else if (opcode == OP(SEND_ONLY_WITH_IMMEDIATE)) | |
413 | goto send_last_imm; | |
414 | /* FALLTHROUGH */ | |
415 | case OP(SEND_MIDDLE): | |
416 | /* Check for invalid length PMTU or posted rwqe len. */ | |
417 | if (unlikely(tlen != (hdrsize + pmtu + 4))) | |
418 | goto rewind; | |
419 | qp->r_rcv_len += pmtu; | |
420 | if (unlikely(qp->r_rcv_len > qp->r_len)) | |
421 | goto rewind; | |
7b0b01aa | 422 | hfi1_copy_sge(&qp->r_sge, data, pmtu, 0, 0); |
77241056 MM |
423 | break; |
424 | ||
425 | case OP(SEND_LAST_WITH_IMMEDIATE): | |
426 | send_last_imm: | |
427 | wc.ex.imm_data = ohdr->u.imm_data; | |
428 | wc.wc_flags = IB_WC_WITH_IMM; | |
429 | goto send_last; | |
430 | case OP(SEND_LAST): | |
431 | no_immediate_data: | |
432 | wc.ex.imm_data = 0; | |
433 | wc.wc_flags = 0; | |
434 | send_last: | |
435 | /* Get the number of bytes the message was padded by. */ | |
436 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
437 | /* Check for invalid length. */ | |
438 | /* LAST len should be >= 1 */ | |
439 | if (unlikely(tlen < (hdrsize + pad + 4))) | |
440 | goto rewind; | |
441 | /* Don't count the CRC. */ | |
442 | tlen -= (hdrsize + pad + 4); | |
443 | wc.byte_len = tlen + qp->r_rcv_len; | |
444 | if (unlikely(wc.byte_len > qp->r_len)) | |
445 | goto rewind; | |
446 | wc.opcode = IB_WC_RECV; | |
7b0b01aa | 447 | hfi1_copy_sge(&qp->r_sge, data, tlen, 0, 0); |
ec4274f1 | 448 | rvt_put_ss(&qp->s_rdma_read_sge); |
77241056 MM |
449 | last_imm: |
450 | wc.wr_id = qp->r_wr_id; | |
451 | wc.status = IB_WC_SUCCESS; | |
452 | wc.qp = &qp->ibqp; | |
453 | wc.src_qp = qp->remote_qpn; | |
454 | wc.slid = qp->remote_ah_attr.dlid; | |
455 | /* | |
456 | * It seems that IB mandates the presence of an SL in a | |
457 | * work completion only for the UD transport (see section | |
458 | * 11.4.2 of IBTA Vol. 1). | |
459 | * | |
460 | * However, the way the SL is chosen below is consistent | |
461 | * with the way that IB/qib works and is trying avoid | |
462 | * introducing incompatibilities. | |
463 | * | |
464 | * See also OPA Vol. 1, section 9.7.6, and table 9-17. | |
465 | */ | |
466 | wc.sl = qp->remote_ah_attr.sl; | |
467 | /* zero fields that are N/A */ | |
468 | wc.vendor_err = 0; | |
469 | wc.pkey_index = 0; | |
470 | wc.dlid_path_bits = 0; | |
471 | wc.port_num = 0; | |
472 | /* Signal completion event if the solicited bit is set. */ | |
abd712da DD |
473 | rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, |
474 | (ohdr->bth[0] & | |
475 | cpu_to_be32(IB_BTH_SOLICITED)) != 0); | |
77241056 MM |
476 | break; |
477 | ||
478 | case OP(RDMA_WRITE_FIRST): | |
479 | case OP(RDMA_WRITE_ONLY): | |
480 | case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): /* consume RWQE */ | |
481 | rdma_first: | |
482 | if (unlikely(!(qp->qp_access_flags & | |
483 | IB_ACCESS_REMOTE_WRITE))) { | |
484 | goto drop; | |
485 | } | |
486 | reth = &ohdr->u.rc.reth; | |
487 | qp->r_len = be32_to_cpu(reth->length); | |
488 | qp->r_rcv_len = 0; | |
489 | qp->r_sge.sg_list = NULL; | |
490 | if (qp->r_len != 0) { | |
491 | u32 rkey = be32_to_cpu(reth->rkey); | |
492 | u64 vaddr = be64_to_cpu(reth->vaddr); | |
493 | int ok; | |
494 | ||
495 | /* Check rkey */ | |
895420dd DD |
496 | ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, |
497 | vaddr, rkey, IB_ACCESS_REMOTE_WRITE); | |
77241056 MM |
498 | if (unlikely(!ok)) |
499 | goto drop; | |
500 | qp->r_sge.num_sge = 1; | |
501 | } else { | |
502 | qp->r_sge.num_sge = 0; | |
503 | qp->r_sge.sge.mr = NULL; | |
504 | qp->r_sge.sge.vaddr = NULL; | |
505 | qp->r_sge.sge.length = 0; | |
506 | qp->r_sge.sge.sge_length = 0; | |
507 | } | |
e490974e | 508 | if (opcode == OP(RDMA_WRITE_ONLY)) { |
77241056 | 509 | goto rdma_last; |
e490974e | 510 | } else if (opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) { |
77241056 MM |
511 | wc.ex.imm_data = ohdr->u.rc.imm_data; |
512 | goto rdma_last_imm; | |
513 | } | |
514 | /* FALLTHROUGH */ | |
515 | case OP(RDMA_WRITE_MIDDLE): | |
516 | /* Check for invalid length PMTU or posted rwqe len. */ | |
517 | if (unlikely(tlen != (hdrsize + pmtu + 4))) | |
518 | goto drop; | |
519 | qp->r_rcv_len += pmtu; | |
520 | if (unlikely(qp->r_rcv_len > qp->r_len)) | |
521 | goto drop; | |
7b0b01aa | 522 | hfi1_copy_sge(&qp->r_sge, data, pmtu, 1, 0); |
77241056 MM |
523 | break; |
524 | ||
525 | case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): | |
526 | wc.ex.imm_data = ohdr->u.imm_data; | |
527 | rdma_last_imm: | |
528 | wc.wc_flags = IB_WC_WITH_IMM; | |
529 | ||
530 | /* Get the number of bytes the message was padded by. */ | |
531 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
532 | /* Check for invalid length. */ | |
533 | /* LAST len should be >= 1 */ | |
534 | if (unlikely(tlen < (hdrsize + pad + 4))) | |
535 | goto drop; | |
536 | /* Don't count the CRC. */ | |
537 | tlen -= (hdrsize + pad + 4); | |
538 | if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) | |
539 | goto drop; | |
e490974e | 540 | if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) { |
ec4274f1 | 541 | rvt_put_ss(&qp->s_rdma_read_sge); |
e490974e | 542 | } else { |
ec4274f1 | 543 | ret = hfi1_rvt_get_rwqe(qp, 1); |
77241056 MM |
544 | if (ret < 0) |
545 | goto op_err; | |
546 | if (!ret) | |
547 | goto drop; | |
548 | } | |
549 | wc.byte_len = qp->r_len; | |
550 | wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; | |
7b0b01aa | 551 | hfi1_copy_sge(&qp->r_sge, data, tlen, 1, 0); |
ec4274f1 | 552 | rvt_put_ss(&qp->r_sge); |
77241056 MM |
553 | goto last_imm; |
554 | ||
555 | case OP(RDMA_WRITE_LAST): | |
556 | rdma_last: | |
557 | /* Get the number of bytes the message was padded by. */ | |
558 | pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; | |
559 | /* Check for invalid length. */ | |
560 | /* LAST len should be >= 1 */ | |
561 | if (unlikely(tlen < (hdrsize + pad + 4))) | |
562 | goto drop; | |
563 | /* Don't count the CRC. */ | |
564 | tlen -= (hdrsize + pad + 4); | |
565 | if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) | |
566 | goto drop; | |
7b0b01aa | 567 | hfi1_copy_sge(&qp->r_sge, data, tlen, 1, 0); |
ec4274f1 | 568 | rvt_put_ss(&qp->r_sge); |
77241056 MM |
569 | break; |
570 | ||
571 | default: | |
572 | /* Drop packet for unknown opcodes. */ | |
573 | goto drop; | |
574 | } | |
575 | qp->r_psn++; | |
576 | qp->r_state = opcode; | |
577 | return; | |
578 | ||
579 | rewind: | |
54d10c1e | 580 | set_bit(RVT_R_REWIND_SGE, &qp->r_aflags); |
77241056 MM |
581 | qp->r_sge.num_sge = 0; |
582 | drop: | |
4eb06882 | 583 | ibp->rvp.n_pkt_drops++; |
77241056 MM |
584 | return; |
585 | ||
586 | op_err: | |
587 | hfi1_rc_error(qp, IB_WC_LOC_QP_OP_ERR); | |
77241056 | 588 | } |