1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
6 #include <linux/errno.h>
7 #include <linux/types.h>
9 #include <linux/scatterlist.h>
10 #include <linux/llist.h>
11 #include <asm/barrier.h>
15 #include "siw_verbs.h"
18 static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = {
19 [SIW_QP_STATE_IDLE] = "IDLE",
20 [SIW_QP_STATE_RTR] = "RTR",
21 [SIW_QP_STATE_RTS] = "RTS",
22 [SIW_QP_STATE_CLOSING] = "CLOSING",
23 [SIW_QP_STATE_TERMINATE] = "TERMINATE",
24 [SIW_QP_STATE_ERROR] = "ERROR"
28 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a
29 * per-RDMAP message basis. Please keep order of initializer. All MPA len
30 * is initialized to minimum packet size.
32 struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = {
33 { /* RDMAP_RDMA_WRITE */
34 .hdr_len = sizeof(struct iwarp_rdma_write),
35 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
36 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
37 cpu_to_be16(DDP_VERSION << 8) |
38 cpu_to_be16(RDMAP_VERSION << 6) |
39 cpu_to_be16(RDMAP_RDMA_WRITE),
40 .rx_data = siw_proc_write },
41 { /* RDMAP_RDMA_READ_REQ */
42 .hdr_len = sizeof(struct iwarp_rdma_rreq),
43 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
44 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
45 cpu_to_be16(RDMAP_VERSION << 6) |
46 cpu_to_be16(RDMAP_RDMA_READ_REQ),
47 .rx_data = siw_proc_rreq },
48 { /* RDMAP_RDMA_READ_RESP */
49 .hdr_len = sizeof(struct iwarp_rdma_rresp),
50 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
51 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
52 cpu_to_be16(DDP_VERSION << 8) |
53 cpu_to_be16(RDMAP_VERSION << 6) |
54 cpu_to_be16(RDMAP_RDMA_READ_RESP),
55 .rx_data = siw_proc_rresp },
57 .hdr_len = sizeof(struct iwarp_send),
58 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
59 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
60 cpu_to_be16(RDMAP_VERSION << 6) |
61 cpu_to_be16(RDMAP_SEND),
62 .rx_data = siw_proc_send },
63 { /* RDMAP_SEND_INVAL */
64 .hdr_len = sizeof(struct iwarp_send_inv),
65 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
66 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
67 cpu_to_be16(RDMAP_VERSION << 6) |
68 cpu_to_be16(RDMAP_SEND_INVAL),
69 .rx_data = siw_proc_send },
71 .hdr_len = sizeof(struct iwarp_send),
72 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
73 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
74 cpu_to_be16(RDMAP_VERSION << 6) |
75 cpu_to_be16(RDMAP_SEND_SE),
76 .rx_data = siw_proc_send },
77 { /* RDMAP_SEND_SE_INVAL */
78 .hdr_len = sizeof(struct iwarp_send_inv),
79 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
80 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
81 cpu_to_be16(RDMAP_VERSION << 6) |
82 cpu_to_be16(RDMAP_SEND_SE_INVAL),
83 .rx_data = siw_proc_send },
84 { /* RDMAP_TERMINATE */
85 .hdr_len = sizeof(struct iwarp_terminate),
86 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
87 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
88 cpu_to_be16(RDMAP_VERSION << 6) |
89 cpu_to_be16(RDMAP_TERMINATE),
90 .rx_data = siw_proc_terminate }
93 void siw_qp_llp_data_ready(struct sock *sk)
97 read_lock(&sk->sk_callback_lock);
99 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk)))
104 if (likely(!qp->rx_stream.rx_suspend &&
105 down_read_trylock(&qp->state_lock))) {
106 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };
108 if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
110 * Implements data receive operation during
111 * socket callback. TCP gracefully catches
112 * the case where there is nothing to receive
113 * (not calling siw_tcp_rx_data() then).
115 tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
117 up_read(&qp->state_lock);
119 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",
120 qp->rx_stream.rx_suspend);
123 read_unlock(&sk->sk_callback_lock);
126 void siw_qp_llp_close(struct siw_qp *qp)
128 siw_dbg_qp(qp, "enter llp close, state = %s\n",
129 siw_qp_state_to_string[qp->attrs.state]);
131 down_write(&qp->state_lock);
133 qp->rx_stream.rx_suspend = 1;
134 qp->tx_ctx.tx_suspend = 1;
137 switch (qp->attrs.state) {
138 case SIW_QP_STATE_RTS:
139 case SIW_QP_STATE_RTR:
140 case SIW_QP_STATE_IDLE:
141 case SIW_QP_STATE_TERMINATE:
142 qp->attrs.state = SIW_QP_STATE_ERROR;
145 * SIW_QP_STATE_CLOSING:
147 * This is a forced close. shall the QP be moved to
150 case SIW_QP_STATE_CLOSING:
151 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
152 qp->attrs.state = SIW_QP_STATE_ERROR;
154 qp->attrs.state = SIW_QP_STATE_IDLE;
158 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",
159 siw_qp_state_to_string[qp->attrs.state]);
166 * Dereference closing CEP
169 siw_cep_put(qp->cep);
173 up_write(&qp->state_lock);
175 siw_dbg_qp(qp, "llp close exit: state %s\n",
176 siw_qp_state_to_string[qp->attrs.state]);
180 * socket callback routine informing about newly available send space.
181 * Function schedules SQ work for processing SQ items.
183 void siw_qp_llp_write_space(struct sock *sk)
185 struct siw_cep *cep = sk_to_cep(sk);
187 cep->sk_write_space(sk);
189 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
190 (void)siw_sq_start(cep->qp);
193 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)
195 irq_size = roundup_pow_of_two(irq_size);
196 orq_size = roundup_pow_of_two(orq_size);
198 qp->attrs.irq_size = irq_size;
199 qp->attrs.orq_size = orq_size;
201 qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe));
203 siw_dbg_qp(qp, "irq malloc for %d failed\n", irq_size);
204 qp->attrs.irq_size = 0;
207 qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe));
209 siw_dbg_qp(qp, "orq malloc for %d failed\n", orq_size);
210 qp->attrs.orq_size = 0;
211 qp->attrs.irq_size = 0;
215 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);
219 static int siw_qp_enable_crc(struct siw_qp *qp)
221 struct siw_rx_stream *c_rx = &qp->rx_stream;
222 struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
225 if (siw_crypto_shash == NULL)
228 size = crypto_shash_descsize(siw_crypto_shash) +
229 sizeof(struct shash_desc);
231 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL);
232 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL);
233 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) {
234 kfree(c_tx->mpa_crc_hd);
235 kfree(c_rx->mpa_crc_hd);
236 c_tx->mpa_crc_hd = NULL;
237 c_rx->mpa_crc_hd = NULL;
240 c_tx->mpa_crc_hd->tfm = siw_crypto_shash;
241 c_rx->mpa_crc_hd->tfm = siw_crypto_shash;
247 * Send a non signalled READ or WRITE to peer side as negotiated
248 * with MPAv2 P2P setup protocol. The work request is only created
249 * as a current active WR and does not consume Send Queue space.
251 * Caller must hold QP state lock.
253 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)
255 struct siw_wqe *wqe = tx_wqe(qp);
259 spin_lock_irqsave(&qp->sq_lock, flags);
261 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) {
262 spin_unlock_irqrestore(&qp->sq_lock, flags);
265 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
267 wqe->wr_status = SIW_WR_QUEUED;
269 wqe->sqe.num_sge = 1;
270 wqe->sqe.sge[0].length = 0;
271 wqe->sqe.sge[0].laddr = 0;
272 wqe->sqe.sge[0].lkey = 0;
274 * While it must not be checked for inbound zero length
275 * READ/WRITE, some HW may treat STag 0 special.
281 if (ctrl & MPA_V2_RDMA_WRITE_RTR)
282 wqe->sqe.opcode = SIW_OP_WRITE;
283 else if (ctrl & MPA_V2_RDMA_READ_RTR) {
284 struct siw_sqe *rreq;
286 wqe->sqe.opcode = SIW_OP_READ;
288 spin_lock(&qp->orq_lock);
290 rreq = orq_get_free(qp);
292 siw_read_to_orq(rreq, &wqe->sqe);
297 spin_unlock(&qp->orq_lock);
302 wqe->wr_status = SIW_WR_IDLE;
304 spin_unlock_irqrestore(&qp->sq_lock, flags);
307 rv = siw_sq_start(qp);
313 * Map memory access error to DDP tagged error
315 enum ddp_ecode siw_tagged_error(enum siw_access_state state)
319 return DDP_ECODE_T_INVALID_STAG;
321 return DDP_ECODE_T_BASE_BOUNDS;
323 return DDP_ECODE_T_STAG_NOT_ASSOC;
326 * RFC 5041 (DDP) lacks an ecode for insufficient access
327 * permissions. 'Invalid STag' seem to be the closest
330 return DDP_ECODE_T_INVALID_STAG;
333 return DDP_ECODE_T_INVALID_STAG;
338 * Map memory access error to RDMAP protection error
340 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state)
344 return RDMAP_ECODE_INVALID_STAG;
346 return RDMAP_ECODE_BASE_BOUNDS;
348 return RDMAP_ECODE_STAG_NOT_ASSOC;
350 return RDMAP_ECODE_ACCESS_RIGHTS;
352 return RDMAP_ECODE_UNSPECIFIED;
356 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,
359 if (!qp->term_info.valid) {
360 memset(&qp->term_info, 0, sizeof(qp->term_info));
361 qp->term_info.layer = layer;
362 qp->term_info.etype = etype;
363 qp->term_info.ecode = ecode;
364 qp->term_info.in_tx = in_tx;
365 qp->term_info.valid = 1;
367 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
368 layer, etype, ecode, in_tx ? "yes" : "no");
372 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581.
373 * Sending TERMINATE messages is best effort - such messages
374 * can only be send if the QP is still connected and it does
375 * not have another outbound message in-progress, i.e. the
376 * TERMINATE message must not interfer with an incomplete current
377 * transmit operation.
379 void siw_send_terminate(struct siw_qp *qp)
382 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
383 struct iwarp_terminate *term = NULL;
384 union iwarp_hdr *err_hdr = NULL;
385 struct socket *s = qp->attrs.sk;
386 struct siw_rx_stream *srx = &qp->rx_stream;
387 union iwarp_hdr *rx_hdr = &srx->hdr;
389 int num_frags, len_terminate, rv;
391 if (!qp->term_info.valid)
394 qp->term_info.valid = 0;
396 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
397 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",
398 tx_type(tx_wqe(qp)));
402 /* QP not yet in RTS. Take socket from connection end point */
406 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");
410 term = kzalloc(sizeof(*term), GFP_KERNEL);
414 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE);
416 term->ddp_msn = cpu_to_be32(1);
418 iov[0].iov_base = term;
419 iov[0].iov_len = sizeof(*term);
421 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
422 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
423 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
424 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL);
430 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl,
431 sizeof(struct iwarp_ctrl));
433 __rdmap_term_set_layer(term, qp->term_info.layer);
434 __rdmap_term_set_etype(term, qp->term_info.etype);
435 __rdmap_term_set_ecode(term, qp->term_info.ecode);
437 switch (qp->term_info.layer) {
438 case TERM_ERROR_LAYER_RDMAP:
439 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
440 /* No additional DDP/RDMAP header to be included */
443 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
445 * Complete RDMAP frame will get attached, and
446 * DDP segment length is valid
452 if (qp->term_info.in_tx) {
453 struct iwarp_rdma_rreq *rreq;
454 struct siw_wqe *wqe = tx_wqe(qp);
456 /* Inbound RREQ error, detected during
457 * RRESP creation. Take state from
458 * current TX work queue element to
459 * reconstruct peers RREQ.
461 rreq = (struct iwarp_rdma_rreq *)err_hdr;
464 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
465 sizeof(struct iwarp_ctrl));
469 htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
471 /* Provide RREQ's MSN as kept aside */
472 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);
474 rreq->ddp_mo = htonl(wqe->processed);
475 rreq->sink_stag = htonl(wqe->sqe.rkey);
476 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);
477 rreq->read_size = htonl(wqe->sqe.sge[0].length);
478 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);
480 cpu_to_be64(wqe->sqe.sge[0].laddr);
482 iov[1].iov_base = rreq;
483 iov[1].iov_len = sizeof(*rreq);
485 rx_hdr = (union iwarp_hdr *)rreq;
487 /* Take RDMAP/DDP information from
488 * current (failed) inbound frame.
490 iov[1].iov_base = rx_hdr;
492 if (__rdmap_get_opcode(&rx_hdr->ctrl) ==
495 sizeof(struct iwarp_rdma_rreq);
498 sizeof(struct iwarp_send);
501 /* Do not report DDP hdr information if packet
504 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
505 (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
508 iov[1].iov_base = rx_hdr;
510 /* Only DDP frame will get attached */
511 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
513 sizeof(struct iwarp_rdma_write);
515 iov[1].iov_len = sizeof(struct iwarp_send);
520 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len);
523 case TERM_ERROR_LAYER_DDP:
524 /* Report error encountered while DDP processing.
525 * This can only happen as a result of inbound
529 /* Do not report DDP hdr information if packet
532 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
533 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
534 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
535 (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
538 iov[1].iov_base = rx_hdr;
540 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
541 iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged);
543 iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged);
552 if (term->flag_m || term->flag_d || term->flag_r) {
553 iov[2].iov_base = &crc;
554 iov[2].iov_len = sizeof(crc);
555 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE;
558 iov[1].iov_base = &crc;
559 iov[1].iov_len = sizeof(crc);
560 len_terminate = sizeof(*term) + MPA_CRC_SIZE;
564 /* Adjust DDP Segment Length parameter, if valid */
566 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len);
567 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl);
569 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE;
570 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len);
574 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE));
575 if (qp->tx_ctx.mpa_crc_hd) {
576 crypto_shash_init(qp->tx_ctx.mpa_crc_hd);
577 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
578 (u8 *)iov[0].iov_base,
582 if (num_frags == 3) {
583 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
584 (u8 *)iov[1].iov_base,
588 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc);
591 rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate);
592 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
593 rv == len_terminate ? "success" : "failure",
594 __rdmap_term_layer(term), __rdmap_term_etype(term),
595 __rdmap_term_ecode(term), rv);
602 * Handle all attrs other than state
604 static void siw_qp_modify_nonstate(struct siw_qp *qp,
605 struct siw_qp_attrs *attrs,
606 enum siw_qp_attr_mask mask)
608 if (mask & SIW_QP_ATTR_ACCESS_FLAGS) {
609 if (attrs->flags & SIW_RDMA_BIND_ENABLED)
610 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
612 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
614 if (attrs->flags & SIW_RDMA_WRITE_ENABLED)
615 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
617 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
619 if (attrs->flags & SIW_RDMA_READ_ENABLED)
620 qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
622 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
626 static int siw_qp_nextstate_from_idle(struct siw_qp *qp,
627 struct siw_qp_attrs *attrs,
628 enum siw_qp_attr_mask mask)
632 switch (attrs->state) {
633 case SIW_QP_STATE_RTS:
634 if (attrs->flags & SIW_MPA_CRC) {
635 rv = siw_qp_enable_crc(qp);
639 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) {
640 siw_dbg_qp(qp, "no socket\n");
644 if (!(mask & SIW_QP_ATTR_MPA)) {
645 siw_dbg_qp(qp, "no MPA\n");
650 * Initialize iWARP TX state
652 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
653 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
654 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
657 * Initialize iWARP RX state
659 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
660 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
661 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
664 * init IRD free queue, caller has already checked
667 rv = siw_qp_readq_init(qp, attrs->irq_size,
672 qp->attrs.sk = attrs->sk;
673 qp->attrs.state = SIW_QP_STATE_RTS;
675 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",
676 attrs->flags & SIW_MPA_CRC ? "y" : "n",
677 qp->attrs.orq_size, qp->attrs.irq_size);
680 case SIW_QP_STATE_ERROR:
682 qp->attrs.state = SIW_QP_STATE_ERROR;
684 siw_cep_put(qp->cep);
695 static int siw_qp_nextstate_from_rts(struct siw_qp *qp,
696 struct siw_qp_attrs *attrs)
700 switch (attrs->state) {
701 case SIW_QP_STATE_CLOSING:
703 * Verbs: move to IDLE if SQ and ORQ are empty.
704 * Move to ERROR otherwise. But first of all we must
705 * close the connection. So we keep CLOSING or ERROR
706 * as a transient state, schedule connection drop work
707 * and wait for the socket state change upcall to
710 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
711 qp->attrs.state = SIW_QP_STATE_CLOSING;
713 qp->attrs.state = SIW_QP_STATE_ERROR;
721 case SIW_QP_STATE_TERMINATE:
722 qp->attrs.state = SIW_QP_STATE_TERMINATE;
724 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
725 RDMAP_ETYPE_CATASTROPHIC,
726 RDMAP_ECODE_UNSPECIFIED, 1);
730 case SIW_QP_STATE_ERROR:
732 * This is an emergency close.
734 * Any in progress transmit operation will get
736 * This will likely result in a protocol failure,
737 * if a TX operation is in transit. The caller
738 * could unconditional wait to give the current
739 * operation a chance to complete.
740 * Esp., how to handle the non-empty IRQ case?
741 * The peer was asking for data transfer at a valid
746 qp->attrs.state = SIW_QP_STATE_ERROR;
756 static void siw_qp_nextstate_from_term(struct siw_qp *qp,
757 struct siw_qp_attrs *attrs)
759 switch (attrs->state) {
760 case SIW_QP_STATE_ERROR:
762 qp->attrs.state = SIW_QP_STATE_ERROR;
764 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
773 static int siw_qp_nextstate_from_close(struct siw_qp *qp,
774 struct siw_qp_attrs *attrs)
778 switch (attrs->state) {
779 case SIW_QP_STATE_IDLE:
780 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
781 qp->attrs.state = SIW_QP_STATE_IDLE;
784 case SIW_QP_STATE_CLOSING:
786 * The LLP may already moved the QP to closing
787 * due to graceful peer close init
791 case SIW_QP_STATE_ERROR:
793 * QP was moved to CLOSING by LLP event
794 * not yet seen by user.
796 qp->attrs.state = SIW_QP_STATE_ERROR;
798 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
805 siw_dbg_qp(qp, "state transition undefined: %s => %s\n",
806 siw_qp_state_to_string[qp->attrs.state],
807 siw_qp_state_to_string[attrs->state]);
815 * Caller must hold qp->state_lock
817 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,
818 enum siw_qp_attr_mask mask)
820 int drop_conn = 0, rv = 0;
825 siw_dbg_qp(qp, "state: %s => %s\n",
826 siw_qp_state_to_string[qp->attrs.state],
827 siw_qp_state_to_string[attrs->state]);
829 if (mask != SIW_QP_ATTR_STATE)
830 siw_qp_modify_nonstate(qp, attrs, mask);
832 if (!(mask & SIW_QP_ATTR_STATE))
835 switch (qp->attrs.state) {
836 case SIW_QP_STATE_IDLE:
837 case SIW_QP_STATE_RTR:
838 rv = siw_qp_nextstate_from_idle(qp, attrs, mask);
841 case SIW_QP_STATE_RTS:
842 drop_conn = siw_qp_nextstate_from_rts(qp, attrs);
845 case SIW_QP_STATE_TERMINATE:
846 siw_qp_nextstate_from_term(qp, attrs);
849 case SIW_QP_STATE_CLOSING:
850 siw_qp_nextstate_from_close(qp, attrs);
856 siw_qp_cm_drop(qp, 0);
861 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)
864 rreq->opcode = sqe->opcode;
865 rreq->sge[0].laddr = sqe->sge[0].laddr;
866 rreq->sge[0].length = sqe->sge[0].length;
867 rreq->sge[0].lkey = sqe->sge[0].lkey;
868 rreq->sge[1].lkey = sqe->sge[1].lkey;
869 rreq->flags = sqe->flags | SIW_WQE_VALID;
874 * Must be called with SQ locked.
875 * To avoid complete SQ starvation by constant inbound READ requests,
876 * the active IRQ will not be served after qp->irq_burst, if the
877 * SQ has pending work.
879 int siw_activate_tx(struct siw_qp *qp)
881 struct siw_sqe *irqe, *sqe;
882 struct siw_wqe *wqe = tx_wqe(qp);
885 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
887 if (irqe->flags & SIW_WQE_VALID) {
888 sqe = sq_get_next(qp);
891 * Avoid local WQE processing starvation in case
892 * of constant inbound READ request stream
894 if (sqe && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
898 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
899 wqe->wr_status = SIW_WR_QUEUED;
901 /* start READ RESPONSE */
902 wqe->sqe.opcode = SIW_OP_READ_RESPONSE;
905 wqe->sqe.num_sge = 1;
906 wqe->sqe.sge[0].length = irqe->sge[0].length;
907 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;
908 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;
910 wqe->sqe.num_sge = 0;
913 /* Retain original RREQ's message sequence number for
914 * potential error reporting cases.
916 wqe->sqe.sge[1].length = irqe->sge[1].length;
918 wqe->sqe.rkey = irqe->rkey;
919 wqe->sqe.raddr = irqe->raddr;
924 /* mark current IRQ entry free */
925 smp_store_mb(irqe->flags, 0);
929 sqe = sq_get_next(qp);
932 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
933 wqe->wr_status = SIW_WR_QUEUED;
935 /* First copy SQE to kernel private memory */
936 memcpy(&wqe->sqe, sqe, sizeof(*sqe));
938 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {
942 if (wqe->sqe.flags & SIW_WQE_INLINE) {
943 if (wqe->sqe.opcode != SIW_OP_SEND &&
944 wqe->sqe.opcode != SIW_OP_WRITE) {
948 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {
952 wqe->sqe.sge[0].laddr = (u64)&wqe->sqe.sge[1];
953 wqe->sqe.sge[0].lkey = 0;
954 wqe->sqe.num_sge = 1;
956 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {
957 /* A READ cannot be fenced */
958 if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||
960 SIW_OP_READ_LOCAL_INV)) {
961 siw_dbg_qp(qp, "cannot fence read\n");
965 spin_lock(&qp->orq_lock);
967 if (!siw_orq_empty(qp)) {
968 qp->tx_ctx.orq_fence = 1;
971 spin_unlock(&qp->orq_lock);
973 } else if (wqe->sqe.opcode == SIW_OP_READ ||
974 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
975 struct siw_sqe *rreq;
977 wqe->sqe.num_sge = 1;
979 spin_lock(&qp->orq_lock);
981 rreq = orq_get_free(qp);
984 * Make an immediate copy in ORQ to be ready
985 * to process loopback READ reply
987 siw_read_to_orq(rreq, &wqe->sqe);
990 qp->tx_ctx.orq_fence = 1;
993 spin_unlock(&qp->orq_lock);
996 /* Clear SQE, can be re-used by application */
997 smp_store_mb(sqe->flags, 0);
1003 if (unlikely(rv < 0)) {
1004 siw_dbg_qp(qp, "error %d\n", rv);
1005 wqe->wr_status = SIW_WR_IDLE;
1011 * Check if current CQ state qualifies for calling CQ completion
1012 * handler. Must be called with CQ lock held.
1014 static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags)
1018 if (!cq->base_cq.comp_handler)
1021 /* Read application shared notification state */
1022 cq_notify = READ_ONCE(cq->notify->flags);
1024 if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) ||
1025 ((cq_notify & SIW_NOTIFY_SOLICITED) &&
1026 (flags & SIW_WQE_SOLICITED))) {
1028 * CQ notification is one-shot: Since the
1029 * current CQE causes user notification,
1030 * the CQ gets dis-aremd and must be re-aremd
1031 * by the user for a new notification.
1033 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT);
1040 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
1041 enum siw_wc_status status)
1043 struct siw_cq *cq = qp->scq;
1047 u32 sqe_flags = sqe->flags;
1048 struct siw_cqe *cqe;
1050 unsigned long flags;
1052 spin_lock_irqsave(&cq->lock, flags);
1054 idx = cq->cq_put % cq->num_cqe;
1055 cqe = &cq->queue[idx];
1057 if (!READ_ONCE(cqe->flags)) {
1061 cqe->opcode = sqe->opcode;
1062 cqe->status = status;
1066 if (cq->kernel_verbs)
1067 cqe->base_qp = qp->ib_qp;
1069 cqe->qp_id = qp_id(qp);
1071 /* mark CQE valid for application */
1072 WRITE_ONCE(cqe->flags, SIW_WQE_VALID);
1074 smp_store_mb(sqe->flags, 0);
1077 notify = siw_cq_notify_now(cq, sqe_flags);
1079 spin_unlock_irqrestore(&cq->lock, flags);
1082 siw_dbg_cq(cq, "Call completion handler\n");
1083 cq->base_cq.comp_handler(&cq->base_cq,
1084 cq->base_cq.cq_context);
1087 spin_unlock_irqrestore(&cq->lock, flags);
1089 siw_cq_event(cq, IB_EVENT_CQ_ERR);
1093 smp_store_mb(sqe->flags, 0);
1098 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
1099 u32 inval_stag, enum siw_wc_status status)
1101 struct siw_cq *cq = qp->rcq;
1105 struct siw_cqe *cqe;
1107 unsigned long flags;
1109 spin_lock_irqsave(&cq->lock, flags);
1111 idx = cq->cq_put % cq->num_cqe;
1112 cqe = &cq->queue[idx];
1114 if (!READ_ONCE(cqe->flags)) {
1116 u8 cqe_flags = SIW_WQE_VALID;
1119 cqe->opcode = SIW_OP_RECEIVE;
1120 cqe->status = status;
1124 if (cq->kernel_verbs) {
1125 cqe->base_qp = qp->ib_qp;
1127 cqe_flags |= SIW_WQE_REM_INVAL;
1128 cqe->inval_stag = inval_stag;
1131 cqe->qp_id = qp_id(qp);
1133 /* mark CQE valid for application */
1134 WRITE_ONCE(cqe->flags, cqe_flags);
1136 smp_store_mb(rqe->flags, 0);
1139 notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED);
1141 spin_unlock_irqrestore(&cq->lock, flags);
1144 siw_dbg_cq(cq, "Call completion handler\n");
1145 cq->base_cq.comp_handler(&cq->base_cq,
1146 cq->base_cq.cq_context);
1149 spin_unlock_irqrestore(&cq->lock, flags);
1151 siw_cq_event(cq, IB_EVENT_CQ_ERR);
1155 smp_store_mb(rqe->flags, 0);
1163 * Flush SQ and ORRQ entries to CQ.
1165 * Must be called with QP state write lock held.
1166 * Therefore, SQ and ORQ lock must not be taken.
1168 void siw_sq_flush(struct siw_qp *qp)
1170 struct siw_sqe *sqe;
1171 struct siw_wqe *wqe = tx_wqe(qp);
1172 int async_event = 0;
1175 * Start with completing any work currently on the ORQ
1177 while (qp->attrs.orq_size) {
1178 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1179 if (!READ_ONCE(sqe->flags))
1182 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1185 WRITE_ONCE(sqe->flags, 0);
1189 * Flush an in-progress WQE if present
1191 if (wqe->wr_status != SIW_WR_IDLE) {
1192 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",
1193 tx_type(wqe), wqe->wr_status);
1195 siw_wqe_put_mem(wqe, tx_type(wqe));
1197 if (tx_type(wqe) != SIW_OP_READ_RESPONSE &&
1198 ((tx_type(wqe) != SIW_OP_READ &&
1199 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) ||
1200 wqe->wr_status == SIW_WR_QUEUED))
1202 * An in-progress Read Request is already in
1205 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1206 SIW_WC_WR_FLUSH_ERR);
1208 wqe->wr_status = SIW_WR_IDLE;
1211 * Flush the Send Queue
1213 while (qp->attrs.sq_size) {
1214 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1215 if (!READ_ONCE(sqe->flags))
1219 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1221 * Shall IB_EVENT_SQ_DRAINED be supressed if work
1226 WRITE_ONCE(sqe->flags, 0);
1230 siw_qp_event(qp, IB_EVENT_SQ_DRAINED);
1236 * Flush recv queue entries to CQ. Also
1237 * takes care of pending active tagged and untagged
1238 * inbound transfers, which have target memory
1241 * Must be called with QP state write lock held.
1242 * Therefore, RQ lock must not be taken.
1244 void siw_rq_flush(struct siw_qp *qp)
1246 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1249 * Flush an in-progress untagged operation if present
1251 if (wqe->wr_status != SIW_WR_IDLE) {
1252 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",
1253 rx_type(wqe), wqe->wr_status);
1255 siw_wqe_put_mem(wqe, rx_type(wqe));
1257 if (rx_type(wqe) == SIW_OP_RECEIVE) {
1258 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1259 0, SIW_WC_WR_FLUSH_ERR);
1260 } else if (rx_type(wqe) != SIW_OP_READ &&
1261 rx_type(wqe) != SIW_OP_READ_RESPONSE &&
1262 rx_type(wqe) != SIW_OP_WRITE) {
1263 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1265 wqe->wr_status = SIW_WR_IDLE;
1267 wqe = &qp->rx_tagged.wqe_active;
1269 if (wqe->wr_status != SIW_WR_IDLE) {
1270 siw_wqe_put_mem(wqe, rx_type(wqe));
1271 wqe->wr_status = SIW_WR_IDLE;
1274 * Flush the Receive Queue
1276 while (qp->attrs.rq_size) {
1277 struct siw_rqe *rqe =
1278 &qp->recvq[qp->rq_get % qp->attrs.rq_size];
1280 if (!READ_ONCE(rqe->flags))
1283 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1286 WRITE_ONCE(rqe->flags, 0);
1291 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)
1293 int rv = xa_alloc(&sdev->qp_xa, &qp->ib_qp->qp_num, qp, xa_limit_32b,
1297 kref_init(&qp->ref);
1299 qp->qp_num = qp->ib_qp->qp_num;
1300 siw_dbg_qp(qp, "new QP\n");
1305 void siw_free_qp(struct kref *ref)
1307 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);
1308 struct siw_device *sdev = qp->sdev;
1309 unsigned long flags;
1312 siw_cep_put(qp->cep);
1314 found = xa_erase(&sdev->qp_xa, qp_id(qp));
1315 WARN_ON(found != qp);
1316 spin_lock_irqsave(&sdev->lock, flags);
1317 list_del(&qp->devq);
1318 spin_unlock_irqrestore(&sdev->lock, flags);
1325 siw_put_tx_cpu(qp->tx_cpu);
1327 atomic_dec(&sdev->num_qp);
1328 siw_dbg_qp(qp, "free QP\n");