IB/hfi1: Remove dependence on qp->s_cur_size

The qp->s_cur_size field assumes that the S_BUSY bit protects
the field from modification after the slock is dropped. Scaling the
send engine to multiple cores would break that assumption.

Correct the issue by carrying the payload size in the txreq structure.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Don Hiatt <don.hiatt@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index aaca8588e1996dbc9eef7d1ab42615b5ca9d094b..e69161e007dba3c6eb743f8a11183a24cb26ccbb 100644 (file)
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -351,7 +351,7 @@ normal:
        qp->s_rdma_ack_cnt++;
        qp->s_hdrwords = hwords;
        ps->s_txreq->sde = priv->s_sde;
-       qp->s_cur_size = len;
+       ps->s_txreq->s_cur_size = len;
        hfi1_make_ruc_header(qp, ohdr, bth0, bth2, middle, ps);
        /* pbc */
        ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
@@ -802,7 +802,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        qp->s_hdrwords = hwords;
        ps->s_txreq->sde = priv->s_sde;
        qp->s_cur_sge = ss;
-       qp->s_cur_size = len;
+       ps->s_txreq->s_cur_size = len;
        hfi1_make_ruc_header(
                qp,
                ohdr,

diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index 5a70e91b5191d00121348de577f1c4933d0465fb..2ec3e908691bf69c6b6248945b9c08ef90e0d73a 100644 (file)
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -767,8 +767,8 @@ void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
        u32 bth1;
 
        /* Construct the header. */
-       extra_bytes = -qp->s_cur_size & 3;
-       nwords = (qp->s_cur_size + extra_bytes) >> 2;
+       extra_bytes = -ps->s_txreq->s_cur_size & 3;
+       nwords = (ps->s_txreq->s_cur_size + extra_bytes) >> 2;
        lrh0 = HFI1_LRH_BTH;
        if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
                qp->s_hdrwords += hfi1_make_grh(ibp,

diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index 5e6d1bac4914a3ae1ad4d3111155141a28b8a59d..d062c3537f7cc2541fddedebaa28a8e2a52ae872 100644 (file)
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -259,7 +259,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        qp->s_hdrwords = hwords;
        ps->s_txreq->sde = priv->s_sde;
        qp->s_cur_sge = &qp->s_sge;
-       qp->s_cur_size = len;
+       ps->s_txreq->s_cur_size = len;
        hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24),
                             mask_psn(qp->s_psn++), middle, ps);
        /* pbc */

diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c
index 97ae24b6314cfdae3bd1492c921f27e993099948..d742ac1af9477f0fae1166f0563db039a6a473eb 100644 (file)
--- a/drivers/infiniband/hw/hfi1/ud.c
+++ b/drivers/infiniband/hw/hfi1/ud.c
@@ -354,7 +354,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 
        /* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
        qp->s_hdrwords = 7;
-       qp->s_cur_size = wqe->length;
+       ps->s_txreq->s_cur_size = wqe->length;
        qp->s_cur_sge = &qp->s_sge;
        qp->s_srate = ah_attr->static_rate;
        qp->srate_mbps = ib_rate_to_mbps(qp->s_srate);

diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 09132582e09cef3121d9c4482783a7a8eed2085f..65f2f1d76fc8b27d172310ab50a46dd94f93caf0 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -895,7 +895,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
        struct hfi1_ahg_info *ahg_info = priv->s_ahg;
        u32 hdrwords = qp->s_hdrwords;
        struct rvt_sge_state *ss = qp->s_cur_sge;
-       u32 len = qp->s_cur_size;
+       u32 len = ps->s_txreq->s_cur_size;
        u32 plen = hdrwords + ((len + 3) >> 2) + 2; /* includes pbc */
        struct hfi1_ibdev *dev = ps->dev;
        struct hfi1_pportdata *ppd = ps->ppd;
@@ -1012,7 +1012,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
        struct hfi1_qp_priv *priv = qp->priv;
        u32 hdrwords = qp->s_hdrwords;
        struct rvt_sge_state *ss = qp->s_cur_sge;
-       u32 len = qp->s_cur_size;
+       u32 len = ps->s_txreq->s_cur_size;
        u32 dwords = (len + 3) >> 2;
        u32 plen = hdrwords + dwords + 2; /* includes pbc */
        struct hfi1_pportdata *ppd = ps->ppd;
@@ -1240,7 +1240,7 @@ static inline send_routine get_send_routine(struct rvt_qp *qp,
                u8 op = get_opcode(h);
 
                if (piothreshold &&
-                   qp->s_cur_size <= min(piothreshold, qp->pmtu) &&
+                   tx->s_cur_size <= min(piothreshold, qp->pmtu) &&
                    (BIT(op & OPMASK) & pio_opmask[op >> 5]) &&
                    iowait_sdma_pending(&priv->s_iowait) == 0 &&
                    !sdma_txreq_built(&tx->txreq))

diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h
index 5660897593ba4444f43661cba45d1ee06102ad20..76216f2ef35a5159bbd6a47c8497a206ee7afc74 100644 (file)
--- a/drivers/infiniband/hw/hfi1/verbs_txreq.h
+++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h
@@ -65,6 +65,7 @@ struct verbs_txreq {
        struct sdma_engine     *sde;
        struct send_context     *psc;
        u16                     hdr_dwords;
+       u16                     s_cur_size;
 };
 
 struct hfi1_ibdev;