drivers/infiniband/hw/mlx5/wr.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
   4  */
   5
   6 #include <linux/gfp.h>
   7 #include <linux/mlx5/qp.h>
   8 #include <linux/mlx5/driver.h>
   9 #include "wr.h"
  10 #include "umr.h"
  11
  12 static const u32 mlx5_ib_opcode[] = {
  13         [IB_WR_SEND]                            = MLX5_OPCODE_SEND,
  14         [IB_WR_LSO]                             = MLX5_OPCODE_LSO,
  15         [IB_WR_SEND_WITH_IMM]                   = MLX5_OPCODE_SEND_IMM,
  16         [IB_WR_RDMA_WRITE]                      = MLX5_OPCODE_RDMA_WRITE,
  17         [IB_WR_RDMA_WRITE_WITH_IMM]             = MLX5_OPCODE_RDMA_WRITE_IMM,
  18         [IB_WR_RDMA_READ]                       = MLX5_OPCODE_RDMA_READ,
  19         [IB_WR_ATOMIC_CMP_AND_SWP]              = MLX5_OPCODE_ATOMIC_CS,
  20         [IB_WR_ATOMIC_FETCH_AND_ADD]            = MLX5_OPCODE_ATOMIC_FA,
  21         [IB_WR_SEND_WITH_INV]                   = MLX5_OPCODE_SEND_INVAL,
  22         [IB_WR_LOCAL_INV]                       = MLX5_OPCODE_UMR,
  23         [IB_WR_REG_MR]                          = MLX5_OPCODE_UMR,
  24         [IB_WR_MASKED_ATOMIC_CMP_AND_SWP]       = MLX5_OPCODE_ATOMIC_MASKED_CS,
  25         [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]     = MLX5_OPCODE_ATOMIC_MASKED_FA,
  26         [MLX5_IB_WR_UMR]                        = MLX5_OPCODE_UMR,
  27 };
  28
  29 int mlx5r_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
  30 {
  31         struct mlx5_ib_cq *cq;
  32         unsigned int cur;
  33
  34         cur = wq->head - wq->tail;
  35         if (likely(cur + nreq < wq->max_post))
  36                 return 0;
  37
  38         cq = to_mcq(ib_cq);
  39         spin_lock(&cq->lock);
  40         cur = wq->head - wq->tail;
  41         spin_unlock(&cq->lock);
  42
  43         return cur + nreq >= wq->max_post;
  44 }
  45
  46 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
  47                                           u64 remote_addr, u32 rkey)
  48 {
  49         rseg->raddr    = cpu_to_be64(remote_addr);
  50         rseg->rkey     = cpu_to_be32(rkey);
  51         rseg->reserved = 0;
  52 }
  53
  54 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
  55                         void **seg, int *size, void **cur_edge)
  56 {
  57         struct mlx5_wqe_eth_seg *eseg = *seg;
  58
  59         memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
  60
  61         if (wr->send_flags & IB_SEND_IP_CSUM)
  62                 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
  63                                  MLX5_ETH_WQE_L4_CSUM;
  64
  65         if (wr->opcode == IB_WR_LSO) {
  66                 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
  67                 size_t left, copysz;
  68                 void *pdata = ud_wr->header;
  69                 size_t stride;
  70
  71                 left = ud_wr->hlen;
  72                 eseg->mss = cpu_to_be16(ud_wr->mss);
  73                 eseg->inline_hdr.sz = cpu_to_be16(left);
  74
  75                 /* mlx5r_memcpy_send_wqe should get a 16B align address. Hence,
  76                  * we first copy up to the current edge and then, if needed,
  77                  * continue to mlx5r_memcpy_send_wqe.
  78                  */
  79                 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
  80                                left);
  81                 memcpy(eseg->inline_hdr.data, pdata, copysz);
  82                 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
  83                                sizeof(eseg->inline_hdr.start) + copysz, 16);
  84                 *size += stride / 16;
  85                 *seg += stride;
  86
  87                 if (copysz < left) {
  88                         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
  89                         left -= copysz;
  90                         pdata += copysz;
  91                         mlx5r_memcpy_send_wqe(&qp->sq, cur_edge, seg, size,
  92                                               pdata, left);
  93                 }
  94
  95                 return;
  96         }
  97
  98         *seg += sizeof(struct mlx5_wqe_eth_seg);
  99         *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
 100 }
 101
 102 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
 103                              const struct ib_send_wr *wr)
 104 {
 105         memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
 106         dseg->av.dqp_dct =
 107                 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
 108         dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
 109 }
 110
 111 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
 112 {
 113         dseg->byte_count = cpu_to_be32(sg->length);
 114         dseg->lkey       = cpu_to_be32(sg->lkey);
 115         dseg->addr       = cpu_to_be64(sg->addr);
 116 }
 117
 118 static __be64 frwr_mkey_mask(bool atomic)
 119 {
 120         u64 result;
 121
 122         result = MLX5_MKEY_MASK_LEN             |
 123                 MLX5_MKEY_MASK_PAGE_SIZE        |
 124                 MLX5_MKEY_MASK_START_ADDR       |
 125                 MLX5_MKEY_MASK_EN_RINVAL        |
 126                 MLX5_MKEY_MASK_KEY              |
 127                 MLX5_MKEY_MASK_LR               |
 128                 MLX5_MKEY_MASK_LW               |
 129                 MLX5_MKEY_MASK_RR               |
 130                 MLX5_MKEY_MASK_RW               |
 131                 MLX5_MKEY_MASK_SMALL_FENCE      |
 132                 MLX5_MKEY_MASK_FREE;
 133
 134         if (atomic)
 135                 result |= MLX5_MKEY_MASK_A;
 136
 137         return cpu_to_be64(result);
 138 }
 139
 140 static __be64 sig_mkey_mask(void)
 141 {
 142         u64 result;
 143
 144         result = MLX5_MKEY_MASK_LEN             |
 145                 MLX5_MKEY_MASK_PAGE_SIZE        |
 146                 MLX5_MKEY_MASK_START_ADDR       |
 147                 MLX5_MKEY_MASK_EN_SIGERR        |
 148                 MLX5_MKEY_MASK_EN_RINVAL        |
 149                 MLX5_MKEY_MASK_KEY              |
 150                 MLX5_MKEY_MASK_LR               |
 151                 MLX5_MKEY_MASK_LW               |
 152                 MLX5_MKEY_MASK_RR               |
 153                 MLX5_MKEY_MASK_RW               |
 154                 MLX5_MKEY_MASK_SMALL_FENCE      |
 155                 MLX5_MKEY_MASK_FREE             |
 156                 MLX5_MKEY_MASK_BSF_EN;
 157
 158         return cpu_to_be64(result);
 159 }
 160
 161 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
 162                             struct mlx5_ib_mr *mr, u8 flags, bool atomic)
 163 {
 164         int size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
 165
 166         memset(umr, 0, sizeof(*umr));
 167
 168         umr->flags = flags;
 169         umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));
 170         umr->mkey_mask = frwr_mkey_mask(atomic);
 171 }
 172
 173 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
 174 {
 175         memset(umr, 0, sizeof(*umr));
 176         umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 177         umr->flags = MLX5_UMR_INLINE;
 178 }
 179
 180 static u8 get_umr_flags(int acc)
 181 {
 182         return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
 183                (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
 184                (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
 185                (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
 186                 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
 187 }
 188
 189 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
 190                              struct mlx5_ib_mr *mr,
 191                              u32 key, int access)
 192 {
 193         int ndescs = ALIGN(mr->mmkey.ndescs + mr->meta_ndescs, 8) >> 1;
 194
 195         memset(seg, 0, sizeof(*seg));
 196
 197         if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
 198                 seg->log2_page_size = ilog2(mr->ibmr.page_size);
 199         else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
 200                 /* KLMs take twice the size of MTTs */
 201                 ndescs *= 2;
 202
 203         seg->flags = get_umr_flags(access) | mr->access_mode;
 204         seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
 205         seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
 206         seg->start_addr = cpu_to_be64(mr->ibmr.iova);
 207         seg->len = cpu_to_be64(mr->ibmr.length);
 208         seg->xlt_oct_size = cpu_to_be32(ndescs);
 209 }
 210
 211 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
 212 {
 213         memset(seg, 0, sizeof(*seg));
 214         seg->status = MLX5_MKEY_STATUS_FREE;
 215 }
 216
 217 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
 218                              struct mlx5_ib_mr *mr,
 219                              struct mlx5_ib_pd *pd)
 220 {
 221         int bcount = mr->desc_size * (mr->mmkey.ndescs + mr->meta_ndescs);
 222
 223         dseg->addr = cpu_to_be64(mr->desc_map);
 224         dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
 225         dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
 226 }
 227
 228 static __be32 send_ieth(const struct ib_send_wr *wr)
 229 {
 230         switch (wr->opcode) {
 231         case IB_WR_SEND_WITH_IMM:
 232         case IB_WR_RDMA_WRITE_WITH_IMM:
 233                 return wr->ex.imm_data;
 234
 235         case IB_WR_SEND_WITH_INV:
 236                 return cpu_to_be32(wr->ex.invalidate_rkey);
 237
 238         default:
 239                 return 0;
 240         }
 241 }
 242
 243 static u8 calc_sig(void *wqe, int size)
 244 {
 245         u8 *p = wqe;
 246         u8 res = 0;
 247         int i;
 248
 249         for (i = 0; i < size; i++)
 250                 res ^= p[i];
 251
 252         return ~res;
 253 }
 254
 255 static u8 wq_sig(void *wqe)
 256 {
 257         return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
 258 }
 259
 260 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
 261                             void **wqe, int *wqe_sz, void **cur_edge)
 262 {
 263         struct mlx5_wqe_inline_seg *seg;
 264         size_t offset;
 265         int inl = 0;
 266         int i;
 267
 268         seg = *wqe;
 269         *wqe += sizeof(*seg);
 270         offset = sizeof(*seg);
 271
 272         for (i = 0; i < wr->num_sge; i++) {
 273                 size_t len  = wr->sg_list[i].length;
 274                 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
 275
 276                 inl += len;
 277
 278                 if (unlikely(inl > qp->max_inline_data))
 279                         return -ENOMEM;
 280
 281                 while (likely(len)) {
 282                         size_t leftlen;
 283                         size_t copysz;
 284
 285                         handle_post_send_edge(&qp->sq, wqe,
 286                                               *wqe_sz + (offset >> 4),
 287                                               cur_edge);
 288
 289                         leftlen = *cur_edge - *wqe;
 290                         copysz = min_t(size_t, leftlen, len);
 291
 292                         memcpy(*wqe, addr, copysz);
 293                         len -= copysz;
 294                         addr += copysz;
 295                         *wqe += copysz;
 296                         offset += copysz;
 297                 }
 298         }
 299
 300         seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
 301
 302         *wqe_sz +=  ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
 303
 304         return 0;
 305 }
 306
 307 static u16 prot_field_size(enum ib_signature_type type)
 308 {
 309         switch (type) {
 310         case IB_SIG_TYPE_T10_DIF:
 311                 return MLX5_DIF_SIZE;
 312         default:
 313                 return 0;
 314         }
 315 }
 316
 317 static u8 bs_selector(int block_size)
 318 {
 319         switch (block_size) {
 320         case 512:           return 0x1;
 321         case 520:           return 0x2;
 322         case 4096:          return 0x3;
 323         case 4160:          return 0x4;
 324         case 1073741824:    return 0x5;
 325         default:            return 0;
 326         }
 327 }
 328
 329 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
 330                               struct mlx5_bsf_inl *inl)
 331 {
 332         /* Valid inline section and allow BSF refresh */
 333         inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
 334                                        MLX5_BSF_REFRESH_DIF);
 335         inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
 336         inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
 337         /* repeating block */
 338         inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
 339         inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
 340                         MLX5_DIF_CRC : MLX5_DIF_IPCS;
 341
 342         if (domain->sig.dif.ref_remap)
 343                 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
 344
 345         if (domain->sig.dif.app_escape) {
 346                 if (domain->sig.dif.ref_escape)
 347                         inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
 348                 else
 349                         inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
 350         }
 351
 352         inl->dif_app_bitmask_check =
 353                 cpu_to_be16(domain->sig.dif.apptag_check_mask);
 354 }
 355
 356 static int mlx5_set_bsf(struct ib_mr *sig_mr,
 357                         struct ib_sig_attrs *sig_attrs,
 358                         struct mlx5_bsf *bsf, u32 data_size)
 359 {
 360         struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
 361         struct mlx5_bsf_basic *basic = &bsf->basic;
 362         struct ib_sig_domain *mem = &sig_attrs->mem;
 363         struct ib_sig_domain *wire = &sig_attrs->wire;
 364
 365         memset(bsf, 0, sizeof(*bsf));
 366
 367         /* Basic + Extended + Inline */
 368         basic->bsf_size_sbs = 1 << 7;
 369         /* Input domain check byte mask */
 370         basic->check_byte_mask = sig_attrs->check_mask;
 371         basic->raw_data_size = cpu_to_be32(data_size);
 372
 373         /* Memory domain */
 374         switch (sig_attrs->mem.sig_type) {
 375         case IB_SIG_TYPE_NONE:
 376                 break;
 377         case IB_SIG_TYPE_T10_DIF:
 378                 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
 379                 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
 380                 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
 381                 break;
 382         default:
 383                 return -EINVAL;
 384         }
 385
 386         /* Wire domain */
 387         switch (sig_attrs->wire.sig_type) {
 388         case IB_SIG_TYPE_NONE:
 389                 break;
 390         case IB_SIG_TYPE_T10_DIF:
 391                 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
 392                     mem->sig_type == wire->sig_type) {
 393                         /* Same block structure */
 394                         basic->bsf_size_sbs |= 1 << 4;
 395                         if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
 396                                 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
 397                         if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
 398                                 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
 399                         if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
 400                                 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
 401                 } else
 402                         basic->wire.bs_selector =
 403                                 bs_selector(wire->sig.dif.pi_interval);
 404
 405                 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
 406                 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
 407                 break;
 408         default:
 409                 return -EINVAL;
 410         }
 411
 412         return 0;
 413 }
 414
 415
 416 static int set_sig_data_segment(const struct ib_send_wr *send_wr,
 417                                 struct ib_mr *sig_mr,
 418                                 struct ib_sig_attrs *sig_attrs,
 419                                 struct mlx5_ib_qp *qp, void **seg, int *size,
 420                                 void **cur_edge)
 421 {
 422         struct mlx5_bsf *bsf;
 423         u32 data_len;
 424         u32 data_key;
 425         u64 data_va;
 426         u32 prot_len = 0;
 427         u32 prot_key = 0;
 428         u64 prot_va = 0;
 429         bool prot = false;
 430         int ret;
 431         int wqe_size;
 432         struct mlx5_ib_mr *mr = to_mmr(sig_mr);
 433         struct mlx5_ib_mr *pi_mr = mr->pi_mr;
 434
 435         data_len = pi_mr->data_length;
 436         data_key = pi_mr->ibmr.lkey;
 437         data_va = pi_mr->data_iova;
 438         if (pi_mr->meta_ndescs) {
 439                 prot_len = pi_mr->meta_length;
 440                 prot_key = pi_mr->ibmr.lkey;
 441                 prot_va = pi_mr->pi_iova;
 442                 prot = true;
 443         }
 444
 445         if (!prot || (data_key == prot_key && data_va == prot_va &&
 446                       data_len == prot_len)) {
 447                 /**
 448                  * Source domain doesn't contain signature information
 449                  * or data and protection are interleaved in memory.
 450                  * So need construct:
 451                  *                  ------------------
 452                  *                 |     data_klm     |
 453                  *                  ------------------
 454                  *                 |       BSF        |
 455                  *                  ------------------
 456                  **/
 457                 struct mlx5_klm *data_klm = *seg;
 458
 459                 data_klm->bcount = cpu_to_be32(data_len);
 460                 data_klm->key = cpu_to_be32(data_key);
 461                 data_klm->va = cpu_to_be64(data_va);
 462                 wqe_size = ALIGN(sizeof(*data_klm), 64);
 463         } else {
 464                 /**
 465                  * Source domain contains signature information
 466                  * So need construct a strided block format:
 467                  *               ---------------------------
 468                  *              |     stride_block_ctrl     |
 469                  *               ---------------------------
 470                  *              |          data_klm         |
 471                  *               ---------------------------
 472                  *              |          prot_klm         |
 473                  *               ---------------------------
 474                  *              |             BSF           |
 475                  *               ---------------------------
 476                  **/
 477                 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
 478                 struct mlx5_stride_block_entry *data_sentry;
 479                 struct mlx5_stride_block_entry *prot_sentry;
 480                 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
 481                 int prot_size;
 482
 483                 sblock_ctrl = *seg;
 484                 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
 485                 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
 486
 487                 prot_size = prot_field_size(sig_attrs->mem.sig_type);
 488                 if (!prot_size) {
 489                         pr_err("Bad block size given: %u\n", block_size);
 490                         return -EINVAL;
 491                 }
 492                 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
 493                                                             prot_size);
 494                 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
 495                 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
 496                 sblock_ctrl->num_entries = cpu_to_be16(2);
 497
 498                 data_sentry->bcount = cpu_to_be16(block_size);
 499                 data_sentry->key = cpu_to_be32(data_key);
 500                 data_sentry->va = cpu_to_be64(data_va);
 501                 data_sentry->stride = cpu_to_be16(block_size);
 502
 503                 prot_sentry->bcount = cpu_to_be16(prot_size);
 504                 prot_sentry->key = cpu_to_be32(prot_key);
 505                 prot_sentry->va = cpu_to_be64(prot_va);
 506                 prot_sentry->stride = cpu_to_be16(prot_size);
 507
 508                 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
 509                                  sizeof(*prot_sentry), 64);
 510         }
 511
 512         *seg += wqe_size;
 513         *size += wqe_size / 16;
 514         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 515
 516         bsf = *seg;
 517         ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
 518         if (ret)
 519                 return -EINVAL;
 520
 521         *seg += sizeof(*bsf);
 522         *size += sizeof(*bsf) / 16;
 523         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 524
 525         return 0;
 526 }
 527
 528 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
 529                                  struct ib_mr *sig_mr, int access_flags,
 530                                  u32 size, u32 length, u32 pdn)
 531 {
 532         u32 sig_key = sig_mr->rkey;
 533         u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
 534
 535         memset(seg, 0, sizeof(*seg));
 536
 537         seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
 538         seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
 539         seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
 540                                     MLX5_MKEY_BSF_EN | pdn);
 541         seg->len = cpu_to_be64(length);
 542         seg->xlt_oct_size = cpu_to_be32(mlx5r_umr_get_xlt_octo(size));
 543         seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
 544 }
 545
 546 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
 547                                 u32 size)
 548 {
 549         memset(umr, 0, sizeof(*umr));
 550
 551         umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
 552         umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));
 553         umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
 554         umr->mkey_mask = sig_mkey_mask();
 555 }
 556
 557 static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
 558                          struct mlx5_ib_qp *qp, void **seg, int *size,
 559                          void **cur_edge)
 560 {
 561         const struct ib_reg_wr *wr = reg_wr(send_wr);
 562         struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
 563         struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
 564         struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
 565         u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
 566         u32 xlt_size;
 567         int region_len, ret;
 568
 569         if (unlikely(send_wr->num_sge != 0) ||
 570             unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
 571             unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
 572             unlikely(!sig_mr->sig->sig_status_checked))
 573                 return -EINVAL;
 574
 575         /* length of the protected region, data + protection */
 576         region_len = pi_mr->ibmr.length;
 577
 578         /**
 579          * KLM octoword size - if protection was provided
 580          * then we use strided block format (3 octowords),
 581          * else we use single KLM (1 octoword)
 582          **/
 583         if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
 584                 xlt_size = 0x30;
 585         else
 586                 xlt_size = sizeof(struct mlx5_klm);
 587
 588         set_sig_umr_segment(*seg, xlt_size);
 589         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 590         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 591         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 592
 593         set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
 594                              pdn);
 595         *seg += sizeof(struct mlx5_mkey_seg);
 596         *size += sizeof(struct mlx5_mkey_seg) / 16;
 597         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 598
 599         ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
 600                                    cur_edge);
 601         if (ret)
 602                 return ret;
 603
 604         sig_mr->sig->sig_status_checked = false;
 605         return 0;
 606 }
 607
 608 static int set_psv_wr(struct ib_sig_domain *domain,
 609                       u32 psv_idx, void **seg, int *size)
 610 {
 611         struct mlx5_seg_set_psv *psv_seg = *seg;
 612
 613         memset(psv_seg, 0, sizeof(*psv_seg));
 614         psv_seg->psv_num = cpu_to_be32(psv_idx);
 615         switch (domain->sig_type) {
 616         case IB_SIG_TYPE_NONE:
 617                 break;
 618         case IB_SIG_TYPE_T10_DIF:
 619                 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
 620                                                      domain->sig.dif.app_tag);
 621                 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
 622                 break;
 623         default:
 624                 pr_err("Bad signature type (%d) is given.\n",
 625                        domain->sig_type);
 626                 return -EINVAL;
 627         }
 628
 629         *seg += sizeof(*psv_seg);
 630         *size += sizeof(*psv_seg) / 16;
 631
 632         return 0;
 633 }
 634
 635 static int set_reg_wr(struct mlx5_ib_qp *qp,
 636                       const struct ib_reg_wr *wr,
 637                       void **seg, int *size, void **cur_edge,
 638                       bool check_not_free)
 639 {
 640         struct mlx5_ib_mr *mr = to_mmr(wr->mr);
 641         struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
 642         struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
 643         int mr_list_size = (mr->mmkey.ndescs + mr->meta_ndescs) * mr->desc_size;
 644         bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
 645         bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
 646         u8 flags = 0;
 647
 648         /* Matches access in mlx5_set_umr_free_mkey().
 649          * Relaxed Ordering is set implicitly in mlx5_set_umr_free_mkey() and
 650          * kernel ULPs are not aware of it, so we don't set it here.
 651          */
 652         if (!mlx5r_umr_can_reconfig(dev, 0, wr->access)) {
 653                 mlx5_ib_warn(
 654                         to_mdev(qp->ibqp.device),
 655                         "Fast update for MR access flags is not possible\n");
 656                 return -EINVAL;
 657         }
 658
 659         if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
 660                 mlx5_ib_warn(to_mdev(qp->ibqp.device),
 661                              "Invalid IB_SEND_INLINE send flag\n");
 662                 return -EINVAL;
 663         }
 664
 665         if (check_not_free)
 666                 flags |= MLX5_UMR_CHECK_NOT_FREE;
 667         if (umr_inline)
 668                 flags |= MLX5_UMR_INLINE;
 669
 670         set_reg_umr_seg(*seg, mr, flags, atomic);
 671         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 672         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 673         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 674
 675         set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
 676         *seg += sizeof(struct mlx5_mkey_seg);
 677         *size += sizeof(struct mlx5_mkey_seg) / 16;
 678         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 679
 680         if (umr_inline) {
 681                 mlx5r_memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
 682                                       mr_list_size);
 683                 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
 684         } else {
 685                 set_reg_data_seg(*seg, mr, pd);
 686                 *seg += sizeof(struct mlx5_wqe_data_seg);
 687                 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
 688         }
 689         return 0;
 690 }
 691
 692 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
 693                         void **cur_edge)
 694 {
 695         set_linv_umr_seg(*seg);
 696         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 697         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 698         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 699         set_linv_mkey_seg(*seg);
 700         *seg += sizeof(struct mlx5_mkey_seg);
 701         *size += sizeof(struct mlx5_mkey_seg) / 16;
 702         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 703 }
 704
 705 static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
 706 {
 707         __be32 *p = NULL;
 708         int i, j;
 709
 710         pr_debug("dump WQE index %u:\n", idx);
 711         for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
 712                 if ((i & 0xf) == 0) {
 713                         p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
 714                         pr_debug("WQBB at %p:\n", (void *)p);
 715                         j = 0;
 716                         idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
 717                 }
 718                 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
 719                          be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
 720                          be32_to_cpu(p[j + 3]));
 721         }
 722 }
 723
 724 int mlx5r_begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 725                     struct mlx5_wqe_ctrl_seg **ctrl, unsigned int *idx,
 726                     int *size, void **cur_edge, int nreq, __be32 general_id,
 727                     bool send_signaled, bool solicited)
 728 {
 729         if (unlikely(mlx5r_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
 730                 return -ENOMEM;
 731
 732         *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
 733         *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
 734         *ctrl = *seg;
 735         *(uint32_t *)(*seg + 8) = 0;
 736         (*ctrl)->general_id = general_id;
 737         (*ctrl)->fm_ce_se = qp->sq_signal_bits |
 738                             (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
 739                             (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
 740
 741         *seg += sizeof(**ctrl);
 742         *size = sizeof(**ctrl) / 16;
 743         *cur_edge = qp->sq.cur_edge;
 744
 745         return 0;
 746 }
 747
 748 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 749                      struct mlx5_wqe_ctrl_seg **ctrl,
 750                      const struct ib_send_wr *wr, unsigned int *idx, int *size,
 751                      void **cur_edge, int nreq)
 752 {
 753         return mlx5r_begin_wqe(qp, seg, ctrl, idx, size, cur_edge, nreq,
 754                                send_ieth(wr), wr->send_flags & IB_SEND_SIGNALED,
 755                                wr->send_flags & IB_SEND_SOLICITED);
 756 }
 757
 758 void mlx5r_finish_wqe(struct mlx5_ib_qp *qp, struct mlx5_wqe_ctrl_seg *ctrl,
 759                       void *seg, u8 size, void *cur_edge, unsigned int idx,
 760                       u64 wr_id, int nreq, u8 fence, u32 mlx5_opcode)
 761 {
 762         u8 opmod = 0;
 763
 764         ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
 765                                              mlx5_opcode | ((u32)opmod << 24));
 766         ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
 767         ctrl->fm_ce_se |= fence;
 768         if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
 769                 ctrl->signature = wq_sig(ctrl);
 770
 771         qp->sq.wrid[idx] = wr_id;
 772         qp->sq.w_list[idx].opcode = mlx5_opcode;
 773         qp->sq.wqe_head[idx] = qp->sq.head + nreq;
 774         qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
 775         qp->sq.w_list[idx].next = qp->sq.cur_post;
 776
 777         /* We save the edge which was possibly updated during the WQE
 778          * construction, into SQ's cache.
 779          */
 780         seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
 781         qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
 782                           get_sq_edge(&qp->sq, qp->sq.cur_post &
 783                                       (qp->sq.wqe_cnt - 1)) :
 784                           cur_edge;
 785 }
 786
 787 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
 788 {
 789         set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
 790         *seg += sizeof(struct mlx5_wqe_raddr_seg);
 791         *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
 792 }
 793
 794 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
 795                              struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
 796                              int *size, void **cur_edge, unsigned int idx)
 797 {
 798         qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
 799         (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
 800         set_linv_wr(qp, seg, size, cur_edge);
 801 }
 802
 803 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
 804                          struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
 805                          void **cur_edge, unsigned int idx)
 806 {
 807         qp->sq.wr_data[idx] = IB_WR_REG_MR;
 808         (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
 809         return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
 810 }
 811
 812 static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 813                       const struct ib_send_wr *wr,
 814                       struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
 815                       void **cur_edge, unsigned int *idx, int nreq,
 816                       struct ib_sig_domain *domain, u32 psv_index,
 817                       u8 next_fence)
 818 {
 819         int err;
 820
 821         /*
 822          * SET_PSV WQEs are not signaled and solicited on error.
 823          */
 824         err = mlx5r_begin_wqe(qp, seg, ctrl, idx, size, cur_edge, nreq,
 825                               send_ieth(wr), false, true);
 826         if (unlikely(err)) {
 827                 mlx5_ib_warn(dev, "\n");
 828                 err = -ENOMEM;
 829                 goto out;
 830         }
 831         err = set_psv_wr(domain, psv_index, seg, size);
 832         if (unlikely(err)) {
 833                 mlx5_ib_warn(dev, "\n");
 834                 goto out;
 835         }
 836         mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
 837                          nreq, next_fence, MLX5_OPCODE_SET_PSV);
 838
 839 out:
 840         return err;
 841 }
 842
 843 static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
 844                                    struct mlx5_ib_qp *qp,
 845                                    const struct ib_send_wr *wr,
 846                                    struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
 847                                    int *size, void **cur_edge,
 848                                    unsigned int *idx, int nreq, u8 fence,
 849                                    u8 next_fence)
 850 {
 851         struct mlx5_ib_mr *mr;
 852         struct mlx5_ib_mr *pi_mr;
 853         struct mlx5_ib_mr pa_pi_mr;
 854         struct ib_sig_attrs *sig_attrs;
 855         struct ib_reg_wr reg_pi_wr;
 856         int err;
 857
 858         qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
 859
 860         mr = to_mmr(reg_wr(wr)->mr);
 861         pi_mr = mr->pi_mr;
 862
 863         if (pi_mr) {
 864                 memset(&reg_pi_wr, 0,
 865                        sizeof(struct ib_reg_wr));
 866
 867                 reg_pi_wr.mr = &pi_mr->ibmr;
 868                 reg_pi_wr.access = reg_wr(wr)->access;
 869                 reg_pi_wr.key = pi_mr->ibmr.rkey;
 870
 871                 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
 872                 /* UMR for data + prot registration */
 873                 err = set_reg_wr(qp, &reg_pi_wr, seg, size, cur_edge, false);
 874                 if (unlikely(err))
 875                         goto out;
 876
 877                 mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx,
 878                                  wr->wr_id, nreq, fence, MLX5_OPCODE_UMR);
 879
 880                 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
 881                 if (unlikely(err)) {
 882                         mlx5_ib_warn(dev, "\n");
 883                         err = -ENOMEM;
 884                         goto out;
 885                 }
 886         } else {
 887                 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
 888                 /* No UMR, use local_dma_lkey */
 889                 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
 890                 pa_pi_mr.mmkey.ndescs = mr->mmkey.ndescs;
 891                 pa_pi_mr.data_length = mr->data_length;
 892                 pa_pi_mr.data_iova = mr->data_iova;
 893                 if (mr->meta_ndescs) {
 894                         pa_pi_mr.meta_ndescs = mr->meta_ndescs;
 895                         pa_pi_mr.meta_length = mr->meta_length;
 896                         pa_pi_mr.pi_iova = mr->pi_iova;
 897                 }
 898
 899                 pa_pi_mr.ibmr.length = mr->ibmr.length;
 900                 mr->pi_mr = &pa_pi_mr;
 901         }
 902         (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
 903         /* UMR for sig MR */
 904         err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
 905         if (unlikely(err)) {
 906                 mlx5_ib_warn(dev, "\n");
 907                 goto out;
 908         }
 909         mlx5r_finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
 910                          nreq, fence, MLX5_OPCODE_UMR);
 911
 912         sig_attrs = mr->ibmr.sig_attrs;
 913         err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
 914                          &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
 915                          next_fence);
 916         if (unlikely(err))
 917                 goto out;
 918
 919         err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
 920                          &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
 921                          next_fence);
 922         if (unlikely(err))
 923                 goto out;
 924
 925         qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
 926
 927 out:
 928         return err;
 929 }
 930
 931 static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 932                          const struct ib_send_wr *wr,
 933                          struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
 934                          void **cur_edge, unsigned int *idx, int nreq, u8 fence,
 935                          u8 next_fence, int *num_sge)
 936 {
 937         int err = 0;
 938
 939         switch (wr->opcode) {
 940         case IB_WR_RDMA_READ:
 941         case IB_WR_RDMA_WRITE:
 942         case IB_WR_RDMA_WRITE_WITH_IMM:
 943                 handle_rdma_op(wr, seg, size);
 944                 break;
 945
 946         case IB_WR_ATOMIC_CMP_AND_SWP:
 947         case IB_WR_ATOMIC_FETCH_AND_ADD:
 948         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
 949                 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
 950                 err = -EOPNOTSUPP;
 951                 goto out;
 952
 953         case IB_WR_LOCAL_INV:
 954                 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
 955                 *num_sge = 0;
 956                 break;
 957
 958         case IB_WR_REG_MR:
 959                 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
 960                 if (unlikely(err))
 961                         goto out;
 962                 *num_sge = 0;
 963                 break;
 964
 965         case IB_WR_REG_MR_INTEGRITY:
 966                 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
 967                                               cur_edge, idx, nreq, fence,
 968                                               next_fence);
 969                 if (unlikely(err))
 970                         goto out;
 971                 *num_sge = 0;
 972                 break;
 973
 974         default:
 975                 break;
 976         }
 977
 978 out:
 979         return err;
 980 }
 981
 982 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
 983 {
 984         switch (wr->opcode) {
 985         case IB_WR_RDMA_WRITE:
 986         case IB_WR_RDMA_WRITE_WITH_IMM:
 987                 handle_rdma_op(wr, seg, size);
 988                 break;
 989         default:
 990                 break;
 991         }
 992 }
 993
 994 static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
 995                               const struct ib_send_wr *wr, void **seg,
 996                               int *size, void **cur_edge)
 997 {
 998         set_datagram_seg(*seg, wr);
 999         *seg += sizeof(struct mlx5_wqe_datagram_seg);
1000         *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1001         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1002 }
1003
1004 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1005                           void **seg, int *size, void **cur_edge)
1006 {
1007         set_datagram_seg(*seg, wr);
1008         *seg += sizeof(struct mlx5_wqe_datagram_seg);
1009         *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1010         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1011
1012         /* handle qp that supports ud offload */
1013         if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1014                 struct mlx5_wqe_eth_pad *pad;
1015
1016                 pad = *seg;
1017                 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1018                 *seg += sizeof(struct mlx5_wqe_eth_pad);
1019                 *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1020                 set_eth_seg(wr, qp, seg, size, cur_edge);
1021                 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1022         }
1023 }
1024
1025 void mlx5r_ring_db(struct mlx5_ib_qp *qp, unsigned int nreq,
1026                    struct mlx5_wqe_ctrl_seg *ctrl)
1027 {
1028         struct mlx5_bf *bf = &qp->bf;
1029
1030         qp->sq.head += nreq;
1031
1032         /* Make sure that descriptors are written before
1033          * updating doorbell record and ringing the doorbell
1034          */
1035         wmb();
1036
1037         qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1038
1039         /* Make sure doorbell record is visible to the HCA before
1040          * we hit doorbell.
1041          */
1042         wmb();
1043
1044         mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1045         /* Make sure doorbells don't leak out of SQ spinlock
1046          * and reach the HCA out of order.
1047          */
1048         bf->offset ^= bf->buf_size;
1049 }
1050
1051 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1052                       const struct ib_send_wr **bad_wr, bool drain)
1053 {
1054         struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */
1055         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1056         struct mlx5_core_dev *mdev = dev->mdev;
1057         struct mlx5_ib_qp *qp = to_mqp(ibqp);
1058         struct mlx5_wqe_xrc_seg *xrc;
1059         void *cur_edge;
1060         int size;
1061         unsigned long flags;
1062         unsigned int idx;
1063         int err = 0;
1064         int num_sge;
1065         void *seg;
1066         int nreq;
1067         int i;
1068         u8 next_fence = 0;
1069         u8 fence;
1070
1071         if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1072                      !drain)) {
1073                 *bad_wr = wr;
1074                 return -EIO;
1075         }
1076
1077         if (qp->type == IB_QPT_GSI)
1078                 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1079
1080         spin_lock_irqsave(&qp->sq.lock, flags);
1081
1082         for (nreq = 0; wr; nreq++, wr = wr->next) {
1083                 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1084                         mlx5_ib_warn(dev, "\n");
1085                         err = -EINVAL;
1086                         *bad_wr = wr;
1087                         goto out;
1088                 }
1089
1090                 num_sge = wr->num_sge;
1091                 if (unlikely(num_sge > qp->sq.max_gs)) {
1092                         mlx5_ib_warn(dev, "\n");
1093                         err = -EINVAL;
1094                         *bad_wr = wr;
1095                         goto out;
1096                 }
1097
1098                 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1099                                 nreq);
1100                 if (err) {
1101                         mlx5_ib_warn(dev, "\n");
1102                         err = -ENOMEM;
1103                         *bad_wr = wr;
1104                         goto out;
1105                 }
1106
1107                 if (wr->opcode == IB_WR_REG_MR ||
1108                     wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1109                         fence = dev->umr_fence;
1110                         next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1111                 } else  {
1112                         if (wr->send_flags & IB_SEND_FENCE) {
1113                                 if (qp->next_fence)
1114                                         fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1115                                 else
1116                                         fence = MLX5_FENCE_MODE_FENCE;
1117                         } else {
1118                                 fence = qp->next_fence;
1119                         }
1120                 }
1121
1122                 switch (qp->type) {
1123                 case IB_QPT_XRC_INI:
1124                         xrc = seg;
1125                         seg += sizeof(*xrc);
1126                         size += sizeof(*xrc) / 16;
1127                         fallthrough;
1128                 case IB_QPT_RC:
1129                         err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1130                                             &cur_edge, &idx, nreq, fence,
1131                                             next_fence, &num_sge);
1132                         if (unlikely(err)) {
1133                                 *bad_wr = wr;
1134                                 goto out;
1135                         } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1136                                 goto skip_psv;
1137                         }
1138                         break;
1139
1140                 case IB_QPT_UC:
1141                         handle_qpt_uc(wr, &seg, &size);
1142                         break;
1143                 case IB_QPT_SMI:
1144                         if (unlikely(!dev->port_caps[qp->port - 1].has_smi)) {
1145                                 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1146                                 err = -EPERM;
1147                                 *bad_wr = wr;
1148                                 goto out;
1149                         }
1150                         fallthrough;
1151                 case MLX5_IB_QPT_HW_GSI:
1152                         handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1153                         break;
1154                 case IB_QPT_UD:
1155                         handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1156                         break;
1157
1158                 default:
1159                         break;
1160                 }
1161
1162                 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1163                         err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1164                         if (unlikely(err)) {
1165                                 mlx5_ib_warn(dev, "\n");
1166                                 *bad_wr = wr;
1167                                 goto out;
1168                         }
1169                 } else {
1170                         for (i = 0; i < num_sge; i++) {
1171                                 handle_post_send_edge(&qp->sq, &seg, size,
1172                                                       &cur_edge);
1173                                 if (unlikely(!wr->sg_list[i].length))
1174                                         continue;
1175
1176                                 set_data_ptr_seg(
1177                                         (struct mlx5_wqe_data_seg *)seg,
1178                                         wr->sg_list + i);
1179                                 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1180                                 seg += sizeof(struct mlx5_wqe_data_seg);
1181                         }
1182                 }
1183
1184                 qp->next_fence = next_fence;
1185                 mlx5r_finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id,
1186                                  nreq, fence, mlx5_ib_opcode[wr->opcode]);
1187 skip_psv:
1188                 if (0)
1189                         dump_wqe(qp, idx, size);
1190         }
1191
1192 out:
1193         if (likely(nreq))
1194                 mlx5r_ring_db(qp, nreq, ctrl);
1195
1196         spin_unlock_irqrestore(&qp->sq.lock, flags);
1197
1198         return err;
1199 }
1200
1201 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1202 {
1203          sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1204 }
1205
1206 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1207                       const struct ib_recv_wr **bad_wr, bool drain)
1208 {
1209         struct mlx5_ib_qp *qp = to_mqp(ibqp);
1210         struct mlx5_wqe_data_seg *scat;
1211         struct mlx5_rwqe_sig *sig;
1212         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1213         struct mlx5_core_dev *mdev = dev->mdev;
1214         unsigned long flags;
1215         int err = 0;
1216         int nreq;
1217         int ind;
1218         int i;
1219
1220         if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1221                      !drain)) {
1222                 *bad_wr = wr;
1223                 return -EIO;
1224         }
1225
1226         if (qp->type == IB_QPT_GSI)
1227                 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1228
1229         spin_lock_irqsave(&qp->rq.lock, flags);
1230
1231         ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1232
1233         for (nreq = 0; wr; nreq++, wr = wr->next) {
1234                 if (mlx5r_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1235                         err = -ENOMEM;
1236                         *bad_wr = wr;
1237                         goto out;
1238                 }
1239
1240                 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1241                         err = -EINVAL;
1242                         *bad_wr = wr;
1243                         goto out;
1244                 }
1245
1246                 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1247                 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1248                         scat++;
1249
1250                 for (i = 0; i < wr->num_sge; i++)
1251                         set_data_ptr_seg(scat + i, wr->sg_list + i);
1252
1253                 if (i < qp->rq.max_gs) {
1254                         scat[i].byte_count = 0;
1255                         scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey;
1256                         scat[i].addr       = 0;
1257                 }
1258
1259                 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1260                         sig = (struct mlx5_rwqe_sig *)scat;
1261                         set_sig_seg(sig, qp->rq.max_gs);
1262                 }
1263
1264                 qp->rq.wrid[ind] = wr->wr_id;
1265
1266                 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1267         }
1268
1269 out:
1270         if (likely(nreq)) {
1271                 qp->rq.head += nreq;
1272
1273                 /* Make sure that descriptors are written before
1274                  * doorbell record.
1275                  */
1276                 wmb();
1277
1278                 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1279         }
1280
1281         spin_unlock_irqrestore(&qp->rq.lock, flags);
1282
1283         return err;
1284 }