drivers/infiniband/hw/mlx5/wr.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
   4  */
   5
   6 #include <linux/gfp.h>
   7 #include <linux/mlx5/qp.h>
   8 #include <linux/mlx5/driver.h>
   9 #include "wr.h"
  10
  11 static const u32 mlx5_ib_opcode[] = {
  12         [IB_WR_SEND]                            = MLX5_OPCODE_SEND,
  13         [IB_WR_LSO]                             = MLX5_OPCODE_LSO,
  14         [IB_WR_SEND_WITH_IMM]                   = MLX5_OPCODE_SEND_IMM,
  15         [IB_WR_RDMA_WRITE]                      = MLX5_OPCODE_RDMA_WRITE,
  16         [IB_WR_RDMA_WRITE_WITH_IMM]             = MLX5_OPCODE_RDMA_WRITE_IMM,
  17         [IB_WR_RDMA_READ]                       = MLX5_OPCODE_RDMA_READ,
  18         [IB_WR_ATOMIC_CMP_AND_SWP]              = MLX5_OPCODE_ATOMIC_CS,
  19         [IB_WR_ATOMIC_FETCH_AND_ADD]            = MLX5_OPCODE_ATOMIC_FA,
  20         [IB_WR_SEND_WITH_INV]                   = MLX5_OPCODE_SEND_INVAL,
  21         [IB_WR_LOCAL_INV]                       = MLX5_OPCODE_UMR,
  22         [IB_WR_REG_MR]                          = MLX5_OPCODE_UMR,
  23         [IB_WR_MASKED_ATOMIC_CMP_AND_SWP]       = MLX5_OPCODE_ATOMIC_MASKED_CS,
  24         [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]     = MLX5_OPCODE_ATOMIC_MASKED_FA,
  25         [MLX5_IB_WR_UMR]                        = MLX5_OPCODE_UMR,
  26 };
  27
  28 /* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
  29  * next nearby edge and get new address translation for current WQE position.
  30  * @sq - SQ buffer.
  31  * @seg: Current WQE position (16B aligned).
  32  * @wqe_sz: Total current WQE size [16B].
  33  * @cur_edge: Updated current edge.
  34  */
  35 static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
  36                                          u32 wqe_sz, void **cur_edge)
  37 {
  38         u32 idx;
  39
  40         if (likely(*seg != *cur_edge))
  41                 return;
  42
  43         idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
  44         *cur_edge = get_sq_edge(sq, idx);
  45
  46         *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
  47 }
  48
  49 /* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's
  50  * pointers. At the end @seg is aligned to 16B regardless the copied size.
  51  * @sq - SQ buffer.
  52  * @cur_edge: Updated current edge.
  53  * @seg: Current WQE position (16B aligned).
  54  * @wqe_sz: Total current WQE size [16B].
  55  * @src: Pointer to copy from.
  56  * @n: Number of bytes to copy.
  57  */
  58 static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
  59                                    void **seg, u32 *wqe_sz, const void *src,
  60                                    size_t n)
  61 {
  62         while (likely(n)) {
  63                 size_t leftlen = *cur_edge - *seg;
  64                 size_t copysz = min_t(size_t, leftlen, n);
  65                 size_t stride;
  66
  67                 memcpy(*seg, src, copysz);
  68
  69                 n -= copysz;
  70                 src += copysz;
  71                 stride = !n ? ALIGN(copysz, 16) : copysz;
  72                 *seg += stride;
  73                 *wqe_sz += stride >> 4;
  74                 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
  75         }
  76 }
  77
  78 static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq,
  79                             struct ib_cq *ib_cq)
  80 {
  81         struct mlx5_ib_cq *cq;
  82         unsigned int cur;
  83
  84         cur = wq->head - wq->tail;
  85         if (likely(cur + nreq < wq->max_post))
  86                 return 0;
  87
  88         cq = to_mcq(ib_cq);
  89         spin_lock(&cq->lock);
  90         cur = wq->head - wq->tail;
  91         spin_unlock(&cq->lock);
  92
  93         return cur + nreq >= wq->max_post;
  94 }
  95
  96 static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
  97                                           u64 remote_addr, u32 rkey)
  98 {
  99         rseg->raddr    = cpu_to_be64(remote_addr);
 100         rseg->rkey     = cpu_to_be32(rkey);
 101         rseg->reserved = 0;
 102 }
 103
 104 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
 105                         void **seg, int *size, void **cur_edge)
 106 {
 107         struct mlx5_wqe_eth_seg *eseg = *seg;
 108
 109         memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
 110
 111         if (wr->send_flags & IB_SEND_IP_CSUM)
 112                 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
 113                                  MLX5_ETH_WQE_L4_CSUM;
 114
 115         if (wr->opcode == IB_WR_LSO) {
 116                 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
 117                 size_t left, copysz;
 118                 void *pdata = ud_wr->header;
 119                 size_t stride;
 120
 121                 left = ud_wr->hlen;
 122                 eseg->mss = cpu_to_be16(ud_wr->mss);
 123                 eseg->inline_hdr.sz = cpu_to_be16(left);
 124
 125                 /* memcpy_send_wqe should get a 16B align address. Hence, we
 126                  * first copy up to the current edge and then, if needed,
 127                  * continue to memcpy_send_wqe.
 128                  */
 129                 copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
 130                                left);
 131                 memcpy(eseg->inline_hdr.start, pdata, copysz);
 132                 stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
 133                                sizeof(eseg->inline_hdr.start) + copysz, 16);
 134                 *size += stride / 16;
 135                 *seg += stride;
 136
 137                 if (copysz < left) {
 138                         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 139                         left -= copysz;
 140                         pdata += copysz;
 141                         memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
 142                                         left);
 143                 }
 144
 145                 return;
 146         }
 147
 148         *seg += sizeof(struct mlx5_wqe_eth_seg);
 149         *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
 150 }
 151
 152 static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
 153                              const struct ib_send_wr *wr)
 154 {
 155         memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
 156         dseg->av.dqp_dct =
 157                 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
 158         dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
 159 }
 160
 161 static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
 162 {
 163         dseg->byte_count = cpu_to_be32(sg->length);
 164         dseg->lkey       = cpu_to_be32(sg->lkey);
 165         dseg->addr       = cpu_to_be64(sg->addr);
 166 }
 167
 168 static u64 get_xlt_octo(u64 bytes)
 169 {
 170         return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
 171                MLX5_IB_UMR_OCTOWORD;
 172 }
 173
 174 static __be64 frwr_mkey_mask(bool atomic)
 175 {
 176         u64 result;
 177
 178         result = MLX5_MKEY_MASK_LEN             |
 179                 MLX5_MKEY_MASK_PAGE_SIZE        |
 180                 MLX5_MKEY_MASK_START_ADDR       |
 181                 MLX5_MKEY_MASK_EN_RINVAL        |
 182                 MLX5_MKEY_MASK_KEY              |
 183                 MLX5_MKEY_MASK_LR               |
 184                 MLX5_MKEY_MASK_LW               |
 185                 MLX5_MKEY_MASK_RR               |
 186                 MLX5_MKEY_MASK_RW               |
 187                 MLX5_MKEY_MASK_SMALL_FENCE      |
 188                 MLX5_MKEY_MASK_FREE;
 189
 190         if (atomic)
 191                 result |= MLX5_MKEY_MASK_A;
 192
 193         return cpu_to_be64(result);
 194 }
 195
 196 static __be64 sig_mkey_mask(void)
 197 {
 198         u64 result;
 199
 200         result = MLX5_MKEY_MASK_LEN             |
 201                 MLX5_MKEY_MASK_PAGE_SIZE        |
 202                 MLX5_MKEY_MASK_START_ADDR       |
 203                 MLX5_MKEY_MASK_EN_SIGERR        |
 204                 MLX5_MKEY_MASK_EN_RINVAL        |
 205                 MLX5_MKEY_MASK_KEY              |
 206                 MLX5_MKEY_MASK_LR               |
 207                 MLX5_MKEY_MASK_LW               |
 208                 MLX5_MKEY_MASK_RR               |
 209                 MLX5_MKEY_MASK_RW               |
 210                 MLX5_MKEY_MASK_SMALL_FENCE      |
 211                 MLX5_MKEY_MASK_FREE             |
 212                 MLX5_MKEY_MASK_BSF_EN;
 213
 214         return cpu_to_be64(result);
 215 }
 216
 217 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
 218                             struct mlx5_ib_mr *mr, u8 flags, bool atomic)
 219 {
 220         int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
 221
 222         memset(umr, 0, sizeof(*umr));
 223
 224         umr->flags = flags;
 225         umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
 226         umr->mkey_mask = frwr_mkey_mask(atomic);
 227 }
 228
 229 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
 230 {
 231         memset(umr, 0, sizeof(*umr));
 232         umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 233         umr->flags = MLX5_UMR_INLINE;
 234 }
 235
 236 static __be64 get_umr_enable_mr_mask(void)
 237 {
 238         u64 result;
 239
 240         result = MLX5_MKEY_MASK_KEY |
 241                  MLX5_MKEY_MASK_FREE;
 242
 243         return cpu_to_be64(result);
 244 }
 245
 246 static __be64 get_umr_disable_mr_mask(void)
 247 {
 248         u64 result;
 249
 250         result = MLX5_MKEY_MASK_FREE;
 251
 252         return cpu_to_be64(result);
 253 }
 254
 255 static __be64 get_umr_update_translation_mask(void)
 256 {
 257         u64 result;
 258
 259         result = MLX5_MKEY_MASK_LEN |
 260                  MLX5_MKEY_MASK_PAGE_SIZE |
 261                  MLX5_MKEY_MASK_START_ADDR;
 262
 263         return cpu_to_be64(result);
 264 }
 265
 266 static __be64 get_umr_update_access_mask(int atomic)
 267 {
 268         u64 result;
 269
 270         result = MLX5_MKEY_MASK_LR |
 271                  MLX5_MKEY_MASK_LW |
 272                  MLX5_MKEY_MASK_RR |
 273                  MLX5_MKEY_MASK_RW;
 274
 275         if (atomic)
 276                 result |= MLX5_MKEY_MASK_A;
 277
 278         return cpu_to_be64(result);
 279 }
 280
 281 static __be64 get_umr_update_pd_mask(void)
 282 {
 283         u64 result;
 284
 285         result = MLX5_MKEY_MASK_PD;
 286
 287         return cpu_to_be64(result);
 288 }
 289
 290 static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
 291 {
 292         if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
 293              MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
 294             (mask & MLX5_MKEY_MASK_A &&
 295              MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
 296                 return -EPERM;
 297         return 0;
 298 }
 299
 300 static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
 301                                struct mlx5_wqe_umr_ctrl_seg *umr,
 302                                const struct ib_send_wr *wr, int atomic)
 303 {
 304         const struct mlx5_umr_wr *umrwr = umr_wr(wr);
 305
 306         memset(umr, 0, sizeof(*umr));
 307
 308         if (!umrwr->ignore_free_state) {
 309                 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
 310                          /* fail if free */
 311                         umr->flags = MLX5_UMR_CHECK_FREE;
 312                 else
 313                         /* fail if not free */
 314                         umr->flags = MLX5_UMR_CHECK_NOT_FREE;
 315         }
 316
 317         umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
 318         if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
 319                 u64 offset = get_xlt_octo(umrwr->offset);
 320
 321                 umr->xlt_offset = cpu_to_be16(offset & 0xffff);
 322                 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
 323                 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
 324         }
 325         if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
 326                 umr->mkey_mask |= get_umr_update_translation_mask();
 327         if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
 328                 umr->mkey_mask |= get_umr_update_access_mask(atomic);
 329                 umr->mkey_mask |= get_umr_update_pd_mask();
 330         }
 331         if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
 332                 umr->mkey_mask |= get_umr_enable_mr_mask();
 333         if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
 334                 umr->mkey_mask |= get_umr_disable_mr_mask();
 335
 336         if (!wr->num_sge)
 337                 umr->flags |= MLX5_UMR_INLINE;
 338
 339         return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
 340 }
 341
 342 static u8 get_umr_flags(int acc)
 343 {
 344         return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
 345                (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
 346                (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
 347                (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
 348                 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
 349 }
 350
 351 static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
 352                              struct mlx5_ib_mr *mr,
 353                              u32 key, int access)
 354 {
 355         int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
 356
 357         memset(seg, 0, sizeof(*seg));
 358
 359         if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
 360                 seg->log2_page_size = ilog2(mr->ibmr.page_size);
 361         else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
 362                 /* KLMs take twice the size of MTTs */
 363                 ndescs *= 2;
 364
 365         seg->flags = get_umr_flags(access) | mr->access_mode;
 366         seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
 367         seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
 368         seg->start_addr = cpu_to_be64(mr->ibmr.iova);
 369         seg->len = cpu_to_be64(mr->ibmr.length);
 370         seg->xlt_oct_size = cpu_to_be32(ndescs);
 371 }
 372
 373 static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
 374 {
 375         memset(seg, 0, sizeof(*seg));
 376         seg->status = MLX5_MKEY_STATUS_FREE;
 377 }
 378
 379 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,
 380                                  const struct ib_send_wr *wr)
 381 {
 382         const struct mlx5_umr_wr *umrwr = umr_wr(wr);
 383
 384         memset(seg, 0, sizeof(*seg));
 385         if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
 386                 seg->status = MLX5_MKEY_STATUS_FREE;
 387
 388         seg->flags = convert_access(umrwr->access_flags);
 389         if (umrwr->pd)
 390                 seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
 391         if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
 392             !umrwr->length)
 393                 seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
 394
 395         seg->start_addr = cpu_to_be64(umrwr->virt_addr);
 396         seg->len = cpu_to_be64(umrwr->length);
 397         seg->log2_page_size = umrwr->page_shift;
 398         seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
 399                                        mlx5_mkey_variant(umrwr->mkey));
 400 }
 401
 402 static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
 403                              struct mlx5_ib_mr *mr,
 404                              struct mlx5_ib_pd *pd)
 405 {
 406         int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
 407
 408         dseg->addr = cpu_to_be64(mr->desc_map);
 409         dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
 410         dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
 411 }
 412
 413 static __be32 send_ieth(const struct ib_send_wr *wr)
 414 {
 415         switch (wr->opcode) {
 416         case IB_WR_SEND_WITH_IMM:
 417         case IB_WR_RDMA_WRITE_WITH_IMM:
 418                 return wr->ex.imm_data;
 419
 420         case IB_WR_SEND_WITH_INV:
 421                 return cpu_to_be32(wr->ex.invalidate_rkey);
 422
 423         default:
 424                 return 0;
 425         }
 426 }
 427
 428 static u8 calc_sig(void *wqe, int size)
 429 {
 430         u8 *p = wqe;
 431         u8 res = 0;
 432         int i;
 433
 434         for (i = 0; i < size; i++)
 435                 res ^= p[i];
 436
 437         return ~res;
 438 }
 439
 440 static u8 wq_sig(void *wqe)
 441 {
 442         return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
 443 }
 444
 445 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
 446                             void **wqe, int *wqe_sz, void **cur_edge)
 447 {
 448         struct mlx5_wqe_inline_seg *seg;
 449         size_t offset;
 450         int inl = 0;
 451         int i;
 452
 453         seg = *wqe;
 454         *wqe += sizeof(*seg);
 455         offset = sizeof(*seg);
 456
 457         for (i = 0; i < wr->num_sge; i++) {
 458                 size_t len  = wr->sg_list[i].length;
 459                 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
 460
 461                 inl += len;
 462
 463                 if (unlikely(inl > qp->max_inline_data))
 464                         return -ENOMEM;
 465
 466                 while (likely(len)) {
 467                         size_t leftlen;
 468                         size_t copysz;
 469
 470                         handle_post_send_edge(&qp->sq, wqe,
 471                                               *wqe_sz + (offset >> 4),
 472                                               cur_edge);
 473
 474                         leftlen = *cur_edge - *wqe;
 475                         copysz = min_t(size_t, leftlen, len);
 476
 477                         memcpy(*wqe, addr, copysz);
 478                         len -= copysz;
 479                         addr += copysz;
 480                         *wqe += copysz;
 481                         offset += copysz;
 482                 }
 483         }
 484
 485         seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
 486
 487         *wqe_sz +=  ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
 488
 489         return 0;
 490 }
 491
 492 static u16 prot_field_size(enum ib_signature_type type)
 493 {
 494         switch (type) {
 495         case IB_SIG_TYPE_T10_DIF:
 496                 return MLX5_DIF_SIZE;
 497         default:
 498                 return 0;
 499         }
 500 }
 501
 502 static u8 bs_selector(int block_size)
 503 {
 504         switch (block_size) {
 505         case 512:           return 0x1;
 506         case 520:           return 0x2;
 507         case 4096:          return 0x3;
 508         case 4160:          return 0x4;
 509         case 1073741824:    return 0x5;
 510         default:            return 0;
 511         }
 512 }
 513
 514 static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
 515                               struct mlx5_bsf_inl *inl)
 516 {
 517         /* Valid inline section and allow BSF refresh */
 518         inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
 519                                        MLX5_BSF_REFRESH_DIF);
 520         inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
 521         inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
 522         /* repeating block */
 523         inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
 524         inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
 525                         MLX5_DIF_CRC : MLX5_DIF_IPCS;
 526
 527         if (domain->sig.dif.ref_remap)
 528                 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
 529
 530         if (domain->sig.dif.app_escape) {
 531                 if (domain->sig.dif.ref_escape)
 532                         inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
 533                 else
 534                         inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
 535         }
 536
 537         inl->dif_app_bitmask_check =
 538                 cpu_to_be16(domain->sig.dif.apptag_check_mask);
 539 }
 540
 541 static int mlx5_set_bsf(struct ib_mr *sig_mr,
 542                         struct ib_sig_attrs *sig_attrs,
 543                         struct mlx5_bsf *bsf, u32 data_size)
 544 {
 545         struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
 546         struct mlx5_bsf_basic *basic = &bsf->basic;
 547         struct ib_sig_domain *mem = &sig_attrs->mem;
 548         struct ib_sig_domain *wire = &sig_attrs->wire;
 549
 550         memset(bsf, 0, sizeof(*bsf));
 551
 552         /* Basic + Extended + Inline */
 553         basic->bsf_size_sbs = 1 << 7;
 554         /* Input domain check byte mask */
 555         basic->check_byte_mask = sig_attrs->check_mask;
 556         basic->raw_data_size = cpu_to_be32(data_size);
 557
 558         /* Memory domain */
 559         switch (sig_attrs->mem.sig_type) {
 560         case IB_SIG_TYPE_NONE:
 561                 break;
 562         case IB_SIG_TYPE_T10_DIF:
 563                 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
 564                 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
 565                 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
 566                 break;
 567         default:
 568                 return -EINVAL;
 569         }
 570
 571         /* Wire domain */
 572         switch (sig_attrs->wire.sig_type) {
 573         case IB_SIG_TYPE_NONE:
 574                 break;
 575         case IB_SIG_TYPE_T10_DIF:
 576                 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
 577                     mem->sig_type == wire->sig_type) {
 578                         /* Same block structure */
 579                         basic->bsf_size_sbs |= 1 << 4;
 580                         if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
 581                                 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
 582                         if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
 583                                 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
 584                         if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
 585                                 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
 586                 } else
 587                         basic->wire.bs_selector =
 588                                 bs_selector(wire->sig.dif.pi_interval);
 589
 590                 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
 591                 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
 592                 break;
 593         default:
 594                 return -EINVAL;
 595         }
 596
 597         return 0;
 598 }
 599
 600
 601 static int set_sig_data_segment(const struct ib_send_wr *send_wr,
 602                                 struct ib_mr *sig_mr,
 603                                 struct ib_sig_attrs *sig_attrs,
 604                                 struct mlx5_ib_qp *qp, void **seg, int *size,
 605                                 void **cur_edge)
 606 {
 607         struct mlx5_bsf *bsf;
 608         u32 data_len;
 609         u32 data_key;
 610         u64 data_va;
 611         u32 prot_len = 0;
 612         u32 prot_key = 0;
 613         u64 prot_va = 0;
 614         bool prot = false;
 615         int ret;
 616         int wqe_size;
 617         struct mlx5_ib_mr *mr = to_mmr(sig_mr);
 618         struct mlx5_ib_mr *pi_mr = mr->pi_mr;
 619
 620         data_len = pi_mr->data_length;
 621         data_key = pi_mr->ibmr.lkey;
 622         data_va = pi_mr->data_iova;
 623         if (pi_mr->meta_ndescs) {
 624                 prot_len = pi_mr->meta_length;
 625                 prot_key = pi_mr->ibmr.lkey;
 626                 prot_va = pi_mr->pi_iova;
 627                 prot = true;
 628         }
 629
 630         if (!prot || (data_key == prot_key && data_va == prot_va &&
 631                       data_len == prot_len)) {
 632                 /**
 633                  * Source domain doesn't contain signature information
 634                  * or data and protection are interleaved in memory.
 635                  * So need construct:
 636                  *                  ------------------
 637                  *                 |     data_klm     |
 638                  *                  ------------------
 639                  *                 |       BSF        |
 640                  *                  ------------------
 641                  **/
 642                 struct mlx5_klm *data_klm = *seg;
 643
 644                 data_klm->bcount = cpu_to_be32(data_len);
 645                 data_klm->key = cpu_to_be32(data_key);
 646                 data_klm->va = cpu_to_be64(data_va);
 647                 wqe_size = ALIGN(sizeof(*data_klm), 64);
 648         } else {
 649                 /**
 650                  * Source domain contains signature information
 651                  * So need construct a strided block format:
 652                  *               ---------------------------
 653                  *              |     stride_block_ctrl     |
 654                  *               ---------------------------
 655                  *              |          data_klm         |
 656                  *               ---------------------------
 657                  *              |          prot_klm         |
 658                  *               ---------------------------
 659                  *              |             BSF           |
 660                  *               ---------------------------
 661                  **/
 662                 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
 663                 struct mlx5_stride_block_entry *data_sentry;
 664                 struct mlx5_stride_block_entry *prot_sentry;
 665                 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
 666                 int prot_size;
 667
 668                 sblock_ctrl = *seg;
 669                 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
 670                 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
 671
 672                 prot_size = prot_field_size(sig_attrs->mem.sig_type);
 673                 if (!prot_size) {
 674                         pr_err("Bad block size given: %u\n", block_size);
 675                         return -EINVAL;
 676                 }
 677                 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
 678                                                             prot_size);
 679                 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
 680                 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
 681                 sblock_ctrl->num_entries = cpu_to_be16(2);
 682
 683                 data_sentry->bcount = cpu_to_be16(block_size);
 684                 data_sentry->key = cpu_to_be32(data_key);
 685                 data_sentry->va = cpu_to_be64(data_va);
 686                 data_sentry->stride = cpu_to_be16(block_size);
 687
 688                 prot_sentry->bcount = cpu_to_be16(prot_size);
 689                 prot_sentry->key = cpu_to_be32(prot_key);
 690                 prot_sentry->va = cpu_to_be64(prot_va);
 691                 prot_sentry->stride = cpu_to_be16(prot_size);
 692
 693                 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
 694                                  sizeof(*prot_sentry), 64);
 695         }
 696
 697         *seg += wqe_size;
 698         *size += wqe_size / 16;
 699         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 700
 701         bsf = *seg;
 702         ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
 703         if (ret)
 704                 return -EINVAL;
 705
 706         *seg += sizeof(*bsf);
 707         *size += sizeof(*bsf) / 16;
 708         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 709
 710         return 0;
 711 }
 712
 713 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
 714                                  struct ib_mr *sig_mr, int access_flags,
 715                                  u32 size, u32 length, u32 pdn)
 716 {
 717         u32 sig_key = sig_mr->rkey;
 718         u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
 719
 720         memset(seg, 0, sizeof(*seg));
 721
 722         seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
 723         seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
 724         seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
 725                                     MLX5_MKEY_BSF_EN | pdn);
 726         seg->len = cpu_to_be64(length);
 727         seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
 728         seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
 729 }
 730
 731 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
 732                                 u32 size)
 733 {
 734         memset(umr, 0, sizeof(*umr));
 735
 736         umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
 737         umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
 738         umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
 739         umr->mkey_mask = sig_mkey_mask();
 740 }
 741
 742 static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
 743                          struct mlx5_ib_qp *qp, void **seg, int *size,
 744                          void **cur_edge)
 745 {
 746         const struct ib_reg_wr *wr = reg_wr(send_wr);
 747         struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
 748         struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
 749         struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
 750         u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
 751         u32 xlt_size;
 752         int region_len, ret;
 753
 754         if (unlikely(send_wr->num_sge != 0) ||
 755             unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
 756             unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
 757             unlikely(!sig_mr->sig->sig_status_checked))
 758                 return -EINVAL;
 759
 760         /* length of the protected region, data + protection */
 761         region_len = pi_mr->ibmr.length;
 762
 763         /**
 764          * KLM octoword size - if protection was provided
 765          * then we use strided block format (3 octowords),
 766          * else we use single KLM (1 octoword)
 767          **/
 768         if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
 769                 xlt_size = 0x30;
 770         else
 771                 xlt_size = sizeof(struct mlx5_klm);
 772
 773         set_sig_umr_segment(*seg, xlt_size);
 774         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 775         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 776         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 777
 778         set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
 779                              pdn);
 780         *seg += sizeof(struct mlx5_mkey_seg);
 781         *size += sizeof(struct mlx5_mkey_seg) / 16;
 782         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 783
 784         ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
 785                                    cur_edge);
 786         if (ret)
 787                 return ret;
 788
 789         sig_mr->sig->sig_status_checked = false;
 790         return 0;
 791 }
 792
 793 static int set_psv_wr(struct ib_sig_domain *domain,
 794                       u32 psv_idx, void **seg, int *size)
 795 {
 796         struct mlx5_seg_set_psv *psv_seg = *seg;
 797
 798         memset(psv_seg, 0, sizeof(*psv_seg));
 799         psv_seg->psv_num = cpu_to_be32(psv_idx);
 800         switch (domain->sig_type) {
 801         case IB_SIG_TYPE_NONE:
 802                 break;
 803         case IB_SIG_TYPE_T10_DIF:
 804                 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
 805                                                      domain->sig.dif.app_tag);
 806                 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
 807                 break;
 808         default:
 809                 pr_err("Bad signature type (%d) is given.\n",
 810                        domain->sig_type);
 811                 return -EINVAL;
 812         }
 813
 814         *seg += sizeof(*psv_seg);
 815         *size += sizeof(*psv_seg) / 16;
 816
 817         return 0;
 818 }
 819
 820 static int set_reg_wr(struct mlx5_ib_qp *qp,
 821                       const struct ib_reg_wr *wr,
 822                       void **seg, int *size, void **cur_edge,
 823                       bool check_not_free)
 824 {
 825         struct mlx5_ib_mr *mr = to_mmr(wr->mr);
 826         struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
 827         struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
 828         int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
 829         bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
 830         bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
 831         u8 flags = 0;
 832
 833         if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) {
 834                 mlx5_ib_warn(to_mdev(qp->ibqp.device),
 835                              "Fast update of %s for MR is disabled\n",
 836                              (MLX5_CAP_GEN(dev->mdev,
 837                                            umr_modify_entity_size_disabled)) ?
 838                                      "entity size" :
 839                                      "atomic access");
 840                 return -EINVAL;
 841         }
 842
 843         if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
 844                 mlx5_ib_warn(to_mdev(qp->ibqp.device),
 845                              "Invalid IB_SEND_INLINE send flag\n");
 846                 return -EINVAL;
 847         }
 848
 849         if (check_not_free)
 850                 flags |= MLX5_UMR_CHECK_NOT_FREE;
 851         if (umr_inline)
 852                 flags |= MLX5_UMR_INLINE;
 853
 854         set_reg_umr_seg(*seg, mr, flags, atomic);
 855         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 856         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 857         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 858
 859         set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
 860         *seg += sizeof(struct mlx5_mkey_seg);
 861         *size += sizeof(struct mlx5_mkey_seg) / 16;
 862         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 863
 864         if (umr_inline) {
 865                 memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
 866                                 mr_list_size);
 867                 *size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
 868         } else {
 869                 set_reg_data_seg(*seg, mr, pd);
 870                 *seg += sizeof(struct mlx5_wqe_data_seg);
 871                 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
 872         }
 873         return 0;
 874 }
 875
 876 static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
 877                         void **cur_edge)
 878 {
 879         set_linv_umr_seg(*seg);
 880         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
 881         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
 882         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 883         set_linv_mkey_seg(*seg);
 884         *seg += sizeof(struct mlx5_mkey_seg);
 885         *size += sizeof(struct mlx5_mkey_seg) / 16;
 886         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
 887 }
 888
 889 static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
 890 {
 891         __be32 *p = NULL;
 892         int i, j;
 893
 894         pr_debug("dump WQE index %u:\n", idx);
 895         for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
 896                 if ((i & 0xf) == 0) {
 897                         p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
 898                         pr_debug("WQBB at %p:\n", (void *)p);
 899                         j = 0;
 900                         idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
 901                 }
 902                 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
 903                          be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
 904                          be32_to_cpu(p[j + 3]));
 905         }
 906 }
 907
 908 static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 909                        struct mlx5_wqe_ctrl_seg **ctrl,
 910                        const struct ib_send_wr *wr, unsigned int *idx,
 911                        int *size, void **cur_edge, int nreq,
 912                        bool send_signaled, bool solicited)
 913 {
 914         if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
 915                 return -ENOMEM;
 916
 917         *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
 918         *seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
 919         *ctrl = *seg;
 920         *(uint32_t *)(*seg + 8) = 0;
 921         (*ctrl)->imm = send_ieth(wr);
 922         (*ctrl)->fm_ce_se = qp->sq_signal_bits |
 923                 (send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
 924                 (solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
 925
 926         *seg += sizeof(**ctrl);
 927         *size = sizeof(**ctrl) / 16;
 928         *cur_edge = qp->sq.cur_edge;
 929
 930         return 0;
 931 }
 932
 933 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 934                      struct mlx5_wqe_ctrl_seg **ctrl,
 935                      const struct ib_send_wr *wr, unsigned int *idx, int *size,
 936                      void **cur_edge, int nreq)
 937 {
 938         return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
 939                            wr->send_flags & IB_SEND_SIGNALED,
 940                            wr->send_flags & IB_SEND_SOLICITED);
 941 }
 942
 943 static void finish_wqe(struct mlx5_ib_qp *qp,
 944                        struct mlx5_wqe_ctrl_seg *ctrl,
 945                        void *seg, u8 size, void *cur_edge,
 946                        unsigned int idx, u64 wr_id, int nreq, u8 fence,
 947                        u32 mlx5_opcode)
 948 {
 949         u8 opmod = 0;
 950
 951         ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
 952                                              mlx5_opcode | ((u32)opmod << 24));
 953         ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
 954         ctrl->fm_ce_se |= fence;
 955         if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
 956                 ctrl->signature = wq_sig(ctrl);
 957
 958         qp->sq.wrid[idx] = wr_id;
 959         qp->sq.w_list[idx].opcode = mlx5_opcode;
 960         qp->sq.wqe_head[idx] = qp->sq.head + nreq;
 961         qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
 962         qp->sq.w_list[idx].next = qp->sq.cur_post;
 963
 964         /* We save the edge which was possibly updated during the WQE
 965          * construction, into SQ's cache.
 966          */
 967         seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
 968         qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
 969                           get_sq_edge(&qp->sq, qp->sq.cur_post &
 970                                       (qp->sq.wqe_cnt - 1)) :
 971                           cur_edge;
 972 }
 973
 974 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
 975 {
 976         set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
 977         *seg += sizeof(struct mlx5_wqe_raddr_seg);
 978         *size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
 979 }
 980
 981 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
 982                              struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
 983                              int *size, void **cur_edge, unsigned int idx)
 984 {
 985         qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
 986         (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
 987         set_linv_wr(qp, seg, size, cur_edge);
 988 }
 989
 990 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
 991                          struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
 992                          void **cur_edge, unsigned int idx)
 993 {
 994         qp->sq.wr_data[idx] = IB_WR_REG_MR;
 995         (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
 996         return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
 997 }
 998
 999 static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1000                       const struct ib_send_wr *wr,
1001                       struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1002                       void **cur_edge, unsigned int *idx, int nreq,
1003                       struct ib_sig_domain *domain, u32 psv_index,
1004                       u8 next_fence)
1005 {
1006         int err;
1007
1008         /*
1009          * SET_PSV WQEs are not signaled and solicited on error.
1010          */
1011         err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
1012                           false, true);
1013         if (unlikely(err)) {
1014                 mlx5_ib_warn(dev, "\n");
1015                 err = -ENOMEM;
1016                 goto out;
1017         }
1018         err = set_psv_wr(domain, psv_index, seg, size);
1019         if (unlikely(err)) {
1020                 mlx5_ib_warn(dev, "\n");
1021                 goto out;
1022         }
1023         finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1024                    next_fence, MLX5_OPCODE_SET_PSV);
1025
1026 out:
1027         return err;
1028 }
1029
1030 static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
1031                                    struct mlx5_ib_qp *qp,
1032                                    const struct ib_send_wr *wr,
1033                                    struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1034                                    int *size, void **cur_edge,
1035                                    unsigned int *idx, int nreq, u8 fence,
1036                                    u8 next_fence)
1037 {
1038         struct mlx5_ib_mr *mr;
1039         struct mlx5_ib_mr *pi_mr;
1040         struct mlx5_ib_mr pa_pi_mr;
1041         struct ib_sig_attrs *sig_attrs;
1042         struct ib_reg_wr reg_pi_wr;
1043         int err;
1044
1045         qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
1046
1047         mr = to_mmr(reg_wr(wr)->mr);
1048         pi_mr = mr->pi_mr;
1049
1050         if (pi_mr) {
1051                 memset(&reg_pi_wr, 0,
1052                        sizeof(struct ib_reg_wr));
1053
1054                 reg_pi_wr.mr = &pi_mr->ibmr;
1055                 reg_pi_wr.access = reg_wr(wr)->access;
1056                 reg_pi_wr.key = pi_mr->ibmr.rkey;
1057
1058                 (*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
1059                 /* UMR for data + prot registration */
1060                 err = set_reg_wr(qp, &reg_pi_wr, seg, size, cur_edge, false);
1061                 if (unlikely(err))
1062                         goto out;
1063
1064                 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
1065                            nreq, fence, MLX5_OPCODE_UMR);
1066
1067                 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
1068                 if (unlikely(err)) {
1069                         mlx5_ib_warn(dev, "\n");
1070                         err = -ENOMEM;
1071                         goto out;
1072                 }
1073         } else {
1074                 memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
1075                 /* No UMR, use local_dma_lkey */
1076                 pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
1077                 pa_pi_mr.ndescs = mr->ndescs;
1078                 pa_pi_mr.data_length = mr->data_length;
1079                 pa_pi_mr.data_iova = mr->data_iova;
1080                 if (mr->meta_ndescs) {
1081                         pa_pi_mr.meta_ndescs = mr->meta_ndescs;
1082                         pa_pi_mr.meta_length = mr->meta_length;
1083                         pa_pi_mr.pi_iova = mr->pi_iova;
1084                 }
1085
1086                 pa_pi_mr.ibmr.length = mr->ibmr.length;
1087                 mr->pi_mr = &pa_pi_mr;
1088         }
1089         (*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
1090         /* UMR for sig MR */
1091         err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
1092         if (unlikely(err)) {
1093                 mlx5_ib_warn(dev, "\n");
1094                 goto out;
1095         }
1096         finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
1097                    fence, MLX5_OPCODE_UMR);
1098
1099         sig_attrs = mr->ibmr.sig_attrs;
1100         err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1101                          &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
1102                          next_fence);
1103         if (unlikely(err))
1104                 goto out;
1105
1106         err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
1107                          &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
1108                          next_fence);
1109         if (unlikely(err))
1110                 goto out;
1111
1112         qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1113
1114 out:
1115         return err;
1116 }
1117
1118 static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1119                          const struct ib_send_wr *wr,
1120                          struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
1121                          void **cur_edge, unsigned int *idx, int nreq, u8 fence,
1122                          u8 next_fence, int *num_sge)
1123 {
1124         int err = 0;
1125
1126         switch (wr->opcode) {
1127         case IB_WR_RDMA_READ:
1128         case IB_WR_RDMA_WRITE:
1129         case IB_WR_RDMA_WRITE_WITH_IMM:
1130                 handle_rdma_op(wr, seg, size);
1131                 break;
1132
1133         case IB_WR_ATOMIC_CMP_AND_SWP:
1134         case IB_WR_ATOMIC_FETCH_AND_ADD:
1135         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
1136                 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
1137                 err = -EOPNOTSUPP;
1138                 goto out;
1139
1140         case IB_WR_LOCAL_INV:
1141                 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
1142                 *num_sge = 0;
1143                 break;
1144
1145         case IB_WR_REG_MR:
1146                 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
1147                 if (unlikely(err))
1148                         goto out;
1149                 *num_sge = 0;
1150                 break;
1151
1152         case IB_WR_REG_MR_INTEGRITY:
1153                 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
1154                                               cur_edge, idx, nreq, fence,
1155                                               next_fence);
1156                 if (unlikely(err))
1157                         goto out;
1158                 *num_sge = 0;
1159                 break;
1160
1161         default:
1162                 break;
1163         }
1164
1165 out:
1166         return err;
1167 }
1168
1169 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
1170 {
1171         switch (wr->opcode) {
1172         case IB_WR_RDMA_WRITE:
1173         case IB_WR_RDMA_WRITE_WITH_IMM:
1174                 handle_rdma_op(wr, seg, size);
1175                 break;
1176         default:
1177                 break;
1178         }
1179 }
1180
1181 static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
1182                               const struct ib_send_wr *wr, void **seg,
1183                               int *size, void **cur_edge)
1184 {
1185         set_datagram_seg(*seg, wr);
1186         *seg += sizeof(struct mlx5_wqe_datagram_seg);
1187         *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1188         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1189 }
1190
1191 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
1192                           void **seg, int *size, void **cur_edge)
1193 {
1194         set_datagram_seg(*seg, wr);
1195         *seg += sizeof(struct mlx5_wqe_datagram_seg);
1196         *size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
1197         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1198
1199         /* handle qp that supports ud offload */
1200         if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
1201                 struct mlx5_wqe_eth_pad *pad;
1202
1203                 pad = *seg;
1204                 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
1205                 *seg += sizeof(struct mlx5_wqe_eth_pad);
1206                 *size += sizeof(struct mlx5_wqe_eth_pad) / 16;
1207                 set_eth_seg(wr, qp, seg, size, cur_edge);
1208                 handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1209         }
1210 }
1211
1212 static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1213                               const struct ib_send_wr *wr,
1214                               struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
1215                               int *size, void **cur_edge, unsigned int idx)
1216 {
1217         int err = 0;
1218
1219         if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) {
1220                 err = -EINVAL;
1221                 mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode);
1222                 goto out;
1223         }
1224
1225         qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
1226         (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
1227         err = set_reg_umr_segment(dev, *seg, wr,
1228                                   !!(MLX5_CAP_GEN(dev->mdev, atomic)));
1229         if (unlikely(err))
1230                 goto out;
1231         *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
1232         *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
1233         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1234         set_reg_mkey_segment(*seg, wr);
1235         *seg += sizeof(struct mlx5_mkey_seg);
1236         *size += sizeof(struct mlx5_mkey_seg) / 16;
1237         handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
1238 out:
1239         return err;
1240 }
1241
1242 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
1243                       const struct ib_send_wr **bad_wr, bool drain)
1244 {
1245         struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */
1246         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1247         struct mlx5_core_dev *mdev = dev->mdev;
1248         struct mlx5_ib_qp *qp;
1249         struct mlx5_wqe_xrc_seg *xrc;
1250         struct mlx5_bf *bf;
1251         void *cur_edge;
1252         int uninitialized_var(size);
1253         unsigned long flags;
1254         unsigned int idx;
1255         int err = 0;
1256         int num_sge;
1257         void *seg;
1258         int nreq;
1259         int i;
1260         u8 next_fence = 0;
1261         u8 fence;
1262
1263         if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1264                      !drain)) {
1265                 *bad_wr = wr;
1266                 return -EIO;
1267         }
1268
1269         if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1270                 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
1271
1272         qp = to_mqp(ibqp);
1273         bf = &qp->bf;
1274
1275         spin_lock_irqsave(&qp->sq.lock, flags);
1276
1277         for (nreq = 0; wr; nreq++, wr = wr->next) {
1278                 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
1279                         mlx5_ib_warn(dev, "\n");
1280                         err = -EINVAL;
1281                         *bad_wr = wr;
1282                         goto out;
1283                 }
1284
1285                 num_sge = wr->num_sge;
1286                 if (unlikely(num_sge > qp->sq.max_gs)) {
1287                         mlx5_ib_warn(dev, "\n");
1288                         err = -EINVAL;
1289                         *bad_wr = wr;
1290                         goto out;
1291                 }
1292
1293                 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
1294                                 nreq);
1295                 if (err) {
1296                         mlx5_ib_warn(dev, "\n");
1297                         err = -ENOMEM;
1298                         *bad_wr = wr;
1299                         goto out;
1300                 }
1301
1302                 if (wr->opcode == IB_WR_REG_MR ||
1303                     wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1304                         fence = dev->umr_fence;
1305                         next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
1306                 } else  {
1307                         if (wr->send_flags & IB_SEND_FENCE) {
1308                                 if (qp->next_fence)
1309                                         fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
1310                                 else
1311                                         fence = MLX5_FENCE_MODE_FENCE;
1312                         } else {
1313                                 fence = qp->next_fence;
1314                         }
1315                 }
1316
1317                 switch (ibqp->qp_type) {
1318                 case IB_QPT_XRC_INI:
1319                         xrc = seg;
1320                         seg += sizeof(*xrc);
1321                         size += sizeof(*xrc) / 16;
1322                         fallthrough;
1323                 case IB_QPT_RC:
1324                         err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
1325                                             &cur_edge, &idx, nreq, fence,
1326                                             next_fence, &num_sge);
1327                         if (unlikely(err)) {
1328                                 *bad_wr = wr;
1329                                 goto out;
1330                         } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
1331                                 goto skip_psv;
1332                         }
1333                         break;
1334
1335                 case IB_QPT_UC:
1336                         handle_qpt_uc(wr, &seg, &size);
1337                         break;
1338                 case IB_QPT_SMI:
1339                         if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
1340                                 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
1341                                 err = -EPERM;
1342                                 *bad_wr = wr;
1343                                 goto out;
1344                         }
1345                         fallthrough;
1346                 case MLX5_IB_QPT_HW_GSI:
1347                         handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
1348                         break;
1349                 case IB_QPT_UD:
1350                         handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
1351                         break;
1352                 case MLX5_IB_QPT_REG_UMR:
1353                         err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg,
1354                                                        &size, &cur_edge, idx);
1355                         if (unlikely(err))
1356                                 goto out;
1357                         break;
1358
1359                 default:
1360                         break;
1361                 }
1362
1363                 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
1364                         err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
1365                         if (unlikely(err)) {
1366                                 mlx5_ib_warn(dev, "\n");
1367                                 *bad_wr = wr;
1368                                 goto out;
1369                         }
1370                 } else {
1371                         for (i = 0; i < num_sge; i++) {
1372                                 handle_post_send_edge(&qp->sq, &seg, size,
1373                                                       &cur_edge);
1374                                 if (unlikely(!wr->sg_list[i].length))
1375                                         continue;
1376
1377                                 set_data_ptr_seg(
1378                                         (struct mlx5_wqe_data_seg *)seg,
1379                                         wr->sg_list + i);
1380                                 size += sizeof(struct mlx5_wqe_data_seg) / 16;
1381                                 seg += sizeof(struct mlx5_wqe_data_seg);
1382                         }
1383                 }
1384
1385                 qp->next_fence = next_fence;
1386                 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,
1387                            fence, mlx5_ib_opcode[wr->opcode]);
1388 skip_psv:
1389                 if (0)
1390                         dump_wqe(qp, idx, size);
1391         }
1392
1393 out:
1394         if (likely(nreq)) {
1395                 qp->sq.head += nreq;
1396
1397                 /* Make sure that descriptors are written before
1398                  * updating doorbell record and ringing the doorbell
1399                  */
1400                 wmb();
1401
1402                 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
1403
1404                 /* Make sure doorbell record is visible to the HCA before
1405                  * we hit doorbell.
1406                  */
1407                 wmb();
1408
1409                 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
1410                 /* Make sure doorbells don't leak out of SQ spinlock
1411                  * and reach the HCA out of order.
1412                  */
1413                 bf->offset ^= bf->buf_size;
1414         }
1415
1416         spin_unlock_irqrestore(&qp->sq.lock, flags);
1417
1418         return err;
1419 }
1420
1421 static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
1422 {
1423          sig->signature = calc_sig(sig, (max_gs + 1) << 2);
1424 }
1425
1426 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1427                       const struct ib_recv_wr **bad_wr, bool drain)
1428 {
1429         struct mlx5_ib_qp *qp = to_mqp(ibqp);
1430         struct mlx5_wqe_data_seg *scat;
1431         struct mlx5_rwqe_sig *sig;
1432         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1433         struct mlx5_core_dev *mdev = dev->mdev;
1434         unsigned long flags;
1435         int err = 0;
1436         int nreq;
1437         int ind;
1438         int i;
1439
1440         if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1441                      !drain)) {
1442                 *bad_wr = wr;
1443                 return -EIO;
1444         }
1445
1446         if (unlikely(ibqp->qp_type == IB_QPT_GSI))
1447                 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
1448
1449         spin_lock_irqsave(&qp->rq.lock, flags);
1450
1451         ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1452
1453         for (nreq = 0; wr; nreq++, wr = wr->next) {
1454                 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1455                         err = -ENOMEM;
1456                         *bad_wr = wr;
1457                         goto out;
1458                 }
1459
1460                 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1461                         err = -EINVAL;
1462                         *bad_wr = wr;
1463                         goto out;
1464                 }
1465
1466                 scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
1467                 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
1468                         scat++;
1469
1470                 for (i = 0; i < wr->num_sge; i++)
1471                         set_data_ptr_seg(scat + i, wr->sg_list + i);
1472
1473                 if (i < qp->rq.max_gs) {
1474                         scat[i].byte_count = 0;
1475                         scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
1476                         scat[i].addr       = 0;
1477                 }
1478
1479                 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
1480                         sig = (struct mlx5_rwqe_sig *)scat;
1481                         set_sig_seg(sig, qp->rq.max_gs);
1482                 }
1483
1484                 qp->rq.wrid[ind] = wr->wr_id;
1485
1486                 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1487         }
1488
1489 out:
1490         if (likely(nreq)) {
1491                 qp->rq.head += nreq;
1492
1493                 /* Make sure that descriptors are written before
1494                  * doorbell record.
1495                  */
1496                 wmb();
1497
1498                 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1499         }
1500
1501         spin_unlock_irqrestore(&qp->rq.lock, flags);
1502
1503         return err;
1504 }