drivers/infiniband/hw/mlx4/qp.c

   1 /*
   2  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/log2.h>
  35 #include <linux/etherdevice.h>
  36 #include <net/ip.h>
  37 #include <linux/slab.h>
  38 #include <linux/netdevice.h>
  39
  40 #include <rdma/ib_cache.h>
  41 #include <rdma/ib_pack.h>
  42 #include <rdma/ib_addr.h>
  43 #include <rdma/ib_mad.h>
  44
  45 #include <linux/mlx4/driver.h>
  46 #include <linux/mlx4/qp.h>
  47
  48 #include "mlx4_ib.h"
  49 #include <rdma/mlx4-abi.h>
  50
  51 static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
  52                              struct mlx4_ib_cq *recv_cq);
  53 static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
  54                                struct mlx4_ib_cq *recv_cq);
  55 static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state);
  56
  57 enum {
  58         MLX4_IB_ACK_REQ_FREQ    = 8,
  59 };
  60
  61 enum {
  62         MLX4_IB_DEFAULT_SCHED_QUEUE     = 0x83,
  63         MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
  64         MLX4_IB_LINK_TYPE_IB            = 0,
  65         MLX4_IB_LINK_TYPE_ETH           = 1
  66 };
  67
  68 enum {
  69         /*
  70          * Largest possible UD header: send with GRH and immediate
  71          * data plus 18 bytes for an Ethernet header with VLAN/802.1Q
  72          * tag.  (LRH would only use 8 bytes, so Ethernet is the
  73          * biggest case)
  74          */
  75         MLX4_IB_UD_HEADER_SIZE          = 82,
  76         MLX4_IB_LSO_HEADER_SPARE        = 128,
  77 };
  78
  79 struct mlx4_ib_sqp {
  80         struct mlx4_ib_qp       qp;
  81         int                     pkey_index;
  82         u32                     qkey;
  83         u32                     send_psn;
  84         struct ib_ud_header     ud_header;
  85         u8                      header_buf[MLX4_IB_UD_HEADER_SIZE];
  86         struct ib_qp            *roce_v2_gsi;
  87 };
  88
  89 enum {
  90         MLX4_IB_MIN_SQ_STRIDE   = 6,
  91         MLX4_IB_CACHE_LINE_SIZE = 64,
  92 };
  93
  94 enum {
  95         MLX4_RAW_QP_MTU         = 7,
  96         MLX4_RAW_QP_MSGMAX      = 31,
  97 };
  98
  99 #ifndef ETH_ALEN
 100 #define ETH_ALEN        6
 101 #endif
 102
 103 static const __be32 mlx4_ib_opcode[] = {
 104         [IB_WR_SEND]                            = cpu_to_be32(MLX4_OPCODE_SEND),
 105         [IB_WR_LSO]                             = cpu_to_be32(MLX4_OPCODE_LSO),
 106         [IB_WR_SEND_WITH_IMM]                   = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
 107         [IB_WR_RDMA_WRITE]                      = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
 108         [IB_WR_RDMA_WRITE_WITH_IMM]             = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
 109         [IB_WR_RDMA_READ]                       = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
 110         [IB_WR_ATOMIC_CMP_AND_SWP]              = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
 111         [IB_WR_ATOMIC_FETCH_AND_ADD]            = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
 112         [IB_WR_SEND_WITH_INV]                   = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
 113         [IB_WR_LOCAL_INV]                       = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
 114         [IB_WR_REG_MR]                          = cpu_to_be32(MLX4_OPCODE_FMR),
 115         [IB_WR_MASKED_ATOMIC_CMP_AND_SWP]       = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
 116         [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]     = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
 117 };
 118
 119 enum mlx4_ib_source_type {
 120         MLX4_IB_QP_SRC  = 0,
 121         MLX4_IB_RWQ_SRC = 1,
 122 };
 123
 124 static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
 125 {
 126         return container_of(mqp, struct mlx4_ib_sqp, qp);
 127 }
 128
 129 static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 130 {
 131         if (!mlx4_is_master(dev->dev))
 132                 return 0;
 133
 134         return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
 135                qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
 136                 8 * MLX4_MFUNC_MAX;
 137 }
 138
 139 static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 140 {
 141         int proxy_sqp = 0;
 142         int real_sqp = 0;
 143         int i;
 144         /* PPF or Native -- real SQP */
 145         real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
 146                     qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
 147                     qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
 148         if (real_sqp)
 149                 return 1;
 150         /* VF or PF -- proxy SQP */
 151         if (mlx4_is_mfunc(dev->dev)) {
 152                 for (i = 0; i < dev->dev->caps.num_ports; i++) {
 153                         if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy ||
 154                             qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) {
 155                                 proxy_sqp = 1;
 156                                 break;
 157                         }
 158                 }
 159         }
 160         if (proxy_sqp)
 161                 return 1;
 162
 163         return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
 164 }
 165
 166 /* used for INIT/CLOSE port logic */
 167 static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 168 {
 169         int proxy_qp0 = 0;
 170         int real_qp0 = 0;
 171         int i;
 172         /* PPF or Native -- real QP0 */
 173         real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
 174                     qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
 175                     qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
 176         if (real_qp0)
 177                 return 1;
 178         /* VF or PF -- proxy QP0 */
 179         if (mlx4_is_mfunc(dev->dev)) {
 180                 for (i = 0; i < dev->dev->caps.num_ports; i++) {
 181                         if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) {
 182                                 proxy_qp0 = 1;
 183                                 break;
 184                         }
 185                 }
 186         }
 187         return proxy_qp0;
 188 }
 189
 190 static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
 191 {
 192         return mlx4_buf_offset(&qp->buf, offset);
 193 }
 194
 195 static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
 196 {
 197         return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
 198 }
 199
 200 static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
 201 {
 202         return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
 203 }
 204
 205 /*
 206  * Stamp a SQ WQE so that it is invalid if prefetched by marking the
 207  * first four bytes of every 64 byte chunk with
 208  *     0x7FFFFFF | (invalid_ownership_value << 31).
 209  *
 210  * When the max work request size is less than or equal to the WQE
 211  * basic block size, as an optimization, we can stamp all WQEs with
 212  * 0xffffffff, and skip the very first chunk of each WQE.
 213  */
 214 static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
 215 {
 216         __be32 *wqe;
 217         int i;
 218         int s;
 219         int ind;
 220         void *buf;
 221         __be32 stamp;
 222         struct mlx4_wqe_ctrl_seg *ctrl;
 223
 224         if (qp->sq_max_wqes_per_wr > 1) {
 225                 s = roundup(size, 1U << qp->sq.wqe_shift);
 226                 for (i = 0; i < s; i += 64) {
 227                         ind = (i >> qp->sq.wqe_shift) + n;
 228                         stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
 229                                                        cpu_to_be32(0xffffffff);
 230                         buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 231                         wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
 232                         *wqe = stamp;
 233                 }
 234         } else {
 235                 ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
 236                 s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4;
 237                 for (i = 64; i < s; i += 64) {
 238                         wqe = buf + i;
 239                         *wqe = cpu_to_be32(0xffffffff);
 240                 }
 241         }
 242 }
 243
 244 static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
 245 {
 246         struct mlx4_wqe_ctrl_seg *ctrl;
 247         struct mlx4_wqe_inline_seg *inl;
 248         void *wqe;
 249         int s;
 250
 251         ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
 252         s = sizeof(struct mlx4_wqe_ctrl_seg);
 253
 254         if (qp->ibqp.qp_type == IB_QPT_UD) {
 255                 struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
 256                 struct mlx4_av *av = (struct mlx4_av *)dgram->av;
 257                 memset(dgram, 0, sizeof *dgram);
 258                 av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
 259                 s += sizeof(struct mlx4_wqe_datagram_seg);
 260         }
 261
 262         /* Pad the remainder of the WQE with an inline data segment. */
 263         if (size > s) {
 264                 inl = wqe + s;
 265                 inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
 266         }
 267         ctrl->srcrb_flags = 0;
 268         ctrl->qpn_vlan.fence_size = size / 16;
 269         /*
 270          * Make sure descriptor is fully written before setting ownership bit
 271          * (because HW can start executing as soon as we do).
 272          */
 273         wmb();
 274
 275         ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
 276                 (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
 277
 278         stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
 279 }
 280
 281 /* Post NOP WQE to prevent wrap-around in the middle of WR */
 282 static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
 283 {
 284         unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
 285         if (unlikely(s < qp->sq_max_wqes_per_wr)) {
 286                 post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
 287                 ind += s;
 288         }
 289         return ind;
 290 }
 291
 292 static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
 293 {
 294         struct ib_event event;
 295         struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
 296
 297         if (type == MLX4_EVENT_TYPE_PATH_MIG)
 298                 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
 299
 300         if (ibqp->event_handler) {
 301                 event.device     = ibqp->device;
 302                 event.element.qp = ibqp;
 303                 switch (type) {
 304                 case MLX4_EVENT_TYPE_PATH_MIG:
 305                         event.event = IB_EVENT_PATH_MIG;
 306                         break;
 307                 case MLX4_EVENT_TYPE_COMM_EST:
 308                         event.event = IB_EVENT_COMM_EST;
 309                         break;
 310                 case MLX4_EVENT_TYPE_SQ_DRAINED:
 311                         event.event = IB_EVENT_SQ_DRAINED;
 312                         break;
 313                 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
 314                         event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 315                         break;
 316                 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
 317                         event.event = IB_EVENT_QP_FATAL;
 318                         break;
 319                 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
 320                         event.event = IB_EVENT_PATH_MIG_ERR;
 321                         break;
 322                 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 323                         event.event = IB_EVENT_QP_REQ_ERR;
 324                         break;
 325                 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
 326                         event.event = IB_EVENT_QP_ACCESS_ERR;
 327                         break;
 328                 default:
 329                         pr_warn("Unexpected event type %d "
 330                                "on QP %06x\n", type, qp->qpn);
 331                         return;
 332                 }
 333
 334                 ibqp->event_handler(&event, ibqp->qp_context);
 335         }
 336 }
 337
 338 static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
 339 {
 340         pr_warn_ratelimited("Unexpected event type %d on WQ 0x%06x. Events are not supported for WQs\n",
 341                             type, qp->qpn);
 342 }
 343
 344 static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
 345 {
 346         /*
 347          * UD WQEs must have a datagram segment.
 348          * RC and UC WQEs might have a remote address segment.
 349          * MLX WQEs need two extra inline data segments (for the UD
 350          * header and space for the ICRC).
 351          */
 352         switch (type) {
 353         case MLX4_IB_QPT_UD:
 354                 return sizeof (struct mlx4_wqe_ctrl_seg) +
 355                         sizeof (struct mlx4_wqe_datagram_seg) +
 356                         ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
 357         case MLX4_IB_QPT_PROXY_SMI_OWNER:
 358         case MLX4_IB_QPT_PROXY_SMI:
 359         case MLX4_IB_QPT_PROXY_GSI:
 360                 return sizeof (struct mlx4_wqe_ctrl_seg) +
 361                         sizeof (struct mlx4_wqe_datagram_seg) + 64;
 362         case MLX4_IB_QPT_TUN_SMI_OWNER:
 363         case MLX4_IB_QPT_TUN_GSI:
 364                 return sizeof (struct mlx4_wqe_ctrl_seg) +
 365                         sizeof (struct mlx4_wqe_datagram_seg);
 366
 367         case MLX4_IB_QPT_UC:
 368                 return sizeof (struct mlx4_wqe_ctrl_seg) +
 369                         sizeof (struct mlx4_wqe_raddr_seg);
 370         case MLX4_IB_QPT_RC:
 371                 return sizeof (struct mlx4_wqe_ctrl_seg) +
 372                         sizeof (struct mlx4_wqe_masked_atomic_seg) +
 373                         sizeof (struct mlx4_wqe_raddr_seg);
 374         case MLX4_IB_QPT_SMI:
 375         case MLX4_IB_QPT_GSI:
 376                 return sizeof (struct mlx4_wqe_ctrl_seg) +
 377                         ALIGN(MLX4_IB_UD_HEADER_SIZE +
 378                               DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
 379                                            MLX4_INLINE_ALIGN) *
 380                               sizeof (struct mlx4_wqe_inline_seg),
 381                               sizeof (struct mlx4_wqe_data_seg)) +
 382                         ALIGN(4 +
 383                               sizeof (struct mlx4_wqe_inline_seg),
 384                               sizeof (struct mlx4_wqe_data_seg));
 385         default:
 386                 return sizeof (struct mlx4_wqe_ctrl_seg);
 387         }
 388 }
 389
 390 static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
 391                        int is_user, int has_rq, struct mlx4_ib_qp *qp,
 392                        u32 inl_recv_sz)
 393 {
 394         /* Sanity check RQ size before proceeding */
 395         if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
 396             cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
 397                 return -EINVAL;
 398
 399         if (!has_rq) {
 400                 if (cap->max_recv_wr || inl_recv_sz)
 401                         return -EINVAL;
 402
 403                 qp->rq.wqe_cnt = qp->rq.max_gs = 0;
 404         } else {
 405                 u32 max_inl_recv_sz = dev->dev->caps.max_rq_sg *
 406                         sizeof(struct mlx4_wqe_data_seg);
 407                 u32 wqe_size;
 408
 409                 /* HW requires >= 1 RQ entry with >= 1 gather entry */
 410                 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge ||
 411                                 inl_recv_sz > max_inl_recv_sz))
 412                         return -EINVAL;
 413
 414                 qp->rq.wqe_cnt   = roundup_pow_of_two(max(1U, cap->max_recv_wr));
 415                 qp->rq.max_gs    = roundup_pow_of_two(max(1U, cap->max_recv_sge));
 416                 wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg);
 417                 qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz));
 418         }
 419
 420         /* leave userspace return values as they were, so as not to break ABI */
 421         if (is_user) {
 422                 cap->max_recv_wr  = qp->rq.max_post = qp->rq.wqe_cnt;
 423                 cap->max_recv_sge = qp->rq.max_gs;
 424         } else {
 425                 cap->max_recv_wr  = qp->rq.max_post =
 426                         min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
 427                 cap->max_recv_sge = min(qp->rq.max_gs,
 428                                         min(dev->dev->caps.max_sq_sg,
 429                                             dev->dev->caps.max_rq_sg));
 430         }
 431
 432         return 0;
 433 }
 434
 435 static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
 436                               enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp,
 437                               bool shrink_wqe)
 438 {
 439         int s;
 440
 441         /* Sanity check SQ size before proceeding */
 442         if (cap->max_send_wr  > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
 443             cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
 444             cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
 445             sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
 446                 return -EINVAL;
 447
 448         /*
 449          * For MLX transport we need 2 extra S/G entries:
 450          * one for the header and one for the checksum at the end
 451          */
 452         if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
 453              type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
 454             cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
 455                 return -EINVAL;
 456
 457         s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
 458                 cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
 459                 send_wqe_overhead(type, qp->flags);
 460
 461         if (s > dev->dev->caps.max_sq_desc_sz)
 462                 return -EINVAL;
 463
 464         /*
 465          * Hermon supports shrinking WQEs, such that a single work
 466          * request can include multiple units of 1 << wqe_shift.  This
 467          * way, work requests can differ in size, and do not have to
 468          * be a power of 2 in size, saving memory and speeding up send
 469          * WR posting.  Unfortunately, if we do this then the
 470          * wqe_index field in CQEs can't be used to look up the WR ID
 471          * anymore, so we do this only if selective signaling is off.
 472          *
 473          * Further, on 32-bit platforms, we can't use vmap() to make
 474          * the QP buffer virtually contiguous.  Thus we have to use
 475          * constant-sized WRs to make sure a WR is always fully within
 476          * a single page-sized chunk.
 477          *
 478          * Finally, we use NOP work requests to pad the end of the
 479          * work queue, to avoid wrap-around in the middle of WR.  We
 480          * set NEC bit to avoid getting completions with error for
 481          * these NOP WRs, but since NEC is only supported starting
 482          * with firmware 2.2.232, we use constant-sized WRs for older
 483          * firmware.
 484          *
 485          * And, since MLX QPs only support SEND, we use constant-sized
 486          * WRs in this case.
 487          *
 488          * We look for the smallest value of wqe_shift such that the
 489          * resulting number of wqes does not exceed device
 490          * capabilities.
 491          *
 492          * We set WQE size to at least 64 bytes, this way stamping
 493          * invalidates each WQE.
 494          */
 495         if (shrink_wqe && dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
 496             qp->sq_signal_bits && BITS_PER_LONG == 64 &&
 497             type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI &&
 498             !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI |
 499                       MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER)))
 500                 qp->sq.wqe_shift = ilog2(64);
 501         else
 502                 qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
 503
 504         for (;;) {
 505                 qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
 506
 507                 /*
 508                  * We need to leave 2 KB + 1 WR of headroom in the SQ to
 509                  * allow HW to prefetch.
 510                  */
 511                 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
 512                 qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
 513                                                     qp->sq_max_wqes_per_wr +
 514                                                     qp->sq_spare_wqes);
 515
 516                 if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
 517                         break;
 518
 519                 if (qp->sq_max_wqes_per_wr <= 1)
 520                         return -EINVAL;
 521
 522                 ++qp->sq.wqe_shift;
 523         }
 524
 525         qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
 526                              (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
 527                          send_wqe_overhead(type, qp->flags)) /
 528                 sizeof (struct mlx4_wqe_data_seg);
 529
 530         qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 531                 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
 532         if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
 533                 qp->rq.offset = 0;
 534                 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 535         } else {
 536                 qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
 537                 qp->sq.offset = 0;
 538         }
 539
 540         cap->max_send_wr  = qp->sq.max_post =
 541                 (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
 542         cap->max_send_sge = min(qp->sq.max_gs,
 543                                 min(dev->dev->caps.max_sq_sg,
 544                                     dev->dev->caps.max_rq_sg));
 545         /* We don't support inline sends for kernel QPs (yet) */
 546         cap->max_inline_data = 0;
 547
 548         return 0;
 549 }
 550
 551 static int set_user_sq_size(struct mlx4_ib_dev *dev,
 552                             struct mlx4_ib_qp *qp,
 553                             struct mlx4_ib_create_qp *ucmd)
 554 {
 555         /* Sanity check SQ size before proceeding */
 556         if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes       ||
 557             ucmd->log_sq_stride >
 558                 ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
 559             ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
 560                 return -EINVAL;
 561
 562         qp->sq.wqe_cnt   = 1 << ucmd->log_sq_bb_count;
 563         qp->sq.wqe_shift = ucmd->log_sq_stride;
 564
 565         qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 566                 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
 567
 568         return 0;
 569 }
 570
 571 static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
 572 {
 573         int i;
 574
 575         qp->sqp_proxy_rcv =
 576                 kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt,
 577                         GFP_KERNEL);
 578         if (!qp->sqp_proxy_rcv)
 579                 return -ENOMEM;
 580         for (i = 0; i < qp->rq.wqe_cnt; i++) {
 581                 qp->sqp_proxy_rcv[i].addr =
 582                         kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
 583                                 GFP_KERNEL);
 584                 if (!qp->sqp_proxy_rcv[i].addr)
 585                         goto err;
 586                 qp->sqp_proxy_rcv[i].map =
 587                         ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
 588                                           sizeof (struct mlx4_ib_proxy_sqp_hdr),
 589                                           DMA_FROM_DEVICE);
 590                 if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) {
 591                         kfree(qp->sqp_proxy_rcv[i].addr);
 592                         goto err;
 593                 }
 594         }
 595         return 0;
 596
 597 err:
 598         while (i > 0) {
 599                 --i;
 600                 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
 601                                     sizeof (struct mlx4_ib_proxy_sqp_hdr),
 602                                     DMA_FROM_DEVICE);
 603                 kfree(qp->sqp_proxy_rcv[i].addr);
 604         }
 605         kfree(qp->sqp_proxy_rcv);
 606         qp->sqp_proxy_rcv = NULL;
 607         return -ENOMEM;
 608 }
 609
 610 static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
 611 {
 612         int i;
 613
 614         for (i = 0; i < qp->rq.wqe_cnt; i++) {
 615                 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
 616                                     sizeof (struct mlx4_ib_proxy_sqp_hdr),
 617                                     DMA_FROM_DEVICE);
 618                 kfree(qp->sqp_proxy_rcv[i].addr);
 619         }
 620         kfree(qp->sqp_proxy_rcv);
 621 }
 622
 623 static int qp_has_rq(struct ib_qp_init_attr *attr)
 624 {
 625         if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
 626                 return 0;
 627
 628         return !attr->srq;
 629 }
 630
 631 static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
 632 {
 633         int i;
 634         for (i = 0; i < dev->caps.num_ports; i++) {
 635                 if (qpn == dev->caps.spec_qps[i].qp0_proxy)
 636                         return !!dev->caps.spec_qps[i].qp0_qkey;
 637         }
 638         return 0;
 639 }
 640
 641 static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
 642                                     struct mlx4_ib_qp *qp)
 643 {
 644         mutex_lock(&dev->counters_table[qp->port - 1].mutex);
 645         mlx4_counter_free(dev->dev, qp->counter_index->index);
 646         list_del(&qp->counter_index->list);
 647         mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
 648
 649         kfree(qp->counter_index);
 650         qp->counter_index = NULL;
 651 }
 652
 653 static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
 654                       struct ib_qp_init_attr *init_attr,
 655                       struct mlx4_ib_create_qp_rss *ucmd)
 656 {
 657         rss_ctx->base_qpn_tbl_sz = init_attr->rwq_ind_tbl->ind_tbl[0]->wq_num |
 658                 (init_attr->rwq_ind_tbl->log_ind_tbl_size << 24);
 659
 660         if ((ucmd->rx_hash_function == MLX4_IB_RX_HASH_FUNC_TOEPLITZ) &&
 661             (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP)) {
 662                 memcpy(rss_ctx->rss_key, ucmd->rx_hash_key,
 663                        MLX4_EN_RSS_KEY_SIZE);
 664         } else {
 665                 pr_debug("RX Hash function is not supported\n");
 666                 return (-EOPNOTSUPP);
 667         }
 668
 669         if (ucmd->rx_hash_fields_mask & ~(MLX4_IB_RX_HASH_SRC_IPV4      |
 670                                           MLX4_IB_RX_HASH_DST_IPV4      |
 671                                           MLX4_IB_RX_HASH_SRC_IPV6      |
 672                                           MLX4_IB_RX_HASH_DST_IPV6      |
 673                                           MLX4_IB_RX_HASH_SRC_PORT_TCP  |
 674                                           MLX4_IB_RX_HASH_DST_PORT_TCP  |
 675                                           MLX4_IB_RX_HASH_SRC_PORT_UDP  |
 676                                           MLX4_IB_RX_HASH_DST_PORT_UDP)) {
 677                 pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",
 678                          ucmd->rx_hash_fields_mask);
 679                 return (-EOPNOTSUPP);
 680         }
 681
 682         if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) &&
 683             (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
 684                 rss_ctx->flags = MLX4_RSS_IPV4;
 685         } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) ||
 686                    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
 687                 pr_debug("RX Hash fields_mask is not supported - both IPv4 SRC and DST must be set\n");
 688                 return (-EOPNOTSUPP);
 689         }
 690
 691         if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) &&
 692             (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
 693                 rss_ctx->flags |= MLX4_RSS_IPV6;
 694         } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) ||
 695                    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
 696                 pr_debug("RX Hash fields_mask is not supported - both IPv6 SRC and DST must be set\n");
 697                 return (-EOPNOTSUPP);
 698         }
 699
 700         if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) &&
 701             (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
 702                 if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) {
 703                         pr_debug("RX Hash fields_mask for UDP is not supported\n");
 704                         return (-EOPNOTSUPP);
 705                 }
 706
 707                 if (rss_ctx->flags & MLX4_RSS_IPV4)
 708                         rss_ctx->flags |= MLX4_RSS_UDP_IPV4;
 709                 if (rss_ctx->flags & MLX4_RSS_IPV6)
 710                         rss_ctx->flags |= MLX4_RSS_UDP_IPV6;
 711                 if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
 712                         pr_debug("RX Hash fields_mask is not supported - UDP must be set with IPv4 or IPv6\n");
 713                         return (-EOPNOTSUPP);
 714                 }
 715         } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) ||
 716                    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
 717                 pr_debug("RX Hash fields_mask is not supported - both UDP SRC and DST must be set\n");
 718                 return (-EOPNOTSUPP);
 719         }
 720
 721         if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) &&
 722             (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
 723                 if (rss_ctx->flags & MLX4_RSS_IPV4)
 724                         rss_ctx->flags |= MLX4_RSS_TCP_IPV4;
 725                 if (rss_ctx->flags & MLX4_RSS_IPV6)
 726                         rss_ctx->flags |= MLX4_RSS_TCP_IPV6;
 727                 if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
 728                         pr_debug("RX Hash fields_mask is not supported - TCP must be set with IPv4 or IPv6\n");
 729                         return (-EOPNOTSUPP);
 730                 }
 731         } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) ||
 732                    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
 733                 pr_debug("RX Hash fields_mask is not supported - both TCP SRC and DST must be set\n");
 734                 return (-EOPNOTSUPP);
 735         }
 736
 737         if (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_INNER) {
 738                 if (dev->dev->caps.tunnel_offload_mode ==
 739                     MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
 740                         /*
 741                          * Hash according to inner headers if exist, otherwise
 742                          * according to outer headers.
 743                          */
 744                         rss_ctx->flags |= MLX4_RSS_BY_INNER_HEADERS_IPONLY;
 745                 } else {
 746                         pr_debug("RSS Hash for inner headers isn't supported\n");
 747                         return (-EOPNOTSUPP);
 748                 }
 749         }
 750
 751         return 0;
 752 }
 753
 754 static int create_qp_rss(struct mlx4_ib_dev *dev,
 755                          struct ib_qp_init_attr *init_attr,
 756                          struct mlx4_ib_create_qp_rss *ucmd,
 757                          struct mlx4_ib_qp *qp)
 758 {
 759         int qpn;
 760         int err;
 761
 762         qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
 763
 764         err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage);
 765         if (err)
 766                 return err;
 767
 768         err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
 769         if (err)
 770                 goto err_qpn;
 771
 772         mutex_init(&qp->mutex);
 773
 774         INIT_LIST_HEAD(&qp->gid_list);
 775         INIT_LIST_HEAD(&qp->steering_rules);
 776
 777         qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
 778         qp->state = IB_QPS_RESET;
 779
 780         /* Set dummy send resources to be compatible with HV and PRM */
 781         qp->sq_no_prefetch = 1;
 782         qp->sq.wqe_cnt = 1;
 783         qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
 784         qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE;
 785         qp->mtt = (to_mqp(
 786                    (struct ib_qp *)init_attr->rwq_ind_tbl->ind_tbl[0]))->mtt;
 787
 788         qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL);
 789         if (!qp->rss_ctx) {
 790                 err = -ENOMEM;
 791                 goto err_qp_alloc;
 792         }
 793
 794         err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd);
 795         if (err)
 796                 goto err;
 797
 798         return 0;
 799
 800 err:
 801         kfree(qp->rss_ctx);
 802
 803 err_qp_alloc:
 804         mlx4_qp_remove(dev->dev, &qp->mqp);
 805         mlx4_qp_free(dev->dev, &qp->mqp);
 806
 807 err_qpn:
 808         mlx4_qp_release_range(dev->dev, qpn, 1);
 809         return err;
 810 }
 811
 812 static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,
 813                                             struct ib_qp_init_attr *init_attr,
 814                                             struct ib_udata *udata)
 815 {
 816         struct mlx4_ib_qp *qp;
 817         struct mlx4_ib_create_qp_rss ucmd = {};
 818         size_t required_cmd_sz;
 819         int err;
 820
 821         if (!udata) {
 822                 pr_debug("RSS QP with NULL udata\n");
 823                 return ERR_PTR(-EINVAL);
 824         }
 825
 826         if (udata->outlen)
 827                 return ERR_PTR(-EOPNOTSUPP);
 828
 829         required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
 830                                         sizeof(ucmd.reserved1);
 831         if (udata->inlen < required_cmd_sz) {
 832                 pr_debug("invalid inlen\n");
 833                 return ERR_PTR(-EINVAL);
 834         }
 835
 836         if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
 837                 pr_debug("copy failed\n");
 838                 return ERR_PTR(-EFAULT);
 839         }
 840
 841         if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
 842                 return ERR_PTR(-EOPNOTSUPP);
 843
 844         if (ucmd.comp_mask || ucmd.reserved1)
 845                 return ERR_PTR(-EOPNOTSUPP);
 846
 847         if (udata->inlen > sizeof(ucmd) &&
 848             !ib_is_udata_cleared(udata, sizeof(ucmd),
 849                                  udata->inlen - sizeof(ucmd))) {
 850                 pr_debug("inlen is not supported\n");
 851                 return ERR_PTR(-EOPNOTSUPP);
 852         }
 853
 854         if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
 855                 pr_debug("RSS QP with unsupported QP type %d\n",
 856                          init_attr->qp_type);
 857                 return ERR_PTR(-EOPNOTSUPP);
 858         }
 859
 860         if (init_attr->create_flags) {
 861                 pr_debug("RSS QP doesn't support create flags\n");
 862                 return ERR_PTR(-EOPNOTSUPP);
 863         }
 864
 865         if (init_attr->send_cq || init_attr->cap.max_send_wr) {
 866                 pr_debug("RSS QP with unsupported send attributes\n");
 867                 return ERR_PTR(-EOPNOTSUPP);
 868         }
 869
 870         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
 871         if (!qp)
 872                 return ERR_PTR(-ENOMEM);
 873
 874         qp->pri.vid = 0xFFFF;
 875         qp->alt.vid = 0xFFFF;
 876
 877         err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
 878         if (err) {
 879                 kfree(qp);
 880                 return ERR_PTR(err);
 881         }
 882
 883         qp->ibqp.qp_num = qp->mqp.qpn;
 884
 885         return &qp->ibqp;
 886 }
 887
 888 /*
 889  * This function allocates a WQN from a range which is consecutive and aligned
 890  * to its size. In case the range is full, then it creates a new range and
 891  * allocates WQN from it. The new range will be used for following allocations.
 892  */
 893 static int mlx4_ib_alloc_wqn(struct mlx4_ib_ucontext *context,
 894                              struct mlx4_ib_qp *qp, int range_size, int *wqn)
 895 {
 896         struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
 897         struct mlx4_wqn_range *range;
 898         int err = 0;
 899
 900         mutex_lock(&context->wqn_ranges_mutex);
 901
 902         range = list_first_entry_or_null(&context->wqn_ranges_list,
 903                                          struct mlx4_wqn_range, list);
 904
 905         if (!range || (range->refcount == range->size) || range->dirty) {
 906                 range = kzalloc(sizeof(*range), GFP_KERNEL);
 907                 if (!range) {
 908                         err = -ENOMEM;
 909                         goto out;
 910                 }
 911
 912                 err = mlx4_qp_reserve_range(dev->dev, range_size,
 913                                             range_size, &range->base_wqn, 0,
 914                                             qp->mqp.usage);
 915                 if (err) {
 916                         kfree(range);
 917                         goto out;
 918                 }
 919
 920                 range->size = range_size;
 921                 list_add(&range->list, &context->wqn_ranges_list);
 922         } else if (range_size != 1) {
 923                 /*
 924                  * Requesting a new range (>1) when last range is still open, is
 925                  * not valid.
 926                  */
 927                 err = -EINVAL;
 928                 goto out;
 929         }
 930
 931         qp->wqn_range = range;
 932
 933         *wqn = range->base_wqn + range->refcount;
 934
 935         range->refcount++;
 936
 937 out:
 938         mutex_unlock(&context->wqn_ranges_mutex);
 939
 940         return err;
 941 }
 942
 943 static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,
 944                                 struct mlx4_ib_qp *qp, bool dirty_release)
 945 {
 946         struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
 947         struct mlx4_wqn_range *range;
 948
 949         mutex_lock(&context->wqn_ranges_mutex);
 950
 951         range = qp->wqn_range;
 952
 953         range->refcount--;
 954         if (!range->refcount) {
 955                 mlx4_qp_release_range(dev->dev, range->base_wqn,
 956                                       range->size);
 957                 list_del(&range->list);
 958                 kfree(range);
 959         } else if (dirty_release) {
 960         /*
 961          * A range which one of its WQNs is destroyed, won't be able to be
 962          * reused for further WQN allocations.
 963          * The next created WQ will allocate a new range.
 964          */
 965                 range->dirty = 1;
 966         }
 967
 968         mutex_unlock(&context->wqn_ranges_mutex);
 969 }
 970
 971 static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 972                             enum mlx4_ib_source_type src,
 973                             struct ib_qp_init_attr *init_attr,
 974                             struct ib_udata *udata, int sqpn,
 975                             struct mlx4_ib_qp **caller_qp)
 976 {
 977         int qpn;
 978         int err;
 979         struct ib_qp_cap backup_cap;
 980         struct mlx4_ib_sqp *sqp = NULL;
 981         struct mlx4_ib_qp *qp;
 982         enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
 983         struct mlx4_ib_cq *mcq;
 984         unsigned long flags;
 985         int range_size = 0;
 986
 987         /* When tunneling special qps, we use a plain UD qp */
 988         if (sqpn) {
 989                 if (mlx4_is_mfunc(dev->dev) &&
 990                     (!mlx4_is_master(dev->dev) ||
 991                      !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
 992                         if (init_attr->qp_type == IB_QPT_GSI)
 993                                 qp_type = MLX4_IB_QPT_PROXY_GSI;
 994                         else {
 995                                 if (mlx4_is_master(dev->dev) ||
 996                                     qp0_enabled_vf(dev->dev, sqpn))
 997                                         qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
 998                                 else
 999                                         qp_type = MLX4_IB_QPT_PROXY_SMI;
1000                         }
1001                 }
1002                 qpn = sqpn;
1003                 /* add extra sg entry for tunneling */
1004                 init_attr->cap.max_recv_sge++;
1005         } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
1006                 struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
1007                         container_of(init_attr,
1008                                      struct mlx4_ib_qp_tunnel_init_attr, init_attr);
1009                 if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
1010                      tnl_init->proxy_qp_type != IB_QPT_GSI)   ||
1011                     !mlx4_is_master(dev->dev))
1012                         return -EINVAL;
1013                 if (tnl_init->proxy_qp_type == IB_QPT_GSI)
1014                         qp_type = MLX4_IB_QPT_TUN_GSI;
1015                 else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
1016                          mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
1017                                              tnl_init->port))
1018                         qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
1019                 else
1020                         qp_type = MLX4_IB_QPT_TUN_SMI;
1021                 /* we are definitely in the PPF here, since we are creating
1022                  * tunnel QPs. base_tunnel_sqpn is therefore valid. */
1023                 qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
1024                         + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
1025                 sqpn = qpn;
1026         }
1027
1028         if (!*caller_qp) {
1029                 if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||
1030                     (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
1031                                 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
1032                         sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
1033                         if (!sqp)
1034                                 return -ENOMEM;
1035                         qp = &sqp->qp;
1036                         qp->pri.vid = 0xFFFF;
1037                         qp->alt.vid = 0xFFFF;
1038                 } else {
1039                         qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);
1040                         if (!qp)
1041                                 return -ENOMEM;
1042                         qp->pri.vid = 0xFFFF;
1043                         qp->alt.vid = 0xFFFF;
1044                 }
1045         } else
1046                 qp = *caller_qp;
1047
1048         qp->mlx4_ib_qp_type = qp_type;
1049
1050         mutex_init(&qp->mutex);
1051         spin_lock_init(&qp->sq.lock);
1052         spin_lock_init(&qp->rq.lock);
1053         INIT_LIST_HEAD(&qp->gid_list);
1054         INIT_LIST_HEAD(&qp->steering_rules);
1055
1056         qp->state        = IB_QPS_RESET;
1057         if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1058                 qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1059
1060
1061         if (pd->uobject) {
1062                 union {
1063                         struct mlx4_ib_create_qp qp;
1064                         struct mlx4_ib_create_wq wq;
1065                 } ucmd;
1066                 size_t copy_len;
1067                 int shift;
1068                 int n;
1069
1070                 copy_len = (src == MLX4_IB_QP_SRC) ?
1071                            sizeof(struct mlx4_ib_create_qp) :
1072                            min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
1073
1074                 if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
1075                         err = -EFAULT;
1076                         goto err;
1077                 }
1078
1079                 if (src == MLX4_IB_RWQ_SRC) {
1080                         if (ucmd.wq.comp_mask || ucmd.wq.reserved[0] ||
1081                             ucmd.wq.reserved[1] || ucmd.wq.reserved[2]) {
1082                                 pr_debug("user command isn't supported\n");
1083                                 err = -EOPNOTSUPP;
1084                                 goto err;
1085                         }
1086
1087                         if (ucmd.wq.log_range_size >
1088                             ilog2(dev->dev->caps.max_rss_tbl_sz)) {
1089                                 pr_debug("WQN range size must be equal or smaller than %d\n",
1090                                          dev->dev->caps.max_rss_tbl_sz);
1091                                 err = -EOPNOTSUPP;
1092                                 goto err;
1093                         }
1094                         range_size = 1 << ucmd.wq.log_range_size;
1095                 } else {
1096                         qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
1097                 }
1098
1099                 if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1100                         if (!(dev->dev->caps.flags &
1101                               MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
1102                                 pr_debug("scatter FCS is unsupported\n");
1103                                 err = -EOPNOTSUPP;
1104                                 goto err;
1105                         }
1106
1107                         qp->flags |= MLX4_IB_QP_SCATTER_FCS;
1108                 }
1109
1110                 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
1111                                   qp_has_rq(init_attr), qp, qp->inl_recv_sz);
1112                 if (err)
1113                         goto err;
1114
1115                 if (src == MLX4_IB_QP_SRC) {
1116                         qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;
1117
1118                         err = set_user_sq_size(dev, qp,
1119                                                (struct mlx4_ib_create_qp *)
1120                                                &ucmd);
1121                         if (err)
1122                                 goto err;
1123                 } else {
1124                         qp->sq_no_prefetch = 1;
1125                         qp->sq.wqe_cnt = 1;
1126                         qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
1127                         /* Allocated buffer expects to have at least that SQ
1128                          * size.
1129                          */
1130                         qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
1131                                 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
1132                 }
1133
1134                 qp->umem = ib_umem_get(pd->uobject->context,
1135                                 (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
1136                                 ucmd.wq.buf_addr, qp->buf_size, 0, 0);
1137                 if (IS_ERR(qp->umem)) {
1138                         err = PTR_ERR(qp->umem);
1139                         goto err;
1140                 }
1141
1142                 n = ib_umem_page_count(qp->umem);
1143                 shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
1144                 err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
1145
1146                 if (err)
1147                         goto err_buf;
1148
1149                 err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
1150                 if (err)
1151                         goto err_mtt;
1152
1153                 if (qp_has_rq(init_attr)) {
1154                         err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
1155                                 (src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
1156                                 ucmd.wq.db_addr, &qp->db);
1157                         if (err)
1158                                 goto err_mtt;
1159                 }
1160                 qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
1161         } else {
1162                 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
1163                                   qp_has_rq(init_attr), qp, 0);
1164                 if (err)
1165                         goto err;
1166
1167                 qp->sq_no_prefetch = 0;
1168
1169                 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
1170                         qp->flags |= MLX4_IB_QP_LSO;
1171
1172                 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1173                         if (dev->steering_support ==
1174                             MLX4_STEERING_MODE_DEVICE_MANAGED)
1175                                 qp->flags |= MLX4_IB_QP_NETIF;
1176                         else
1177                                 goto err;
1178                 }
1179
1180                 memcpy(&backup_cap, &init_attr->cap, sizeof(backup_cap));
1181                 err = set_kernel_sq_size(dev, &init_attr->cap,
1182                                          qp_type, qp, true);
1183                 if (err)
1184                         goto err;
1185
1186                 if (qp_has_rq(init_attr)) {
1187                         err = mlx4_db_alloc(dev->dev, &qp->db, 0);
1188                         if (err)
1189                                 goto err;
1190
1191                         *qp->db.db = 0;
1192                 }
1193
1194                 if (mlx4_buf_alloc(dev->dev, qp->buf_size, qp->buf_size,
1195                                    &qp->buf)) {
1196                         memcpy(&init_attr->cap, &backup_cap,
1197                                sizeof(backup_cap));
1198                         err = set_kernel_sq_size(dev, &init_attr->cap, qp_type,
1199                                                  qp, false);
1200                         if (err)
1201                                 goto err_db;
1202
1203                         if (mlx4_buf_alloc(dev->dev, qp->buf_size,
1204                                            PAGE_SIZE * 2, &qp->buf)) {
1205                                 err = -ENOMEM;
1206                                 goto err_db;
1207                         }
1208                 }
1209
1210                 err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
1211                                     &qp->mtt);
1212                 if (err)
1213                         goto err_buf;
1214
1215                 err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
1216                 if (err)
1217                         goto err_mtt;
1218
1219                 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1220                                              sizeof(u64), GFP_KERNEL);
1221                 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1222                                              sizeof(u64), GFP_KERNEL);
1223                 if (!qp->sq.wrid || !qp->rq.wrid) {
1224                         err = -ENOMEM;
1225                         goto err_wrid;
1226                 }
1227                 qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
1228         }
1229
1230         if (sqpn) {
1231                 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1232                     MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
1233                         if (alloc_proxy_bufs(pd->device, qp)) {
1234                                 err = -ENOMEM;
1235                                 goto err_wrid;
1236                         }
1237                 }
1238         } else if (src == MLX4_IB_RWQ_SRC) {
1239                 err = mlx4_ib_alloc_wqn(to_mucontext(pd->uobject->context), qp,
1240                                         range_size, &qpn);
1241                 if (err)
1242                         goto err_wrid;
1243         } else {
1244                 /* Raw packet QPNs may not have bits 6,7 set in their qp_num;
1245                  * otherwise, the WQE BlueFlame setup flow wrongly causes
1246                  * VLAN insertion. */
1247                 if (init_attr->qp_type == IB_QPT_RAW_PACKET)
1248                         err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
1249                                                     (init_attr->cap.max_send_wr ?
1250                                                      MLX4_RESERVE_ETH_BF_QP : 0) |
1251                                                     (init_attr->cap.max_recv_wr ?
1252                                                      MLX4_RESERVE_A0_QP : 0),
1253                                                     qp->mqp.usage);
1254                 else
1255                         if (qp->flags & MLX4_IB_QP_NETIF)
1256                                 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
1257                         else
1258                                 err = mlx4_qp_reserve_range(dev->dev, 1, 1,
1259                                                             &qpn, 0, qp->mqp.usage);
1260                 if (err)
1261                         goto err_proxy;
1262         }
1263
1264         if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
1265                 qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1266
1267         err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
1268         if (err)
1269                 goto err_qpn;
1270
1271         if (init_attr->qp_type == IB_QPT_XRC_TGT)
1272                 qp->mqp.qpn |= (1 << 23);
1273
1274         /*
1275          * Hardware wants QPN written in big-endian order (after
1276          * shifting) for send doorbell.  Precompute this value to save
1277          * a little bit when posting sends.
1278          */
1279         qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
1280
1281         qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
1282                                                   mlx4_ib_wq_event;
1283
1284         if (!*caller_qp)
1285                 *caller_qp = qp;
1286
1287         spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1288         mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
1289                          to_mcq(init_attr->recv_cq));
1290         /* Maintain device to QPs access, needed for further handling
1291          * via reset flow
1292          */
1293         list_add_tail(&qp->qps_list, &dev->qp_list);
1294         /* Maintain CQ to QPs access, needed for further handling
1295          * via reset flow
1296          */
1297         mcq = to_mcq(init_attr->send_cq);
1298         list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
1299         mcq = to_mcq(init_attr->recv_cq);
1300         list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
1301         mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
1302                            to_mcq(init_attr->recv_cq));
1303         spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1304         return 0;
1305
1306 err_qpn:
1307         if (!sqpn) {
1308                 if (qp->flags & MLX4_IB_QP_NETIF)
1309                         mlx4_ib_steer_qp_free(dev, qpn, 1);
1310                 else if (src == MLX4_IB_RWQ_SRC)
1311                         mlx4_ib_release_wqn(to_mucontext(pd->uobject->context),
1312                                             qp, 0);
1313                 else
1314                         mlx4_qp_release_range(dev->dev, qpn, 1);
1315         }
1316 err_proxy:
1317         if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1318                 free_proxy_bufs(pd->device, qp);
1319 err_wrid:
1320         if (pd->uobject) {
1321                 if (qp_has_rq(init_attr))
1322                         mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
1323         } else {
1324                 kvfree(qp->sq.wrid);
1325                 kvfree(qp->rq.wrid);
1326         }
1327
1328 err_mtt:
1329         mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1330
1331 err_buf:
1332         if (pd->uobject)
1333                 ib_umem_release(qp->umem);
1334         else
1335                 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1336
1337 err_db:
1338         if (!pd->uobject && qp_has_rq(init_attr))
1339                 mlx4_db_free(dev->dev, &qp->db);
1340
1341 err:
1342         if (sqp)
1343                 kfree(sqp);
1344         else if (!*caller_qp)
1345                 kfree(qp);
1346         return err;
1347 }
1348
1349 static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
1350 {
1351         switch (state) {
1352         case IB_QPS_RESET:      return MLX4_QP_STATE_RST;
1353         case IB_QPS_INIT:       return MLX4_QP_STATE_INIT;
1354         case IB_QPS_RTR:        return MLX4_QP_STATE_RTR;
1355         case IB_QPS_RTS:        return MLX4_QP_STATE_RTS;
1356         case IB_QPS_SQD:        return MLX4_QP_STATE_SQD;
1357         case IB_QPS_SQE:        return MLX4_QP_STATE_SQER;
1358         case IB_QPS_ERR:        return MLX4_QP_STATE_ERR;
1359         default:                return -1;
1360         }
1361 }
1362
1363 static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1364         __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1365 {
1366         if (send_cq == recv_cq) {
1367                 spin_lock(&send_cq->lock);
1368                 __acquire(&recv_cq->lock);
1369         } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1370                 spin_lock(&send_cq->lock);
1371                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1372         } else {
1373                 spin_lock(&recv_cq->lock);
1374                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1375         }
1376 }
1377
1378 static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1379         __releases(&send_cq->lock) __releases(&recv_cq->lock)
1380 {
1381         if (send_cq == recv_cq) {
1382                 __release(&recv_cq->lock);
1383                 spin_unlock(&send_cq->lock);
1384         } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1385                 spin_unlock(&recv_cq->lock);
1386                 spin_unlock(&send_cq->lock);
1387         } else {
1388                 spin_unlock(&send_cq->lock);
1389                 spin_unlock(&recv_cq->lock);
1390         }
1391 }
1392
1393 static void del_gid_entries(struct mlx4_ib_qp *qp)
1394 {
1395         struct mlx4_ib_gid_entry *ge, *tmp;
1396
1397         list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1398                 list_del(&ge->list);
1399                 kfree(ge);
1400         }
1401 }
1402
1403 static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
1404 {
1405         if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
1406                 return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
1407         else
1408                 return to_mpd(qp->ibqp.pd);
1409 }
1410
1411 static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src,
1412                     struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
1413 {
1414         switch (qp->ibqp.qp_type) {
1415         case IB_QPT_XRC_TGT:
1416                 *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
1417                 *recv_cq = *send_cq;
1418                 break;
1419         case IB_QPT_XRC_INI:
1420                 *send_cq = to_mcq(qp->ibqp.send_cq);
1421                 *recv_cq = *send_cq;
1422                 break;
1423         default:
1424                 *recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) :
1425                                                      to_mcq(qp->ibwq.cq);
1426                 *send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) :
1427                                                      *recv_cq;
1428                 break;
1429         }
1430 }
1431
1432 static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1433 {
1434         if (qp->state != IB_QPS_RESET) {
1435                 int i;
1436
1437                 for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size);
1438                      i++) {
1439                         struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i];
1440                         struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
1441
1442                         mutex_lock(&wq->mutex);
1443
1444                         wq->rss_usecnt--;
1445
1446                         mutex_unlock(&wq->mutex);
1447                 }
1448
1449                 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1450                                    MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1451                         pr_warn("modify QP %06x to RESET failed.\n",
1452                                 qp->mqp.qpn);
1453         }
1454
1455         mlx4_qp_remove(dev->dev, &qp->mqp);
1456         mlx4_qp_free(dev->dev, &qp->mqp);
1457         mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1458         del_gid_entries(qp);
1459         kfree(qp->rss_ctx);
1460 }
1461
1462 static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
1463                               enum mlx4_ib_source_type src, int is_user)
1464 {
1465         struct mlx4_ib_cq *send_cq, *recv_cq;
1466         unsigned long flags;
1467
1468         if (qp->state != IB_QPS_RESET) {
1469                 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1470                                    MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1471                         pr_warn("modify QP %06x to RESET failed.\n",
1472                                qp->mqp.qpn);
1473                 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
1474                         mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1475                         qp->pri.smac = 0;
1476                         qp->pri.smac_port = 0;
1477                 }
1478                 if (qp->alt.smac) {
1479                         mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1480                         qp->alt.smac = 0;
1481                 }
1482                 if (qp->pri.vid < 0x1000) {
1483                         mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
1484                         qp->pri.vid = 0xFFFF;
1485                         qp->pri.candidate_vid = 0xFFFF;
1486                         qp->pri.update_vid = 0;
1487                 }
1488                 if (qp->alt.vid < 0x1000) {
1489                         mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
1490                         qp->alt.vid = 0xFFFF;
1491                         qp->alt.candidate_vid = 0xFFFF;
1492                         qp->alt.update_vid = 0;
1493                 }
1494         }
1495
1496         get_cqs(qp, src, &send_cq, &recv_cq);
1497
1498         spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1499         mlx4_ib_lock_cqs(send_cq, recv_cq);
1500
1501         /* del from lists under both locks above to protect reset flow paths */
1502         list_del(&qp->qps_list);
1503         list_del(&qp->cq_send_list);
1504         list_del(&qp->cq_recv_list);
1505         if (!is_user) {
1506                 __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1507                                  qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1508                 if (send_cq != recv_cq)
1509                         __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1510         }
1511
1512         mlx4_qp_remove(dev->dev, &qp->mqp);
1513
1514         mlx4_ib_unlock_cqs(send_cq, recv_cq);
1515         spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1516
1517         mlx4_qp_free(dev->dev, &qp->mqp);
1518
1519         if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
1520                 if (qp->flags & MLX4_IB_QP_NETIF)
1521                         mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1522                 else if (src == MLX4_IB_RWQ_SRC)
1523                         mlx4_ib_release_wqn(to_mucontext(
1524                                             qp->ibwq.uobject->context), qp, 1);
1525                 else
1526                         mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1527         }
1528
1529         mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1530
1531         if (is_user) {
1532                 if (qp->rq.wqe_cnt) {
1533                         struct mlx4_ib_ucontext *mcontext = !src ?
1534                                 to_mucontext(qp->ibqp.uobject->context) :
1535                                 to_mucontext(qp->ibwq.uobject->context);
1536                         mlx4_ib_db_unmap_user(mcontext, &qp->db);
1537                 }
1538                 ib_umem_release(qp->umem);
1539         } else {
1540                 kvfree(qp->sq.wrid);
1541                 kvfree(qp->rq.wrid);
1542                 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1543                     MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
1544                         free_proxy_bufs(&dev->ib_dev, qp);
1545                 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1546                 if (qp->rq.wqe_cnt)
1547                         mlx4_db_free(dev->dev, &qp->db);
1548         }
1549
1550         del_gid_entries(qp);
1551 }
1552
1553 static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
1554 {
1555         /* Native or PPF */
1556         if (!mlx4_is_mfunc(dev->dev) ||
1557             (mlx4_is_master(dev->dev) &&
1558              attr->create_flags & MLX4_IB_SRIOV_SQP)) {
1559                 return  dev->dev->phys_caps.base_sqpn +
1560                         (attr->qp_type == IB_QPT_SMI ? 0 : 2) +
1561                         attr->port_num - 1;
1562         }
1563         /* PF or VF -- creating proxies */
1564         if (attr->qp_type == IB_QPT_SMI)
1565                 return dev->dev->caps.spec_qps[attr->port_num - 1].qp0_proxy;
1566         else
1567                 return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
1568 }
1569
1570 static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
1571                                         struct ib_qp_init_attr *init_attr,
1572                                         struct ib_udata *udata)
1573 {
1574         struct mlx4_ib_qp *qp = NULL;
1575         int err;
1576         int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1577         u16 xrcdn = 0;
1578
1579         if (init_attr->rwq_ind_tbl)
1580                 return _mlx4_ib_create_qp_rss(pd, init_attr, udata);
1581
1582         /*
1583          * We only support LSO, vendor flag1, and multicast loopback blocking,
1584          * and only for kernel UD QPs.
1585          */
1586         if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
1587                                         MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
1588                                         MLX4_IB_SRIOV_TUNNEL_QP |
1589                                         MLX4_IB_SRIOV_SQP |
1590                                         MLX4_IB_QP_NETIF |
1591                                         MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1592                 return ERR_PTR(-EINVAL);
1593
1594         if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1595                 if (init_attr->qp_type != IB_QPT_UD)
1596                         return ERR_PTR(-EINVAL);
1597         }
1598
1599         if (init_attr->create_flags) {
1600                 if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1601                         return ERR_PTR(-EINVAL);
1602
1603                 if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
1604                                                  MLX4_IB_QP_CREATE_ROCE_V2_GSI  |
1605                                                  MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
1606                      init_attr->qp_type != IB_QPT_UD) ||
1607                     (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
1608                      init_attr->qp_type > IB_QPT_GSI) ||
1609                     (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1610                      init_attr->qp_type != IB_QPT_GSI))
1611                         return ERR_PTR(-EINVAL);
1612         }
1613
1614         switch (init_attr->qp_type) {
1615         case IB_QPT_XRC_TGT:
1616                 pd = to_mxrcd(init_attr->xrcd)->pd;
1617                 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1618                 init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1619                 /* fall through */
1620         case IB_QPT_XRC_INI:
1621                 if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1622                         return ERR_PTR(-ENOSYS);
1623                 init_attr->recv_cq = init_attr->send_cq;
1624                 /* fall through */
1625         case IB_QPT_RC:
1626         case IB_QPT_UC:
1627         case IB_QPT_RAW_PACKET:
1628                 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1629                 if (!qp)
1630                         return ERR_PTR(-ENOMEM);
1631                 qp->pri.vid = 0xFFFF;
1632                 qp->alt.vid = 0xFFFF;
1633                 /* fall through */
1634         case IB_QPT_UD:
1635         {
1636                 err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1637                                        init_attr, udata, 0, &qp);
1638                 if (err) {
1639                         kfree(qp);
1640                         return ERR_PTR(err);
1641                 }
1642
1643                 qp->ibqp.qp_num = qp->mqp.qpn;
1644                 qp->xrcdn = xrcdn;
1645
1646                 break;
1647         }
1648         case IB_QPT_SMI:
1649         case IB_QPT_GSI:
1650         {
1651                 int sqpn;
1652
1653                 /* Userspace is not allowed to create special QPs: */
1654                 if (udata)
1655                         return ERR_PTR(-EINVAL);
1656                 if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1657                         int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1658                                                         1, 1, &sqpn, 0,
1659                                                         MLX4_RES_USAGE_DRIVER);
1660
1661                         if (res)
1662                                 return ERR_PTR(res);
1663                 } else {
1664                         sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1665                 }
1666
1667                 err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1668                                        init_attr, udata, sqpn, &qp);
1669                 if (err)
1670                         return ERR_PTR(err);
1671
1672                 qp->port        = init_attr->port_num;
1673                 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
1674                         init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
1675                 break;
1676         }
1677         default:
1678                 /* Don't support raw QPs */
1679                 return ERR_PTR(-EINVAL);
1680         }
1681
1682         return &qp->ibqp;
1683 }
1684
1685 struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
1686                                 struct ib_qp_init_attr *init_attr,
1687                                 struct ib_udata *udata) {
1688         struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
1689         struct ib_qp *ibqp;
1690         struct mlx4_ib_dev *dev = to_mdev(device);
1691
1692         ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
1693
1694         if (!IS_ERR(ibqp) &&
1695             (init_attr->qp_type == IB_QPT_GSI) &&
1696             !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1697                 struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
1698                 int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1699
1700                 if (is_eth &&
1701                     dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
1702                         init_attr->create_flags |= MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1703                         sqp->roce_v2_gsi = ib_create_qp(pd, init_attr);
1704
1705                         if (IS_ERR(sqp->roce_v2_gsi)) {
1706                                 pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1707                                 sqp->roce_v2_gsi = NULL;
1708                         } else {
1709                                 sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
1710                                 sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP;
1711                         }
1712
1713                         init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1714                 }
1715         }
1716         return ibqp;
1717 }
1718
1719 static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
1720 {
1721         struct mlx4_ib_dev *dev = to_mdev(qp->device);
1722         struct mlx4_ib_qp *mqp = to_mqp(qp);
1723
1724         if (is_qp0(dev, mqp))
1725                 mlx4_CLOSE_PORT(dev->dev, mqp->port);
1726
1727         if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI &&
1728             dev->qp1_proxy[mqp->port - 1] == mqp) {
1729                 mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
1730                 dev->qp1_proxy[mqp->port - 1] = NULL;
1731                 mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
1732         }
1733
1734         if (mqp->counter_index)
1735                 mlx4_ib_free_qp_counter(dev, mqp);
1736
1737         if (qp->rwq_ind_tbl) {
1738                 destroy_qp_rss(dev, mqp);
1739         } else {
1740                 struct mlx4_ib_pd *pd;
1741
1742                 pd = get_pd(mqp);
1743                 destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, !!pd->ibpd.uobject);
1744         }
1745
1746         if (is_sqp(dev, mqp))
1747                 kfree(to_msqp(mqp));
1748         else
1749                 kfree(mqp);
1750
1751         return 0;
1752 }
1753
1754 int mlx4_ib_destroy_qp(struct ib_qp *qp)
1755 {
1756         struct mlx4_ib_qp *mqp = to_mqp(qp);
1757
1758         if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1759                 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
1760
1761                 if (sqp->roce_v2_gsi)
1762                         ib_destroy_qp(sqp->roce_v2_gsi);
1763         }
1764
1765         return _mlx4_ib_destroy_qp(qp);
1766 }
1767
1768 static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
1769 {
1770         switch (type) {
1771         case MLX4_IB_QPT_RC:            return MLX4_QP_ST_RC;
1772         case MLX4_IB_QPT_UC:            return MLX4_QP_ST_UC;
1773         case MLX4_IB_QPT_UD:            return MLX4_QP_ST_UD;
1774         case MLX4_IB_QPT_XRC_INI:
1775         case MLX4_IB_QPT_XRC_TGT:       return MLX4_QP_ST_XRC;
1776         case MLX4_IB_QPT_SMI:
1777         case MLX4_IB_QPT_GSI:
1778         case MLX4_IB_QPT_RAW_PACKET:    return MLX4_QP_ST_MLX;
1779
1780         case MLX4_IB_QPT_PROXY_SMI_OWNER:
1781         case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ?
1782                                                 MLX4_QP_ST_MLX : -1);
1783         case MLX4_IB_QPT_PROXY_SMI:
1784         case MLX4_IB_QPT_TUN_SMI:
1785         case MLX4_IB_QPT_PROXY_GSI:
1786         case MLX4_IB_QPT_TUN_GSI:       return (mlx4_is_mfunc(dev->dev) ?
1787                                                 MLX4_QP_ST_UD : -1);
1788         default:                        return -1;
1789         }
1790 }
1791
1792 static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
1793                                    int attr_mask)
1794 {
1795         u8 dest_rd_atomic;
1796         u32 access_flags;
1797         u32 hw_access_flags = 0;
1798
1799         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1800                 dest_rd_atomic = attr->max_dest_rd_atomic;
1801         else
1802                 dest_rd_atomic = qp->resp_depth;
1803
1804         if (attr_mask & IB_QP_ACCESS_FLAGS)
1805                 access_flags = attr->qp_access_flags;
1806         else
1807                 access_flags = qp->atomic_rd_en;
1808
1809         if (!dest_rd_atomic)
1810                 access_flags &= IB_ACCESS_REMOTE_WRITE;
1811
1812         if (access_flags & IB_ACCESS_REMOTE_READ)
1813                 hw_access_flags |= MLX4_QP_BIT_RRE;
1814         if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
1815                 hw_access_flags |= MLX4_QP_BIT_RAE;
1816         if (access_flags & IB_ACCESS_REMOTE_WRITE)
1817                 hw_access_flags |= MLX4_QP_BIT_RWE;
1818
1819         return cpu_to_be32(hw_access_flags);
1820 }
1821
1822 static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
1823                             int attr_mask)
1824 {
1825         if (attr_mask & IB_QP_PKEY_INDEX)
1826                 sqp->pkey_index = attr->pkey_index;
1827         if (attr_mask & IB_QP_QKEY)
1828                 sqp->qkey = attr->qkey;
1829         if (attr_mask & IB_QP_SQ_PSN)
1830                 sqp->send_psn = attr->sq_psn;
1831 }
1832
1833 static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
1834 {
1835         path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
1836 }
1837
1838 static int _mlx4_set_path(struct mlx4_ib_dev *dev,
1839                           const struct rdma_ah_attr *ah,
1840                           u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
1841                           struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
1842 {
1843         int vidx;
1844         int smac_index;
1845         int err;
1846
1847         path->grh_mylmc = rdma_ah_get_path_bits(ah) & 0x7f;
1848         path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
1849         if (rdma_ah_get_static_rate(ah)) {
1850                 path->static_rate = rdma_ah_get_static_rate(ah) +
1851                                     MLX4_STAT_RATE_OFFSET;
1852                 while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
1853                        !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
1854                         --path->static_rate;
1855         } else
1856                 path->static_rate = 0;
1857
1858         if (rdma_ah_get_ah_flags(ah) & IB_AH_GRH) {
1859                 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
1860                 int real_sgid_index =
1861                         mlx4_ib_gid_index_to_real_index(dev, port,
1862                                                         grh->sgid_index);
1863
1864                 if (real_sgid_index < 0)
1865                         return real_sgid_index;
1866                 if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) {
1867                         pr_err("sgid_index (%u) too large. max is %d\n",
1868                                real_sgid_index, dev->dev->caps.gid_table_len[port] - 1);
1869                         return -1;
1870                 }
1871
1872                 path->grh_mylmc |= 1 << 7;
1873                 path->mgid_index = real_sgid_index;
1874                 path->hop_limit  = grh->hop_limit;
1875                 path->tclass_flowlabel =
1876                         cpu_to_be32((grh->traffic_class << 20) |
1877                                     (grh->flow_label));
1878                 memcpy(path->rgid, grh->dgid.raw, 16);
1879         }
1880
1881         if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
1882                 if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH))
1883                         return -1;
1884
1885                 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1886                         ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 7) << 3);
1887
1888                 path->feup |= MLX4_FEUP_FORCE_ETH_UP;
1889                 if (vlan_tag < 0x1000) {
1890                         if (smac_info->vid < 0x1000) {
1891                                 /* both valid vlan ids */
1892                                 if (smac_info->vid != vlan_tag) {
1893                                         /* different VIDs.  unreg old and reg new */
1894                                         err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1895                                         if (err)
1896                                                 return err;
1897                                         smac_info->candidate_vid = vlan_tag;
1898                                         smac_info->candidate_vlan_index = vidx;
1899                                         smac_info->candidate_vlan_port = port;
1900                                         smac_info->update_vid = 1;
1901                                         path->vlan_index = vidx;
1902                                 } else {
1903                                         path->vlan_index = smac_info->vlan_index;
1904                                 }
1905                         } else {
1906                                 /* no current vlan tag in qp */
1907                                 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1908                                 if (err)
1909                                         return err;
1910                                 smac_info->candidate_vid = vlan_tag;
1911                                 smac_info->candidate_vlan_index = vidx;
1912                                 smac_info->candidate_vlan_port = port;
1913                                 smac_info->update_vid = 1;
1914                                 path->vlan_index = vidx;
1915                         }
1916                         path->feup |= MLX4_FVL_FORCE_ETH_VLAN;
1917                         path->fl = 1 << 6;
1918                 } else {
1919                         /* have current vlan tag. unregister it at modify-qp success */
1920                         if (smac_info->vid < 0x1000) {
1921                                 smac_info->candidate_vid = 0xFFFF;
1922                                 smac_info->update_vid = 1;
1923                         }
1924                 }
1925
1926                 /* get smac_index for RoCE use.
1927                  * If no smac was yet assigned, register one.
1928                  * If one was already assigned, but the new mac differs,
1929                  * unregister the old one and register the new one.
1930                 */
1931                 if ((!smac_info->smac && !smac_info->smac_port) ||
1932                     smac_info->smac != smac) {
1933                         /* register candidate now, unreg if needed, after success */
1934                         smac_index = mlx4_register_mac(dev->dev, port, smac);
1935                         if (smac_index >= 0) {
1936                                 smac_info->candidate_smac_index = smac_index;
1937                                 smac_info->candidate_smac = smac;
1938                                 smac_info->candidate_smac_port = port;
1939                         } else {
1940                                 return -EINVAL;
1941                         }
1942                 } else {
1943                         smac_index = smac_info->smac_index;
1944                 }
1945                 memcpy(path->dmac, ah->roce.dmac, 6);
1946                 path->ackto = MLX4_IB_LINK_TYPE_ETH;
1947                 /* put MAC table smac index for IBoE */
1948                 path->grh_mylmc = (u8) (smac_index) | 0x80;
1949         } else {
1950                 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1951                         ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 0xf) << 2);
1952         }
1953
1954         return 0;
1955 }
1956
1957 static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,
1958                          enum ib_qp_attr_mask qp_attr_mask,
1959                          struct mlx4_ib_qp *mqp,
1960                          struct mlx4_qp_path *path, u8 port,
1961                          u16 vlan_id, u8 *smac)
1962 {
1963         return _mlx4_set_path(dev, &qp->ah_attr,
1964                               mlx4_mac_to_u64(smac),
1965                               vlan_id,
1966                               path, &mqp->pri, port);
1967 }
1968
1969 static int mlx4_set_alt_path(struct mlx4_ib_dev *dev,
1970                              const struct ib_qp_attr *qp,
1971                              enum ib_qp_attr_mask qp_attr_mask,
1972                              struct mlx4_ib_qp *mqp,
1973                              struct mlx4_qp_path *path, u8 port)
1974 {
1975         return _mlx4_set_path(dev, &qp->alt_ah_attr,
1976                               0,
1977                               0xffff,
1978                               path, &mqp->alt, port);
1979 }
1980
1981 static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1982 {
1983         struct mlx4_ib_gid_entry *ge, *tmp;
1984
1985         list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1986                 if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
1987                         ge->added = 1;
1988                         ge->port = qp->port;
1989                 }
1990         }
1991 }
1992
1993 static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev,
1994                                     struct mlx4_ib_qp *qp,
1995                                     struct mlx4_qp_context *context)
1996 {
1997         u64 u64_mac;
1998         int smac_index;
1999
2000         u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]);
2001
2002         context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6);
2003         if (!qp->pri.smac && !qp->pri.smac_port) {
2004                 smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac);
2005                 if (smac_index >= 0) {
2006                         qp->pri.candidate_smac_index = smac_index;
2007                         qp->pri.candidate_smac = u64_mac;
2008                         qp->pri.candidate_smac_port = qp->port;
2009                         context->pri_path.grh_mylmc = 0x80 | (u8) smac_index;
2010                 } else {
2011                         return -ENOENT;
2012                 }
2013         }
2014         return 0;
2015 }
2016
2017 static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
2018 {
2019         struct counter_index *new_counter_index;
2020         int err;
2021         u32 tmp_idx;
2022
2023         if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) !=
2024             IB_LINK_LAYER_ETHERNET ||
2025             !(qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) ||
2026             !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
2027                 return 0;
2028
2029         err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
2030         if (err)
2031                 return err;
2032
2033         new_counter_index = kmalloc(sizeof(*new_counter_index), GFP_KERNEL);
2034         if (!new_counter_index) {
2035                 mlx4_counter_free(dev->dev, tmp_idx);
2036                 return -ENOMEM;
2037         }
2038
2039         new_counter_index->index = tmp_idx;
2040         new_counter_index->allocated = 1;
2041         qp->counter_index = new_counter_index;
2042
2043         mutex_lock(&dev->counters_table[qp->port - 1].mutex);
2044         list_add_tail(&new_counter_index->list,
2045                       &dev->counters_table[qp->port - 1].counters_list);
2046         mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
2047
2048         return 0;
2049 }
2050
2051 enum {
2052         MLX4_QPC_ROCE_MODE_1 = 0,
2053         MLX4_QPC_ROCE_MODE_2 = 2,
2054         MLX4_QPC_ROCE_MODE_UNDEFINED = 0xff
2055 };
2056
2057 static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
2058 {
2059         switch (gid_type) {
2060         case IB_GID_TYPE_ROCE:
2061                 return MLX4_QPC_ROCE_MODE_1;
2062         case IB_GID_TYPE_ROCE_UDP_ENCAP:
2063                 return MLX4_QPC_ROCE_MODE_2;
2064         default:
2065                 return MLX4_QPC_ROCE_MODE_UNDEFINED;
2066         }
2067 }
2068
2069 /*
2070  * Go over all RSS QP's childes (WQs) and apply their HW state according to
2071  * their logic state if the RSS QP is the first RSS QP associated for the WQ.
2072  */
2073 static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num)
2074 {
2075         int err = 0;
2076         int i;
2077
2078         for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2079                 struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2080                 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2081
2082                 mutex_lock(&wq->mutex);
2083
2084                 /* Mlx4_ib restrictions:
2085                  * WQ's is associated to a port according to the RSS QP it is
2086                  * associates to.
2087                  * In case the WQ is associated to a different port by another
2088                  * RSS QP, return a failure.
2089                  */
2090                 if ((wq->rss_usecnt > 0) && (wq->port != port_num)) {
2091                         err = -EINVAL;
2092                         mutex_unlock(&wq->mutex);
2093                         break;
2094                 }
2095                 wq->port = port_num;
2096                 if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
2097                         err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY);
2098                         if (err) {
2099                                 mutex_unlock(&wq->mutex);
2100                                 break;
2101                         }
2102                 }
2103                 wq->rss_usecnt++;
2104
2105                 mutex_unlock(&wq->mutex);
2106         }
2107
2108         if (i && err) {
2109                 int j;
2110
2111                 for (j = (i - 1); j >= 0; j--) {
2112                         struct ib_wq *ibwq = ind_tbl->ind_tbl[j];
2113                         struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2114
2115                         mutex_lock(&wq->mutex);
2116
2117                         if ((wq->rss_usecnt == 1) &&
2118                             (ibwq->state == IB_WQS_RDY))
2119                                 if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
2120                                         pr_warn("failed to reverse WQN=0x%06x\n",
2121                                                 ibwq->wq_num);
2122                         wq->rss_usecnt--;
2123
2124                         mutex_unlock(&wq->mutex);
2125                 }
2126         }
2127
2128         return err;
2129 }
2130
2131 static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl)
2132 {
2133         int i;
2134
2135         for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2136                 struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2137                 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2138
2139                 mutex_lock(&wq->mutex);
2140
2141                 if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
2142                         if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
2143                                 pr_warn("failed to reverse WQN=%x\n",
2144                                         ibwq->wq_num);
2145                 wq->rss_usecnt--;
2146
2147                 mutex_unlock(&wq->mutex);
2148         }
2149 }
2150
2151 static void fill_qp_rss_context(struct mlx4_qp_context *context,
2152                                 struct mlx4_ib_qp *qp)
2153 {
2154         struct mlx4_rss_context *rss_context;
2155
2156         rss_context = (void *)context + offsetof(struct mlx4_qp_context,
2157                         pri_path) + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
2158
2159         rss_context->base_qpn = cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz);
2160         rss_context->default_qpn =
2161                 cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz & 0xffffff);
2162         if (qp->rss_ctx->flags & (MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6))
2163                 rss_context->base_qpn_udp = rss_context->default_qpn;
2164         rss_context->flags = qp->rss_ctx->flags;
2165         /* Currently support just toeplitz */
2166         rss_context->hash_fn = MLX4_RSS_HASH_TOP;
2167
2168         memcpy(rss_context->rss_key, qp->rss_ctx->rss_key,
2169                MLX4_EN_RSS_KEY_SIZE);
2170 }
2171
2172 static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
2173                                const struct ib_qp_attr *attr, int attr_mask,
2174                                enum ib_qp_state cur_state, enum ib_qp_state new_state)
2175 {
2176         struct ib_uobject *ibuobject;
2177         struct ib_srq  *ibsrq;
2178         struct ib_rwq_ind_table *rwq_ind_tbl;
2179         enum ib_qp_type qp_type;
2180         struct mlx4_ib_dev *dev;
2181         struct mlx4_ib_qp *qp;
2182         struct mlx4_ib_pd *pd;
2183         struct mlx4_ib_cq *send_cq, *recv_cq;
2184         struct mlx4_qp_context *context;
2185         enum mlx4_qp_optpar optpar = 0;
2186         int sqd_event;
2187         int steer_qp = 0;
2188         int err = -EINVAL;
2189         int counter_index;
2190
2191         if (src_type == MLX4_IB_RWQ_SRC) {
2192                 struct ib_wq *ibwq;
2193
2194                 ibwq        = (struct ib_wq *)src;
2195                 ibuobject   = ibwq->uobject;
2196                 ibsrq       = NULL;
2197                 rwq_ind_tbl = NULL;
2198                 qp_type     = IB_QPT_RAW_PACKET;
2199                 qp          = to_mqp((struct ib_qp *)ibwq);
2200                 dev         = to_mdev(ibwq->device);
2201                 pd          = to_mpd(ibwq->pd);
2202         } else {
2203                 struct ib_qp *ibqp;
2204
2205                 ibqp        = (struct ib_qp *)src;
2206                 ibuobject   = ibqp->uobject;
2207                 ibsrq       = ibqp->srq;
2208                 rwq_ind_tbl = ibqp->rwq_ind_tbl;
2209                 qp_type     = ibqp->qp_type;
2210                 qp          = to_mqp(ibqp);
2211                 dev         = to_mdev(ibqp->device);
2212                 pd          = get_pd(qp);
2213         }
2214
2215         /* APM is not supported under RoCE */
2216         if (attr_mask & IB_QP_ALT_PATH &&
2217             rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2218             IB_LINK_LAYER_ETHERNET)
2219                 return -ENOTSUPP;
2220
2221         context = kzalloc(sizeof *context, GFP_KERNEL);
2222         if (!context)
2223                 return -ENOMEM;
2224
2225         context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
2226                                      (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
2227
2228         if (!(attr_mask & IB_QP_PATH_MIG_STATE))
2229                 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2230         else {
2231                 optpar |= MLX4_QP_OPTPAR_PM_STATE;
2232                 switch (attr->path_mig_state) {
2233                 case IB_MIG_MIGRATED:
2234                         context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2235                         break;
2236                 case IB_MIG_REARM:
2237                         context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
2238                         break;
2239                 case IB_MIG_ARMED:
2240                         context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
2241                         break;
2242                 }
2243         }
2244
2245         if (qp->inl_recv_sz)
2246                 context->param3 |= cpu_to_be32(1 << 25);
2247
2248         if (qp->flags & MLX4_IB_QP_SCATTER_FCS)
2249                 context->param3 |= cpu_to_be32(1 << 29);
2250
2251         if (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI)
2252                 context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
2253         else if (qp_type == IB_QPT_RAW_PACKET)
2254                 context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
2255         else if (qp_type == IB_QPT_UD) {
2256                 if (qp->flags & MLX4_IB_QP_LSO)
2257                         context->mtu_msgmax = (IB_MTU_4096 << 5) |
2258                                               ilog2(dev->dev->caps.max_gso_sz);
2259                 else
2260                         context->mtu_msgmax = (IB_MTU_4096 << 5) | 13;
2261         } else if (attr_mask & IB_QP_PATH_MTU) {
2262                 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
2263                         pr_err("path MTU (%u) is invalid\n",
2264                                attr->path_mtu);
2265                         goto out;
2266                 }
2267                 context->mtu_msgmax = (attr->path_mtu << 5) |
2268                         ilog2(dev->dev->caps.max_msg_sz);
2269         }
2270
2271         if (!rwq_ind_tbl) { /* PRM RSS receive side should be left zeros */
2272                 if (qp->rq.wqe_cnt)
2273                         context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
2274                 context->rq_size_stride |= qp->rq.wqe_shift - 4;
2275         }
2276
2277         if (qp->sq.wqe_cnt)
2278                 context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
2279         context->sq_size_stride |= qp->sq.wqe_shift - 4;
2280
2281         if (new_state == IB_QPS_RESET && qp->counter_index)
2282                 mlx4_ib_free_qp_counter(dev, qp);
2283
2284         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2285                 context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
2286                 context->xrcd = cpu_to_be32((u32) qp->xrcdn);
2287                 if (qp_type == IB_QPT_RAW_PACKET)
2288                         context->param3 |= cpu_to_be32(1 << 30);
2289         }
2290
2291         if (ibuobject)
2292                 context->usr_page = cpu_to_be32(
2293                         mlx4_to_hw_uar_index(dev->dev,
2294                                              to_mucontext(ibuobject->context)
2295                                              ->uar.index));
2296         else
2297                 context->usr_page = cpu_to_be32(
2298                         mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
2299
2300         if (attr_mask & IB_QP_DEST_QPN)
2301                 context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
2302
2303         if (attr_mask & IB_QP_PORT) {
2304                 if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
2305                     !(attr_mask & IB_QP_AV)) {
2306                         mlx4_set_sched(&context->pri_path, attr->port_num);
2307                         optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
2308                 }
2309         }
2310
2311         if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2312                 err = create_qp_lb_counter(dev, qp);
2313                 if (err)
2314                         goto out;
2315
2316                 counter_index =
2317                         dev->counters_table[qp->port - 1].default_counter;
2318                 if (qp->counter_index)
2319                         counter_index = qp->counter_index->index;
2320
2321                 if (counter_index != -1) {
2322                         context->pri_path.counter_index = counter_index;
2323                         optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
2324                         if (qp->counter_index) {
2325                                 context->pri_path.fl |=
2326                                         MLX4_FL_ETH_SRC_CHECK_MC_LB;
2327                                 context->pri_path.vlan_control |=
2328                                         MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
2329                         }
2330                 } else
2331                         context->pri_path.counter_index =
2332                                 MLX4_SINK_COUNTER_INDEX(dev->dev);
2333
2334                 if (qp->flags & MLX4_IB_QP_NETIF) {
2335                         mlx4_ib_steer_qp_reg(dev, qp, 1);
2336                         steer_qp = 1;
2337                 }
2338
2339                 if (qp_type == IB_QPT_GSI) {
2340                         enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
2341                                 IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
2342                         u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
2343
2344                         context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2345                 }
2346         }
2347
2348         if (attr_mask & IB_QP_PKEY_INDEX) {
2349                 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2350                         context->pri_path.disable_pkey_check = 0x40;
2351                 context->pri_path.pkey_index = attr->pkey_index;
2352                 optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
2353         }
2354
2355         if (attr_mask & IB_QP_AV) {
2356                 u8 port_num = mlx4_is_bonded(dev->dev) ? 1 :
2357                         attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2358                 union ib_gid gid;
2359                 struct ib_gid_attr gid_attr = {.gid_type = IB_GID_TYPE_IB};
2360                 u16 vlan = 0xffff;
2361                 u8 smac[ETH_ALEN];
2362                 int status = 0;
2363                 int is_eth =
2364                         rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
2365                         rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
2366
2367                 if (is_eth) {
2368                         int index =
2369                                 rdma_ah_read_grh(&attr->ah_attr)->sgid_index;
2370
2371                         status = ib_get_cached_gid(&dev->ib_dev, port_num,
2372                                                    index, &gid, &gid_attr);
2373                         if (!status && !memcmp(&gid, &zgid, sizeof(gid)))
2374                                 status = -ENOENT;
2375                         if (!status && gid_attr.ndev) {
2376                                 vlan = rdma_vlan_dev_vlan_id(gid_attr.ndev);
2377                                 memcpy(smac, gid_attr.ndev->dev_addr, ETH_ALEN);
2378                                 dev_put(gid_attr.ndev);
2379                         }
2380                 }
2381                 if (status)
2382                         goto out;
2383
2384                 if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
2385                                   port_num, vlan, smac))
2386                         goto out;
2387
2388                 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
2389                            MLX4_QP_OPTPAR_SCHED_QUEUE);
2390
2391                 if (is_eth &&
2392                     (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)) {
2393                         u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
2394
2395                         if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_UNDEFINED) {
2396                                 err = -EINVAL;
2397                                 goto out;
2398                         }
2399                         context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2400                 }
2401
2402         }
2403
2404         if (attr_mask & IB_QP_TIMEOUT) {
2405                 context->pri_path.ackto |= attr->timeout << 3;
2406                 optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
2407         }
2408
2409         if (attr_mask & IB_QP_ALT_PATH) {
2410                 if (attr->alt_port_num == 0 ||
2411                     attr->alt_port_num > dev->dev->caps.num_ports)
2412                         goto out;
2413
2414                 if (attr->alt_pkey_index >=
2415                     dev->dev->caps.pkey_table_len[attr->alt_port_num])
2416                         goto out;
2417
2418                 if (mlx4_set_alt_path(dev, attr, attr_mask, qp,
2419                                       &context->alt_path,
2420                                       attr->alt_port_num))
2421                         goto out;
2422
2423                 context->alt_path.pkey_index = attr->alt_pkey_index;
2424                 context->alt_path.ackto = attr->alt_timeout << 3;
2425                 optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
2426         }
2427
2428         context->pd = cpu_to_be32(pd->pdn);
2429
2430         if (!rwq_ind_tbl) {
2431                 context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
2432                 get_cqs(qp, src_type, &send_cq, &recv_cq);
2433         } else { /* Set dummy CQs to be compatible with HV and PRM */
2434                 send_cq = to_mcq(rwq_ind_tbl->ind_tbl[0]->cq);
2435                 recv_cq = send_cq;
2436         }
2437         context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
2438         context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
2439
2440         /* Set "fast registration enabled" for all kernel QPs */
2441         if (!ibuobject)
2442                 context->params1 |= cpu_to_be32(1 << 11);
2443
2444         if (attr_mask & IB_QP_RNR_RETRY) {
2445                 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
2446                 optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
2447         }
2448
2449         if (attr_mask & IB_QP_RETRY_CNT) {
2450                 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
2451                 optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
2452         }
2453
2454         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2455                 if (attr->max_rd_atomic)
2456                         context->params1 |=
2457                                 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
2458                 optpar |= MLX4_QP_OPTPAR_SRA_MAX;
2459         }
2460
2461         if (attr_mask & IB_QP_SQ_PSN)
2462                 context->next_send_psn = cpu_to_be32(attr->sq_psn);
2463
2464         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2465                 if (attr->max_dest_rd_atomic)
2466                         context->params2 |=
2467                                 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
2468                 optpar |= MLX4_QP_OPTPAR_RRA_MAX;
2469         }
2470
2471         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
2472                 context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
2473                 optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
2474         }
2475
2476         if (ibsrq)
2477                 context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
2478
2479         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2480                 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
2481                 optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
2482         }
2483         if (attr_mask & IB_QP_RQ_PSN)
2484                 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
2485
2486         /* proxy and tunnel qp qkeys will be changed in modify-qp wrappers */
2487         if (attr_mask & IB_QP_QKEY) {
2488                 if (qp->mlx4_ib_qp_type &
2489                     (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))
2490                         context->qkey = cpu_to_be32(IB_QP_SET_QKEY);
2491                 else {
2492                         if (mlx4_is_mfunc(dev->dev) &&
2493                             !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) &&
2494                             (attr->qkey & MLX4_RESERVED_QKEY_MASK) ==
2495                             MLX4_RESERVED_QKEY_BASE) {
2496                                 pr_err("Cannot use reserved QKEY"
2497                                        " 0x%x (range 0xffff0000..0xffffffff"
2498                                        " is reserved)\n", attr->qkey);
2499                                 err = -EINVAL;
2500                                 goto out;
2501                         }
2502                         context->qkey = cpu_to_be32(attr->qkey);
2503                 }
2504                 optpar |= MLX4_QP_OPTPAR_Q_KEY;
2505         }
2506
2507         if (ibsrq)
2508                 context->srqn = cpu_to_be32(1 << 24 |
2509                                             to_msrq(ibsrq)->msrq.srqn);
2510
2511         if (qp->rq.wqe_cnt &&
2512             cur_state == IB_QPS_RESET &&
2513             new_state == IB_QPS_INIT)
2514                 context->db_rec_addr = cpu_to_be64(qp->db.dma);
2515
2516         if (cur_state == IB_QPS_INIT &&
2517             new_state == IB_QPS_RTR  &&
2518             (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI ||
2519              qp_type == IB_QPT_UD || qp_type == IB_QPT_RAW_PACKET)) {
2520                 context->pri_path.sched_queue = (qp->port - 1) << 6;
2521                 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
2522                     qp->mlx4_ib_qp_type &
2523                     (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) {
2524                         context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
2525                         if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI)
2526                                 context->pri_path.fl = 0x80;
2527                 } else {
2528                         if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2529                                 context->pri_path.fl = 0x80;
2530                         context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
2531                 }
2532                 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2533                     IB_LINK_LAYER_ETHERNET) {
2534                         if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI ||
2535                             qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI)
2536                                 context->pri_path.feup = 1 << 7; /* don't fsm */
2537                         /* handle smac_index */
2538                         if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
2539                             qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI ||
2540                             qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
2541                                 err = handle_eth_ud_smac_index(dev, qp, context);
2542                                 if (err) {
2543                                         err = -EINVAL;
2544                                         goto out;
2545                                 }
2546                                 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
2547                                         dev->qp1_proxy[qp->port - 1] = qp;
2548                         }
2549                 }
2550         }
2551
2552         if (qp_type == IB_QPT_RAW_PACKET) {
2553                 context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
2554                                         MLX4_IB_LINK_TYPE_ETH;
2555                 if (dev->dev->caps.tunnel_offload_mode ==  MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
2556                         /* set QP to receive both tunneled & non-tunneled packets */
2557                         if (!rwq_ind_tbl)
2558                                 context->srqn = cpu_to_be32(7 << 28);
2559                 }
2560         }
2561
2562         if (qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
2563                 int is_eth = rdma_port_get_link_layer(
2564                                 &dev->ib_dev, qp->port) ==
2565                                 IB_LINK_LAYER_ETHERNET;
2566                 if (is_eth) {
2567                         context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH;
2568                         optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH;
2569                 }
2570         }
2571
2572         if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD  &&
2573             attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
2574                 sqd_event = 1;
2575         else
2576                 sqd_event = 0;
2577
2578         if (!ibuobject &&
2579             cur_state == IB_QPS_RESET &&
2580             new_state == IB_QPS_INIT)
2581                 context->rlkey_roce_mode |= (1 << 4);
2582
2583         /*
2584          * Before passing a kernel QP to the HW, make sure that the
2585          * ownership bits of the send queue are set and the SQ
2586          * headroom is stamped so that the hardware doesn't start
2587          * processing stale work requests.
2588          */
2589         if (!ibuobject &&
2590             cur_state == IB_QPS_RESET &&
2591             new_state == IB_QPS_INIT) {
2592                 struct mlx4_wqe_ctrl_seg *ctrl;
2593                 int i;
2594
2595                 for (i = 0; i < qp->sq.wqe_cnt; ++i) {
2596                         ctrl = get_send_wqe(qp, i);
2597                         ctrl->owner_opcode = cpu_to_be32(1 << 31);
2598                         if (qp->sq_max_wqes_per_wr == 1)
2599                                 ctrl->qpn_vlan.fence_size =
2600                                                 1 << (qp->sq.wqe_shift - 4);
2601
2602                         stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
2603                 }
2604         }
2605
2606         if (rwq_ind_tbl &&
2607             cur_state == IB_QPS_RESET &&
2608             new_state == IB_QPS_INIT) {
2609                 fill_qp_rss_context(context, qp);
2610                 context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET);
2611         }
2612
2613         err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
2614                              to_mlx4_state(new_state), context, optpar,
2615                              sqd_event, &qp->mqp);
2616         if (err)
2617                 goto out;
2618
2619         qp->state = new_state;
2620
2621         if (attr_mask & IB_QP_ACCESS_FLAGS)
2622                 qp->atomic_rd_en = attr->qp_access_flags;
2623         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2624                 qp->resp_depth = attr->max_dest_rd_atomic;
2625         if (attr_mask & IB_QP_PORT) {
2626                 qp->port = attr->port_num;
2627                 update_mcg_macs(dev, qp);
2628         }
2629         if (attr_mask & IB_QP_ALT_PATH)
2630                 qp->alt_port = attr->alt_port_num;
2631
2632         if (is_sqp(dev, qp))
2633                 store_sqp_attrs(to_msqp(qp), attr, attr_mask);
2634
2635         /*
2636          * If we moved QP0 to RTR, bring the IB link up; if we moved
2637          * QP0 to RESET or ERROR, bring the link back down.
2638          */
2639         if (is_qp0(dev, qp)) {
2640                 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
2641                         if (mlx4_INIT_PORT(dev->dev, qp->port))
2642                                 pr_warn("INIT_PORT failed for port %d\n",
2643                                        qp->port);
2644
2645                 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
2646                     (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
2647                         mlx4_CLOSE_PORT(dev->dev, qp->port);
2648         }
2649
2650         /*
2651          * If we moved a kernel QP to RESET, clean up all old CQ
2652          * entries and reinitialize the QP.
2653          */
2654         if (new_state == IB_QPS_RESET) {
2655                 if (!ibuobject) {
2656                         mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2657                                          ibsrq ? to_msrq(ibsrq) : NULL);
2658                         if (send_cq != recv_cq)
2659                                 mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
2660
2661                         qp->rq.head = 0;
2662                         qp->rq.tail = 0;
2663                         qp->sq.head = 0;
2664                         qp->sq.tail = 0;
2665                         qp->sq_next_wqe = 0;
2666                         if (qp->rq.wqe_cnt)
2667                                 *qp->db.db  = 0;
2668
2669                         if (qp->flags & MLX4_IB_QP_NETIF)
2670                                 mlx4_ib_steer_qp_reg(dev, qp, 0);
2671                 }
2672                 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
2673                         mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2674                         qp->pri.smac = 0;
2675                         qp->pri.smac_port = 0;
2676                 }
2677                 if (qp->alt.smac) {
2678                         mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2679                         qp->alt.smac = 0;
2680                 }
2681                 if (qp->pri.vid < 0x1000) {
2682                         mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
2683                         qp->pri.vid = 0xFFFF;
2684                         qp->pri.candidate_vid = 0xFFFF;
2685                         qp->pri.update_vid = 0;
2686                 }
2687
2688                 if (qp->alt.vid < 0x1000) {
2689                         mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
2690                         qp->alt.vid = 0xFFFF;
2691                         qp->alt.candidate_vid = 0xFFFF;
2692                         qp->alt.update_vid = 0;
2693                 }
2694         }
2695 out:
2696         if (err && qp->counter_index)
2697                 mlx4_ib_free_qp_counter(dev, qp);
2698         if (err && steer_qp)
2699                 mlx4_ib_steer_qp_reg(dev, qp, 0);
2700         kfree(context);
2701         if (qp->pri.candidate_smac ||
2702             (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) {
2703                 if (err) {
2704                         mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac);
2705                 } else {
2706                         if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port))
2707                                 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2708                         qp->pri.smac = qp->pri.candidate_smac;
2709                         qp->pri.smac_index = qp->pri.candidate_smac_index;
2710                         qp->pri.smac_port = qp->pri.candidate_smac_port;
2711                 }
2712                 qp->pri.candidate_smac = 0;
2713                 qp->pri.candidate_smac_index = 0;
2714                 qp->pri.candidate_smac_port = 0;
2715         }
2716         if (qp->alt.candidate_smac) {
2717                 if (err) {
2718                         mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac);
2719                 } else {
2720                         if (qp->alt.smac)
2721                                 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2722                         qp->alt.smac = qp->alt.candidate_smac;
2723                         qp->alt.smac_index = qp->alt.candidate_smac_index;
2724                         qp->alt.smac_port = qp->alt.candidate_smac_port;
2725                 }
2726                 qp->alt.candidate_smac = 0;
2727                 qp->alt.candidate_smac_index = 0;
2728                 qp->alt.candidate_smac_port = 0;
2729         }
2730
2731         if (qp->pri.update_vid) {
2732                 if (err) {
2733                         if (qp->pri.candidate_vid < 0x1000)
2734                                 mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port,
2735                                                      qp->pri.candidate_vid);
2736                 } else {
2737                         if (qp->pri.vid < 0x1000)
2738                                 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port,
2739                                                      qp->pri.vid);
2740                         qp->pri.vid = qp->pri.candidate_vid;
2741                         qp->pri.vlan_port = qp->pri.candidate_vlan_port;
2742                         qp->pri.vlan_index =  qp->pri.candidate_vlan_index;
2743                 }
2744                 qp->pri.candidate_vid = 0xFFFF;
2745                 qp->pri.update_vid = 0;
2746         }
2747
2748         if (qp->alt.update_vid) {
2749                 if (err) {
2750                         if (qp->alt.candidate_vid < 0x1000)
2751                                 mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port,
2752                                                      qp->alt.candidate_vid);
2753                 } else {
2754                         if (qp->alt.vid < 0x1000)
2755                                 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port,
2756                                                      qp->alt.vid);
2757                         qp->alt.vid = qp->alt.candidate_vid;
2758                         qp->alt.vlan_port = qp->alt.candidate_vlan_port;
2759                         qp->alt.vlan_index =  qp->alt.candidate_vlan_index;
2760                 }
2761                 qp->alt.candidate_vid = 0xFFFF;
2762                 qp->alt.update_vid = 0;
2763         }
2764
2765         return err;
2766 }
2767
2768 enum {
2769         MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK = (IB_QP_STATE       |
2770                                               IB_QP_PORT),
2771 };
2772
2773 static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2774                               int attr_mask, struct ib_udata *udata)
2775 {
2776         enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
2777         struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2778         struct mlx4_ib_qp *qp = to_mqp(ibqp);
2779         enum ib_qp_state cur_state, new_state;
2780         int err = -EINVAL;
2781         mutex_lock(&qp->mutex);
2782
2783         cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2784         new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2785
2786         if (cur_state != new_state || cur_state != IB_QPS_RESET) {
2787                 int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2788                 ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2789         }
2790
2791         if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2792                                 attr_mask, ll)) {
2793                 pr_debug("qpn 0x%x: invalid attribute mask specified "
2794                          "for transition %d to %d. qp_type %d,"
2795                          " attr_mask 0x%x\n",
2796                          ibqp->qp_num, cur_state, new_state,
2797                          ibqp->qp_type, attr_mask);
2798                 goto out;
2799         }
2800
2801         if (ibqp->rwq_ind_tbl) {
2802                 if (!(((cur_state == IB_QPS_RESET) &&
2803                        (new_state == IB_QPS_INIT)) ||
2804                       ((cur_state == IB_QPS_INIT)  &&
2805                        (new_state == IB_QPS_RTR)))) {
2806                         pr_debug("qpn 0x%x: RSS QP unsupported transition %d to %d\n",
2807                                  ibqp->qp_num, cur_state, new_state);
2808
2809                         err = -EOPNOTSUPP;
2810                         goto out;
2811                 }
2812
2813                 if (attr_mask & ~MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK) {
2814                         pr_debug("qpn 0x%x: RSS QP unsupported attribute mask 0x%x for transition %d to %d\n",
2815                                  ibqp->qp_num, attr_mask, cur_state, new_state);
2816
2817                         err = -EOPNOTSUPP;
2818                         goto out;
2819                 }
2820         }
2821
2822         if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT)) {
2823                 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
2824                         if ((ibqp->qp_type == IB_QPT_RC) ||
2825                             (ibqp->qp_type == IB_QPT_UD) ||
2826                             (ibqp->qp_type == IB_QPT_UC) ||
2827                             (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
2828                             (ibqp->qp_type == IB_QPT_XRC_INI)) {
2829                                 attr->port_num = mlx4_ib_bond_next_port(dev);
2830                         }
2831                 } else {
2832                         /* no sense in changing port_num
2833                          * when ports are bonded */
2834                         attr_mask &= ~IB_QP_PORT;
2835                 }
2836         }
2837
2838         if ((attr_mask & IB_QP_PORT) &&
2839             (attr->port_num == 0 || attr->port_num > dev->num_ports)) {
2840                 pr_debug("qpn 0x%x: invalid port number (%d) specified "
2841                          "for transition %d to %d. qp_type %d\n",
2842                          ibqp->qp_num, attr->port_num, cur_state,
2843                          new_state, ibqp->qp_type);
2844                 goto out;
2845         }
2846
2847         if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
2848             (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
2849              IB_LINK_LAYER_ETHERNET))
2850                 goto out;
2851
2852         if (attr_mask & IB_QP_PKEY_INDEX) {
2853                 int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2854                 if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) {
2855                         pr_debug("qpn 0x%x: invalid pkey index (%d) specified "
2856                                  "for transition %d to %d. qp_type %d\n",
2857                                  ibqp->qp_num, attr->pkey_index, cur_state,
2858                                  new_state, ibqp->qp_type);
2859                         goto out;
2860                 }
2861         }
2862
2863         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
2864             attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
2865                 pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. "
2866                          "Transition %d to %d. qp_type %d\n",
2867                          ibqp->qp_num, attr->max_rd_atomic, cur_state,
2868                          new_state, ibqp->qp_type);
2869                 goto out;
2870         }
2871
2872         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
2873             attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
2874                 pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. "
2875                          "Transition %d to %d. qp_type %d\n",
2876                          ibqp->qp_num, attr->max_dest_rd_atomic, cur_state,
2877                          new_state, ibqp->qp_type);
2878                 goto out;
2879         }
2880
2881         if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2882                 err = 0;
2883                 goto out;
2884         }
2885
2886         if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
2887                 err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num);
2888                 if (err)
2889                         goto out;
2890         }
2891
2892         err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
2893                                   cur_state, new_state);
2894
2895         if (ibqp->rwq_ind_tbl && err)
2896                 bring_down_rss_rwqs(ibqp->rwq_ind_tbl);
2897
2898         if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2899                 attr->port_num = 1;
2900
2901 out:
2902         mutex_unlock(&qp->mutex);
2903         return err;
2904 }
2905
2906 int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2907                       int attr_mask, struct ib_udata *udata)
2908 {
2909         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
2910         int ret;
2911
2912         ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2913
2914         if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2915                 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
2916                 int err = 0;
2917
2918                 if (sqp->roce_v2_gsi)
2919                         err = ib_modify_qp(sqp->roce_v2_gsi, attr, attr_mask);
2920                 if (err)
2921                         pr_err("Failed to modify GSI QP for RoCEv2 (%d)\n",
2922                                err);
2923         }
2924         return ret;
2925 }
2926
2927 static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
2928 {
2929         int i;
2930         for (i = 0; i < dev->caps.num_ports; i++) {
2931                 if (qpn == dev->caps.spec_qps[i].qp0_proxy ||
2932                     qpn == dev->caps.spec_qps[i].qp0_tunnel) {
2933                         *qkey = dev->caps.spec_qps[i].qp0_qkey;
2934                         return 0;
2935                 }
2936         }
2937         return -EINVAL;
2938 }
2939
2940 static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
2941                                   struct ib_ud_wr *wr,
2942                                   void *wqe, unsigned *mlx_seg_len)
2943 {
2944         struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
2945         struct ib_device *ib_dev = &mdev->ib_dev;
2946         struct mlx4_wqe_mlx_seg *mlx = wqe;
2947         struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2948         struct mlx4_ib_ah *ah = to_mah(wr->ah);
2949         u16 pkey;
2950         u32 qkey;
2951         int send_size;
2952         int header_size;
2953         int spc;
2954         int i;
2955
2956         if (wr->wr.opcode != IB_WR_SEND)
2957                 return -EINVAL;
2958
2959         send_size = 0;
2960
2961         for (i = 0; i < wr->wr.num_sge; ++i)
2962                 send_size += wr->wr.sg_list[i].length;
2963
2964         /* for proxy-qp0 sends, need to add in size of tunnel header */
2965         /* for tunnel-qp0 sends, tunnel header is already in s/g list */
2966         if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2967                 send_size += sizeof (struct mlx4_ib_tunnel_header);
2968
2969         ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2970
2971         if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2972                 sqp->ud_header.lrh.service_level =
2973                         be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2974                 sqp->ud_header.lrh.destination_lid =
2975                         cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2976                 sqp->ud_header.lrh.source_lid =
2977                         cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2978         }
2979
2980         mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2981
2982         /* force loopback */
2983         mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR);
2984         mlx->rlid = sqp->ud_header.lrh.destination_lid;
2985
2986         sqp->ud_header.lrh.virtual_lane    = 0;
2987         sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2988         ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
2989         sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2990         if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2991                 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2992         else
2993                 sqp->ud_header.bth.destination_qpn =
2994                         cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel);
2995
2996         sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2997         if (mlx4_is_master(mdev->dev)) {
2998                 if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2999                         return -EINVAL;
3000         } else {
3001                 if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
3002                         return -EINVAL;
3003         }
3004         sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
3005         sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
3006
3007         sqp->ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY;
3008         sqp->ud_header.immediate_present = 0;
3009
3010         header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3011
3012         /*
3013          * Inline data segments may not cross a 64 byte boundary.  If
3014          * our UD header is bigger than the space available up to the
3015          * next 64 byte boundary in the WQE, use two inline data
3016          * segments to hold the UD header.
3017          */
3018         spc = MLX4_INLINE_ALIGN -
3019               ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3020         if (header_size <= spc) {
3021                 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
3022                 memcpy(inl + 1, sqp->header_buf, header_size);
3023                 i = 1;
3024         } else {
3025                 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3026                 memcpy(inl + 1, sqp->header_buf, spc);
3027
3028                 inl = (void *) (inl + 1) + spc;
3029                 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3030                 /*
3031                  * Need a barrier here to make sure all the data is
3032                  * visible before the byte_count field is set.
3033                  * Otherwise the HCA prefetcher could grab the 64-byte
3034                  * chunk with this inline segment and get a valid (!=
3035                  * 0xffffffff) byte count but stale data, and end up
3036                  * generating a packet with bad headers.
3037                  *
3038                  * The first inline segment's byte_count field doesn't
3039                  * need a barrier, because it comes after a
3040                  * control/MLX segment and therefore is at an offset
3041                  * of 16 mod 64.
3042                  */
3043                 wmb();
3044                 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3045                 i = 2;
3046         }
3047
3048         *mlx_seg_len =
3049         ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3050         return 0;
3051 }
3052
3053 static u8 sl_to_vl(struct mlx4_ib_dev *dev, u8 sl, int port_num)
3054 {
3055         union sl2vl_tbl_to_u64 tmp_vltab;
3056         u8 vl;
3057
3058         if (sl > 15)
3059                 return 0xf;
3060         tmp_vltab.sl64 = atomic64_read(&dev->sl2vl[port_num - 1]);
3061         vl = tmp_vltab.sl8[sl >> 1];
3062         if (sl & 1)
3063                 vl &= 0x0f;
3064         else
3065                 vl >>= 4;
3066         return vl;
3067 }
3068
3069 static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num,
3070                                 int index, union ib_gid *gid,
3071                                 enum ib_gid_type *gid_type)
3072 {
3073         struct mlx4_ib_iboe *iboe = &ibdev->iboe;
3074         struct mlx4_port_gid_table *port_gid_table;
3075         unsigned long flags;
3076
3077         port_gid_table = &iboe->gids[port_num - 1];
3078         spin_lock_irqsave(&iboe->lock, flags);
3079         memcpy(gid, &port_gid_table->gids[index].gid, sizeof(*gid));
3080         *gid_type = port_gid_table->gids[index].gid_type;
3081         spin_unlock_irqrestore(&iboe->lock, flags);
3082         if (!memcmp(gid, &zgid, sizeof(*gid)))
3083                 return -ENOENT;
3084
3085         return 0;
3086 }
3087
3088 #define MLX4_ROCEV2_QP1_SPORT 0xC000
3089 static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
3090                             void *wqe, unsigned *mlx_seg_len)
3091 {
3092         struct ib_device *ib_dev = sqp->qp.ibqp.device;
3093         struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);
3094         struct mlx4_wqe_mlx_seg *mlx = wqe;
3095         struct mlx4_wqe_ctrl_seg *ctrl = wqe;
3096         struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
3097         struct mlx4_ib_ah *ah = to_mah(wr->ah);
3098         union ib_gid sgid;
3099         u16 pkey;
3100         int send_size;
3101         int header_size;
3102         int spc;
3103         int i;
3104         int err = 0;
3105         u16 vlan = 0xffff;
3106         bool is_eth;
3107         bool is_vlan = false;
3108         bool is_grh;
3109         bool is_udp = false;
3110         int ip_version = 0;
3111
3112         send_size = 0;
3113         for (i = 0; i < wr->wr.num_sge; ++i)
3114                 send_size += wr->wr.sg_list[i].length;
3115
3116         is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
3117         is_grh = mlx4_ib_ah_grh_present(ah);
3118         if (is_eth) {
3119                 enum ib_gid_type gid_type;
3120                 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3121                         /* When multi-function is enabled, the ib_core gid
3122                          * indexes don't necessarily match the hw ones, so
3123                          * we must use our own cache */
3124                         err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev,
3125                                                            be32_to_cpu(ah->av.ib.port_pd) >> 24,
3126                                                            ah->av.ib.gid_index, &sgid.raw[0]);
3127                         if (err)
3128                                 return err;
3129                 } else  {
3130                         err = fill_gid_by_hw_index(ibdev, sqp->qp.port,
3131                                             ah->av.ib.gid_index,
3132                                             &sgid, &gid_type);
3133                         if (!err) {
3134                                 is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
3135                                 if (is_udp) {
3136                                         if (ipv6_addr_v4mapped((struct in6_addr *)&sgid))
3137                                                 ip_version = 4;
3138                                         else
3139                                                 ip_version = 6;
3140                                         is_grh = false;
3141                                 }
3142                         } else {
3143                                 return err;
3144                         }
3145                 }
3146                 if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
3147                         vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
3148                         is_vlan = 1;
3149                 }
3150         }
3151         err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
3152                           ip_version, is_udp, 0, &sqp->ud_header);
3153         if (err)
3154                 return err;
3155
3156         if (!is_eth) {
3157                 sqp->ud_header.lrh.service_level =
3158                         be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
3159                 sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
3160                 sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
3161         }
3162
3163         if (is_grh || (ip_version == 6)) {
3164                 sqp->ud_header.grh.traffic_class =
3165                         (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3166                 sqp->ud_header.grh.flow_label    =
3167                         ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
3168                 sqp->ud_header.grh.hop_limit     = ah->av.ib.hop_limit;
3169                 if (is_eth) {
3170                         memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
3171                 } else {
3172                         if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3173                                 /* When multi-function is enabled, the ib_core gid
3174                                  * indexes don't necessarily match the hw ones, so
3175                                  * we must use our own cache
3176                                  */
3177                                 sqp->ud_header.grh.source_gid.global.subnet_prefix =
3178                                         cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov.
3179                                                                     demux[sqp->qp.port - 1].
3180                                                                     subnet_prefix)));
3181                                 sqp->ud_header.grh.source_gid.global.interface_id =
3182                                         to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
3183                                                        guid_cache[ah->av.ib.gid_index];
3184                         } else {
3185                                 ib_get_cached_gid(ib_dev,
3186                                                   be32_to_cpu(ah->av.ib.port_pd) >> 24,
3187                                                   ah->av.ib.gid_index,
3188                                                   &sqp->ud_header.grh.source_gid, NULL);
3189                         }
3190                 }
3191                 memcpy(sqp->ud_header.grh.destination_gid.raw,
3192                        ah->av.ib.dgid, 16);
3193         }
3194
3195         if (ip_version == 4) {
3196                 sqp->ud_header.ip4.tos =
3197                         (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3198                 sqp->ud_header.ip4.id = 0;
3199                 sqp->ud_header.ip4.frag_off = htons(IP_DF);
3200                 sqp->ud_header.ip4.ttl = ah->av.eth.hop_limit;
3201
3202                 memcpy(&sqp->ud_header.ip4.saddr,
3203                        sgid.raw + 12, 4);
3204                 memcpy(&sqp->ud_header.ip4.daddr, ah->av.ib.dgid + 12, 4);
3205                 sqp->ud_header.ip4.check = ib_ud_ip4_csum(&sqp->ud_header);
3206         }
3207
3208         if (is_udp) {
3209                 sqp->ud_header.udp.dport = htons(ROCE_V2_UDP_DPORT);
3210                 sqp->ud_header.udp.sport = htons(MLX4_ROCEV2_QP1_SPORT);
3211                 sqp->ud_header.udp.csum = 0;
3212         }
3213
3214         mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
3215
3216         if (!is_eth) {
3217                 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
3218                                           (sqp->ud_header.lrh.destination_lid ==
3219                                            IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
3220                                           (sqp->ud_header.lrh.service_level << 8));
3221                 if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
3222                         mlx->flags |= cpu_to_be32(0x1); /* force loopback */
3223                 mlx->rlid = sqp->ud_header.lrh.destination_lid;
3224         }
3225
3226         switch (wr->wr.opcode) {
3227         case IB_WR_SEND:
3228                 sqp->ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY;
3229                 sqp->ud_header.immediate_present = 0;
3230                 break;
3231         case IB_WR_SEND_WITH_IMM:
3232                 sqp->ud_header.bth.opcode        = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
3233                 sqp->ud_header.immediate_present = 1;
3234                 sqp->ud_header.immediate_data    = wr->wr.ex.imm_data;
3235                 break;
3236         default:
3237                 return -EINVAL;
3238         }
3239
3240         if (is_eth) {
3241                 struct in6_addr in6;
3242                 u16 ether_type;
3243                 u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
3244
3245                 ether_type = (!is_udp) ? ETH_P_IBOE:
3246                         (ip_version == 4 ? ETH_P_IP : ETH_P_IPV6);
3247
3248                 mlx->sched_prio = cpu_to_be16(pcp);
3249
3250                 ether_addr_copy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac);
3251                 memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
3252                 memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
3253                 memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
3254                 memcpy(&in6, sgid.raw, sizeof(in6));
3255
3256
3257                 if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
3258                         mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
3259                 if (!is_vlan) {
3260                         sqp->ud_header.eth.type = cpu_to_be16(ether_type);
3261                 } else {
3262                         sqp->ud_header.vlan.type = cpu_to_be16(ether_type);
3263                         sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
3264                 }
3265         } else {
3266                 sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 :
3267                                                         sl_to_vl(to_mdev(ib_dev),
3268                                                                  sqp->ud_header.lrh.service_level,
3269                                                                  sqp->qp.port);
3270                 if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
3271                         return -EINVAL;
3272                 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
3273                         sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
3274         }
3275         sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
3276         if (!sqp->qp.ibqp.qp_num)
3277                 ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
3278         else
3279                 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, &pkey);
3280         sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
3281         sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
3282         sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
3283         sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
3284                                                sqp->qkey : wr->remote_qkey);
3285         sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
3286
3287         header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3288
3289         if (0) {
3290                 pr_err("built UD header of size %d:\n", header_size);
3291                 for (i = 0; i < header_size / 4; ++i) {
3292                         if (i % 8 == 0)
3293                                 pr_err("  [%02x] ", i * 4);
3294                         pr_cont(" %08x",
3295                                 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
3296                         if ((i + 1) % 8 == 0)
3297                                 pr_cont("\n");
3298                 }
3299                 pr_err("\n");
3300         }
3301
3302         /*
3303          * Inline data segments may not cross a 64 byte boundary.  If
3304          * our UD header is bigger than the space available up to the
3305          * next 64 byte boundary in the WQE, use two inline data
3306          * segments to hold the UD header.
3307          */
3308         spc = MLX4_INLINE_ALIGN -
3309                 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3310         if (header_size <= spc) {
3311                 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
3312                 memcpy(inl + 1, sqp->header_buf, header_size);
3313                 i = 1;
3314         } else {
3315                 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3316                 memcpy(inl + 1, sqp->header_buf, spc);
3317
3318                 inl = (void *) (inl + 1) + spc;
3319                 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3320                 /*
3321                  * Need a barrier here to make sure all the data is
3322                  * visible before the byte_count field is set.
3323                  * Otherwise the HCA prefetcher could grab the 64-byte
3324                  * chunk with this inline segment and get a valid (!=
3325                  * 0xffffffff) byte count but stale data, and end up
3326                  * generating a packet with bad headers.
3327                  *
3328                  * The first inline segment's byte_count field doesn't
3329                  * need a barrier, because it comes after a
3330                  * control/MLX segment and therefore is at an offset
3331                  * of 16 mod 64.
3332                  */
3333                 wmb();
3334                 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3335                 i = 2;
3336         }
3337
3338         *mlx_seg_len =
3339                 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3340         return 0;
3341 }
3342
3343 static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
3344 {
3345         unsigned cur;
3346         struct mlx4_ib_cq *cq;
3347
3348         cur = wq->head - wq->tail;
3349         if (likely(cur + nreq < wq->max_post))
3350                 return 0;
3351
3352         cq = to_mcq(ib_cq);
3353         spin_lock(&cq->lock);
3354         cur = wq->head - wq->tail;
3355         spin_unlock(&cq->lock);
3356
3357         return cur + nreq >= wq->max_post;
3358 }
3359
3360 static __be32 convert_access(int acc)
3361 {
3362         return (acc & IB_ACCESS_REMOTE_ATOMIC ?
3363                 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC)       : 0) |
3364                (acc & IB_ACCESS_REMOTE_WRITE  ?
3365                 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) |
3366                (acc & IB_ACCESS_REMOTE_READ   ?
3367                 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ)  : 0) |
3368                (acc & IB_ACCESS_LOCAL_WRITE   ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE)  : 0) |
3369                 cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
3370 }
3371
3372 static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,
3373                         struct ib_reg_wr *wr)
3374 {
3375         struct mlx4_ib_mr *mr = to_mmr(wr->mr);
3376
3377         fseg->flags             = convert_access(wr->access);
3378         fseg->mem_key           = cpu_to_be32(wr->key);
3379         fseg->buf_list          = cpu_to_be64(mr->page_map);
3380         fseg->start_addr        = cpu_to_be64(mr->ibmr.iova);
3381         fseg->reg_len           = cpu_to_be64(mr->ibmr.length);
3382         fseg->offset            = 0; /* XXX -- is this just for ZBVA? */
3383         fseg->page_size         = cpu_to_be32(ilog2(mr->ibmr.page_size));
3384         fseg->reserved[0]       = 0;
3385         fseg->reserved[1]       = 0;
3386 }
3387
3388 static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
3389 {
3390         memset(iseg, 0, sizeof(*iseg));
3391         iseg->mem_key = cpu_to_be32(rkey);
3392 }
3393
3394 static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
3395                                           u64 remote_addr, u32 rkey)
3396 {
3397         rseg->raddr    = cpu_to_be64(remote_addr);
3398         rseg->rkey     = cpu_to_be32(rkey);
3399         rseg->reserved = 0;
3400 }
3401
3402 static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg,
3403                 struct ib_atomic_wr *wr)
3404 {
3405         if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
3406                 aseg->swap_add = cpu_to_be64(wr->swap);
3407                 aseg->compare  = cpu_to_be64(wr->compare_add);
3408         } else if (wr->wr.opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
3409                 aseg->swap_add = cpu_to_be64(wr->compare_add);
3410                 aseg->compare  = cpu_to_be64(wr->compare_add_mask);
3411         } else {
3412                 aseg->swap_add = cpu_to_be64(wr->compare_add);
3413                 aseg->compare  = 0;
3414         }
3415
3416 }
3417
3418 static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
3419                                   struct ib_atomic_wr *wr)
3420 {
3421         aseg->swap_add          = cpu_to_be64(wr->swap);
3422         aseg->swap_add_mask     = cpu_to_be64(wr->swap_mask);
3423         aseg->compare           = cpu_to_be64(wr->compare_add);
3424         aseg->compare_mask      = cpu_to_be64(wr->compare_add_mask);
3425 }
3426
3427 static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
3428                              struct ib_ud_wr *wr)
3429 {
3430         memcpy(dseg->av, &to_mah(wr->ah)->av, sizeof (struct mlx4_av));
3431         dseg->dqpn = cpu_to_be32(wr->remote_qpn);
3432         dseg->qkey = cpu_to_be32(wr->remote_qkey);
3433         dseg->vlan = to_mah(wr->ah)->av.eth.vlan;
3434         memcpy(dseg->mac, to_mah(wr->ah)->av.eth.mac, 6);
3435 }
3436
3437 static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
3438                                     struct mlx4_wqe_datagram_seg *dseg,
3439                                     struct ib_ud_wr *wr,
3440                                     enum mlx4_ib_qp_type qpt)
3441 {
3442         union mlx4_ext_av *av = &to_mah(wr->ah)->av;
3443         struct mlx4_av sqp_av = {0};
3444         int port = *((u8 *) &av->ib.port_pd) & 0x3;
3445
3446         /* force loopback */
3447         sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000);
3448         sqp_av.g_slid = av->ib.g_slid & 0x7f; /* no GRH */
3449         sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel &
3450                         cpu_to_be32(0xf0000000);
3451
3452         memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
3453         if (qpt == MLX4_IB_QPT_PROXY_GSI)
3454                 dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp1_tunnel);
3455         else
3456                 dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp0_tunnel);
3457         /* Use QKEY from the QP context, which is set by master */
3458         dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
3459 }
3460
3461 static void build_tunnel_header(struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len)
3462 {
3463         struct mlx4_wqe_inline_seg *inl = wqe;
3464         struct mlx4_ib_tunnel_header hdr;
3465         struct mlx4_ib_ah *ah = to_mah(wr->ah);
3466         int spc;
3467         int i;
3468
3469         memcpy(&hdr.av, &ah->av, sizeof hdr.av);
3470         hdr.remote_qpn = cpu_to_be32(wr->remote_qpn);
3471         hdr.pkey_index = cpu_to_be16(wr->pkey_index);
3472         hdr.qkey = cpu_to_be32(wr->remote_qkey);
3473         memcpy(hdr.mac, ah->av.eth.mac, 6);
3474         hdr.vlan = ah->av.eth.vlan;
3475
3476         spc = MLX4_INLINE_ALIGN -
3477                 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3478         if (sizeof (hdr) <= spc) {
3479                 memcpy(inl + 1, &hdr, sizeof (hdr));
3480                 wmb();
3481                 inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr));
3482                 i = 1;
3483         } else {
3484                 memcpy(inl + 1, &hdr, spc);
3485                 wmb();
3486                 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3487
3488                 inl = (void *) (inl + 1) + spc;
3489                 memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc);
3490                 wmb();
3491                 inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc));
3492                 i = 2;
3493         }
3494
3495         *mlx_seg_len =
3496                 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16);
3497 }
3498
3499 static void set_mlx_icrc_seg(void *dseg)
3500 {
3501         u32 *t = dseg;
3502         struct mlx4_wqe_inline_seg *iseg = dseg;
3503
3504         t[1] = 0;
3505
3506         /*
3507          * Need a barrier here before writing the byte_count field to
3508          * make sure that all the data is visible before the
3509          * byte_count field is set.  Otherwise, if the segment begins
3510          * a new cacheline, the HCA prefetcher could grab the 64-byte
3511          * chunk and get a valid (!= * 0xffffffff) byte count but
3512          * stale data, and end up sending the wrong data.
3513          */
3514         wmb();
3515
3516         iseg->byte_count = cpu_to_be32((1 << 31) | 4);
3517 }
3518
3519 static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3520 {
3521         dseg->lkey       = cpu_to_be32(sg->lkey);
3522         dseg->addr       = cpu_to_be64(sg->addr);
3523
3524         /*
3525          * Need a barrier here before writing the byte_count field to
3526          * make sure that all the data is visible before the
3527          * byte_count field is set.  Otherwise, if the segment begins
3528          * a new cacheline, the HCA prefetcher could grab the 64-byte
3529          * chunk and get a valid (!= * 0xffffffff) byte count but
3530          * stale data, and end up sending the wrong data.
3531          */
3532         wmb();
3533
3534         dseg->byte_count = cpu_to_be32(sg->length);
3535 }
3536
3537 static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3538 {
3539         dseg->byte_count = cpu_to_be32(sg->length);
3540         dseg->lkey       = cpu_to_be32(sg->lkey);
3541         dseg->addr       = cpu_to_be64(sg->addr);
3542 }
3543
3544 static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_ud_wr *wr,
3545                          struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
3546                          __be32 *lso_hdr_sz, __be32 *blh)
3547 {
3548         unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16);
3549
3550         if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
3551                 *blh = cpu_to_be32(1 << 6);
3552
3553         if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
3554                      wr->wr.num_sge > qp->sq.max_gs - (halign >> 4)))
3555                 return -EINVAL;
3556
3557         memcpy(wqe->header, wr->header, wr->hlen);
3558
3559         *lso_hdr_sz  = cpu_to_be32(wr->mss << 16 | wr->hlen);
3560         *lso_seg_len = halign;
3561         return 0;
3562 }
3563
3564 static __be32 send_ieth(struct ib_send_wr *wr)
3565 {
3566         switch (wr->opcode) {
3567         case IB_WR_SEND_WITH_IMM:
3568         case IB_WR_RDMA_WRITE_WITH_IMM:
3569                 return wr->ex.imm_data;
3570
3571         case IB_WR_SEND_WITH_INV:
3572                 return cpu_to_be32(wr->ex.invalidate_rkey);
3573
3574         default:
3575                 return 0;
3576         }
3577 }
3578
3579 static void add_zero_len_inline(void *wqe)
3580 {
3581         struct mlx4_wqe_inline_seg *inl = wqe;
3582         memset(wqe, 0, 16);
3583         inl->byte_count = cpu_to_be32(1 << 31);
3584 }
3585
3586 int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3587                       struct ib_send_wr **bad_wr)
3588 {
3589         struct mlx4_ib_qp *qp = to_mqp(ibqp);
3590         void *wqe;
3591         struct mlx4_wqe_ctrl_seg *ctrl;
3592         struct mlx4_wqe_data_seg *dseg;
3593         unsigned long flags;
3594         int nreq;
3595         int err = 0;
3596         unsigned ind;
3597         int uninitialized_var(stamp);
3598         int uninitialized_var(size);
3599         unsigned uninitialized_var(seglen);
3600         __be32 dummy;
3601         __be32 *lso_wqe;
3602         __be32 uninitialized_var(lso_hdr_sz);
3603         __be32 blh;
3604         int i;
3605         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3606
3607         if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
3608                 struct mlx4_ib_sqp *sqp = to_msqp(qp);
3609
3610                 if (sqp->roce_v2_gsi) {
3611                         struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
3612                         enum ib_gid_type gid_type;
3613                         union ib_gid gid;
3614
3615                         if (!fill_gid_by_hw_index(mdev, sqp->qp.port,
3616                                            ah->av.ib.gid_index,
3617                                            &gid, &gid_type))
3618                                 qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3619                                                 to_mqp(sqp->roce_v2_gsi) : qp;
3620                         else
3621                                 pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n",
3622                                        ah->av.ib.gid_index);
3623                 }
3624         }
3625
3626         spin_lock_irqsave(&qp->sq.lock, flags);
3627         if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3628                 err = -EIO;
3629                 *bad_wr = wr;
3630                 nreq = 0;
3631                 goto out;
3632         }
3633
3634         ind = qp->sq_next_wqe;
3635
3636         for (nreq = 0; wr; ++nreq, wr = wr->next) {
3637                 lso_wqe = &dummy;
3638                 blh = 0;
3639
3640                 if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
3641                         err = -ENOMEM;
3642                         *bad_wr = wr;
3643                         goto out;
3644                 }
3645
3646                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
3647                         err = -EINVAL;
3648                         *bad_wr = wr;
3649                         goto out;
3650                 }
3651
3652                 ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
3653                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
3654
3655                 ctrl->srcrb_flags =
3656                         (wr->send_flags & IB_SEND_SIGNALED ?
3657                          cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
3658                         (wr->send_flags & IB_SEND_SOLICITED ?
3659                          cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
3660                         ((wr->send_flags & IB_SEND_IP_CSUM) ?
3661                          cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
3662                                      MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
3663                         qp->sq_signal_bits;
3664
3665                 ctrl->imm = send_ieth(wr);
3666
3667                 wqe += sizeof *ctrl;
3668                 size = sizeof *ctrl / 16;
3669
3670                 switch (qp->mlx4_ib_qp_type) {
3671                 case MLX4_IB_QPT_RC:
3672                 case MLX4_IB_QPT_UC:
3673                         switch (wr->opcode) {
3674                         case IB_WR_ATOMIC_CMP_AND_SWP:
3675                         case IB_WR_ATOMIC_FETCH_AND_ADD:
3676                         case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
3677                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3678                                               atomic_wr(wr)->rkey);
3679                                 wqe  += sizeof (struct mlx4_wqe_raddr_seg);
3680
3681                                 set_atomic_seg(wqe, atomic_wr(wr));
3682                                 wqe  += sizeof (struct mlx4_wqe_atomic_seg);
3683
3684                                 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3685                                          sizeof (struct mlx4_wqe_atomic_seg)) / 16;
3686
3687                                 break;
3688
3689                         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
3690                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3691                                               atomic_wr(wr)->rkey);
3692                                 wqe  += sizeof (struct mlx4_wqe_raddr_seg);
3693
3694                                 set_masked_atomic_seg(wqe, atomic_wr(wr));
3695                                 wqe  += sizeof (struct mlx4_wqe_masked_atomic_seg);
3696
3697                                 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3698                                          sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
3699
3700                                 break;
3701
3702                         case IB_WR_RDMA_READ:
3703                         case IB_WR_RDMA_WRITE:
3704                         case IB_WR_RDMA_WRITE_WITH_IMM:
3705                                 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
3706                                               rdma_wr(wr)->rkey);
3707                                 wqe  += sizeof (struct mlx4_wqe_raddr_seg);
3708                                 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
3709                                 break;
3710
3711                         case IB_WR_LOCAL_INV:
3712                                 ctrl->srcrb_flags |=
3713                                         cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3714                                 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
3715                                 wqe  += sizeof (struct mlx4_wqe_local_inval_seg);
3716                                 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
3717                                 break;
3718
3719                         case IB_WR_REG_MR:
3720                                 ctrl->srcrb_flags |=
3721                                         cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3722                                 set_reg_seg(wqe, reg_wr(wr));
3723                                 wqe  += sizeof(struct mlx4_wqe_fmr_seg);
3724                                 size += sizeof(struct mlx4_wqe_fmr_seg) / 16;
3725                                 break;
3726
3727                         default:
3728                                 /* No extra segments required for sends */
3729                                 break;
3730                         }
3731                         break;
3732
3733                 case MLX4_IB_QPT_TUN_SMI_OWNER:
3734                         err =  build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3735                                         ctrl, &seglen);
3736                         if (unlikely(err)) {
3737                                 *bad_wr = wr;
3738                                 goto out;
3739                         }
3740                         wqe  += seglen;
3741                         size += seglen / 16;
3742                         break;
3743                 case MLX4_IB_QPT_TUN_SMI:
3744                 case MLX4_IB_QPT_TUN_GSI:
3745                         /* this is a UD qp used in MAD responses to slaves. */
3746                         set_datagram_seg(wqe, ud_wr(wr));
3747                         /* set the forced-loopback bit in the data seg av */
3748                         *(__be32 *) wqe |= cpu_to_be32(0x80000000);
3749                         wqe  += sizeof (struct mlx4_wqe_datagram_seg);
3750                         size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3751                         break;
3752                 case MLX4_IB_QPT_UD:
3753                         set_datagram_seg(wqe, ud_wr(wr));
3754                         wqe  += sizeof (struct mlx4_wqe_datagram_seg);
3755                         size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3756
3757                         if (wr->opcode == IB_WR_LSO) {
3758                                 err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen,
3759                                                 &lso_hdr_sz, &blh);
3760                                 if (unlikely(err)) {
3761                                         *bad_wr = wr;
3762                                         goto out;
3763                                 }
3764                                 lso_wqe = (__be32 *) wqe;
3765                                 wqe  += seglen;
3766                                 size += seglen / 16;
3767                         }
3768                         break;
3769
3770                 case MLX4_IB_QPT_PROXY_SMI_OWNER:
3771                         err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3772                                         ctrl, &seglen);
3773                         if (unlikely(err)) {
3774                                 *bad_wr = wr;
3775                                 goto out;
3776                         }
3777                         wqe  += seglen;
3778                         size += seglen / 16;
3779                         /* to start tunnel header on a cache-line boundary */
3780                         add_zero_len_inline(wqe);
3781                         wqe += 16;
3782                         size++;
3783                         build_tunnel_header(ud_wr(wr), wqe, &seglen);
3784                         wqe  += seglen;
3785                         size += seglen / 16;
3786                         break;
3787                 case MLX4_IB_QPT_PROXY_SMI:
3788                 case MLX4_IB_QPT_PROXY_GSI:
3789                         /* If we are tunneling special qps, this is a UD qp.
3790                          * In this case we first add a UD segment targeting
3791                          * the tunnel qp, and then add a header with address
3792                          * information */
3793                         set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe,
3794                                                 ud_wr(wr),
3795                                                 qp->mlx4_ib_qp_type);
3796                         wqe  += sizeof (struct mlx4_wqe_datagram_seg);
3797                         size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3798                         build_tunnel_header(ud_wr(wr), wqe, &seglen);
3799                         wqe  += seglen;
3800                         size += seglen / 16;
3801                         break;
3802
3803                 case MLX4_IB_QPT_SMI:
3804                 case MLX4_IB_QPT_GSI:
3805                         err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,
3806                                         &seglen);
3807                         if (unlikely(err)) {
3808                                 *bad_wr = wr;
3809                                 goto out;
3810                         }
3811                         wqe  += seglen;
3812                         size += seglen / 16;
3813                         break;
3814
3815                 default:
3816                         break;
3817                 }
3818
3819                 /*
3820                  * Write data segments in reverse order, so as to
3821                  * overwrite cacheline stamp last within each
3822                  * cacheline.  This avoids issues with WQE
3823                  * prefetching.
3824                  */
3825
3826                 dseg = wqe;
3827                 dseg += wr->num_sge - 1;
3828                 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
3829
3830                 /* Add one more inline data segment for ICRC for MLX sends */
3831                 if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
3832                              qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI ||
3833                              qp->mlx4_ib_qp_type &
3834                              (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) {
3835                         set_mlx_icrc_seg(dseg + 1);
3836                         size += sizeof (struct mlx4_wqe_data_seg) / 16;
3837                 }
3838
3839                 for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
3840                         set_data_seg(dseg, wr->sg_list + i);
3841
3842                 /*
3843                  * Possibly overwrite stamping in cacheline with LSO
3844                  * segment only after making sure all data segments
3845                  * are written.
3846                  */
3847                 wmb();
3848                 *lso_wqe = lso_hdr_sz;
3849
3850                 ctrl->qpn_vlan.fence_size = (wr->send_flags & IB_SEND_FENCE ?
3851                                              MLX4_WQE_CTRL_FENCE : 0) | size;
3852
3853                 /*
3854                  * Make sure descriptor is fully written before
3855                  * setting ownership bit (because HW can start
3856                  * executing as soon as we do).
3857                  */
3858                 wmb();
3859
3860                 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
3861                         *bad_wr = wr;
3862                         err = -EINVAL;
3863                         goto out;
3864                 }
3865
3866                 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
3867                         (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
3868
3869                 stamp = ind + qp->sq_spare_wqes;
3870                 ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
3871
3872                 /*
3873                  * We can improve latency by not stamping the last
3874                  * send queue WQE until after ringing the doorbell, so
3875                  * only stamp here if there are still more WQEs to post.
3876                  *
3877                  * Same optimization applies to padding with NOP wqe
3878                  * in case of WQE shrinking (used to prevent wrap-around
3879                  * in the middle of WR).
3880                  */
3881                 if (wr->next) {
3882                         stamp_send_wqe(qp, stamp, size * 16);
3883                         ind = pad_wraparound(qp, ind);
3884                 }
3885         }
3886
3887 out:
3888         if (likely(nreq)) {
3889                 qp->sq.head += nreq;
3890
3891                 /*
3892                  * Make sure that descriptors are written before
3893                  * doorbell record.
3894                  */
3895                 wmb();
3896
3897                 writel_relaxed(qp->doorbell_qpn,
3898                         to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3899
3900                 /*
3901                  * Make sure doorbells don't leak out of SQ spinlock
3902                  * and reach the HCA out of order.
3903                  */
3904                 mmiowb();
3905
3906                 stamp_send_wqe(qp, stamp, size * 16);
3907
3908                 ind = pad_wraparound(qp, ind);
3909                 qp->sq_next_wqe = ind;
3910         }
3911
3912         spin_unlock_irqrestore(&qp->sq.lock, flags);
3913
3914         return err;
3915 }
3916
3917 int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3918                       struct ib_recv_wr **bad_wr)
3919 {
3920         struct mlx4_ib_qp *qp = to_mqp(ibqp);
3921         struct mlx4_wqe_data_seg *scat;
3922         unsigned long flags;
3923         int err = 0;
3924         int nreq;
3925         int ind;
3926         int max_gs;
3927         int i;
3928         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3929
3930         max_gs = qp->rq.max_gs;
3931         spin_lock_irqsave(&qp->rq.lock, flags);
3932
3933         if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3934                 err = -EIO;
3935                 *bad_wr = wr;
3936                 nreq = 0;
3937                 goto out;
3938         }
3939
3940         ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
3941
3942         for (nreq = 0; wr; ++nreq, wr = wr->next) {
3943                 if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
3944                         err = -ENOMEM;
3945                         *bad_wr = wr;
3946                         goto out;
3947                 }
3948
3949                 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
3950                         err = -EINVAL;
3951                         *bad_wr = wr;
3952                         goto out;
3953                 }
3954
3955                 scat = get_recv_wqe(qp, ind);
3956
3957                 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
3958                     MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
3959                         ib_dma_sync_single_for_device(ibqp->device,
3960                                                       qp->sqp_proxy_rcv[ind].map,
3961                                                       sizeof (struct mlx4_ib_proxy_sqp_hdr),
3962                                                       DMA_FROM_DEVICE);
3963                         scat->byte_count =
3964                                 cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr));
3965                         /* use dma lkey from upper layer entry */
3966                         scat->lkey = cpu_to_be32(wr->sg_list->lkey);
3967                         scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map);
3968                         scat++;
3969                         max_gs--;
3970                 }
3971
3972                 for (i = 0; i < wr->num_sge; ++i)
3973                         __set_data_seg(scat + i, wr->sg_list + i);
3974
3975                 if (i < max_gs) {
3976                         scat[i].byte_count = 0;
3977                         scat[i].lkey       = cpu_to_be32(MLX4_INVALID_LKEY);
3978                         scat[i].addr       = 0;
3979                 }
3980
3981                 qp->rq.wrid[ind] = wr->wr_id;
3982
3983                 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
3984         }
3985
3986 out:
3987         if (likely(nreq)) {
3988                 qp->rq.head += nreq;
3989
3990                 /*
3991                  * Make sure that descriptors are written before
3992                  * doorbell record.
3993                  */
3994                 wmb();
3995
3996                 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
3997         }
3998
3999         spin_unlock_irqrestore(&qp->rq.lock, flags);
4000
4001         return err;
4002 }
4003
4004 static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
4005 {
4006         switch (mlx4_state) {
4007         case MLX4_QP_STATE_RST:      return IB_QPS_RESET;
4008         case MLX4_QP_STATE_INIT:     return IB_QPS_INIT;
4009         case MLX4_QP_STATE_RTR:      return IB_QPS_RTR;
4010         case MLX4_QP_STATE_RTS:      return IB_QPS_RTS;
4011         case MLX4_QP_STATE_SQ_DRAINING:
4012         case MLX4_QP_STATE_SQD:      return IB_QPS_SQD;
4013         case MLX4_QP_STATE_SQER:     return IB_QPS_SQE;
4014         case MLX4_QP_STATE_ERR:      return IB_QPS_ERR;
4015         default:                     return -1;
4016         }
4017 }
4018
4019 static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
4020 {
4021         switch (mlx4_mig_state) {
4022         case MLX4_QP_PM_ARMED:          return IB_MIG_ARMED;
4023         case MLX4_QP_PM_REARM:          return IB_MIG_REARM;
4024         case MLX4_QP_PM_MIGRATED:       return IB_MIG_MIGRATED;
4025         default: return -1;
4026         }
4027 }
4028
4029 static int to_ib_qp_access_flags(int mlx4_flags)
4030 {
4031         int ib_flags = 0;
4032
4033         if (mlx4_flags & MLX4_QP_BIT_RRE)
4034                 ib_flags |= IB_ACCESS_REMOTE_READ;
4035         if (mlx4_flags & MLX4_QP_BIT_RWE)
4036                 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4037         if (mlx4_flags & MLX4_QP_BIT_RAE)
4038                 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4039
4040         return ib_flags;
4041 }
4042
4043 static void to_rdma_ah_attr(struct mlx4_ib_dev *ibdev,
4044                             struct rdma_ah_attr *ah_attr,
4045                             struct mlx4_qp_path *path)
4046 {
4047         struct mlx4_dev *dev = ibdev->dev;
4048         u8 port_num = path->sched_queue & 0x40 ? 2 : 1;
4049
4050         memset(ah_attr, 0, sizeof(*ah_attr));
4051         ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port_num);
4052         if (port_num == 0 || port_num > dev->caps.num_ports)
4053                 return;
4054
4055         if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
4056                 rdma_ah_set_sl(ah_attr, ((path->sched_queue >> 3) & 0x7) |
4057                                ((path->sched_queue & 4) << 1));
4058         else
4059                 rdma_ah_set_sl(ah_attr, (path->sched_queue >> 2) & 0xf);
4060         rdma_ah_set_port_num(ah_attr, port_num);
4061
4062         rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4063         rdma_ah_set_path_bits(ah_attr, path->grh_mylmc & 0x7f);
4064         rdma_ah_set_static_rate(ah_attr,
4065                                 path->static_rate ? path->static_rate - 5 : 0);
4066         if (path->grh_mylmc & (1 << 7)) {
4067                 rdma_ah_set_grh(ah_attr, NULL,
4068                                 be32_to_cpu(path->tclass_flowlabel) & 0xfffff,
4069                                 path->mgid_index,
4070                                 path->hop_limit,
4071                                 (be32_to_cpu(path->tclass_flowlabel)
4072                                  >> 20) & 0xff);
4073                 rdma_ah_set_dgid_raw(ah_attr, path->rgid);
4074         }
4075 }
4076
4077 int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
4078                      struct ib_qp_init_attr *qp_init_attr)
4079 {
4080         struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
4081         struct mlx4_ib_qp *qp = to_mqp(ibqp);
4082         struct mlx4_qp_context context;
4083         int mlx4_state;
4084         int err = 0;
4085
4086         if (ibqp->rwq_ind_tbl)
4087                 return -EOPNOTSUPP;
4088
4089         mutex_lock(&qp->mutex);
4090
4091         if (qp->state == IB_QPS_RESET) {
4092                 qp_attr->qp_state = IB_QPS_RESET;
4093                 goto done;
4094         }
4095
4096         err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
4097         if (err) {
4098                 err = -EINVAL;
4099                 goto out;
4100         }
4101
4102         mlx4_state = be32_to_cpu(context.flags) >> 28;
4103
4104         qp->state                    = to_ib_qp_state(mlx4_state);
4105         qp_attr->qp_state            = qp->state;
4106         qp_attr->path_mtu            = context.mtu_msgmax >> 5;
4107         qp_attr->path_mig_state      =
4108                 to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
4109         qp_attr->qkey                = be32_to_cpu(context.qkey);
4110         qp_attr->rq_psn              = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
4111         qp_attr->sq_psn              = be32_to_cpu(context.next_send_psn) & 0xffffff;
4112         qp_attr->dest_qp_num         = be32_to_cpu(context.remote_qpn) & 0xffffff;
4113         qp_attr->qp_access_flags     =
4114                 to_ib_qp_access_flags(be32_to_cpu(context.params2));
4115
4116         if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
4117                 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
4118                 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
4119                 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
4120                 qp_attr->alt_port_num   =
4121                         rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
4122         }
4123
4124         qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
4125         if (qp_attr->qp_state == IB_QPS_INIT)
4126                 qp_attr->port_num = qp->port;
4127         else
4128                 qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
4129
4130         /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4131         qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
4132
4133         qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
4134
4135         qp_attr->max_dest_rd_atomic =
4136                 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
4137         qp_attr->min_rnr_timer      =
4138                 (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
4139         qp_attr->timeout            = context.pri_path.ackto >> 3;
4140         qp_attr->retry_cnt          = (be32_to_cpu(context.params1) >> 16) & 0x7;
4141         qp_attr->rnr_retry          = (be32_to_cpu(context.params1) >> 13) & 0x7;
4142         qp_attr->alt_timeout        = context.alt_path.ackto >> 3;
4143
4144 done:
4145         qp_attr->cur_qp_state        = qp_attr->qp_state;
4146         qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
4147         qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
4148
4149         if (!ibqp->uobject) {
4150                 qp_attr->cap.max_send_wr  = qp->sq.wqe_cnt;
4151                 qp_attr->cap.max_send_sge = qp->sq.max_gs;
4152         } else {
4153                 qp_attr->cap.max_send_wr  = 0;
4154                 qp_attr->cap.max_send_sge = 0;
4155         }
4156
4157         /*
4158          * We don't support inline sends for kernel QPs (yet), and we
4159          * don't know what userspace's value should be.
4160          */
4161         qp_attr->cap.max_inline_data = 0;
4162
4163         qp_init_attr->cap            = qp_attr->cap;
4164
4165         qp_init_attr->create_flags = 0;
4166         if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
4167                 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
4168
4169         if (qp->flags & MLX4_IB_QP_LSO)
4170                 qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
4171
4172         if (qp->flags & MLX4_IB_QP_NETIF)
4173                 qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP;
4174
4175         qp_init_attr->sq_sig_type =
4176                 qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ?
4177                 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
4178
4179 out:
4180         mutex_unlock(&qp->mutex);
4181         return err;
4182 }
4183
4184 struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
4185                                 struct ib_wq_init_attr *init_attr,
4186                                 struct ib_udata *udata)
4187 {
4188         struct mlx4_ib_dev *dev;
4189         struct ib_qp_init_attr ib_qp_init_attr;
4190         struct mlx4_ib_qp *qp;
4191         struct mlx4_ib_create_wq ucmd;
4192         int err, required_cmd_sz;
4193
4194         if (!(udata && pd->uobject))
4195                 return ERR_PTR(-EINVAL);
4196
4197         required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
4198                           sizeof(ucmd.comp_mask);
4199         if (udata->inlen < required_cmd_sz) {
4200                 pr_debug("invalid inlen\n");
4201                 return ERR_PTR(-EINVAL);
4202         }
4203
4204         if (udata->inlen > sizeof(ucmd) &&
4205             !ib_is_udata_cleared(udata, sizeof(ucmd),
4206                                  udata->inlen - sizeof(ucmd))) {
4207                 pr_debug("inlen is not supported\n");
4208                 return ERR_PTR(-EOPNOTSUPP);
4209         }
4210
4211         if (udata->outlen)
4212                 return ERR_PTR(-EOPNOTSUPP);
4213
4214         dev = to_mdev(pd->device);
4215
4216         if (init_attr->wq_type != IB_WQT_RQ) {
4217                 pr_debug("unsupported wq type %d\n", init_attr->wq_type);
4218                 return ERR_PTR(-EOPNOTSUPP);
4219         }
4220
4221         if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS) {
4222                 pr_debug("unsupported create_flags %u\n",
4223                          init_attr->create_flags);
4224                 return ERR_PTR(-EOPNOTSUPP);
4225         }
4226
4227         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
4228         if (!qp)
4229                 return ERR_PTR(-ENOMEM);
4230
4231         qp->pri.vid = 0xFFFF;
4232         qp->alt.vid = 0xFFFF;
4233
4234         memset(&ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));
4235         ib_qp_init_attr.qp_context = init_attr->wq_context;
4236         ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
4237         ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
4238         ib_qp_init_attr.cap.max_recv_sge = init_attr->max_sge;
4239         ib_qp_init_attr.recv_cq = init_attr->cq;
4240         ib_qp_init_attr.send_cq = ib_qp_init_attr.recv_cq; /* Dummy CQ */
4241
4242         if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS)
4243                 ib_qp_init_attr.create_flags |= IB_QP_CREATE_SCATTER_FCS;
4244
4245         err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &ib_qp_init_attr,
4246                                udata, 0, &qp);
4247         if (err) {
4248                 kfree(qp);
4249                 return ERR_PTR(err);
4250         }
4251
4252         qp->ibwq.event_handler = init_attr->event_handler;
4253         qp->ibwq.wq_num = qp->mqp.qpn;
4254         qp->ibwq.state = IB_WQS_RESET;
4255
4256         return &qp->ibwq;
4257 }
4258
4259 static int ib_wq2qp_state(enum ib_wq_state state)
4260 {
4261         switch (state) {
4262         case IB_WQS_RESET:
4263                 return IB_QPS_RESET;
4264         case IB_WQS_RDY:
4265                 return IB_QPS_RTR;
4266         default:
4267                 return IB_QPS_ERR;
4268         }
4269 }
4270
4271 static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state)
4272 {
4273         struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4274         enum ib_qp_state qp_cur_state;
4275         enum ib_qp_state qp_new_state;
4276         int attr_mask;
4277         int err;
4278
4279         /* ib_qp.state represents the WQ HW state while ib_wq.state represents
4280          * the WQ logic state.
4281          */
4282         qp_cur_state = qp->state;
4283         qp_new_state = ib_wq2qp_state(new_state);
4284
4285         if (ib_wq2qp_state(new_state) == qp_cur_state)
4286                 return 0;
4287
4288         if (new_state == IB_WQS_RDY) {
4289                 struct ib_qp_attr attr = {};
4290
4291                 attr.port_num = qp->port;
4292                 attr_mask = IB_QP_PORT;
4293
4294                 err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
4295                                           attr_mask, IB_QPS_RESET, IB_QPS_INIT);
4296                 if (err) {
4297                         pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
4298                                  ibwq->wq_num);
4299                         return err;
4300                 }
4301
4302                 qp_cur_state = IB_QPS_INIT;
4303         }
4304
4305         attr_mask = 0;
4306         err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
4307                                   qp_cur_state,  qp_new_state);
4308
4309         if (err && (qp_cur_state == IB_QPS_INIT)) {
4310                 qp_new_state = IB_QPS_RESET;
4311                 if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
4312                                         attr_mask, IB_QPS_INIT, IB_QPS_RESET)) {
4313                         pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
4314                                 ibwq->wq_num);
4315                         qp_new_state = IB_QPS_INIT;
4316                 }
4317         }
4318
4319         qp->state = qp_new_state;
4320
4321         return err;
4322 }
4323
4324 int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
4325                       u32 wq_attr_mask, struct ib_udata *udata)
4326 {
4327         struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4328         struct mlx4_ib_modify_wq ucmd = {};
4329         size_t required_cmd_sz;
4330         enum ib_wq_state cur_state, new_state;
4331         int err = 0;
4332
4333         required_cmd_sz = offsetof(typeof(ucmd), reserved) +
4334                                    sizeof(ucmd.reserved);
4335         if (udata->inlen < required_cmd_sz)
4336                 return -EINVAL;
4337
4338         if (udata->inlen > sizeof(ucmd) &&
4339             !ib_is_udata_cleared(udata, sizeof(ucmd),
4340                                  udata->inlen - sizeof(ucmd)))
4341                 return -EOPNOTSUPP;
4342
4343         if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
4344                 return -EFAULT;
4345
4346         if (ucmd.comp_mask || ucmd.reserved)
4347                 return -EOPNOTSUPP;
4348
4349         if (wq_attr_mask & IB_WQ_FLAGS)
4350                 return -EOPNOTSUPP;
4351
4352         cur_state = wq_attr_mask & IB_WQ_CUR_STATE ? wq_attr->curr_wq_state :
4353                                                      ibwq->state;
4354         new_state = wq_attr_mask & IB_WQ_STATE ? wq_attr->wq_state : cur_state;
4355
4356         if (cur_state  < IB_WQS_RESET || cur_state  > IB_WQS_ERR ||
4357             new_state < IB_WQS_RESET || new_state > IB_WQS_ERR)
4358                 return -EINVAL;
4359
4360         if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
4361                 return -EINVAL;
4362
4363         if ((new_state == IB_WQS_ERR) && (cur_state == IB_WQS_RESET))
4364                 return -EINVAL;
4365
4366         /* Need to protect against the parent RSS which also may modify WQ
4367          * state.
4368          */
4369         mutex_lock(&qp->mutex);
4370
4371         /* Can update HW state only if a RSS QP has already associated to this
4372          * WQ, so we can apply its port on the WQ.
4373          */
4374         if (qp->rss_usecnt)
4375                 err = _mlx4_ib_modify_wq(ibwq, new_state);
4376
4377         if (!err)
4378                 ibwq->state = new_state;
4379
4380         mutex_unlock(&qp->mutex);
4381
4382         return err;
4383 }
4384
4385 int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
4386 {
4387         struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
4388         struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4389
4390         if (qp->counter_index)
4391                 mlx4_ib_free_qp_counter(dev, qp);
4392
4393         destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
4394
4395         kfree(qp);
4396
4397         return 0;
4398 }
4399
4400 struct ib_rwq_ind_table
4401 *mlx4_ib_create_rwq_ind_table(struct ib_device *device,
4402                               struct ib_rwq_ind_table_init_attr *init_attr,
4403                               struct ib_udata *udata)
4404 {
4405         struct ib_rwq_ind_table *rwq_ind_table;
4406         struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
4407         unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
4408         unsigned int base_wqn;
4409         size_t min_resp_len;
4410         int i;
4411         int err;
4412
4413         if (udata->inlen > 0 &&
4414             !ib_is_udata_cleared(udata, 0,
4415                                  udata->inlen))
4416                 return ERR_PTR(-EOPNOTSUPP);
4417
4418         min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4419         if (udata->outlen && udata->outlen < min_resp_len)
4420                 return ERR_PTR(-EINVAL);
4421
4422         if (ind_tbl_size >
4423             device->attrs.rss_caps.max_rwq_indirection_table_size) {
4424                 pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
4425                          ind_tbl_size,
4426                          device->attrs.rss_caps.max_rwq_indirection_table_size);
4427                 return ERR_PTR(-EINVAL);
4428         }
4429
4430         base_wqn = init_attr->ind_tbl[0]->wq_num;
4431
4432         if (base_wqn % ind_tbl_size) {
4433                 pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
4434                          base_wqn);
4435                 return ERR_PTR(-EINVAL);
4436         }
4437
4438         for (i = 1; i < ind_tbl_size; i++) {
4439                 if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
4440                         pr_debug("indirection table's WQNs aren't consecutive\n");
4441                         return ERR_PTR(-EINVAL);
4442                 }
4443         }
4444
4445         rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);
4446         if (!rwq_ind_table)
4447                 return ERR_PTR(-ENOMEM);
4448
4449         if (udata->outlen) {
4450                 resp.response_length = offsetof(typeof(resp), response_length) +
4451                                         sizeof(resp.response_length);
4452                 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4453                 if (err)
4454                         goto err;
4455         }
4456
4457         return rwq_ind_table;
4458
4459 err:
4460         kfree(rwq_ind_table);
4461         return ERR_PTR(err);
4462 }
4463
4464 int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
4465 {
4466         kfree(ib_rwq_ind_tbl);
4467         return 0;
4468 }