drivers/infiniband/sw/siw/siw_verbs.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2
   3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4 /* Copyright (c) 2008-2019, IBM Corporation */
   5
   6 #include <linux/errno.h>
   7 #include <linux/types.h>
   8 #include <linux/uaccess.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/xarray.h>
  11
  12 #include <rdma/iw_cm.h>
  13 #include <rdma/ib_verbs.h>
  14 #include <rdma/ib_user_verbs.h>
  15 #include <rdma/uverbs_ioctl.h>
  16
  17 #include "siw.h"
  18 #include "siw_verbs.h"
  19 #include "siw_mem.h"
  20
  21 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
  22         [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
  23         [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
  24         [IB_QPS_RTR] = SIW_QP_STATE_RTR,
  25         [IB_QPS_RTS] = SIW_QP_STATE_RTS,
  26         [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
  27         [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
  28         [IB_QPS_ERR] = SIW_QP_STATE_ERROR
  29 };
  30
  31 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
  32         [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
  33         [IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
  34         [IB_QPS_ERR] = "ERR"
  35 };
  36
  37 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
  38 {
  39         struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
  40
  41         kfree(entry);
  42 }
  43
  44 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
  45 {
  46         struct siw_ucontext *uctx = to_siw_ctx(ctx);
  47         size_t size = vma->vm_end - vma->vm_start;
  48         struct rdma_user_mmap_entry *rdma_entry;
  49         struct siw_user_mmap_entry *entry;
  50         int rv = -EINVAL;
  51
  52         /*
  53          * Must be page aligned
  54          */
  55         if (vma->vm_start & (PAGE_SIZE - 1)) {
  56                 pr_warn("siw: mmap not page aligned\n");
  57                 return -EINVAL;
  58         }
  59         rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
  60         if (!rdma_entry) {
  61                 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
  62                         vma->vm_pgoff, size);
  63                 return -EINVAL;
  64         }
  65         entry = to_siw_mmap_entry(rdma_entry);
  66
  67         rv = remap_vmalloc_range(vma, entry->address, 0);
  68         if (rv) {
  69                 pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
  70                         size);
  71                 goto out;
  72         }
  73 out:
  74         rdma_user_mmap_entry_put(rdma_entry);
  75
  76         return rv;
  77 }
  78
  79 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
  80 {
  81         struct siw_device *sdev = to_siw_dev(base_ctx->device);
  82         struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
  83         struct siw_uresp_alloc_ctx uresp = {};
  84         int rv;
  85
  86         if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
  87                 rv = -ENOMEM;
  88                 goto err_out;
  89         }
  90         ctx->sdev = sdev;
  91
  92         uresp.dev_id = sdev->vendor_part_id;
  93
  94         if (udata->outlen < sizeof(uresp)) {
  95                 rv = -EINVAL;
  96                 goto err_out;
  97         }
  98         rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
  99         if (rv)
 100                 goto err_out;
 101
 102         siw_dbg(base_ctx->device, "success. now %d context(s)\n",
 103                 atomic_read(&sdev->num_ctx));
 104
 105         return 0;
 106
 107 err_out:
 108         atomic_dec(&sdev->num_ctx);
 109         siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
 110                 atomic_read(&sdev->num_ctx));
 111
 112         return rv;
 113 }
 114
 115 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
 116 {
 117         struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
 118
 119         atomic_dec(&uctx->sdev->num_ctx);
 120 }
 121
 122 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 123                      struct ib_udata *udata)
 124 {
 125         struct siw_device *sdev = to_siw_dev(base_dev);
 126
 127         if (udata->inlen || udata->outlen)
 128                 return -EINVAL;
 129
 130         memset(attr, 0, sizeof(*attr));
 131
 132         /* Revisit atomic caps if RFC 7306 gets supported */
 133         attr->atomic_cap = 0;
 134         attr->device_cap_flags =
 135                 IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG;
 136         attr->max_cq = sdev->attrs.max_cq;
 137         attr->max_cqe = sdev->attrs.max_cqe;
 138         attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
 139         attr->max_mr = sdev->attrs.max_mr;
 140         attr->max_mw = sdev->attrs.max_mw;
 141         attr->max_mr_size = ~0ull;
 142         attr->max_pd = sdev->attrs.max_pd;
 143         attr->max_qp = sdev->attrs.max_qp;
 144         attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
 145         attr->max_qp_rd_atom = sdev->attrs.max_ord;
 146         attr->max_qp_wr = sdev->attrs.max_qp_wr;
 147         attr->max_recv_sge = sdev->attrs.max_sge;
 148         attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
 149         attr->max_send_sge = sdev->attrs.max_sge;
 150         attr->max_sge_rd = sdev->attrs.max_sge_rd;
 151         attr->max_srq = sdev->attrs.max_srq;
 152         attr->max_srq_sge = sdev->attrs.max_srq_sge;
 153         attr->max_srq_wr = sdev->attrs.max_srq_wr;
 154         attr->page_size_cap = PAGE_SIZE;
 155         attr->vendor_id = SIW_VENDOR_ID;
 156         attr->vendor_part_id = sdev->vendor_part_id;
 157
 158         memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
 159
 160         return 0;
 161 }
 162
 163 int siw_query_port(struct ib_device *base_dev, u8 port,
 164                    struct ib_port_attr *attr)
 165 {
 166         struct siw_device *sdev = to_siw_dev(base_dev);
 167         int rv;
 168
 169         memset(attr, 0, sizeof(*attr));
 170
 171         rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
 172                          &attr->active_width);
 173         attr->gid_tbl_len = 1;
 174         attr->max_msg_sz = -1;
 175         attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 176         attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 177         attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
 178                 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
 179         attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 180         attr->state = sdev->state;
 181         /*
 182          * All zero
 183          *
 184          * attr->lid = 0;
 185          * attr->bad_pkey_cntr = 0;
 186          * attr->qkey_viol_cntr = 0;
 187          * attr->sm_lid = 0;
 188          * attr->lmc = 0;
 189          * attr->max_vl_num = 0;
 190          * attr->sm_sl = 0;
 191          * attr->subnet_timeout = 0;
 192          * attr->init_type_repy = 0;
 193          */
 194         return rv;
 195 }
 196
 197 int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
 198                            struct ib_port_immutable *port_immutable)
 199 {
 200         struct ib_port_attr attr;
 201         int rv = siw_query_port(base_dev, port, &attr);
 202
 203         if (rv)
 204                 return rv;
 205
 206         port_immutable->gid_tbl_len = attr.gid_tbl_len;
 207         port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 208
 209         return 0;
 210 }
 211
 212 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
 213                   union ib_gid *gid)
 214 {
 215         struct siw_device *sdev = to_siw_dev(base_dev);
 216
 217         /* subnet_prefix == interface_id == 0; */
 218         memset(gid, 0, sizeof(*gid));
 219         memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
 220
 221         return 0;
 222 }
 223
 224 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 225 {
 226         struct siw_device *sdev = to_siw_dev(pd->device);
 227
 228         if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
 229                 atomic_dec(&sdev->num_pd);
 230                 return -ENOMEM;
 231         }
 232         siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
 233
 234         return 0;
 235 }
 236
 237 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 238 {
 239         struct siw_device *sdev = to_siw_dev(pd->device);
 240
 241         siw_dbg_pd(pd, "free PD\n");
 242         atomic_dec(&sdev->num_pd);
 243         return 0;
 244 }
 245
 246 void siw_qp_get_ref(struct ib_qp *base_qp)
 247 {
 248         siw_qp_get(to_siw_qp(base_qp));
 249 }
 250
 251 void siw_qp_put_ref(struct ib_qp *base_qp)
 252 {
 253         siw_qp_put(to_siw_qp(base_qp));
 254 }
 255
 256 static struct rdma_user_mmap_entry *
 257 siw_mmap_entry_insert(struct siw_ucontext *uctx,
 258                       void *address, size_t length,
 259                       u64 *offset)
 260 {
 261         struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 262         int rv;
 263
 264         *offset = SIW_INVAL_UOBJ_KEY;
 265         if (!entry)
 266                 return NULL;
 267
 268         entry->address = address;
 269
 270         rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
 271                                          &entry->rdma_entry,
 272                                          length);
 273         if (rv) {
 274                 kfree(entry);
 275                 return NULL;
 276         }
 277
 278         *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 279
 280         return &entry->rdma_entry;
 281 }
 282
 283 /*
 284  * siw_create_qp()
 285  *
 286  * Create QP of requested size on given device.
 287  *
 288  * @pd:         Protection Domain
 289  * @attrs:      Initial QP attributes.
 290  * @udata:      used to provide QP ID, SQ and RQ size back to user.
 291  */
 292
 293 struct ib_qp *siw_create_qp(struct ib_pd *pd,
 294                             struct ib_qp_init_attr *attrs,
 295                             struct ib_udata *udata)
 296 {
 297         struct siw_qp *qp = NULL;
 298         struct ib_device *base_dev = pd->device;
 299         struct siw_device *sdev = to_siw_dev(base_dev);
 300         struct siw_ucontext *uctx =
 301                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 302                                           base_ucontext);
 303         struct siw_cq *scq = NULL, *rcq = NULL;
 304         unsigned long flags;
 305         int num_sqe, num_rqe, rv = 0;
 306         size_t length;
 307
 308         siw_dbg(base_dev, "create new QP\n");
 309
 310         if (attrs->create_flags)
 311                 return ERR_PTR(-EOPNOTSUPP);
 312
 313         if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
 314                 siw_dbg(base_dev, "too many QP's\n");
 315                 rv = -ENOMEM;
 316                 goto err_out;
 317         }
 318         if (attrs->qp_type != IB_QPT_RC) {
 319                 siw_dbg(base_dev, "only RC QP's supported\n");
 320                 rv = -EOPNOTSUPP;
 321                 goto err_out;
 322         }
 323         if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
 324             (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
 325             (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
 326             (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
 327                 siw_dbg(base_dev, "QP size error\n");
 328                 rv = -EINVAL;
 329                 goto err_out;
 330         }
 331         if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
 332                 siw_dbg(base_dev, "max inline send: %d > %d\n",
 333                         attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
 334                 rv = -EINVAL;
 335                 goto err_out;
 336         }
 337         /*
 338          * NOTE: we allow for zero element SQ and RQ WQE's SGL's
 339          * but not for a QP unable to hold any WQE (SQ + RQ)
 340          */
 341         if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
 342                 siw_dbg(base_dev, "QP must have send or receive queue\n");
 343                 rv = -EINVAL;
 344                 goto err_out;
 345         }
 346         scq = to_siw_cq(attrs->send_cq);
 347         rcq = to_siw_cq(attrs->recv_cq);
 348
 349         if (!scq || (!rcq && !attrs->srq)) {
 350                 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
 351                 rv = -EINVAL;
 352                 goto err_out;
 353         }
 354         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
 355         if (!qp) {
 356                 rv = -ENOMEM;
 357                 goto err_out;
 358         }
 359         init_rwsem(&qp->state_lock);
 360         spin_lock_init(&qp->sq_lock);
 361         spin_lock_init(&qp->rq_lock);
 362         spin_lock_init(&qp->orq_lock);
 363
 364         rv = siw_qp_add(sdev, qp);
 365         if (rv)
 366                 goto err_out;
 367
 368         /* All queue indices are derived from modulo operations
 369          * on a free running 'get' (consumer) and 'put' (producer)
 370          * unsigned counter. Having queue sizes at power of two
 371          * avoids handling counter wrap around.
 372          */
 373         num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
 374         num_rqe = roundup_pow_of_two(attrs->cap.max_recv_wr);
 375
 376         if (udata)
 377                 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
 378         else
 379                 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
 380
 381         if (qp->sendq == NULL) {
 382                 siw_dbg(base_dev, "SQ size %d alloc failed\n", num_sqe);
 383                 rv = -ENOMEM;
 384                 goto err_out_xa;
 385         }
 386         if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
 387                 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 388                         qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
 389                 else {
 390                         rv = -EINVAL;
 391                         goto err_out_xa;
 392                 }
 393         }
 394         qp->pd = pd;
 395         qp->scq = scq;
 396         qp->rcq = rcq;
 397
 398         if (attrs->srq) {
 399                 /*
 400                  * SRQ support.
 401                  * Verbs 6.3.7: ignore RQ size, if SRQ present
 402                  * Verbs 6.3.5: do not check PD of SRQ against PD of QP
 403                  */
 404                 qp->srq = to_siw_srq(attrs->srq);
 405                 qp->attrs.rq_size = 0;
 406                 siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
 407                         qp->base_qp.qp_num);
 408         } else if (num_rqe) {
 409                 if (udata)
 410                         qp->recvq =
 411                                 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
 412                 else
 413                         qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
 414
 415                 if (qp->recvq == NULL) {
 416                         siw_dbg(base_dev, "RQ size %d alloc failed\n", num_rqe);
 417                         rv = -ENOMEM;
 418                         goto err_out_xa;
 419                 }
 420                 qp->attrs.rq_size = num_rqe;
 421         }
 422         qp->attrs.sq_size = num_sqe;
 423         qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
 424         qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
 425
 426         /* Make those two tunables fixed for now. */
 427         qp->tx_ctx.gso_seg_limit = 1;
 428         qp->tx_ctx.zcopy_tx = zcopy_tx;
 429
 430         qp->attrs.state = SIW_QP_STATE_IDLE;
 431
 432         if (udata) {
 433                 struct siw_uresp_create_qp uresp = {};
 434
 435                 uresp.num_sqe = num_sqe;
 436                 uresp.num_rqe = num_rqe;
 437                 uresp.qp_id = qp_id(qp);
 438
 439                 if (qp->sendq) {
 440                         length = num_sqe * sizeof(struct siw_sqe);
 441                         qp->sq_entry =
 442                                 siw_mmap_entry_insert(uctx, qp->sendq,
 443                                                       length, &uresp.sq_key);
 444                         if (!qp->sq_entry) {
 445                                 rv = -ENOMEM;
 446                                 goto err_out_xa;
 447                         }
 448                 }
 449
 450                 if (qp->recvq) {
 451                         length = num_rqe * sizeof(struct siw_rqe);
 452                         qp->rq_entry =
 453                                 siw_mmap_entry_insert(uctx, qp->recvq,
 454                                                       length, &uresp.rq_key);
 455                         if (!qp->rq_entry) {
 456                                 uresp.sq_key = SIW_INVAL_UOBJ_KEY;
 457                                 rv = -ENOMEM;
 458                                 goto err_out_xa;
 459                         }
 460                 }
 461
 462                 if (udata->outlen < sizeof(uresp)) {
 463                         rv = -EINVAL;
 464                         goto err_out_xa;
 465                 }
 466                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 467                 if (rv)
 468                         goto err_out_xa;
 469         }
 470         qp->tx_cpu = siw_get_tx_cpu(sdev);
 471         if (qp->tx_cpu < 0) {
 472                 rv = -EINVAL;
 473                 goto err_out_xa;
 474         }
 475         INIT_LIST_HEAD(&qp->devq);
 476         spin_lock_irqsave(&sdev->lock, flags);
 477         list_add_tail(&qp->devq, &sdev->qp_list);
 478         spin_unlock_irqrestore(&sdev->lock, flags);
 479
 480         return &qp->base_qp;
 481
 482 err_out_xa:
 483         xa_erase(&sdev->qp_xa, qp_id(qp));
 484 err_out:
 485         if (qp) {
 486                 if (uctx) {
 487                         rdma_user_mmap_entry_remove(qp->sq_entry);
 488                         rdma_user_mmap_entry_remove(qp->rq_entry);
 489                 }
 490                 vfree(qp->sendq);
 491                 vfree(qp->recvq);
 492                 kfree(qp);
 493         }
 494         atomic_dec(&sdev->num_qp);
 495
 496         return ERR_PTR(rv);
 497 }
 498
 499 /*
 500  * Minimum siw_query_qp() verb interface.
 501  *
 502  * @qp_attr_mask is not used but all available information is provided
 503  */
 504 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
 505                  int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
 506 {
 507         struct siw_qp *qp;
 508         struct siw_device *sdev;
 509
 510         if (base_qp && qp_attr && qp_init_attr) {
 511                 qp = to_siw_qp(base_qp);
 512                 sdev = to_siw_dev(base_qp->device);
 513         } else {
 514                 return -EINVAL;
 515         }
 516         qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
 517         qp_attr->cap.max_send_wr = qp->attrs.sq_size;
 518         qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
 519         qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
 520         qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
 521         qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 522         qp_attr->max_rd_atomic = qp->attrs.irq_size;
 523         qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
 524
 525         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
 526                                    IB_ACCESS_REMOTE_WRITE |
 527                                    IB_ACCESS_REMOTE_READ;
 528
 529         qp_init_attr->qp_type = base_qp->qp_type;
 530         qp_init_attr->send_cq = base_qp->send_cq;
 531         qp_init_attr->recv_cq = base_qp->recv_cq;
 532         qp_init_attr->srq = base_qp->srq;
 533
 534         qp_init_attr->cap = qp_attr->cap;
 535
 536         return 0;
 537 }
 538
 539 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
 540                         int attr_mask, struct ib_udata *udata)
 541 {
 542         struct siw_qp_attrs new_attrs;
 543         enum siw_qp_attr_mask siw_attr_mask = 0;
 544         struct siw_qp *qp = to_siw_qp(base_qp);
 545         int rv = 0;
 546
 547         if (!attr_mask)
 548                 return 0;
 549
 550         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
 551                 return -EOPNOTSUPP;
 552
 553         memset(&new_attrs, 0, sizeof(new_attrs));
 554
 555         if (attr_mask & IB_QP_ACCESS_FLAGS) {
 556                 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
 557
 558                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
 559                         new_attrs.flags |= SIW_RDMA_READ_ENABLED;
 560                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
 561                         new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 562                 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
 563                         new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
 564         }
 565         if (attr_mask & IB_QP_STATE) {
 566                 siw_dbg_qp(qp, "desired IB QP state: %s\n",
 567                            ib_qp_state_to_string[attr->qp_state]);
 568
 569                 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
 570
 571                 if (new_attrs.state > SIW_QP_STATE_RTS)
 572                         qp->tx_ctx.tx_suspend = 1;
 573
 574                 siw_attr_mask |= SIW_QP_ATTR_STATE;
 575         }
 576         if (!siw_attr_mask)
 577                 goto out;
 578
 579         down_write(&qp->state_lock);
 580
 581         rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
 582
 583         up_write(&qp->state_lock);
 584 out:
 585         return rv;
 586 }
 587
 588 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
 589 {
 590         struct siw_qp *qp = to_siw_qp(base_qp);
 591         struct siw_ucontext *uctx =
 592                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 593                                           base_ucontext);
 594         struct siw_qp_attrs qp_attrs;
 595
 596         siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
 597
 598         /*
 599          * Mark QP as in process of destruction to prevent from
 600          * any async callbacks to RDMA core
 601          */
 602         qp->attrs.flags |= SIW_QP_IN_DESTROY;
 603         qp->rx_stream.rx_suspend = 1;
 604
 605         if (uctx) {
 606                 rdma_user_mmap_entry_remove(qp->sq_entry);
 607                 rdma_user_mmap_entry_remove(qp->rq_entry);
 608         }
 609
 610         down_write(&qp->state_lock);
 611
 612         qp_attrs.state = SIW_QP_STATE_ERROR;
 613         siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
 614
 615         if (qp->cep) {
 616                 siw_cep_put(qp->cep);
 617                 qp->cep = NULL;
 618         }
 619         up_write(&qp->state_lock);
 620
 621         kfree(qp->tx_ctx.mpa_crc_hd);
 622         kfree(qp->rx_stream.mpa_crc_hd);
 623
 624         qp->scq = qp->rcq = NULL;
 625
 626         siw_qp_put(qp);
 627
 628         return 0;
 629 }
 630
 631 /*
 632  * siw_copy_inline_sgl()
 633  *
 634  * Prepare sgl of inlined data for sending. For userland callers
 635  * function checks if given buffer addresses and len's are within
 636  * process context bounds.
 637  * Data from all provided sge's are copied together into the wqe,
 638  * referenced by a single sge.
 639  */
 640 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
 641                                struct siw_sqe *sqe)
 642 {
 643         struct ib_sge *core_sge = core_wr->sg_list;
 644         void *kbuf = &sqe->sge[1];
 645         int num_sge = core_wr->num_sge, bytes = 0;
 646
 647         sqe->sge[0].laddr = (uintptr_t)kbuf;
 648         sqe->sge[0].lkey = 0;
 649
 650         while (num_sge--) {
 651                 if (!core_sge->length) {
 652                         core_sge++;
 653                         continue;
 654                 }
 655                 bytes += core_sge->length;
 656                 if (bytes > SIW_MAX_INLINE) {
 657                         bytes = -EINVAL;
 658                         break;
 659                 }
 660                 memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
 661                        core_sge->length);
 662
 663                 kbuf += core_sge->length;
 664                 core_sge++;
 665         }
 666         sqe->sge[0].length = bytes > 0 ? bytes : 0;
 667         sqe->num_sge = bytes > 0 ? 1 : 0;
 668
 669         return bytes;
 670 }
 671
 672 /* Complete SQ WR's without processing */
 673 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
 674                            const struct ib_send_wr **bad_wr)
 675 {
 676         struct siw_sqe sqe = {};
 677         int rv = 0;
 678
 679         while (wr) {
 680                 sqe.id = wr->wr_id;
 681                 sqe.opcode = wr->opcode;
 682                 rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);
 683                 if (rv) {
 684                         if (bad_wr)
 685                                 *bad_wr = wr;
 686                         break;
 687                 }
 688                 wr = wr->next;
 689         }
 690         return rv;
 691 }
 692
 693 /* Complete RQ WR's without processing */
 694 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
 695                            const struct ib_recv_wr **bad_wr)
 696 {
 697         struct siw_rqe rqe = {};
 698         int rv = 0;
 699
 700         while (wr) {
 701                 rqe.id = wr->wr_id;
 702                 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
 703                 if (rv) {
 704                         if (bad_wr)
 705                                 *bad_wr = wr;
 706                         break;
 707                 }
 708                 wr = wr->next;
 709         }
 710         return rv;
 711 }
 712
 713 /*
 714  * siw_post_send()
 715  *
 716  * Post a list of S-WR's to a SQ.
 717  *
 718  * @base_qp:    Base QP contained in siw QP
 719  * @wr:         Null terminated list of user WR's
 720  * @bad_wr:     Points to failing WR in case of synchronous failure.
 721  */
 722 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
 723                   const struct ib_send_wr **bad_wr)
 724 {
 725         struct siw_qp *qp = to_siw_qp(base_qp);
 726         struct siw_wqe *wqe = tx_wqe(qp);
 727
 728         unsigned long flags;
 729         int rv = 0;
 730
 731         if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
 732                 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
 733                 *bad_wr = wr;
 734                 return -EINVAL;
 735         }
 736
 737         /*
 738          * Try to acquire QP state lock. Must be non-blocking
 739          * to accommodate kernel clients needs.
 740          */
 741         if (!down_read_trylock(&qp->state_lock)) {
 742                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 743                         /*
 744                          * ERROR state is final, so we can be sure
 745                          * this state will not change as long as the QP
 746                          * exists.
 747                          *
 748                          * This handles an ib_drain_sq() call with
 749                          * a concurrent request to set the QP state
 750                          * to ERROR.
 751                          */
 752                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 753                 } else {
 754                         siw_dbg_qp(qp, "QP locked, state %d\n",
 755                                    qp->attrs.state);
 756                         *bad_wr = wr;
 757                         rv = -ENOTCONN;
 758                 }
 759                 return rv;
 760         }
 761         if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
 762                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 763                         /*
 764                          * Immediately flush this WR to CQ, if QP
 765                          * is in ERROR state. SQ is guaranteed to
 766                          * be empty, so WR complets in-order.
 767                          *
 768                          * Typically triggered by ib_drain_sq().
 769                          */
 770                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 771                 } else {
 772                         siw_dbg_qp(qp, "QP out of state %d\n",
 773                                    qp->attrs.state);
 774                         *bad_wr = wr;
 775                         rv = -ENOTCONN;
 776                 }
 777                 up_read(&qp->state_lock);
 778                 return rv;
 779         }
 780         spin_lock_irqsave(&qp->sq_lock, flags);
 781
 782         while (wr) {
 783                 u32 idx = qp->sq_put % qp->attrs.sq_size;
 784                 struct siw_sqe *sqe = &qp->sendq[idx];
 785
 786                 if (sqe->flags) {
 787                         siw_dbg_qp(qp, "sq full\n");
 788                         rv = -ENOMEM;
 789                         break;
 790                 }
 791                 if (wr->num_sge > qp->attrs.sq_max_sges) {
 792                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
 793                         rv = -EINVAL;
 794                         break;
 795                 }
 796                 sqe->id = wr->wr_id;
 797
 798                 if ((wr->send_flags & IB_SEND_SIGNALED) ||
 799                     (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
 800                         sqe->flags |= SIW_WQE_SIGNALLED;
 801
 802                 if (wr->send_flags & IB_SEND_FENCE)
 803                         sqe->flags |= SIW_WQE_READ_FENCE;
 804
 805                 switch (wr->opcode) {
 806                 case IB_WR_SEND:
 807                 case IB_WR_SEND_WITH_INV:
 808                         if (wr->send_flags & IB_SEND_SOLICITED)
 809                                 sqe->flags |= SIW_WQE_SOLICITED;
 810
 811                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 812                                 siw_copy_sgl(wr->sg_list, sqe->sge,
 813                                              wr->num_sge);
 814                                 sqe->num_sge = wr->num_sge;
 815                         } else {
 816                                 rv = siw_copy_inline_sgl(wr, sqe);
 817                                 if (rv <= 0) {
 818                                         rv = -EINVAL;
 819                                         break;
 820                                 }
 821                                 sqe->flags |= SIW_WQE_INLINE;
 822                                 sqe->num_sge = 1;
 823                         }
 824                         if (wr->opcode == IB_WR_SEND)
 825                                 sqe->opcode = SIW_OP_SEND;
 826                         else {
 827                                 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
 828                                 sqe->rkey = wr->ex.invalidate_rkey;
 829                         }
 830                         break;
 831
 832                 case IB_WR_RDMA_READ_WITH_INV:
 833                 case IB_WR_RDMA_READ:
 834                         /*
 835                          * iWarp restricts RREAD sink to SGL containing
 836                          * 1 SGE only. we could relax to SGL with multiple
 837                          * elements referring the SAME ltag or even sending
 838                          * a private per-rreq tag referring to a checked
 839                          * local sgl with MULTIPLE ltag's.
 840                          */
 841                         if (unlikely(wr->num_sge != 1)) {
 842                                 rv = -EINVAL;
 843                                 break;
 844                         }
 845                         siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
 846                         /*
 847                          * NOTE: zero length RREAD is allowed!
 848                          */
 849                         sqe->raddr = rdma_wr(wr)->remote_addr;
 850                         sqe->rkey = rdma_wr(wr)->rkey;
 851                         sqe->num_sge = 1;
 852
 853                         if (wr->opcode == IB_WR_RDMA_READ)
 854                                 sqe->opcode = SIW_OP_READ;
 855                         else
 856                                 sqe->opcode = SIW_OP_READ_LOCAL_INV;
 857                         break;
 858
 859                 case IB_WR_RDMA_WRITE:
 860                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 861                                 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
 862                                              wr->num_sge);
 863                                 sqe->num_sge = wr->num_sge;
 864                         } else {
 865                                 rv = siw_copy_inline_sgl(wr, sqe);
 866                                 if (unlikely(rv < 0)) {
 867                                         rv = -EINVAL;
 868                                         break;
 869                                 }
 870                                 sqe->flags |= SIW_WQE_INLINE;
 871                                 sqe->num_sge = 1;
 872                         }
 873                         sqe->raddr = rdma_wr(wr)->remote_addr;
 874                         sqe->rkey = rdma_wr(wr)->rkey;
 875                         sqe->opcode = SIW_OP_WRITE;
 876                         break;
 877
 878                 case IB_WR_REG_MR:
 879                         sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
 880                         sqe->rkey = reg_wr(wr)->key;
 881                         sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
 882                         sqe->opcode = SIW_OP_REG_MR;
 883                         break;
 884
 885                 case IB_WR_LOCAL_INV:
 886                         sqe->rkey = wr->ex.invalidate_rkey;
 887                         sqe->opcode = SIW_OP_INVAL_STAG;
 888                         break;
 889
 890                 default:
 891                         siw_dbg_qp(qp, "ib wr type %d unsupported\n",
 892                                    wr->opcode);
 893                         rv = -EINVAL;
 894                         break;
 895                 }
 896                 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
 897                            sqe->opcode, sqe->flags,
 898                            (void *)(uintptr_t)sqe->id);
 899
 900                 if (unlikely(rv < 0))
 901                         break;
 902
 903                 /* make SQE only valid after completely written */
 904                 smp_wmb();
 905                 sqe->flags |= SIW_WQE_VALID;
 906
 907                 qp->sq_put++;
 908                 wr = wr->next;
 909         }
 910
 911         /*
 912          * Send directly if SQ processing is not in progress.
 913          * Eventual immediate errors (rv < 0) do not affect the involved
 914          * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
 915          * processing, if new work is already pending. But rv must be passed
 916          * to caller.
 917          */
 918         if (wqe->wr_status != SIW_WR_IDLE) {
 919                 spin_unlock_irqrestore(&qp->sq_lock, flags);
 920                 goto skip_direct_sending;
 921         }
 922         rv = siw_activate_tx(qp);
 923         spin_unlock_irqrestore(&qp->sq_lock, flags);
 924
 925         if (rv <= 0)
 926                 goto skip_direct_sending;
 927
 928         if (rdma_is_kernel_res(&qp->base_qp.res)) {
 929                 rv = siw_sq_start(qp);
 930         } else {
 931                 qp->tx_ctx.in_syscall = 1;
 932
 933                 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
 934                         siw_qp_cm_drop(qp, 0);
 935
 936                 qp->tx_ctx.in_syscall = 0;
 937         }
 938 skip_direct_sending:
 939
 940         up_read(&qp->state_lock);
 941
 942         if (rv >= 0)
 943                 return 0;
 944         /*
 945          * Immediate error
 946          */
 947         siw_dbg_qp(qp, "error %d\n", rv);
 948
 949         *bad_wr = wr;
 950         return rv;
 951 }
 952
 953 /*
 954  * siw_post_receive()
 955  *
 956  * Post a list of R-WR's to a RQ.
 957  *
 958  * @base_qp:    Base QP contained in siw QP
 959  * @wr:         Null terminated list of user WR's
 960  * @bad_wr:     Points to failing WR in case of synchronous failure.
 961  */
 962 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
 963                      const struct ib_recv_wr **bad_wr)
 964 {
 965         struct siw_qp *qp = to_siw_qp(base_qp);
 966         unsigned long flags;
 967         int rv = 0;
 968
 969         if (qp->srq) {
 970                 *bad_wr = wr;
 971                 return -EOPNOTSUPP; /* what else from errno.h? */
 972         }
 973         if (!rdma_is_kernel_res(&qp->base_qp.res)) {
 974                 siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
 975                 *bad_wr = wr;
 976                 return -EINVAL;
 977         }
 978
 979         /*
 980          * Try to acquire QP state lock. Must be non-blocking
 981          * to accommodate kernel clients needs.
 982          */
 983         if (!down_read_trylock(&qp->state_lock)) {
 984                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 985                         /*
 986                          * ERROR state is final, so we can be sure
 987                          * this state will not change as long as the QP
 988                          * exists.
 989                          *
 990                          * This handles an ib_drain_rq() call with
 991                          * a concurrent request to set the QP state
 992                          * to ERROR.
 993                          */
 994                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
 995                 } else {
 996                         siw_dbg_qp(qp, "QP locked, state %d\n",
 997                                    qp->attrs.state);
 998                         *bad_wr = wr;
 999                         rv = -ENOTCONN;
1000                 }
1001                 return rv;
1002         }
1003         if (qp->attrs.state > SIW_QP_STATE_RTS) {
1004                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1005                         /*
1006                          * Immediately flush this WR to CQ, if QP
1007                          * is in ERROR state. RQ is guaranteed to
1008                          * be empty, so WR complets in-order.
1009                          *
1010                          * Typically triggered by ib_drain_rq().
1011                          */
1012                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1013                 } else {
1014                         siw_dbg_qp(qp, "QP out of state %d\n",
1015                                    qp->attrs.state);
1016                         *bad_wr = wr;
1017                         rv = -ENOTCONN;
1018                 }
1019                 up_read(&qp->state_lock);
1020                 return rv;
1021         }
1022         /*
1023          * Serialize potentially multiple producers.
1024          * Not needed for single threaded consumer side.
1025          */
1026         spin_lock_irqsave(&qp->rq_lock, flags);
1027
1028         while (wr) {
1029                 u32 idx = qp->rq_put % qp->attrs.rq_size;
1030                 struct siw_rqe *rqe = &qp->recvq[idx];
1031
1032                 if (rqe->flags) {
1033                         siw_dbg_qp(qp, "RQ full\n");
1034                         rv = -ENOMEM;
1035                         break;
1036                 }
1037                 if (wr->num_sge > qp->attrs.rq_max_sges) {
1038                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1039                         rv = -EINVAL;
1040                         break;
1041                 }
1042                 rqe->id = wr->wr_id;
1043                 rqe->num_sge = wr->num_sge;
1044                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1045
1046                 /* make sure RQE is completely written before valid */
1047                 smp_wmb();
1048
1049                 rqe->flags = SIW_WQE_VALID;
1050
1051                 qp->rq_put++;
1052                 wr = wr->next;
1053         }
1054         spin_unlock_irqrestore(&qp->rq_lock, flags);
1055
1056         up_read(&qp->state_lock);
1057
1058         if (rv < 0) {
1059                 siw_dbg_qp(qp, "error %d\n", rv);
1060                 *bad_wr = wr;
1061         }
1062         return rv > 0 ? 0 : rv;
1063 }
1064
1065 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1066 {
1067         struct siw_cq *cq = to_siw_cq(base_cq);
1068         struct siw_device *sdev = to_siw_dev(base_cq->device);
1069         struct siw_ucontext *ctx =
1070                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1071                                           base_ucontext);
1072
1073         siw_dbg_cq(cq, "free CQ resources\n");
1074
1075         siw_cq_flush(cq);
1076
1077         if (ctx)
1078                 rdma_user_mmap_entry_remove(cq->cq_entry);
1079
1080         atomic_dec(&sdev->num_cq);
1081
1082         vfree(cq->queue);
1083         return 0;
1084 }
1085
1086 /*
1087  * siw_create_cq()
1088  *
1089  * Populate CQ of requested size
1090  *
1091  * @base_cq: CQ as allocated by RDMA midlayer
1092  * @attr: Initial CQ attributes
1093  * @udata: relates to user context
1094  */
1095
1096 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1097                   struct ib_udata *udata)
1098 {
1099         struct siw_device *sdev = to_siw_dev(base_cq->device);
1100         struct siw_cq *cq = to_siw_cq(base_cq);
1101         int rv, size = attr->cqe;
1102
1103         if (attr->flags)
1104                 return -EOPNOTSUPP;
1105
1106         if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1107                 siw_dbg(base_cq->device, "too many CQ's\n");
1108                 rv = -ENOMEM;
1109                 goto err_out;
1110         }
1111         if (size < 1 || size > sdev->attrs.max_cqe) {
1112                 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1113                 rv = -EINVAL;
1114                 goto err_out;
1115         }
1116         size = roundup_pow_of_two(size);
1117         cq->base_cq.cqe = size;
1118         cq->num_cqe = size;
1119
1120         if (udata)
1121                 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1122                                          sizeof(struct siw_cq_ctrl));
1123         else
1124                 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1125                                     sizeof(struct siw_cq_ctrl));
1126
1127         if (cq->queue == NULL) {
1128                 rv = -ENOMEM;
1129                 goto err_out;
1130         }
1131         get_random_bytes(&cq->id, 4);
1132         siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1133
1134         spin_lock_init(&cq->lock);
1135
1136         cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1137
1138         if (udata) {
1139                 struct siw_uresp_create_cq uresp = {};
1140                 struct siw_ucontext *ctx =
1141                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1142                                                   base_ucontext);
1143                 size_t length = size * sizeof(struct siw_cqe) +
1144                         sizeof(struct siw_cq_ctrl);
1145
1146                 cq->cq_entry =
1147                         siw_mmap_entry_insert(ctx, cq->queue,
1148                                               length, &uresp.cq_key);
1149                 if (!cq->cq_entry) {
1150                         rv = -ENOMEM;
1151                         goto err_out;
1152                 }
1153
1154                 uresp.cq_id = cq->id;
1155                 uresp.num_cqe = size;
1156
1157                 if (udata->outlen < sizeof(uresp)) {
1158                         rv = -EINVAL;
1159                         goto err_out;
1160                 }
1161                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1162                 if (rv)
1163                         goto err_out;
1164         }
1165         return 0;
1166
1167 err_out:
1168         siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1169
1170         if (cq && cq->queue) {
1171                 struct siw_ucontext *ctx =
1172                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1173                                                   base_ucontext);
1174                 if (ctx)
1175                         rdma_user_mmap_entry_remove(cq->cq_entry);
1176                 vfree(cq->queue);
1177         }
1178         atomic_dec(&sdev->num_cq);
1179
1180         return rv;
1181 }
1182
1183 /*
1184  * siw_poll_cq()
1185  *
1186  * Reap CQ entries if available and copy work completion status into
1187  * array of WC's provided by caller. Returns number of reaped CQE's.
1188  *
1189  * @base_cq:    Base CQ contained in siw CQ.
1190  * @num_cqe:    Maximum number of CQE's to reap.
1191  * @wc:         Array of work completions to be filled by siw.
1192  */
1193 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1194 {
1195         struct siw_cq *cq = to_siw_cq(base_cq);
1196         int i;
1197
1198         for (i = 0; i < num_cqe; i++) {
1199                 if (!siw_reap_cqe(cq, wc))
1200                         break;
1201                 wc++;
1202         }
1203         return i;
1204 }
1205
1206 /*
1207  * siw_req_notify_cq()
1208  *
1209  * Request notification for new CQE's added to that CQ.
1210  * Defined flags:
1211  * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1212  *   event if a WQE with notification flag set enters the CQ
1213  * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1214  *   event if a WQE enters the CQ.
1215  * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1216  *   number of not reaped CQE's regardless of its notification
1217  *   type and current or new CQ notification settings.
1218  *
1219  * @base_cq:    Base CQ contained in siw CQ.
1220  * @flags:      Requested notification flags.
1221  */
1222 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1223 {
1224         struct siw_cq *cq = to_siw_cq(base_cq);
1225
1226         siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1227
1228         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1229                 /*
1230                  * Enable CQ event for next solicited completion.
1231                  * and make it visible to all associated producers.
1232                  */
1233                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1234         else
1235                 /*
1236                  * Enable CQ event for any signalled completion.
1237                  * and make it visible to all associated producers.
1238                  */
1239                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1240
1241         if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1242                 return cq->cq_put - cq->cq_get;
1243
1244         return 0;
1245 }
1246
1247 /*
1248  * siw_dereg_mr()
1249  *
1250  * Release Memory Region.
1251  *
1252  * @base_mr: Base MR contained in siw MR.
1253  * @udata: points to user context, unused.
1254  */
1255 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1256 {
1257         struct siw_mr *mr = to_siw_mr(base_mr);
1258         struct siw_device *sdev = to_siw_dev(base_mr->device);
1259
1260         siw_dbg_mem(mr->mem, "deregister MR\n");
1261
1262         atomic_dec(&sdev->num_mr);
1263
1264         siw_mr_drop_mem(mr);
1265         kfree_rcu(mr, rcu);
1266
1267         return 0;
1268 }
1269
1270 /*
1271  * siw_reg_user_mr()
1272  *
1273  * Register Memory Region.
1274  *
1275  * @pd:         Protection Domain
1276  * @start:      starting address of MR (virtual address)
1277  * @len:        len of MR
1278  * @rnic_va:    not used by siw
1279  * @rights:     MR access rights
1280  * @udata:      user buffer to communicate STag and Key.
1281  */
1282 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1283                               u64 rnic_va, int rights, struct ib_udata *udata)
1284 {
1285         struct siw_mr *mr = NULL;
1286         struct siw_umem *umem = NULL;
1287         struct siw_ureq_reg_mr ureq;
1288         struct siw_device *sdev = to_siw_dev(pd->device);
1289
1290         unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1291         int rv;
1292
1293         siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1294                    (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1295                    (unsigned long long)len);
1296
1297         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1298                 siw_dbg_pd(pd, "too many mr's\n");
1299                 rv = -ENOMEM;
1300                 goto err_out;
1301         }
1302         if (!len) {
1303                 rv = -EINVAL;
1304                 goto err_out;
1305         }
1306         if (mem_limit != RLIM_INFINITY) {
1307                 unsigned long num_pages =
1308                         (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1309                 mem_limit >>= PAGE_SHIFT;
1310
1311                 if (num_pages > mem_limit - current->mm->locked_vm) {
1312                         siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1313                                    num_pages, mem_limit,
1314                                    current->mm->locked_vm);
1315                         rv = -ENOMEM;
1316                         goto err_out;
1317                 }
1318         }
1319         umem = siw_umem_get(start, len, ib_access_writable(rights));
1320         if (IS_ERR(umem)) {
1321                 rv = PTR_ERR(umem);
1322                 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1323                 umem = NULL;
1324                 goto err_out;
1325         }
1326         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1327         if (!mr) {
1328                 rv = -ENOMEM;
1329                 goto err_out;
1330         }
1331         rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1332         if (rv)
1333                 goto err_out;
1334
1335         if (udata) {
1336                 struct siw_uresp_reg_mr uresp = {};
1337                 struct siw_mem *mem = mr->mem;
1338
1339                 if (udata->inlen < sizeof(ureq)) {
1340                         rv = -EINVAL;
1341                         goto err_out;
1342                 }
1343                 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1344                 if (rv)
1345                         goto err_out;
1346
1347                 mr->base_mr.lkey |= ureq.stag_key;
1348                 mr->base_mr.rkey |= ureq.stag_key;
1349                 mem->stag |= ureq.stag_key;
1350                 uresp.stag = mem->stag;
1351
1352                 if (udata->outlen < sizeof(uresp)) {
1353                         rv = -EINVAL;
1354                         goto err_out;
1355                 }
1356                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1357                 if (rv)
1358                         goto err_out;
1359         }
1360         mr->mem->stag_valid = 1;
1361
1362         return &mr->base_mr;
1363
1364 err_out:
1365         atomic_dec(&sdev->num_mr);
1366         if (mr) {
1367                 if (mr->mem)
1368                         siw_mr_drop_mem(mr);
1369                 kfree_rcu(mr, rcu);
1370         } else {
1371                 if (umem)
1372                         siw_umem_release(umem, false);
1373         }
1374         return ERR_PTR(rv);
1375 }
1376
1377 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1378                            u32 max_sge)
1379 {
1380         struct siw_device *sdev = to_siw_dev(pd->device);
1381         struct siw_mr *mr = NULL;
1382         struct siw_pbl *pbl = NULL;
1383         int rv;
1384
1385         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1386                 siw_dbg_pd(pd, "too many mr's\n");
1387                 rv = -ENOMEM;
1388                 goto err_out;
1389         }
1390         if (mr_type != IB_MR_TYPE_MEM_REG) {
1391                 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1392                 rv = -EOPNOTSUPP;
1393                 goto err_out;
1394         }
1395         if (max_sge > SIW_MAX_SGE_PBL) {
1396                 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1397                 rv = -ENOMEM;
1398                 goto err_out;
1399         }
1400         pbl = siw_pbl_alloc(max_sge);
1401         if (IS_ERR(pbl)) {
1402                 rv = PTR_ERR(pbl);
1403                 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1404                 pbl = NULL;
1405                 goto err_out;
1406         }
1407         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1408         if (!mr) {
1409                 rv = -ENOMEM;
1410                 goto err_out;
1411         }
1412         rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1413         if (rv)
1414                 goto err_out;
1415
1416         mr->mem->is_pbl = 1;
1417
1418         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1419
1420         return &mr->base_mr;
1421
1422 err_out:
1423         atomic_dec(&sdev->num_mr);
1424
1425         if (!mr) {
1426                 kfree(pbl);
1427         } else {
1428                 if (mr->mem)
1429                         siw_mr_drop_mem(mr);
1430                 kfree_rcu(mr, rcu);
1431         }
1432         siw_dbg_pd(pd, "failed: %d\n", rv);
1433
1434         return ERR_PTR(rv);
1435 }
1436
1437 /* Just used to count number of pages being mapped */
1438 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1439 {
1440         return 0;
1441 }
1442
1443 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1444                   unsigned int *sg_off)
1445 {
1446         struct scatterlist *slp;
1447         struct siw_mr *mr = to_siw_mr(base_mr);
1448         struct siw_mem *mem = mr->mem;
1449         struct siw_pbl *pbl = mem->pbl;
1450         struct siw_pble *pble;
1451         unsigned long pbl_size;
1452         int i, rv;
1453
1454         if (!pbl) {
1455                 siw_dbg_mem(mem, "no PBL allocated\n");
1456                 return -EINVAL;
1457         }
1458         pble = pbl->pbe;
1459
1460         if (pbl->max_buf < num_sle) {
1461                 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1462                             mem->pbl->max_buf, num_sle);
1463                 return -ENOMEM;
1464         }
1465         for_each_sg(sl, slp, num_sle, i) {
1466                 if (sg_dma_len(slp) == 0) {
1467                         siw_dbg_mem(mem, "empty SGE\n");
1468                         return -EINVAL;
1469                 }
1470                 if (i == 0) {
1471                         pble->addr = sg_dma_address(slp);
1472                         pble->size = sg_dma_len(slp);
1473                         pble->pbl_off = 0;
1474                         pbl_size = pble->size;
1475                         pbl->num_buf = 1;
1476                 } else {
1477                         /* Merge PBL entries if adjacent */
1478                         if (pble->addr + pble->size == sg_dma_address(slp)) {
1479                                 pble->size += sg_dma_len(slp);
1480                         } else {
1481                                 pble++;
1482                                 pbl->num_buf++;
1483                                 pble->addr = sg_dma_address(slp);
1484                                 pble->size = sg_dma_len(slp);
1485                                 pble->pbl_off = pbl_size;
1486                         }
1487                         pbl_size += sg_dma_len(slp);
1488                 }
1489                 siw_dbg_mem(mem,
1490                         "sge[%d], size %u, addr 0x%p, total %lu\n",
1491                         i, pble->size, (void *)(uintptr_t)pble->addr,
1492                         pbl_size);
1493         }
1494         rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1495         if (rv > 0) {
1496                 mem->len = base_mr->length;
1497                 mem->va = base_mr->iova;
1498                 siw_dbg_mem(mem,
1499                         "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1500                         mem->len, (void *)(uintptr_t)mem->va, num_sle,
1501                         pbl->num_buf);
1502         }
1503         return rv;
1504 }
1505
1506 /*
1507  * siw_get_dma_mr()
1508  *
1509  * Create a (empty) DMA memory region, where no umem is attached.
1510  */
1511 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1512 {
1513         struct siw_device *sdev = to_siw_dev(pd->device);
1514         struct siw_mr *mr = NULL;
1515         int rv;
1516
1517         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1518                 siw_dbg_pd(pd, "too many mr's\n");
1519                 rv = -ENOMEM;
1520                 goto err_out;
1521         }
1522         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1523         if (!mr) {
1524                 rv = -ENOMEM;
1525                 goto err_out;
1526         }
1527         rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1528         if (rv)
1529                 goto err_out;
1530
1531         mr->mem->stag_valid = 1;
1532
1533         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1534
1535         return &mr->base_mr;
1536
1537 err_out:
1538         if (rv)
1539                 kfree(mr);
1540
1541         atomic_dec(&sdev->num_mr);
1542
1543         return ERR_PTR(rv);
1544 }
1545
1546 /*
1547  * siw_create_srq()
1548  *
1549  * Create Shared Receive Queue of attributes @init_attrs
1550  * within protection domain given by @pd.
1551  *
1552  * @base_srq:   Base SRQ contained in siw SRQ.
1553  * @init_attrs: SRQ init attributes.
1554  * @udata:      points to user context
1555  */
1556 int siw_create_srq(struct ib_srq *base_srq,
1557                    struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1558 {
1559         struct siw_srq *srq = to_siw_srq(base_srq);
1560         struct ib_srq_attr *attrs = &init_attrs->attr;
1561         struct siw_device *sdev = to_siw_dev(base_srq->device);
1562         struct siw_ucontext *ctx =
1563                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1564                                           base_ucontext);
1565         int rv;
1566
1567         if (init_attrs->srq_type != IB_SRQT_BASIC)
1568                 return -EOPNOTSUPP;
1569
1570         if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1571                 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1572                 rv = -ENOMEM;
1573                 goto err_out;
1574         }
1575         if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1576             attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1577                 rv = -EINVAL;
1578                 goto err_out;
1579         }
1580         srq->max_sge = attrs->max_sge;
1581         srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1582         srq->limit = attrs->srq_limit;
1583         if (srq->limit)
1584                 srq->armed = true;
1585
1586         srq->is_kernel_res = !udata;
1587
1588         if (udata)
1589                 srq->recvq =
1590                         vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1591         else
1592                 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1593
1594         if (srq->recvq == NULL) {
1595                 rv = -ENOMEM;
1596                 goto err_out;
1597         }
1598         if (udata) {
1599                 struct siw_uresp_create_srq uresp = {};
1600                 size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1601
1602                 srq->srq_entry =
1603                         siw_mmap_entry_insert(ctx, srq->recvq,
1604                                               length, &uresp.srq_key);
1605                 if (!srq->srq_entry) {
1606                         rv = -ENOMEM;
1607                         goto err_out;
1608                 }
1609
1610                 uresp.num_rqe = srq->num_rqe;
1611
1612                 if (udata->outlen < sizeof(uresp)) {
1613                         rv = -EINVAL;
1614                         goto err_out;
1615                 }
1616                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1617                 if (rv)
1618                         goto err_out;
1619         }
1620         spin_lock_init(&srq->lock);
1621
1622         siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1623
1624         return 0;
1625
1626 err_out:
1627         if (srq->recvq) {
1628                 if (ctx)
1629                         rdma_user_mmap_entry_remove(srq->srq_entry);
1630                 vfree(srq->recvq);
1631         }
1632         atomic_dec(&sdev->num_srq);
1633
1634         return rv;
1635 }
1636
1637 /*
1638  * siw_modify_srq()
1639  *
1640  * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1641  * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1642  *
1643  * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1644  * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1645  */
1646 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1647                    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1648 {
1649         struct siw_srq *srq = to_siw_srq(base_srq);
1650         unsigned long flags;
1651         int rv = 0;
1652
1653         spin_lock_irqsave(&srq->lock, flags);
1654
1655         if (attr_mask & IB_SRQ_MAX_WR) {
1656                 /* resize request not yet supported */
1657                 rv = -EOPNOTSUPP;
1658                 goto out;
1659         }
1660         if (attr_mask & IB_SRQ_LIMIT) {
1661                 if (attrs->srq_limit) {
1662                         if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1663                                 rv = -EINVAL;
1664                                 goto out;
1665                         }
1666                         srq->armed = true;
1667                 } else {
1668                         srq->armed = false;
1669                 }
1670                 srq->limit = attrs->srq_limit;
1671         }
1672 out:
1673         spin_unlock_irqrestore(&srq->lock, flags);
1674
1675         return rv;
1676 }
1677
1678 /*
1679  * siw_query_srq()
1680  *
1681  * Query SRQ attributes.
1682  */
1683 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1684 {
1685         struct siw_srq *srq = to_siw_srq(base_srq);
1686         unsigned long flags;
1687
1688         spin_lock_irqsave(&srq->lock, flags);
1689
1690         attrs->max_wr = srq->num_rqe;
1691         attrs->max_sge = srq->max_sge;
1692         attrs->srq_limit = srq->limit;
1693
1694         spin_unlock_irqrestore(&srq->lock, flags);
1695
1696         return 0;
1697 }
1698
1699 /*
1700  * siw_destroy_srq()
1701  *
1702  * Destroy SRQ.
1703  * It is assumed that the SRQ is not referenced by any
1704  * QP anymore - the code trusts the RDMA core environment to keep track
1705  * of QP references.
1706  */
1707 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1708 {
1709         struct siw_srq *srq = to_siw_srq(base_srq);
1710         struct siw_device *sdev = to_siw_dev(base_srq->device);
1711         struct siw_ucontext *ctx =
1712                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1713                                           base_ucontext);
1714
1715         if (ctx)
1716                 rdma_user_mmap_entry_remove(srq->srq_entry);
1717         vfree(srq->recvq);
1718         atomic_dec(&sdev->num_srq);
1719         return 0;
1720 }
1721
1722 /*
1723  * siw_post_srq_recv()
1724  *
1725  * Post a list of receive queue elements to SRQ.
1726  * NOTE: The function does not check or lock a certain SRQ state
1727  *       during the post operation. The code simply trusts the
1728  *       RDMA core environment.
1729  *
1730  * @base_srq:   Base SRQ contained in siw SRQ
1731  * @wr:         List of R-WR's
1732  * @bad_wr:     Updated to failing WR if posting fails.
1733  */
1734 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1735                       const struct ib_recv_wr **bad_wr)
1736 {
1737         struct siw_srq *srq = to_siw_srq(base_srq);
1738         unsigned long flags;
1739         int rv = 0;
1740
1741         if (unlikely(!srq->is_kernel_res)) {
1742                 siw_dbg_pd(base_srq->pd,
1743                            "[SRQ]: no kernel post_recv for mapped srq\n");
1744                 rv = -EINVAL;
1745                 goto out;
1746         }
1747         /*
1748          * Serialize potentially multiple producers.
1749          * Also needed to serialize potentially multiple
1750          * consumers.
1751          */
1752         spin_lock_irqsave(&srq->lock, flags);
1753
1754         while (wr) {
1755                 u32 idx = srq->rq_put % srq->num_rqe;
1756                 struct siw_rqe *rqe = &srq->recvq[idx];
1757
1758                 if (rqe->flags) {
1759                         siw_dbg_pd(base_srq->pd, "SRQ full\n");
1760                         rv = -ENOMEM;
1761                         break;
1762                 }
1763                 if (unlikely(wr->num_sge > srq->max_sge)) {
1764                         siw_dbg_pd(base_srq->pd,
1765                                    "[SRQ]: too many sge's: %d\n", wr->num_sge);
1766                         rv = -EINVAL;
1767                         break;
1768                 }
1769                 rqe->id = wr->wr_id;
1770                 rqe->num_sge = wr->num_sge;
1771                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1772
1773                 /* Make sure S-RQE is completely written before valid */
1774                 smp_wmb();
1775
1776                 rqe->flags = SIW_WQE_VALID;
1777
1778                 srq->rq_put++;
1779                 wr = wr->next;
1780         }
1781         spin_unlock_irqrestore(&srq->lock, flags);
1782 out:
1783         if (unlikely(rv < 0)) {
1784                 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1785                 *bad_wr = wr;
1786         }
1787         return rv;
1788 }
1789
1790 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1791 {
1792         struct ib_event event;
1793         struct ib_qp *base_qp = &qp->base_qp;
1794
1795         /*
1796          * Do not report asynchronous errors on QP which gets
1797          * destroyed via verbs interface (siw_destroy_qp())
1798          */
1799         if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1800                 return;
1801
1802         event.event = etype;
1803         event.device = base_qp->device;
1804         event.element.qp = base_qp;
1805
1806         if (base_qp->event_handler) {
1807                 siw_dbg_qp(qp, "reporting event %d\n", etype);
1808                 base_qp->event_handler(&event, base_qp->qp_context);
1809         }
1810 }
1811
1812 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1813 {
1814         struct ib_event event;
1815         struct ib_cq *base_cq = &cq->base_cq;
1816
1817         event.event = etype;
1818         event.device = base_cq->device;
1819         event.element.cq = base_cq;
1820
1821         if (base_cq->event_handler) {
1822                 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1823                 base_cq->event_handler(&event, base_cq->cq_context);
1824         }
1825 }
1826
1827 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1828 {
1829         struct ib_event event;
1830         struct ib_srq *base_srq = &srq->base_srq;
1831
1832         event.event = etype;
1833         event.device = base_srq->device;
1834         event.element.srq = base_srq;
1835
1836         if (base_srq->event_handler) {
1837                 siw_dbg_pd(srq->base_srq.pd,
1838                            "reporting SRQ event %d\n", etype);
1839                 base_srq->event_handler(&event, base_srq->srq_context);
1840         }
1841 }
1842
1843 void siw_port_event(struct siw_device *sdev, u8 port, enum ib_event_type etype)
1844 {
1845         struct ib_event event;
1846
1847         event.event = etype;
1848         event.device = &sdev->base_dev;
1849         event.element.port_num = port;
1850
1851         siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1852
1853         ib_dispatch_event(&event);
1854 }