drivers/infiniband/hw/hns/hns_roce_qp.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/pci.h>
  35 #include <rdma/ib_addr.h>
  36 #include <rdma/ib_umem.h>
  37 #include <rdma/uverbs_ioctl.h>
  38 #include "hns_roce_common.h"
  39 #include "hns_roce_device.h"
  40 #include "hns_roce_hem.h"
  41
  42 static void flush_work_handle(struct work_struct *work)
  43 {
  44         struct hns_roce_work *flush_work = container_of(work,
  45                                         struct hns_roce_work, work);
  46         struct hns_roce_qp *hr_qp = container_of(flush_work,
  47                                         struct hns_roce_qp, flush_work);
  48         struct device *dev = flush_work->hr_dev->dev;
  49         struct ib_qp_attr attr;
  50         int attr_mask;
  51         int ret;
  52
  53         attr_mask = IB_QP_STATE;
  54         attr.qp_state = IB_QPS_ERR;
  55
  56         if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) {
  57                 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL);
  58                 if (ret)
  59                         dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n",
  60                                 ret);
  61         }
  62
  63         /*
  64          * make sure we signal QP destroy leg that flush QP was completed
  65          * so that it can safely proceed ahead now and destroy QP
  66          */
  67         if (refcount_dec_and_test(&hr_qp->refcount))
  68                 complete(&hr_qp->free);
  69 }
  70
  71 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
  72 {
  73         struct hns_roce_work *flush_work = &hr_qp->flush_work;
  74
  75         flush_work->hr_dev = hr_dev;
  76         INIT_WORK(&flush_work->work, flush_work_handle);
  77         refcount_inc(&hr_qp->refcount);
  78         queue_work(hr_dev->irq_workq, &flush_work->work);
  79 }
  80
  81 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
  82 {
  83         /*
  84          * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state
  85          * gets into errored mode. Hence, as a workaround to this
  86          * hardware limitation, driver needs to assist in flushing. But
  87          * the flushing operation uses mailbox to convey the QP state to
  88          * the hardware and which can sleep due to the mutex protection
  89          * around the mailbox calls. Hence, use the deferred flush for
  90          * now.
  91          */
  92         if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
  93                 init_flush_work(dev, qp);
  94 }
  95
  96 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
  97 {
  98         struct device *dev = hr_dev->dev;
  99         struct hns_roce_qp *qp;
 100
 101         xa_lock(&hr_dev->qp_table_xa);
 102         qp = __hns_roce_qp_lookup(hr_dev, qpn);
 103         if (qp)
 104                 refcount_inc(&qp->refcount);
 105         xa_unlock(&hr_dev->qp_table_xa);
 106
 107         if (!qp) {
 108                 dev_warn(dev, "Async event for bogus QP %08x\n", qpn);
 109                 return;
 110         }
 111
 112         if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR ||
 113             event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR ||
 114             event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR ||
 115             event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION ||
 116             event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) {
 117                 qp->state = IB_QPS_ERR;
 118
 119                 flush_cqe(hr_dev, qp);
 120         }
 121
 122         qp->event(qp, (enum hns_roce_event)event_type);
 123
 124         if (refcount_dec_and_test(&qp->refcount))
 125                 complete(&qp->free);
 126 }
 127
 128 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
 129                                  enum hns_roce_event type)
 130 {
 131         struct ib_qp *ibqp = &hr_qp->ibqp;
 132         struct ib_event event;
 133
 134         if (ibqp->event_handler) {
 135                 event.device = ibqp->device;
 136                 event.element.qp = ibqp;
 137                 switch (type) {
 138                 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
 139                         event.event = IB_EVENT_PATH_MIG;
 140                         break;
 141                 case HNS_ROCE_EVENT_TYPE_COMM_EST:
 142                         event.event = IB_EVENT_COMM_EST;
 143                         break;
 144                 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
 145                         event.event = IB_EVENT_SQ_DRAINED;
 146                         break;
 147                 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
 148                         event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 149                         break;
 150                 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
 151                         event.event = IB_EVENT_QP_FATAL;
 152                         break;
 153                 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
 154                         event.event = IB_EVENT_PATH_MIG_ERR;
 155                         break;
 156                 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
 157                         event.event = IB_EVENT_QP_REQ_ERR;
 158                         break;
 159                 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
 160                 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
 161                 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
 162                         event.event = IB_EVENT_QP_ACCESS_ERR;
 163                         break;
 164                 default:
 165                         dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
 166                                 type, hr_qp->qpn);
 167                         return;
 168                 }
 169                 ibqp->event_handler(&event, ibqp->qp_context);
 170         }
 171 }
 172
 173 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank)
 174 {
 175         u32 least_load = bank[0].inuse;
 176         u8 bankid = 0;
 177         u32 bankcnt;
 178         u8 i;
 179
 180         for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) {
 181                 bankcnt = bank[i].inuse;
 182                 if (bankcnt < least_load) {
 183                         least_load = bankcnt;
 184                         bankid = i;
 185                 }
 186         }
 187
 188         return bankid;
 189 }
 190
 191 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid,
 192                                  unsigned long *qpn)
 193 {
 194         int id;
 195
 196         id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL);
 197         if (id < 0) {
 198                 id = ida_alloc_range(&bank->ida, bank->min, bank->max,
 199                                      GFP_KERNEL);
 200                 if (id < 0)
 201                         return id;
 202         }
 203
 204         /* the QPN should keep increasing until the max value is reached. */
 205         bank->next = (id + 1) > bank->max ? bank->min : id + 1;
 206
 207         /* the lower 3 bits is bankid */
 208         *qpn = (id << 3) | bankid;
 209
 210         return 0;
 211 }
 212 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 213 {
 214         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 215         unsigned long num = 0;
 216         u8 bankid;
 217         int ret;
 218
 219         if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
 220                 num = 1;
 221                 hr_qp->doorbell_qpn = 1;
 222         } else {
 223                 mutex_lock(&qp_table->bank_mutex);
 224                 bankid = get_least_load_bankid_for_qp(qp_table->bank);
 225
 226                 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid,
 227                                             &num);
 228                 if (ret) {
 229                         ibdev_err(&hr_dev->ib_dev,
 230                                   "failed to alloc QPN, ret = %d\n", ret);
 231                         mutex_unlock(&qp_table->bank_mutex);
 232                         return ret;
 233                 }
 234
 235                 qp_table->bank[bankid].inuse++;
 236                 mutex_unlock(&qp_table->bank_mutex);
 237
 238                 hr_qp->doorbell_qpn = (u32)num;
 239         }
 240
 241         hr_qp->qpn = num;
 242
 243         return 0;
 244 }
 245
 246 static void add_qp_to_list(struct hns_roce_dev *hr_dev,
 247                            struct hns_roce_qp *hr_qp,
 248                            struct ib_cq *send_cq, struct ib_cq *recv_cq)
 249 {
 250         struct hns_roce_cq *hr_send_cq, *hr_recv_cq;
 251         unsigned long flags;
 252
 253         hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL;
 254         hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL;
 255
 256         spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
 257         hns_roce_lock_cqs(hr_send_cq, hr_recv_cq);
 258
 259         list_add_tail(&hr_qp->node, &hr_dev->qp_list);
 260         if (hr_send_cq)
 261                 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list);
 262         if (hr_recv_cq)
 263                 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list);
 264
 265         hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq);
 266         spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
 267 }
 268
 269 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev,
 270                              struct hns_roce_qp *hr_qp,
 271                              struct ib_qp_init_attr *init_attr)
 272 {
 273         struct xarray *xa = &hr_dev->qp_table_xa;
 274         int ret;
 275
 276         if (!hr_qp->qpn)
 277                 return -EINVAL;
 278
 279         ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL));
 280         if (ret)
 281                 dev_err(hr_dev->dev, "Failed to xa store for QPC\n");
 282         else
 283                 /* add QP to device's QP list for softwc */
 284                 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq,
 285                                init_attr->recv_cq);
 286
 287         return ret;
 288 }
 289
 290 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 291 {
 292         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 293         struct device *dev = hr_dev->dev;
 294         int ret;
 295
 296         if (!hr_qp->qpn)
 297                 return -EINVAL;
 298
 299         /* Alloc memory for QPC */
 300         ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
 301         if (ret) {
 302                 dev_err(dev, "Failed to get QPC table\n");
 303                 goto err_out;
 304         }
 305
 306         /* Alloc memory for IRRL */
 307         ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
 308         if (ret) {
 309                 dev_err(dev, "Failed to get IRRL table\n");
 310                 goto err_put_qp;
 311         }
 312
 313         if (hr_dev->caps.trrl_entry_sz) {
 314                 /* Alloc memory for TRRL */
 315                 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
 316                                          hr_qp->qpn);
 317                 if (ret) {
 318                         dev_err(dev, "Failed to get TRRL table\n");
 319                         goto err_put_irrl;
 320                 }
 321         }
 322
 323         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
 324                 /* Alloc memory for SCC CTX */
 325                 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
 326                                          hr_qp->qpn);
 327                 if (ret) {
 328                         dev_err(dev, "Failed to get SCC CTX table\n");
 329                         goto err_put_trrl;
 330                 }
 331         }
 332
 333         return 0;
 334
 335 err_put_trrl:
 336         if (hr_dev->caps.trrl_entry_sz)
 337                 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
 338
 339 err_put_irrl:
 340         hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
 341
 342 err_put_qp:
 343         hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
 344
 345 err_out:
 346         return ret;
 347 }
 348
 349 static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)
 350 {
 351         rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry);
 352 }
 353
 354 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 355 {
 356         struct xarray *xa = &hr_dev->qp_table_xa;
 357         unsigned long flags;
 358
 359         list_del(&hr_qp->node);
 360
 361         if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
 362                 list_del(&hr_qp->sq_node);
 363
 364         if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI &&
 365             hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
 366                 list_del(&hr_qp->rq_node);
 367
 368         xa_lock_irqsave(xa, flags);
 369         __xa_erase(xa, hr_qp->qpn);
 370         xa_unlock_irqrestore(xa, flags);
 371 }
 372
 373 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 374 {
 375         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 376
 377         if (hr_dev->caps.trrl_entry_sz)
 378                 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
 379         hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
 380 }
 381
 382 static inline u8 get_qp_bankid(unsigned long qpn)
 383 {
 384         /* The lower 3 bits of QPN are used to hash to different banks */
 385         return (u8)(qpn & GENMASK(2, 0));
 386 }
 387
 388 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 389 {
 390         u8 bankid;
 391
 392         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
 393                 return;
 394
 395         if (hr_qp->qpn < hr_dev->caps.reserved_qps)
 396                 return;
 397
 398         bankid = get_qp_bankid(hr_qp->qpn);
 399
 400         ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3);
 401
 402         mutex_lock(&hr_dev->qp_table.bank_mutex);
 403         hr_dev->qp_table.bank[bankid].inuse--;
 404         mutex_unlock(&hr_dev->qp_table.bank_mutex);
 405 }
 406
 407 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp,
 408                        bool user)
 409 {
 410         u32 max_sge = dev->caps.max_rq_sg;
 411
 412         if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 413                 return max_sge;
 414
 415         /* Reserve SGEs only for HIP08 in kernel; The userspace driver will
 416          * calculate number of max_sge with reserved SGEs when allocating wqe
 417          * buf, so there is no need to do this again in kernel. But the number
 418          * may exceed the capacity of SGEs recorded in the firmware, so the
 419          * kernel driver should just adapt the value accordingly.
 420          */
 421         if (user)
 422                 max_sge = roundup_pow_of_two(max_sge + 1);
 423         else
 424                 hr_qp->rq.rsv_sge = 1;
 425
 426         return max_sge;
 427 }
 428
 429 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
 430                        struct hns_roce_qp *hr_qp, int has_rq, bool user)
 431 {
 432         u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user);
 433         u32 cnt;
 434
 435         /* If srq exist, set zero for relative number of rq */
 436         if (!has_rq) {
 437                 hr_qp->rq.wqe_cnt = 0;
 438                 hr_qp->rq.max_gs = 0;
 439                 hr_qp->rq_inl_buf.wqe_cnt = 0;
 440                 cap->max_recv_wr = 0;
 441                 cap->max_recv_sge = 0;
 442
 443                 return 0;
 444         }
 445
 446         /* Check the validity of QP support capacity */
 447         if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes ||
 448             cap->max_recv_sge > max_sge) {
 449                 ibdev_err(&hr_dev->ib_dev,
 450                           "RQ config error, depth = %u, sge = %u\n",
 451                           cap->max_recv_wr, cap->max_recv_sge);
 452                 return -EINVAL;
 453         }
 454
 455         cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes));
 456         if (cnt > hr_dev->caps.max_wqes) {
 457                 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n",
 458                           cap->max_recv_wr);
 459                 return -EINVAL;
 460         }
 461
 462         hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) +
 463                                               hr_qp->rq.rsv_sge);
 464
 465         hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz *
 466                                     hr_qp->rq.max_gs);
 467
 468         hr_qp->rq.wqe_cnt = cnt;
 469         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE &&
 470             hr_qp->ibqp.qp_type != IB_QPT_UD &&
 471             hr_qp->ibqp.qp_type != IB_QPT_GSI)
 472                 hr_qp->rq_inl_buf.wqe_cnt = cnt;
 473         else
 474                 hr_qp->rq_inl_buf.wqe_cnt = 0;
 475
 476         cap->max_recv_wr = cnt;
 477         cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
 478
 479         return 0;
 480 }
 481
 482 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp)
 483 {
 484         /* GSI/UD QP only has extended sge */
 485         if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD)
 486                 return qp->sq.max_gs;
 487
 488         if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE)
 489                 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE;
 490
 491         return 0;
 492 }
 493
 494 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
 495                               struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap)
 496 {
 497         u32 total_sge_cnt;
 498         u32 wqe_sge_cnt;
 499
 500         hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
 501
 502         hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
 503
 504         wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp);
 505
 506         /* If the number of extended sge is not zero, they MUST use the
 507          * space of HNS_HW_PAGE_SIZE at least.
 508          */
 509         if (wqe_sge_cnt) {
 510                 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt);
 511                 hr_qp->sge.sge_cnt = max(total_sge_cnt,
 512                                 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
 513         }
 514 }
 515
 516 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
 517                                         struct ib_qp_cap *cap,
 518                                         struct hns_roce_ib_create_qp *ucmd)
 519 {
 520         u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
 521         u8 max_sq_stride = ilog2(roundup_sq_stride);
 522
 523         /* Sanity check SQ size before proceeding */
 524         if (ucmd->log_sq_stride > max_sq_stride ||
 525             ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
 526                 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n");
 527                 return -EINVAL;
 528         }
 529
 530         if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
 531                 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n",
 532                           cap->max_send_sge);
 533                 return -EINVAL;
 534         }
 535
 536         return 0;
 537 }
 538
 539 static int set_user_sq_size(struct hns_roce_dev *hr_dev,
 540                             struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,
 541                             struct hns_roce_ib_create_qp *ucmd)
 542 {
 543         struct ib_device *ibdev = &hr_dev->ib_dev;
 544         u32 cnt = 0;
 545         int ret;
 546
 547         if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
 548             cnt > hr_dev->caps.max_wqes)
 549                 return -EINVAL;
 550
 551         ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
 552         if (ret) {
 553                 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",
 554                           ret);
 555                 return ret;
 556         }
 557
 558         set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
 559
 560         hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
 561         hr_qp->sq.wqe_cnt = cnt;
 562
 563         return 0;
 564 }
 565
 566 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
 567                             struct hns_roce_qp *hr_qp,
 568                             struct hns_roce_buf_attr *buf_attr)
 569 {
 570         int buf_size;
 571         int idx = 0;
 572
 573         hr_qp->buff_size = 0;
 574
 575         /* SQ WQE */
 576         hr_qp->sq.offset = 0;
 577         buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt,
 578                                           hr_qp->sq.wqe_shift);
 579         if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
 580                 buf_attr->region[idx].size = buf_size;
 581                 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num;
 582                 idx++;
 583                 hr_qp->buff_size += buf_size;
 584         }
 585
 586         /* extend SGE WQE in SQ */
 587         hr_qp->sge.offset = hr_qp->buff_size;
 588         buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt,
 589                                           hr_qp->sge.sge_shift);
 590         if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
 591                 buf_attr->region[idx].size = buf_size;
 592                 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num;
 593                 idx++;
 594                 hr_qp->buff_size += buf_size;
 595         }
 596
 597         /* RQ WQE */
 598         hr_qp->rq.offset = hr_qp->buff_size;
 599         buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt,
 600                                           hr_qp->rq.wqe_shift);
 601         if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
 602                 buf_attr->region[idx].size = buf_size;
 603                 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num;
 604                 idx++;
 605                 hr_qp->buff_size += buf_size;
 606         }
 607
 608         if (hr_qp->buff_size < 1)
 609                 return -EINVAL;
 610
 611         buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
 612         buf_attr->region_count = idx;
 613
 614         return 0;
 615 }
 616
 617 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,
 618                               struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)
 619 {
 620         struct ib_device *ibdev = &hr_dev->ib_dev;
 621         u32 cnt;
 622
 623         if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
 624             cap->max_send_sge > hr_dev->caps.max_sq_sg) {
 625                 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n");
 626                 return -EINVAL;
 627         }
 628
 629         cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
 630         if (cnt > hr_dev->caps.max_wqes) {
 631                 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n",
 632                           cnt);
 633                 return -EINVAL;
 634         }
 635
 636         hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
 637         hr_qp->sq.wqe_cnt = cnt;
 638
 639         set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
 640
 641         /* sync the parameters of kernel QP to user's configuration */
 642         cap->max_send_wr = cnt;
 643         cap->max_send_sge = hr_qp->sq.max_gs;
 644
 645         return 0;
 646 }
 647
 648 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
 649 {
 650         if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
 651                 return 0;
 652
 653         return 1;
 654 }
 655
 656 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
 657 {
 658         if (attr->qp_type == IB_QPT_XRC_INI ||
 659             attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
 660             !attr->cap.max_recv_wr)
 661                 return 0;
 662
 663         return 1;
 664 }
 665
 666 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
 667                                struct ib_qp_init_attr *init_attr)
 668 {
 669         u32 max_recv_sge = init_attr->cap.max_recv_sge;
 670         u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt;
 671         struct hns_roce_rinl_wqe *wqe_list;
 672         int i;
 673
 674         /* allocate recv inline buf */
 675         wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
 676                            GFP_KERNEL);
 677         if (!wqe_list)
 678                 goto err;
 679
 680         /* Allocate a continuous buffer for all inline sge we need */
 681         wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
 682                                       sizeof(struct hns_roce_rinl_sge)),
 683                                       GFP_KERNEL);
 684         if (!wqe_list[0].sg_list)
 685                 goto err_wqe_list;
 686
 687         /* Assign buffers of sg_list to each inline wqe */
 688         for (i = 1; i < wqe_cnt; i++)
 689                 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
 690
 691         hr_qp->rq_inl_buf.wqe_list = wqe_list;
 692
 693         return 0;
 694
 695 err_wqe_list:
 696         kfree(wqe_list);
 697
 698 err:
 699         return -ENOMEM;
 700 }
 701
 702 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
 703 {
 704         if (hr_qp->rq_inl_buf.wqe_list)
 705                 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
 706         kfree(hr_qp->rq_inl_buf.wqe_list);
 707 }
 708
 709 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 710                         struct ib_qp_init_attr *init_attr,
 711                         struct ib_udata *udata, unsigned long addr)
 712 {
 713         struct ib_device *ibdev = &hr_dev->ib_dev;
 714         struct hns_roce_buf_attr buf_attr = {};
 715         int ret;
 716
 717         if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
 718                 ret = alloc_rq_inline_buf(hr_qp, init_attr);
 719                 if (ret) {
 720                         ibdev_err(ibdev,
 721                                   "failed to alloc inline buf, ret = %d.\n",
 722                                   ret);
 723                         return ret;
 724                 }
 725         } else {
 726                 hr_qp->rq_inl_buf.wqe_list = NULL;
 727         }
 728
 729         ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
 730         if (ret) {
 731                 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
 732                 goto err_inline;
 733         }
 734         ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
 735                                   PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
 736                                   udata, addr);
 737         if (ret) {
 738                 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
 739                 goto err_inline;
 740         }
 741
 742         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
 743                 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;
 744
 745         return 0;
 746
 747 err_inline:
 748         free_rq_inline_buf(hr_qp);
 749
 750         return ret;
 751 }
 752
 753 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 754 {
 755         hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
 756         free_rq_inline_buf(hr_qp);
 757 }
 758
 759 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
 760                                    struct ib_qp_init_attr *init_attr,
 761                                    struct ib_udata *udata,
 762                                    struct hns_roce_ib_create_qp_resp *resp,
 763                                    struct hns_roce_ib_create_qp *ucmd)
 764 {
 765         return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
 766                 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
 767                 hns_roce_qp_has_sq(init_attr) &&
 768                 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr));
 769 }
 770
 771 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev,
 772                                    struct ib_qp_init_attr *init_attr,
 773                                    struct ib_udata *udata,
 774                                    struct hns_roce_ib_create_qp_resp *resp)
 775 {
 776         return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
 777                 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
 778                 hns_roce_qp_has_rq(init_attr));
 779 }
 780
 781 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,
 782                                      struct ib_qp_init_attr *init_attr)
 783 {
 784         return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
 785                 hns_roce_qp_has_rq(init_attr));
 786 }
 787
 788 static int qp_mmap_entry(struct hns_roce_qp *hr_qp,
 789                          struct hns_roce_dev *hr_dev,
 790                          struct ib_udata *udata,
 791                          struct hns_roce_ib_create_qp_resp *resp)
 792 {
 793         struct hns_roce_ucontext *uctx =
 794                 rdma_udata_to_drv_context(udata,
 795                         struct hns_roce_ucontext, ibucontext);
 796         struct rdma_user_mmap_entry *rdma_entry;
 797         u64 address;
 798
 799         address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE;
 800
 801         hr_qp->dwqe_mmap_entry =
 802                 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address,
 803                                                 HNS_ROCE_DWQE_SIZE,
 804                                                 HNS_ROCE_MMAP_TYPE_DWQE);
 805
 806         if (!hr_qp->dwqe_mmap_entry) {
 807                 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n");
 808                 return -ENOMEM;
 809         }
 810
 811         rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry;
 812         resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
 813
 814         return 0;
 815 }
 816
 817 static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
 818                             struct hns_roce_qp *hr_qp,
 819                             struct ib_qp_init_attr *init_attr,
 820                             struct ib_udata *udata,
 821                             struct hns_roce_ib_create_qp *ucmd,
 822                             struct hns_roce_ib_create_qp_resp *resp)
 823 {
 824         struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
 825                 struct hns_roce_ucontext, ibucontext);
 826         struct ib_device *ibdev = &hr_dev->ib_dev;
 827         int ret;
 828
 829         if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) {
 830                 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb);
 831                 if (ret) {
 832                         ibdev_err(ibdev,
 833                                   "failed to map user SQ doorbell, ret = %d.\n",
 834                                   ret);
 835                         goto err_out;
 836                 }
 837                 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
 838         }
 839
 840         if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
 841                 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb);
 842                 if (ret) {
 843                         ibdev_err(ibdev,
 844                                   "failed to map user RQ doorbell, ret = %d.\n",
 845                                   ret);
 846                         goto err_sdb;
 847                 }
 848                 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 849         }
 850
 851         return 0;
 852
 853 err_sdb:
 854         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
 855                 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
 856 err_out:
 857         return ret;
 858 }
 859
 860 static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev,
 861                               struct hns_roce_qp *hr_qp,
 862                               struct ib_qp_init_attr *init_attr)
 863 {
 864         struct ib_device *ibdev = &hr_dev->ib_dev;
 865         int ret;
 866
 867         if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
 868                 hr_qp->sq.db_reg = hr_dev->mem_base +
 869                                    HNS_ROCE_DWQE_SIZE * hr_qp->qpn;
 870         else
 871                 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset +
 872                                    DB_REG_OFFSET * hr_dev->priv_uar.index;
 873
 874         hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset +
 875                            DB_REG_OFFSET * hr_dev->priv_uar.index;
 876
 877         if (kernel_qp_has_rdb(hr_dev, init_attr)) {
 878                 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
 879                 if (ret) {
 880                         ibdev_err(ibdev,
 881                                   "failed to alloc kernel RQ doorbell, ret = %d.\n",
 882                                   ret);
 883                         return ret;
 884                 }
 885                 *hr_qp->rdb.db_record = 0;
 886                 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 887         }
 888
 889         return 0;
 890 }
 891
 892 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 893                        struct ib_qp_init_attr *init_attr,
 894                        struct ib_udata *udata,
 895                        struct hns_roce_ib_create_qp *ucmd,
 896                        struct hns_roce_ib_create_qp_resp *resp)
 897 {
 898         int ret;
 899
 900         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE)
 901                 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
 902
 903         if (udata) {
 904                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) {
 905                         ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);
 906                         if (ret)
 907                                 return ret;
 908                 }
 909
 910                 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,
 911                                        resp);
 912                 if (ret)
 913                         goto err_remove_qp;
 914         } else {
 915                 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);
 916                 if (ret)
 917                         return ret;
 918         }
 919
 920         return 0;
 921
 922 err_remove_qp:
 923         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
 924                 qp_user_mmap_entry_remove(hr_qp);
 925
 926         return ret;
 927 }
 928
 929 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 930                        struct ib_udata *udata)
 931 {
 932         struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
 933                 udata, struct hns_roce_ucontext, ibucontext);
 934
 935         if (udata) {
 936                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 937                         hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
 938                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
 939                         hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
 940                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
 941                         qp_user_mmap_entry_remove(hr_qp);
 942         } else {
 943                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 944                         hns_roce_free_db(hr_dev, &hr_qp->rdb);
 945         }
 946 }
 947
 948 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
 949                              struct hns_roce_qp *hr_qp)
 950 {
 951         struct ib_device *ibdev = &hr_dev->ib_dev;
 952         u64 *sq_wrid = NULL;
 953         u64 *rq_wrid = NULL;
 954         int ret;
 955
 956         sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
 957         if (ZERO_OR_NULL_PTR(sq_wrid)) {
 958                 ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
 959                 return -ENOMEM;
 960         }
 961
 962         if (hr_qp->rq.wqe_cnt) {
 963                 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
 964                 if (ZERO_OR_NULL_PTR(rq_wrid)) {
 965                         ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
 966                         ret = -ENOMEM;
 967                         goto err_sq;
 968                 }
 969         }
 970
 971         hr_qp->sq.wrid = sq_wrid;
 972         hr_qp->rq.wrid = rq_wrid;
 973         return 0;
 974 err_sq:
 975         kfree(sq_wrid);
 976
 977         return ret;
 978 }
 979
 980 static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
 981 {
 982         kfree(hr_qp->rq.wrid);
 983         kfree(hr_qp->sq.wrid);
 984 }
 985
 986 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 987                         struct ib_qp_init_attr *init_attr,
 988                         struct ib_udata *udata,
 989                         struct hns_roce_ib_create_qp *ucmd)
 990 {
 991         struct ib_device *ibdev = &hr_dev->ib_dev;
 992         int ret;
 993
 994         if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline)
 995                 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline;
 996
 997         hr_qp->max_inline_data = init_attr->cap.max_inline_data;
 998
 999         if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1000                 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
1001         else
1002                 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
1003
1004         ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp,
1005                           hns_roce_qp_has_rq(init_attr), !!udata);
1006         if (ret) {
1007                 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",
1008                           ret);
1009                 return ret;
1010         }
1011
1012         if (udata) {
1013                 ret = ib_copy_from_udata(ucmd, udata,
1014                                          min(udata->inlen, sizeof(*ucmd)));
1015                 if (ret) {
1016                         ibdev_err(ibdev,
1017                                   "failed to copy QP ucmd, ret = %d\n", ret);
1018                         return ret;
1019                 }
1020
1021                 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
1022                 if (ret)
1023                         ibdev_err(ibdev,
1024                                   "failed to set user SQ size, ret = %d.\n",
1025                                   ret);
1026         } else {
1027                 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
1028                 if (ret)
1029                         ibdev_err(ibdev,
1030                                   "failed to set kernel SQ size, ret = %d.\n",
1031                                   ret);
1032         }
1033
1034         return ret;
1035 }
1036
1037 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
1038                                      struct ib_pd *ib_pd,
1039                                      struct ib_qp_init_attr *init_attr,
1040                                      struct ib_udata *udata,
1041                                      struct hns_roce_qp *hr_qp)
1042 {
1043         struct hns_roce_ib_create_qp_resp resp = {};
1044         struct ib_device *ibdev = &hr_dev->ib_dev;
1045         struct hns_roce_ib_create_qp ucmd;
1046         int ret;
1047
1048         mutex_init(&hr_qp->mutex);
1049         spin_lock_init(&hr_qp->sq.lock);
1050         spin_lock_init(&hr_qp->rq.lock);
1051
1052         hr_qp->state = IB_QPS_RESET;
1053         hr_qp->flush_flag = 0;
1054
1055         if (init_attr->create_flags)
1056                 return -EOPNOTSUPP;
1057
1058         ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd);
1059         if (ret) {
1060                 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret);
1061                 return ret;
1062         }
1063
1064         if (!udata) {
1065                 ret = alloc_kernel_wrid(hr_dev, hr_qp);
1066                 if (ret) {
1067                         ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n",
1068                                   ret);
1069                         return ret;
1070                 }
1071         }
1072
1073         ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
1074         if (ret) {
1075                 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);
1076                 goto err_buf;
1077         }
1078
1079         ret = alloc_qpn(hr_dev, hr_qp);
1080         if (ret) {
1081                 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
1082                 goto err_qpn;
1083         }
1084
1085         ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
1086         if (ret) {
1087                 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
1088                           ret);
1089                 goto err_db;
1090         }
1091
1092         ret = alloc_qpc(hr_dev, hr_qp);
1093         if (ret) {
1094                 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",
1095                           ret);
1096                 goto err_qpc;
1097         }
1098
1099         ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
1100         if (ret) {
1101                 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret);
1102                 goto err_store;
1103         }
1104
1105         if (udata) {
1106                 resp.cap_flags = hr_qp->en_flags;
1107                 ret = ib_copy_to_udata(udata, &resp,
1108                                        min(udata->outlen, sizeof(resp)));
1109                 if (ret) {
1110                         ibdev_err(ibdev, "copy qp resp failed!\n");
1111                         goto err_store;
1112                 }
1113         }
1114
1115         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
1116                 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
1117                 if (ret)
1118                         goto err_flow_ctrl;
1119         }
1120
1121         hr_qp->ibqp.qp_num = hr_qp->qpn;
1122         hr_qp->event = hns_roce_ib_qp_event;
1123         refcount_set(&hr_qp->refcount, 1);
1124         init_completion(&hr_qp->free);
1125
1126         return 0;
1127
1128 err_flow_ctrl:
1129         hns_roce_qp_remove(hr_dev, hr_qp);
1130 err_store:
1131         free_qpc(hr_dev, hr_qp);
1132 err_qpc:
1133         free_qp_db(hr_dev, hr_qp, udata);
1134 err_db:
1135         free_qpn(hr_dev, hr_qp);
1136 err_qpn:
1137         free_qp_buf(hr_dev, hr_qp);
1138 err_buf:
1139         free_kernel_wrid(hr_qp);
1140         return ret;
1141 }
1142
1143 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1144                          struct ib_udata *udata)
1145 {
1146         if (refcount_dec_and_test(&hr_qp->refcount))
1147                 complete(&hr_qp->free);
1148         wait_for_completion(&hr_qp->free);
1149
1150         free_qpc(hr_dev, hr_qp);
1151         free_qpn(hr_dev, hr_qp);
1152         free_qp_buf(hr_dev, hr_qp);
1153         free_kernel_wrid(hr_qp);
1154         free_qp_db(hr_dev, hr_qp, udata);
1155 }
1156
1157 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,
1158                          bool is_user)
1159 {
1160         switch (type) {
1161         case IB_QPT_XRC_INI:
1162         case IB_QPT_XRC_TGT:
1163                 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC))
1164                         goto out;
1165                 break;
1166         case IB_QPT_UD:
1167                 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 &&
1168                     is_user)
1169                         goto out;
1170                 break;
1171         case IB_QPT_RC:
1172         case IB_QPT_GSI:
1173                 break;
1174         default:
1175                 goto out;
1176         }
1177
1178         return 0;
1179
1180 out:
1181         ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type);
1182
1183         return -EOPNOTSUPP;
1184 }
1185
1186 int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,
1187                        struct ib_udata *udata)
1188 {
1189         struct ib_device *ibdev = qp->device;
1190         struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
1191         struct hns_roce_qp *hr_qp = to_hr_qp(qp);
1192         struct ib_pd *pd = qp->pd;
1193         int ret;
1194
1195         ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata);
1196         if (ret)
1197                 return ret;
1198
1199         if (init_attr->qp_type == IB_QPT_XRC_TGT)
1200                 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn;
1201
1202         if (init_attr->qp_type == IB_QPT_GSI) {
1203                 hr_qp->port = init_attr->port_num - 1;
1204                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1205         }
1206
1207         ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp);
1208         if (ret)
1209                 ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n",
1210                           init_attr->qp_type, ret);
1211
1212         return ret;
1213 }
1214
1215 int to_hr_qp_type(int qp_type)
1216 {
1217         switch (qp_type) {
1218         case IB_QPT_RC:
1219                 return SERV_TYPE_RC;
1220         case IB_QPT_UD:
1221         case IB_QPT_GSI:
1222                 return SERV_TYPE_UD;
1223         case IB_QPT_XRC_INI:
1224         case IB_QPT_XRC_TGT:
1225                 return SERV_TYPE_XRC;
1226         default:
1227                 return -1;
1228         }
1229 }
1230
1231 static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1232                               struct hns_roce_qp *hr_qp,
1233                               struct ib_qp_attr *attr, int attr_mask)
1234 {
1235         enum ib_mtu active_mtu;
1236         int p;
1237
1238         p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1239         active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1240
1241         if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1242             attr->path_mtu > hr_dev->caps.max_mtu) ||
1243             attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1244                 ibdev_err(&hr_dev->ib_dev,
1245                         "attr path_mtu(%d)invalid while modify qp",
1246                         attr->path_mtu);
1247                 return -EINVAL;
1248         }
1249
1250         return 0;
1251 }
1252
1253 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1254                                   int attr_mask)
1255 {
1256         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1257         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1258         int p;
1259
1260         if ((attr_mask & IB_QP_PORT) &&
1261             (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1262                 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n",
1263                           attr->port_num);
1264                 return -EINVAL;
1265         }
1266
1267         if (attr_mask & IB_QP_PKEY_INDEX) {
1268                 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1269                 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1270                         ibdev_err(&hr_dev->ib_dev,
1271                                   "invalid attr, pkey_index = %u.\n",
1272                                   attr->pkey_index);
1273                         return -EINVAL;
1274                 }
1275         }
1276
1277         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1278             attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1279                 ibdev_err(&hr_dev->ib_dev,
1280                           "invalid attr, max_rd_atomic = %u.\n",
1281                           attr->max_rd_atomic);
1282                 return -EINVAL;
1283         }
1284
1285         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1286             attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1287                 ibdev_err(&hr_dev->ib_dev,
1288                           "invalid attr, max_dest_rd_atomic = %u.\n",
1289                           attr->max_dest_rd_atomic);
1290                 return -EINVAL;
1291         }
1292
1293         if (attr_mask & IB_QP_PATH_MTU)
1294                 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1295
1296         return 0;
1297 }
1298
1299 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1300                        int attr_mask, struct ib_udata *udata)
1301 {
1302         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1303         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1304         enum ib_qp_state cur_state, new_state;
1305         int ret = -EINVAL;
1306
1307         mutex_lock(&hr_qp->mutex);
1308
1309         if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state)
1310                 goto out;
1311
1312         cur_state = hr_qp->state;
1313         new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1314
1315         if (ibqp->uobject &&
1316             (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1317                 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) {
1318                         hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1319
1320                         if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
1321                                 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1322                 } else {
1323                         ibdev_warn(&hr_dev->ib_dev,
1324                                   "flush cqe is not supported in userspace!\n");
1325                         goto out;
1326                 }
1327         }
1328
1329         if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1330                                 attr_mask)) {
1331                 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1332                 goto out;
1333         }
1334
1335         ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1336         if (ret)
1337                 goto out;
1338
1339         if (cur_state == new_state && cur_state == IB_QPS_RESET)
1340                 goto out;
1341
1342         ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1343                                     new_state);
1344
1345 out:
1346         mutex_unlock(&hr_qp->mutex);
1347
1348         return ret;
1349 }
1350
1351 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1352                        __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1353 {
1354         if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1355                 __acquire(&send_cq->lock);
1356                 __acquire(&recv_cq->lock);
1357         } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1358                 spin_lock_irq(&send_cq->lock);
1359                 __acquire(&recv_cq->lock);
1360         } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1361                 spin_lock_irq(&recv_cq->lock);
1362                 __acquire(&send_cq->lock);
1363         } else if (send_cq == recv_cq) {
1364                 spin_lock_irq(&send_cq->lock);
1365                 __acquire(&recv_cq->lock);
1366         } else if (send_cq->cqn < recv_cq->cqn) {
1367                 spin_lock_irq(&send_cq->lock);
1368                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1369         } else {
1370                 spin_lock_irq(&recv_cq->lock);
1371                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1372         }
1373 }
1374
1375 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1376                          struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1377                          __releases(&recv_cq->lock)
1378 {
1379         if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1380                 __release(&recv_cq->lock);
1381                 __release(&send_cq->lock);
1382         } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1383                 __release(&recv_cq->lock);
1384                 spin_unlock(&send_cq->lock);
1385         } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1386                 __release(&send_cq->lock);
1387                 spin_unlock(&recv_cq->lock);
1388         } else if (send_cq == recv_cq) {
1389                 __release(&recv_cq->lock);
1390                 spin_unlock_irq(&send_cq->lock);
1391         } else if (send_cq->cqn < recv_cq->cqn) {
1392                 spin_unlock(&recv_cq->lock);
1393                 spin_unlock_irq(&send_cq->lock);
1394         } else {
1395                 spin_unlock(&send_cq->lock);
1396                 spin_unlock_irq(&recv_cq->lock);
1397         }
1398 }
1399
1400 static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset)
1401 {
1402         return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
1403 }
1404
1405 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1406 {
1407         return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1408 }
1409
1410 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1411 {
1412         return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1413 }
1414
1415 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n)
1416 {
1417         return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
1418 }
1419
1420 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
1421                           struct ib_cq *ib_cq)
1422 {
1423         struct hns_roce_cq *hr_cq;
1424         u32 cur;
1425
1426         cur = hr_wq->head - hr_wq->tail;
1427         if (likely(cur + nreq < hr_wq->wqe_cnt))
1428                 return false;
1429
1430         hr_cq = to_hr_cq(ib_cq);
1431         spin_lock(&hr_cq->lock);
1432         cur = hr_wq->head - hr_wq->tail;
1433         spin_unlock(&hr_cq->lock);
1434
1435         return cur + nreq >= hr_wq->wqe_cnt;
1436 }
1437
1438 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1439 {
1440         struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1441         unsigned int reserved_from_bot;
1442         unsigned int i;
1443
1444         qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps,
1445                                         sizeof(u32), GFP_KERNEL);
1446         if (!qp_table->idx_table.spare_idx)
1447                 return -ENOMEM;
1448
1449         mutex_init(&qp_table->scc_mutex);
1450         mutex_init(&qp_table->bank_mutex);
1451         xa_init(&hr_dev->qp_table_xa);
1452
1453         reserved_from_bot = hr_dev->caps.reserved_qps;
1454
1455         for (i = 0; i < reserved_from_bot; i++) {
1456                 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++;
1457                 hr_dev->qp_table.bank[get_qp_bankid(i)].min++;
1458         }
1459
1460         for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
1461                 ida_init(&hr_dev->qp_table.bank[i].ida);
1462                 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps /
1463                                                HNS_ROCE_QP_BANK_NUM - 1;
1464                 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min;
1465         }
1466
1467         return 0;
1468 }
1469
1470 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1471 {
1472         int i;
1473
1474         for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++)
1475                 ida_destroy(&hr_dev->qp_table.bank[i].ida);
1476         kfree(hr_dev->qp_table.idx_table.spare_idx);
1477 }