projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'vfs-6.7.misc' of gitolite.kernel.org:pub/scm/linux/kernel/git/vfs/vfs
[linux-block.git] / drivers / infiniband / hw / bnxt_re / ib_verbs.c
CommitLineData
1ac5a404
SX		 1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: IB Verbs interpreter
37 */
38
39#include <linux/interrupt.h>
40#include <linux/types.h>
41#include <linux/pci.h>
42#include <linux/netdevice.h>
43#include <linux/if_ether.h>
493620b1 44#include <net/addrconf.h>
1ac5a404
SX		 45
46#include <rdma/ib_verbs.h>
47#include <rdma/ib_user_verbs.h>
48#include <rdma/ib_umem.h>
49#include <rdma/ib_addr.h>
50#include <rdma/ib_mad.h>
51#include <rdma/ib_cache.h>
89944450 52#include <rdma/uverbs_ioctl.h>
1ac5a404
SX		 53
54#include "bnxt_ulp.h"
55
56#include "roce_hsi.h"
57#include "qplib_res.h"
58#include "qplib_sp.h"
59#include "qplib_fp.h"
60#include "qplib_rcfw.h"
61
62#include "bnxt_re.h"
63#include "ib_verbs.h"
360da60d
SX		 64
65#include <rdma/uverbs_types.h>
66#include <rdma/uverbs_std_types.h>
67
68#include <rdma/ib_user_ioctl_cmds.h>
69
70#define UVERBS_MODULE_NAME bnxt_re
71#include <rdma/uverbs_named_ioctl.h>
72
1ac5a404
SX		 73#include <rdma/bnxt_re-abi.h>
74
9152e0b7
EW		 75static int __from_ib_access_flags(int iflags)
76{
77 int qflags = 0;
78
79 if (iflags & IB_ACCESS_LOCAL_WRITE)
80 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
81 if (iflags & IB_ACCESS_REMOTE_READ)
82 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
83 if (iflags & IB_ACCESS_REMOTE_WRITE)
84 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
85 if (iflags & IB_ACCESS_REMOTE_ATOMIC)
86 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
87 if (iflags & IB_ACCESS_MW_BIND)
88 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
89 if (iflags & IB_ZERO_BASED)
90 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
91 if (iflags & IB_ACCESS_ON_DEMAND)
92 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
93 return qflags;
94};
95
96static enum ib_access_flags __to_ib_access_flags(int qflags)
97{
98 enum ib_access_flags iflags = 0;
99
100 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
101 iflags |= IB_ACCESS_LOCAL_WRITE;
102 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
103 iflags |= IB_ACCESS_REMOTE_WRITE;
104 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
105 iflags |= IB_ACCESS_REMOTE_READ;
106 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
107 iflags |= IB_ACCESS_REMOTE_ATOMIC;
108 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
109 iflags |= IB_ACCESS_MW_BIND;
110 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
111 iflags |= IB_ZERO_BASED;
112 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
113 iflags |= IB_ACCESS_ON_DEMAND;
114 return iflags;
115};
116
1ac5a404
SX		 117static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
118 struct bnxt_qplib_sge *sg_list, int num)
119{
120 int i, total = 0;
121
122 for (i = 0; i < num; i++) {
123 sg_list[i].addr = ib_sg_list[i].addr;
124 sg_list[i].lkey = ib_sg_list[i].lkey;
125 sg_list[i].size = ib_sg_list[i].length;
126 total += sg_list[i].size;
127 }
128 return total;
129}
130
131/* Device */
1ac5a404
SX		 132int bnxt_re_query_device(struct ib_device *ibdev,
133 struct ib_device_attr *ib_attr,
134 struct ib_udata *udata)
135{
136 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
137 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
138
139 memset(ib_attr, 0, sizeof(*ib_attr));
2fc68543
SX		 140 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
141 min(sizeof(dev_attr->fw_ver),
142 sizeof(ib_attr->fw_ver)));
493620b1
KH		 143 addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid,
144 rdev->netdev->dev_addr);
58d4a671 145 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
6a7296c9 146 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;
1ac5a404
SX		 147
148 ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
149 ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
150 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
151 ib_attr->max_qp = dev_attr->max_qp;
152 ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
153 ib_attr->device_cap_flags =
154 IB_DEVICE_CURR_QP_STATE_MOD
155 | IB_DEVICE_RC_RNR_NAK_GEN
156 | IB_DEVICE_SHUTDOWN_PORT
157 | IB_DEVICE_SYS_IMAGE_GUID
1ac5a404
SX		 158 | IB_DEVICE_RESIZE_MAX_WR
159 | IB_DEVICE_PORT_ACTIVE_EVENT
160 | IB_DEVICE_N_NOTIFY_CQ
161 | IB_DEVICE_MEM_WINDOW
162 | IB_DEVICE_MEM_WINDOW_TYPE_2B
163 | IB_DEVICE_MEM_MGT_EXTENSIONS;
e945c653 164 ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
33023fb8
SW		 165 ib_attr->max_send_sge = dev_attr->max_qp_sges;
166 ib_attr->max_recv_sge = dev_attr->max_qp_sges;
1ac5a404
SX		 167 ib_attr->max_sge_rd = dev_attr->max_qp_sges;
168 ib_attr->max_cq = dev_attr->max_cq;
169 ib_attr->max_cqe = dev_attr->max_cq_wqes;
170 ib_attr->max_mr = dev_attr->max_mr;
171 ib_attr->max_pd = dev_attr->max_pd;
172 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
a25d112f 173 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
7ff662b7
DS		 174 ib_attr->atomic_cap = IB_ATOMIC_NONE;
175 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
35f5ace5
DS		 176 if (dev_attr->is_atomic) {
177 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
178 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
179 }
1ac5a404
SX		 180
181 ib_attr->max_ee_rd_atom = 0;
182 ib_attr->max_res_rd_atom = 0;
183 ib_attr->max_ee_init_rd_atom = 0;
184 ib_attr->max_ee = 0;
185 ib_attr->max_rdd = 0;
186 ib_attr->max_mw = dev_attr->max_mw;
187 ib_attr->max_raw_ipv6_qp = 0;
188 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
189 ib_attr->max_mcast_grp = 0;
190 ib_attr->max_mcast_qp_attach = 0;
191 ib_attr->max_total_mcast_qp_attach = 0;
192 ib_attr->max_ah = dev_attr->max_ah;
193
1ac5a404
SX		 194 ib_attr->max_srq = dev_attr->max_srq;
195 ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
196 ib_attr->max_srq_sge = dev_attr->max_srq_sges;
197
198 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
199
200 ib_attr->max_pkeys = 1;
601577b7 201 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
1ac5a404
SX		 202 return 0;
203}
204
1ac5a404 205/* Port */
1fb7f897 206int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
1ac5a404
SX		 207 struct ib_port_attr *port_attr)
208{
209 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
210 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
18e7e3e4 211 int rc;
1ac5a404
SX		 212
213 memset(port_attr, 0, sizeof(*port_attr));
214
215 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
216 port_attr->state = IB_PORT_ACTIVE;
72a7720f 217 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
1ac5a404
SX		 218 } else {
219 port_attr->state = IB_PORT_DOWN;
72a7720f 220 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
1ac5a404
SX		 221 }
222 port_attr->max_mtu = IB_MTU_4096;
223 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
224 port_attr->gid_tbl_len = dev_attr->max_sgid;
225 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
226 IB_PORT_DEVICE_MGMT_SUP |
2f944c0f
JG		 227 IB_PORT_VENDOR_CLASS_SUP;
228 port_attr->ip_gids = true;
1ac5a404 229
872f3578 230 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
1ac5a404
SX		 231 port_attr->bad_pkey_cntr = 0;
232 port_attr->qkey_viol_cntr = 0;
233 port_attr->pkey_tbl_len = dev_attr->max_pkey;
234 port_attr->lid = 0;
235 port_attr->sm_lid = 0;
236 port_attr->lmc = 0;
237 port_attr->max_vl_num = 4;
238 port_attr->sm_sl = 0;
239 port_attr->subnet_timeout = 0;
240 port_attr->init_type_reply = 0;
18e7e3e4
KH		 241 rc = ib_get_eth_speed(&rdev->ibdev, port_num, &port_attr->active_speed,
242 &port_attr->active_width);
74828b12 243
18e7e3e4 244 return rc;
1ac5a404
SX		 245}
246
1fb7f897 247int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
1ac5a404
SX		 248 struct ib_port_immutable *immutable)
249{
250 struct ib_port_attr port_attr;
251
252 if (bnxt_re_query_port(ibdev, port_num, &port_attr))
253 return -EINVAL;
254
255 immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
256 immutable->gid_tbl_len = port_attr.gid_tbl_len;
257 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
258 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
259 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
260 return 0;
261}
262
2fc68543
SX		 263void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
264{
265 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
266
267 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
268 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1],
269 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]);
270}
271
1fb7f897 272int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
1ac5a404
SX		 273 u16 index, u16 *pkey)
274{
0e938533
KH		 275 if (index > 0)
276 return -EINVAL;
1ac5a404 277
0e938533 278 *pkey = IB_DEFAULT_PKEY_FULL;
1ac5a404 279
0e938533 280 return 0;
1ac5a404
SX		 281}
282
1fb7f897 283int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
1ac5a404
SX		 284 int index, union ib_gid *gid)
285{
286 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
14611b9b 287 int rc;
1ac5a404
SX		 288
289 /* Ignore port_num */
290 memset(gid, 0, sizeof(*gid));
291 rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
292 &rdev->qplib_res.sgid_tbl, index,
293 (struct bnxt_qplib_gid *)gid);
294 return rc;
295}
296
414448d2 297int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
1ac5a404
SX		 298{
299 int rc = 0;
300 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
414448d2 301 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
1ac5a404 302 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
89aaca54 303 struct bnxt_qplib_gid *gid_to_del;
c56b593d 304 u16 vlan_id = 0xFFFF;
1ac5a404
SX		 305
306 /* Delete the entry from the hardware */
307 ctx = *context;
308 if (!ctx)
309 return -EINVAL;
310
311 if (sgid_tbl && sgid_tbl->active) {
312 if (ctx->idx >= sgid_tbl->max)
313 return -EINVAL;
c56b593d
SX		 314 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
315 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
89aaca54
SK		 316 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
317 * or via the ib_unregister_device path. In the former case QP1
318 * may not be destroyed yet, in which case just return as FW
319 * needs that entry to be present and will fail it's deletion.
320 * We could get invoked again after QP1 is destroyed OR get an
321 * ADD_GID call with a different GID value for the same index
322 * where we issue MODIFY_GID cmd to update the GID entry -- TBD
323 */
324 if (ctx->idx == 0 &&
325 rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
8dae419f 326 ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
6ccad848
DS		 327 ibdev_dbg(&rdev->ibdev,
328 "Trying to delete GID0 while QP1 is alive\n");
89aaca54
SK		 329 return -EFAULT;
330 }
1ac5a404
SX		 331 ctx->refcnt--;
332 if (!ctx->refcnt) {
c56b593d
SX		 333 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
334 vlan_id, true);
4a62c5e9 335 if (rc) {
6ccad848
DS		 336 ibdev_err(&rdev->ibdev,
337 "Failed to remove GID: %#x", rc);
4a62c5e9
SX		 338 } else {
339 ctx_tbl = sgid_tbl->ctx;
340 ctx_tbl[ctx->idx] = NULL;
341 kfree(ctx);
342 }
1ac5a404
SX		 343 }
344 } else {
345 return -EINVAL;
346 }
347 return rc;
348}
349
f4df9a7c 350int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
1ac5a404
SX		 351{
352 int rc;
353 u32 tbl_idx = 0;
354 u16 vlan_id = 0xFFFF;
355 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
414448d2 356 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
1ac5a404
SX		 357 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
358
a70c0739
PP		 359 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
360 if (rc)
361 return rc;
1ac5a404 362
f4df9a7c 363 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
1ac5a404
SX		 364 rdev->qplib_res.netdev->dev_addr,
365 vlan_id, true, &tbl_idx);
366 if (rc == -EALREADY) {
367 ctx_tbl = sgid_tbl->ctx;
368 ctx_tbl[tbl_idx]->refcnt++;
369 *context = ctx_tbl[tbl_idx];
370 return 0;
371 }
372
373 if (rc < 0) {
6ccad848 374 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
1ac5a404
SX		 375 return rc;
376 }
377
378 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
379 if (!ctx)
380 return -ENOMEM;
381 ctx_tbl = sgid_tbl->ctx;
382 ctx->idx = tbl_idx;
383 ctx->refcnt = 1;
384 ctx_tbl[tbl_idx] = ctx;
063fb5bd 385 *context = ctx;
1ac5a404
SX		 386
387 return rc;
388}
389
390enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
1fb7f897 391 u32 port_num)
1ac5a404
SX		 392{
393 return IB_LINK_LAYER_ETHERNET;
394}
395
9152e0b7
EW		 396#define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
397
398static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
399{
400 struct bnxt_re_fence_data *fence = &pd->fence;
401 struct ib_mr *ib_mr = &fence->mr->ib_mr;
402 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
403
404 memset(wqe, 0, sizeof(*wqe));
405 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
406 wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
407 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
408 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
409 wqe->bind.zero_based = false;
410 wqe->bind.parent_l_key = ib_mr->lkey;
411 wqe->bind.va = (u64)(unsigned long)fence->va;
412 wqe->bind.length = fence->size;
413 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
414 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
415
416 /* Save the initial rkey in fence structure for now;
417 * wqe->bind.r_key will be set at (re)bind time.
418 */
419 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
420}
421
422static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
423{
424 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
425 qplib_qp);
426 struct ib_pd *ib_pd = qp->ib_qp.pd;
427 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
428 struct bnxt_re_fence_data *fence = &pd->fence;
429 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
430 struct bnxt_qplib_swqe wqe;
431 int rc;
432
433 memcpy(&wqe, fence_wqe, sizeof(wqe));
434 wqe.bind.r_key = fence->bind_rkey;
435 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
436
6ccad848
DS		 437 ibdev_dbg(&qp->rdev->ibdev,
438 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
9152e0b7
EW		 439 wqe.bind.r_key, qp->qplib_qp.id, pd);
440 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
441 if (rc) {
6ccad848 442 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
9152e0b7
EW		 443 return rc;
444 }
445 bnxt_qplib_post_send_db(&qp->qplib_qp);
446
447 return rc;
448}
449
450static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
451{
452 struct bnxt_re_fence_data *fence = &pd->fence;
453 struct bnxt_re_dev *rdev = pd->rdev;
454 struct device *dev = &rdev->en_dev->pdev->dev;
455 struct bnxt_re_mr *mr = fence->mr;
456
457 if (fence->mw) {
458 bnxt_re_dealloc_mw(fence->mw);
459 fence->mw = NULL;
460 }
461 if (mr) {
462 if (mr->ib_mr.rkey)
463 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
464 true);
465 if (mr->ib_mr.lkey)
466 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
467 kfree(mr);
468 fence->mr = NULL;
469 }
470 if (fence->dma_addr) {
471 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
472 DMA_BIDIRECTIONAL);
473 fence->dma_addr = 0;
474 }
475}
476
477static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
478{
479 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
480 struct bnxt_re_fence_data *fence = &pd->fence;
481 struct bnxt_re_dev *rdev = pd->rdev;
482 struct device *dev = &rdev->en_dev->pdev->dev;
483 struct bnxt_re_mr *mr = NULL;
484 dma_addr_t dma_addr = 0;
485 struct ib_mw *mw;
9152e0b7
EW		 486 int rc;
487
488 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
489 DMA_BIDIRECTIONAL);
490 rc = dma_mapping_error(dev, dma_addr);
491 if (rc) {
6ccad848 492 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
9152e0b7
EW		 493 rc = -EIO;
494 fence->dma_addr = 0;
495 goto fail;
496 }
497 fence->dma_addr = dma_addr;
498
499 /* Allocate a MR */
500 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
501 if (!mr) {
502 rc = -ENOMEM;
503 goto fail;
504 }
505 fence->mr = mr;
506 mr->rdev = rdev;
507 mr->qplib_mr.pd = &pd->qplib_pd;
508 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
509 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
510 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
511 if (rc) {
6ccad848 512 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
9152e0b7
EW		 513 goto fail;
514 }
515
516 /* Register MR */
517 mr->ib_mr.lkey = mr->qplib_mr.lkey;
518 mr->qplib_mr.va = (u64)(unsigned long)fence->va;
519 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
f6919d56
SX		 520 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
521 BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
9152e0b7 522 if (rc) {
6ccad848 523 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
9152e0b7
EW		 524 goto fail;
525 }
526 mr->ib_mr.rkey = mr->qplib_mr.rkey;
527
528 /* Create a fence MW only for kernel consumers */
529 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
653f0a71 530 if (IS_ERR(mw)) {
6ccad848
DS		 531 ibdev_err(&rdev->ibdev,
532 "Failed to create fence-MW for PD: %p\n", pd);
653f0a71 533 rc = PTR_ERR(mw);
9152e0b7
EW		 534 goto fail;
535 }
536 fence->mw = mw;
537
538 bnxt_re_create_fence_wqe(pd);
539 return 0;
540
541fail:
542 bnxt_re_destroy_fence_mr(pd);
543 return rc;
544}
545
24ce9478
SX		 546static struct bnxt_re_user_mmap_entry*
547bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
548 enum bnxt_re_mmap_flag mmap_flag, u64 *offset)
549{
550 struct bnxt_re_user_mmap_entry *entry;
551 int ret;
552
553 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
554 if (!entry)
555 return NULL;
556
557 entry->mem_offset = mem_offset;
558 entry->mmap_flag = mmap_flag;
360da60d 559 entry->uctx = uctx;
24ce9478
SX		 560
561 switch (mmap_flag) {
562 case BNXT_RE_MMAP_SH_PAGE:
563 ret = rdma_user_mmap_entry_insert_exact(&uctx->ib_uctx,
564 &entry->rdma_entry, PAGE_SIZE, 0);
565 break;
566 case BNXT_RE_MMAP_UC_DB:
360da60d 567 case BNXT_RE_MMAP_WC_DB:
ea222485
CA		 568 case BNXT_RE_MMAP_DBR_BAR:
569 case BNXT_RE_MMAP_DBR_PAGE:
24ce9478
SX		 570 ret = rdma_user_mmap_entry_insert(&uctx->ib_uctx,
571 &entry->rdma_entry, PAGE_SIZE);
572 break;
573 default:
574 ret = -EINVAL;
575 break;
576 }
577
578 if (ret) {
579 kfree(entry);
580 return NULL;
581 }
582 if (offset)
583 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
584
585 return entry;
586}
587
1ac5a404 588/* Protection Domains */
91a7c58f 589int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
1ac5a404
SX		 590{
591 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
592 struct bnxt_re_dev *rdev = pd->rdev;
1ac5a404 593
24ce9478
SX		 594 if (udata) {
595 rdma_user_mmap_entry_remove(pd->pd_db_mmap);
596 pd->pd_db_mmap = NULL;
597 }
598
9152e0b7 599 bnxt_re_destroy_fence_mr(pd);
1ac5a404 600
9a381f7e
SX		 601 if (pd->qplib_pd.id) {
602 if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res,
603 &rdev->qplib_res.pd_tbl,
604 &pd->qplib_pd))
063975fe 605 atomic_dec(&rdev->stats.res.pd_count);
9a381f7e 606 }
91a7c58f 607 return 0;
1ac5a404
SX		 608}
609
ff23dfa1 610int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
1ac5a404 611{
21a428a0 612 struct ib_device *ibdev = ibpd->device;
1ac5a404 613 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
ff23dfa1
SR		 614 struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
615 udata, struct bnxt_re_ucontext, ib_uctx);
21a428a0 616 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
24ce9478 617 struct bnxt_re_user_mmap_entry *entry = NULL;
cb95709e 618 u32 active_pds;
24ce9478 619 int rc = 0;
1ac5a404 620
1ac5a404 621 pd->rdev = rdev;
213d2b9b 622 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
6ccad848 623 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
1ac5a404
SX		 624 rc = -ENOMEM;
625 goto fail;
626 }
627
628 if (udata) {
24ce9478 629 struct bnxt_re_pd_resp resp = {};
1ac5a404 630
b3b2c7c5 631 if (!ucntx->dpi.dbr) {
1ac5a404
SX		 632 /* Allocate DPI in alloc_pd to avoid failing of
633 * ibv_devinfo and family of application when DPIs
634 * are depleted.
635 */
0ac20faf
SX		 636 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res,
637 &ucntx->dpi, ucntx, BNXT_QPLIB_DPI_TYPE_UC)) {
1ac5a404
SX		 638 rc = -ENOMEM;
639 goto dbfail;
640 }
1ac5a404
SX		 641 }
642
643 resp.pdid = pd->qplib_pd.id;
644 /* Still allow mapping this DBR to the new user PD. */
b3b2c7c5 645 resp.dpi = ucntx->dpi.dpi;
1ac5a404 646
24ce9478
SX		 647 entry = bnxt_re_mmap_entry_insert(ucntx, (u64)ucntx->dpi.umdbr,
648 BNXT_RE_MMAP_UC_DB, &resp.dbr);
649
650 if (!entry) {
651 rc = -ENOMEM;
652 goto dbfail;
653 }
654
655 pd->pd_db_mmap = &entry->rdma_entry;
656
657 rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
1ac5a404 658 if (rc) {
24ce9478
SX		 659 rdma_user_mmap_entry_remove(pd->pd_db_mmap);
660 rc = -EFAULT;
1ac5a404
SX		 661 goto dbfail;
662 }
663 }
664
9152e0b7
EW		 665 if (!udata)
666 if (bnxt_re_create_fence_mr(pd))
6ccad848
DS		 667 ibdev_warn(&rdev->ibdev,
668 "Failed to create Fence-MR\n");
cb95709e
CA		 669 active_pds = atomic_inc_return(&rdev->stats.res.pd_count);
670 if (active_pds > rdev->stats.res.pd_watermark)
671 rdev->stats.res.pd_watermark = active_pds;
9a381f7e 672
21a428a0 673 return 0;
1ac5a404 674dbfail:
21a428a0
LR		 675 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
676 &pd->qplib_pd);
1ac5a404 677fail:
21a428a0 678 return rc;
1ac5a404
SX		 679}
680
681/* Address Handles */
9a9ebf8c 682int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
1ac5a404
SX		 683{
684 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
685 struct bnxt_re_dev *rdev = ah->rdev;
bb8c9361 686 bool block = true;
14611b9b 687 int rc;
1ac5a404 688
bb8c9361
KD		 689 block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
690 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
691 if (BNXT_RE_CHECK_RC(rc)) {
692 if (rc == -ETIMEDOUT)
693 rc = 0;
694 else
695 goto fail;
696 }
063975fe 697 atomic_dec(&rdev->stats.res.ah_count);
bb8c9361
KD		 698fail:
699 return rc;
1ac5a404
SX		 700}
701
374c5285
DS		 702static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
703{
704 u8 nw_type;
705
706 switch (ntype) {
707 case RDMA_NETWORK_IPV4:
708 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
709 break;
710 case RDMA_NETWORK_IPV6:
711 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
712 break;
713 default:
714 nw_type = CMDQ_CREATE_AH_TYPE_V1;
715 break;
716 }
717 return nw_type;
718}
719
fa5d010c
MG		 720int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
721 struct ib_udata *udata)
1ac5a404 722{
d3456914 723 struct ib_pd *ib_pd = ib_ah->pd;
1ac5a404 724 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
fa5d010c 725 struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
374c5285 726 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
1ac5a404 727 struct bnxt_re_dev *rdev = pd->rdev;
374c5285 728 const struct ib_gid_attr *sgid_attr;
0a01623b 729 struct bnxt_re_gid_ctx *ctx;
d3456914 730 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
cb95709e 731 u32 active_ahs;
1ac5a404 732 u8 nw_type;
374c5285 733 int rc;
1ac5a404 734
d8966fcd 735 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
6ccad848 736 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
d3456914 737 return -EINVAL;
1ac5a404 738 }
1ac5a404
SX		 739
740 ah->rdev = rdev;
741 ah->qplib_ah.pd = &pd->qplib_pd;
742
743 /* Supply the configuration for the HW */
d8966fcd 744 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
1ac5a404 745 sizeof(union ib_gid));
0a01623b
SX		 746 sgid_attr = grh->sgid_attr;
747 /* Get the HW context of the GID. The reference
748 * of GID table entry is already taken by the caller.
1ac5a404 749 */
0a01623b
SX		 750 ctx = rdma_read_gid_hw_context(sgid_attr);
751 ah->qplib_ah.sgid_index = ctx->idx;
d8966fcd
DC		 752 ah->qplib_ah.host_sgid_index = grh->sgid_index;
753 ah->qplib_ah.traffic_class = grh->traffic_class;
754 ah->qplib_ah.flow_label = grh->flow_label;
755 ah->qplib_ah.hop_limit = grh->hop_limit;
756 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
1ac5a404 757
374c5285
DS		 758 /* Get network header type for this GID */
759 nw_type = rdma_gid_attr_network_type(sgid_attr);
760 ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
1ac5a404 761
44c58487 762 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
90e3edd8 763 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
fa5d010c
MG		 764 !(init_attr->flags &
765 RDMA_CREATE_AH_SLEEPABLE));
1ac5a404 766 if (rc) {
6ccad848 767 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
d3456914 768 return rc;
1ac5a404
SX		 769 }
770
771 /* Write AVID to shared page. */
e00b64f7 772 if (udata) {
89944450
SR		 773 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
774 udata, struct bnxt_re_ucontext, ib_uctx);
1ac5a404
SX		 775 unsigned long flag;
776 u32 *wrptr;
777
1ac5a404
SX		 778 spin_lock_irqsave(&uctx->sh_lock, flag);
779 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
780 *wrptr = ah->qplib_ah.id;
781 wmb(); /* make sure cache is updated. */
782 spin_unlock_irqrestore(&uctx->sh_lock, flag);
783 }
cb95709e
CA		 784 active_ahs = atomic_inc_return(&rdev->stats.res.ah_count);
785 if (active_ahs > rdev->stats.res.ah_watermark)
786 rdev->stats.res.ah_watermark = active_ahs;
1ac5a404 787
d3456914 788 return 0;
1ac5a404
SX		 789}
790
90898850 791int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
1ac5a404
SX		 792{
793 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
794
44c58487 795 ah_attr->type = ib_ah->type;
d8966fcd 796 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
44c58487 797 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
d8966fcd
DC		 798 rdma_ah_set_grh(ah_attr, NULL, 0,
799 ah->qplib_ah.host_sgid_index,
800 0, ah->qplib_ah.traffic_class);
801 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
802 rdma_ah_set_port_num(ah_attr, 1);
803 rdma_ah_set_static_rate(ah_attr, 0);
1ac5a404
SX		 804 return 0;
805}
806
942c9b6c 807unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
3b921e3b
SX		 808 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
809{
810 unsigned long flags;
811
812 spin_lock_irqsave(&qp->scq->cq_lock, flags);
813 if (qp->rcq != qp->scq)
814 spin_lock(&qp->rcq->cq_lock);
815 else
816 __acquire(&qp->rcq->cq_lock);
817
818 return flags;
819}
820
942c9b6c
SX		 821void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
822 unsigned long flags)
3b921e3b
SX		 823 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
824{
825 if (qp->rcq != qp->scq)
826 spin_unlock(&qp->rcq->cq_lock);
827 else
828 __release(&qp->rcq->cq_lock);
829 spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
830}
831
8dae419f
DS		 832static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
833{
834 struct bnxt_re_qp *gsi_sqp;
835 struct bnxt_re_ah *gsi_sah;
836 struct bnxt_re_dev *rdev;
14611b9b 837 int rc;
8dae419f
DS		 838
839 rdev = qp->rdev;
840 gsi_sqp = rdev->gsi_ctx.gsi_sqp;
841 gsi_sah = rdev->gsi_ctx.gsi_sah;
842
6ccad848 843 ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
8dae419f
DS		 844 bnxt_qplib_destroy_ah(&rdev->qplib_res,
845 &gsi_sah->qplib_ah,
846 true);
063975fe 847 atomic_dec(&rdev->stats.res.ah_count);
8dae419f
DS		 848 bnxt_qplib_clean_qp(&qp->qplib_qp);
849
6ccad848 850 ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
8dae419f
DS		 851 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
852 if (rc) {
6ccad848 853 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
8dae419f
DS		 854 goto fail;
855 }
856 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
857
097a9d23
SX		 858 /* remove from active qp list */
859 mutex_lock(&rdev->qp_lock);
860 list_del(&gsi_sqp->list);
861 mutex_unlock(&rdev->qp_lock);
063975fe 862 atomic_dec(&rdev->stats.res.qp_count);
097a9d23 863
8dae419f
DS		 864 kfree(rdev->gsi_ctx.sqp_tbl);
865 kfree(gsi_sah);
866 kfree(gsi_sqp);
867 rdev->gsi_ctx.gsi_sqp = NULL;
868 rdev->gsi_ctx.gsi_sah = NULL;
869 rdev->gsi_ctx.sqp_tbl = NULL;
870
871 return 0;
872fail:
873 return rc;
874}
875
1ac5a404 876/* Queue Pairs */
c4367a26 877int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
1ac5a404
SX		 878{
879 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
b5bbc655 880 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
1ac5a404 881 struct bnxt_re_dev *rdev = qp->rdev;
b5bbc655
KD		 882 struct bnxt_qplib_nq *scq_nq = NULL;
883 struct bnxt_qplib_nq *rcq_nq = NULL;
3b921e3b 884 unsigned int flags;
c50866e2 885 int rc;
1ac5a404 886
c88a7858 887 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
8dae419f 888
1ac5a404
SX		 889 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
890 if (rc) {
6ccad848 891 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
1ac5a404
SX		 892 return rc;
893 }
3b921e3b 894
0fca467e 895 if (rdma_is_kernel_res(&qp->ib_qp.res)) {
c50866e2
DS		 896 flags = bnxt_re_lock_cqs(qp);
897 bnxt_qplib_clean_qp(&qp->qplib_qp);
898 bnxt_re_unlock_cqs(qp, flags);
899 }
900
3b921e3b
SX		 901 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
902
8dae419f
DS		 903 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
904 rc = bnxt_re_destroy_gsi_sqp(qp);
905 if (rc)
514aee66 906 return rc;
1ac5a404
SX		 907 }
908
097a9d23
SX		 909 mutex_lock(&rdev->qp_lock);
910 list_del(&qp->list);
911 mutex_unlock(&rdev->qp_lock);
063975fe 912 atomic_dec(&rdev->stats.res.qp_count);
a83c6927
SX		 913 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
914 atomic_dec(&rdev->stats.res.rc_qp_count);
915 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
916 atomic_dec(&rdev->stats.res.ud_qp_count);
097a9d23 917
836a0fbb
LR		 918 ib_umem_release(qp->rumem);
919 ib_umem_release(qp->sumem);
1ac5a404 920
b5bbc655
KD		 921 /* Flush all the entries of notification queue associated with
922 * given qp.
923 */
924 scq_nq = qplib_qp->scq->nq;
925 rcq_nq = qplib_qp->rcq->nq;
926 bnxt_re_synchronize_nq(scq_nq);
927 if (scq_nq != rcq_nq)
928 bnxt_re_synchronize_nq(rcq_nq);
929
1ac5a404
SX		 930 return 0;
931}
932
933static u8 __from_ib_qp_type(enum ib_qp_type type)
934{
935 switch (type) {
936 case IB_QPT_GSI:
937 return CMDQ_CREATE_QP1_TYPE_GSI;
938 case IB_QPT_RC:
939 return CMDQ_CREATE_QP_TYPE_RC;
940 case IB_QPT_UD:
941 return CMDQ_CREATE_QP_TYPE_UD;
942 default:
943 return IB_QPT_MAX;
944 }
945}
946
2bb3c32c
DS		 947static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
948 int rsge, int max)
949{
950 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
951 rsge = max;
952 return bnxt_re_get_rwqe_size(rsge);
953}
954
955static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
956{
957 u16 wqe_size, calc_ils;
958
959 wqe_size = bnxt_re_get_swqe_size(nsge);
960 if (ilsize) {
961 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
962 wqe_size = max_t(u16, calc_ils, wqe_size);
963 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
964 }
965 return wqe_size;
966}
967
968static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
969 struct ib_qp_init_attr *init_attr)
970{
971 struct bnxt_qplib_dev_attr *dev_attr;
972 struct bnxt_qplib_qp *qplqp;
973 struct bnxt_re_dev *rdev;
974 struct bnxt_qplib_q *sq;
975 int align, ilsize;
976
977 rdev = qp->rdev;
978 qplqp = &qp->qplib_qp;
979 sq = &qplqp->sq;
980 dev_attr = &rdev->dev_attr;
981
982 align = sizeof(struct sq_send_hdr);
983 ilsize = ALIGN(init_attr->cap.max_inline_data, align);
984
985 sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
986 if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
987 return -EINVAL;
988 /* For gen p4 and gen p5 backward compatibility mode
989 * wqe size is fixed to 128 bytes
990 */
991 if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
992 qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
993 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
994
995 if (init_attr->cap.max_inline_data) {
996 qplqp->max_inline_data = sq->wqe_size -
997 sizeof(struct sq_send_hdr);
998 init_attr->cap.max_inline_data = qplqp->max_inline_data;
999 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1000 sq->max_sge = qplqp->max_inline_data /
1001 sizeof(struct sq_sge);
1002 }
1003
1004 return 0;
1005}
1006
1ac5a404
SX		 1007static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
1008 struct bnxt_re_qp *qp, struct ib_udata *udata)
1009{
2bb3c32c
DS		 1010 struct bnxt_qplib_qp *qplib_qp;
1011 struct bnxt_re_ucontext *cntx;
1ac5a404 1012 struct bnxt_re_qp_req ureq;
37f91cff 1013 int bytes = 0, psn_sz;
2bb3c32c
DS		 1014 struct ib_umem *umem;
1015 int psn_nume;
89944450 1016
2bb3c32c
DS		 1017 qplib_qp = &qp->qplib_qp;
1018 cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
1019 ib_uctx);
1ac5a404
SX		 1020 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1021 return -EFAULT;
1022
fddcbbb0 1023 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
1ac5a404 1024 /* Consider mapping PSN search memory only for RC QPs. */
37f91cff 1025 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
0cfb329d 1026 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
2bb3c32c
DS		 1027 sizeof(struct sq_psn_search_ext) :
1028 sizeof(struct sq_psn_search);
1029 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1030 qplib_qp->sq.max_wqe :
1031 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
1032 sizeof(struct bnxt_qplib_sge));
1033 bytes += (psn_nume * psn_sz);
37f91cff 1034 }
2bb3c32c 1035
1ac5a404 1036 bytes = PAGE_ALIGN(bytes);
c320e527
MS		 1037 umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
1038 IB_ACCESS_LOCAL_WRITE);
1ac5a404
SX		 1039 if (IS_ERR(umem))
1040 return PTR_ERR(umem);
1041
1042 qp->sumem = umem;
6ef999f5 1043 qplib_qp->sq.sg_info.umem = umem;
0c4dcd60
DS		 1044 qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
1045 qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
1ac5a404
SX		 1046 qplib_qp->qp_handle = ureq.qp_handle;
1047
1048 if (!qp->qplib_qp.srq) {
fddcbbb0 1049 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
1ac5a404 1050 bytes = PAGE_ALIGN(bytes);
c320e527 1051 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes,
72b894b0 1052 IB_ACCESS_LOCAL_WRITE);
1ac5a404
SX		 1053 if (IS_ERR(umem))
1054 goto rqfail;
1055 qp->rumem = umem;
6ef999f5 1056 qplib_qp->rq.sg_info.umem = umem;
0c4dcd60
DS		 1057 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
1058 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
1ac5a404
SX		 1059 }
1060
b3b2c7c5 1061 qplib_qp->dpi = &cntx->dpi;
1ac5a404
SX		 1062 return 0;
1063rqfail:
1064 ib_umem_release(qp->sumem);
1065 qp->sumem = NULL;
5aa84840 1066 memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
1ac5a404
SX		 1067
1068 return PTR_ERR(umem);
1069}
1070
1071static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
1072 (struct bnxt_re_pd *pd,
1073 struct bnxt_qplib_res *qp1_res,
1074 struct bnxt_qplib_qp *qp1_qp)
1075{
1076 struct bnxt_re_dev *rdev = pd->rdev;
1077 struct bnxt_re_ah *ah;
1078 union ib_gid sgid;
1079 int rc;
1080
1081 ah = kzalloc(sizeof(*ah), GFP_KERNEL);
1082 if (!ah)
1083 return NULL;
1084
1ac5a404
SX		 1085 ah->rdev = rdev;
1086 ah->qplib_ah.pd = &pd->qplib_pd;
1087
1088 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
1089 if (rc)
1090 goto fail;
1091
1092 /* supply the dgid data same as sgid */
1093 memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
1094 sizeof(union ib_gid));
1095 ah->qplib_ah.sgid_index = 0;
1096
1097 ah->qplib_ah.traffic_class = 0;
1098 ah->qplib_ah.flow_label = 0;
1099 ah->qplib_ah.hop_limit = 1;
1100 ah->qplib_ah.sl = 0;
1101 /* Have DMAC same as SMAC */
1102 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1103
90e3edd8 1104 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1ac5a404 1105 if (rc) {
6ccad848
DS		 1106 ibdev_err(&rdev->ibdev,
1107 "Failed to allocate HW AH for Shadow QP");
1ac5a404
SX		 1108 goto fail;
1109 }
063975fe 1110 atomic_inc(&rdev->stats.res.ah_count);
1ac5a404
SX		 1111
1112 return ah;
1113
1114fail:
1115 kfree(ah);
1116 return NULL;
1117}
1118
1119static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1120 (struct bnxt_re_pd *pd,
1121 struct bnxt_qplib_res *qp1_res,
1122 struct bnxt_qplib_qp *qp1_qp)
1123{
1124 struct bnxt_re_dev *rdev = pd->rdev;
1125 struct bnxt_re_qp *qp;
1126 int rc;
1127
1128 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1129 if (!qp)
1130 return NULL;
1131
1ac5a404
SX		 1132 qp->rdev = rdev;
1133
1134 /* Initialize the shadow QP structure from the QP1 values */
1135 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1136
1137 qp->qplib_qp.pd = &pd->qplib_pd;
1138 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1139 qp->qplib_qp.type = IB_QPT_UD;
1140
1141 qp->qplib_qp.max_inline_data = 0;
1142 qp->qplib_qp.sig_type = true;
1143
1144 /* Shadow QP SQ depth should be same as QP1 RQ depth */
2bb3c32c 1145 qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1ac5a404
SX		 1146 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1147 qp->qplib_qp.sq.max_sge = 2;
9152e0b7
EW		 1148 /* Q full delta can be 1 since it is internal QP */
1149 qp->qplib_qp.sq.q_full_delta = 1;
0c4dcd60
DS		 1150 qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1151 qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1ac5a404
SX		 1152
1153 qp->qplib_qp.scq = qp1_qp->scq;
1154 qp->qplib_qp.rcq = qp1_qp->rcq;
1155
2bb3c32c 1156 qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1ac5a404
SX		 1157 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1158 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
9152e0b7
EW		 1159 /* Q full delta can be 1 since it is internal QP */
1160 qp->qplib_qp.rq.q_full_delta = 1;
0c4dcd60
DS		 1161 qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1162 qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1ac5a404
SX		 1163
1164 qp->qplib_qp.mtu = qp1_qp->mtu;
1165
1166 qp->qplib_qp.sq_hdr_buf_size = 0;
1167 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1168 qp->qplib_qp.dpi = &rdev->dpi_privileged;
1169
1170 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1171 if (rc)
1172 goto fail;
1173
1ac5a404
SX		 1174 spin_lock_init(&qp->sq_lock);
1175 INIT_LIST_HEAD(&qp->list);
1176 mutex_lock(&rdev->qp_lock);
1177 list_add_tail(&qp->list, &rdev->qp_list);
063975fe 1178 atomic_inc(&rdev->stats.res.qp_count);
1ac5a404
SX		 1179 mutex_unlock(&rdev->qp_lock);
1180 return qp;
1181fail:
1182 kfree(qp);
1183 return NULL;
1184}
1185
8dae419f
DS		 1186static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1187 struct ib_qp_init_attr *init_attr)
1ac5a404 1188{
8dae419f
DS		 1189 struct bnxt_qplib_dev_attr *dev_attr;
1190 struct bnxt_qplib_qp *qplqp;
1191 struct bnxt_re_dev *rdev;
fddcbbb0 1192 struct bnxt_qplib_q *rq;
8dae419f 1193 int entries;
1ac5a404 1194
8dae419f
DS		 1195 rdev = qp->rdev;
1196 qplqp = &qp->qplib_qp;
fddcbbb0 1197 rq = &qplqp->rq;
8dae419f 1198 dev_attr = &rdev->dev_attr;
1ac5a404 1199
8dae419f
DS		 1200 if (init_attr->srq) {
1201 struct bnxt_re_srq *srq;
1ac5a404 1202
8dae419f 1203 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
8dae419f 1204 qplqp->srq = &srq->qplib_srq;
fddcbbb0 1205 rq->max_wqe = 0;
8dae419f 1206 } else {
2bb3c32c
DS		 1207 rq->max_sge = init_attr->cap.max_recv_sge;
1208 if (rq->max_sge > dev_attr->max_qp_sges)
1209 rq->max_sge = dev_attr->max_qp_sges;
1210 init_attr->cap.max_recv_sge = rq->max_sge;
1211 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1212 dev_attr->max_qp_sges);
8dae419f
DS		 1213 /* Allocate 1 more than what's provided so posting max doesn't
1214 * mean empty.
1215 */
1216 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
fddcbbb0 1217 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2bb3c32c
DS		 1218 rq->q_full_delta = 0;
1219 rq->sg_info.pgsize = PAGE_SIZE;
1220 rq->sg_info.pgshft = PAGE_SHIFT;
8dae419f
DS		 1221 }
1222
1223 return 0;
1224}
1225
1226static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1227{
1228 struct bnxt_qplib_dev_attr *dev_attr;
1229 struct bnxt_qplib_qp *qplqp;
1230 struct bnxt_re_dev *rdev;
1231
1232 rdev = qp->rdev;
1233 qplqp = &qp->qplib_qp;
1234 dev_attr = &rdev->dev_attr;
1235
2bb3c32c 1236 if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
8dae419f 1237 qplqp->rq.max_sge = dev_attr->max_qp_sges;
2bb3c32c
DS		 1238 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1239 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1240 qplqp->rq.max_sge = 6;
1241 }
8dae419f
DS		 1242}
1243
2bb3c32c
DS		 1244static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1245 struct ib_qp_init_attr *init_attr,
1246 struct ib_udata *udata)
8dae419f
DS		 1247{
1248 struct bnxt_qplib_dev_attr *dev_attr;
1249 struct bnxt_qplib_qp *qplqp;
1250 struct bnxt_re_dev *rdev;
fddcbbb0 1251 struct bnxt_qplib_q *sq;
8dae419f 1252 int entries;
2bb3c32c
DS		 1253 int diff;
1254 int rc;
8dae419f
DS		 1255
1256 rdev = qp->rdev;
1257 qplqp = &qp->qplib_qp;
fddcbbb0 1258 sq = &qplqp->sq;
8dae419f
DS		 1259 dev_attr = &rdev->dev_attr;
1260
fddcbbb0 1261 sq->max_sge = init_attr->cap.max_send_sge;
2bb3c32c 1262 if (sq->max_sge > dev_attr->max_qp_sges) {
fddcbbb0 1263 sq->max_sge = dev_attr->max_qp_sges;
2bb3c32c
DS		 1264 init_attr->cap.max_send_sge = sq->max_sge;
1265 }
1266
1267 rc = bnxt_re_setup_swqe_size(qp, init_attr);
1268 if (rc)
1269 return rc;
1270
8dae419f
DS		 1271 entries = init_attr->cap.max_send_wr;
1272 /* Allocate 128 + 1 more than what's provided */
2bb3c32c
DS		 1273 diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1274 0 : BNXT_QPLIB_RESERVED_QP_WRS;
1275 entries = roundup_pow_of_two(entries + diff + 1);
1276 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1277 sq->q_full_delta = diff + 1;
8dae419f
DS		 1278 /*
1279 * Reserving one slot for Phantom WQE. Application can
1280 * post one extra entry in this case. But allowing this to avoid
1281 * unexpected Queue full condition
1282 */
1283 qplqp->sq.q_full_delta -= 1;
0c4dcd60
DS		 1284 qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1285 qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
2bb3c32c
DS		 1286
1287 return 0;
8dae419f
DS		 1288}
1289
1290static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1291 struct ib_qp_init_attr *init_attr)
1292{
1293 struct bnxt_qplib_dev_attr *dev_attr;
1294 struct bnxt_qplib_qp *qplqp;
1295 struct bnxt_re_dev *rdev;
1296 int entries;
1297
1298 rdev = qp->rdev;
1299 qplqp = &qp->qplib_qp;
1300 dev_attr = &rdev->dev_attr;
1301
2bb3c32c
DS		 1302 if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1303 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
1304 qplqp->sq.max_wqe = min_t(u32, entries,
1305 dev_attr->max_qp_wqes + 1);
1306 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1307 init_attr->cap.max_send_wr;
1308 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1309 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1310 qplqp->sq.max_sge = dev_attr->max_qp_sges;
1311 }
8dae419f
DS		 1312}
1313
1314static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1315 struct ib_qp_init_attr *init_attr)
1316{
1317 struct bnxt_qplib_chip_ctx *chip_ctx;
1318 int qptype;
1319
0cfb329d 1320 chip_ctx = rdev->chip_ctx;
8dae419f
DS		 1321
1322 qptype = __from_ib_qp_type(init_attr->qp_type);
1323 if (qptype == IB_QPT_MAX) {
6ccad848 1324 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
bb8865f4 1325 qptype = -EOPNOTSUPP;
8dae419f 1326 goto out;
1ac5a404 1327 }
374c5285 1328
8dae419f
DS		 1329 if (bnxt_qplib_is_chip_gen_p5(chip_ctx) &&
1330 init_attr->qp_type == IB_QPT_GSI)
1331 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1332out:
1333 return qptype;
1334}
1ac5a404 1335
8dae419f
DS		 1336static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1337 struct ib_qp_init_attr *init_attr,
1338 struct ib_udata *udata)
1339{
1340 struct bnxt_qplib_dev_attr *dev_attr;
1341 struct bnxt_qplib_qp *qplqp;
1342 struct bnxt_re_dev *rdev;
1343 struct bnxt_re_cq *cq;
1344 int rc = 0, qptype;
1345
1346 rdev = qp->rdev;
1347 qplqp = &qp->qplib_qp;
1348 dev_attr = &rdev->dev_attr;
1349
1350 /* Setup misc params */
1351 ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1352 qplqp->pd = &pd->qplib_pd;
1353 qplqp->qp_handle = (u64)qplqp;
1354 qplqp->max_inline_data = init_attr->cap.max_inline_data;
272bba19 1355 qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
8dae419f
DS		 1356 qptype = bnxt_re_init_qp_type(rdev, init_attr);
1357 if (qptype < 0) {
1358 rc = qptype;
1359 goto out;
1360 }
1361 qplqp->type = (u8)qptype;
1da968e0 1362 qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
8dae419f
DS		 1363
1364 if (init_attr->qp_type == IB_QPT_RC) {
1365 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1366 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1367 }
1368 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1369 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1f11a761 1370 if (init_attr->create_flags) {
6ccad848
DS		 1371 ibdev_dbg(&rdev->ibdev,
1372 "QP create flags 0x%x not supported",
1373 init_attr->create_flags);
1f11a761
JG		 1374 return -EOPNOTSUPP;
1375 }
1ac5a404 1376
8dae419f
DS		 1377 /* Setup CQs */
1378 if (init_attr->send_cq) {
1379 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
8dae419f 1380 qplqp->scq = &cq->qplib_cq;
3b921e3b 1381 qp->scq = cq;
1ac5a404
SX		 1382 }
1383
8dae419f
DS		 1384 if (init_attr->recv_cq) {
1385 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
8dae419f 1386 qplqp->rcq = &cq->qplib_cq;
3b921e3b 1387 qp->rcq = cq;
1ac5a404
SX		 1388 }
1389
8dae419f
DS		 1390 /* Setup RQ/SRQ */
1391 rc = bnxt_re_init_rq_attr(qp, init_attr);
1392 if (rc)
1393 goto out;
1394 if (init_attr->qp_type == IB_QPT_GSI)
1395 bnxt_re_adjust_gsi_rq_attr(qp);
1396
1397 /* Setup SQ */
2bb3c32c
DS		 1398 rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
1399 if (rc)
1400 goto out;
8dae419f
DS		 1401 if (init_attr->qp_type == IB_QPT_GSI)
1402 bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
1ac5a404 1403
8dae419f
DS		 1404 if (udata) /* This will update DPI and qp_handle */
1405 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1406out:
1407 return rc;
1408}
9152e0b7 1409
8dae419f
DS		 1410static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1411 struct bnxt_re_pd *pd)
1412{
f1b195ce 1413 struct bnxt_re_sqp_entries *sqp_tbl;
8dae419f
DS		 1414 struct bnxt_re_dev *rdev;
1415 struct bnxt_re_qp *sqp;
1416 struct bnxt_re_ah *sah;
1417 int rc = 0;
1418
1419 rdev = qp->rdev;
1420 /* Create a shadow QP to handle the QP1 traffic */
f1b195ce 1421 sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),
8dae419f
DS		 1422 GFP_KERNEL);
1423 if (!sqp_tbl)
1424 return -ENOMEM;
1425 rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1426
1427 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1428 if (!sqp) {
1429 rc = -ENODEV;
6ccad848 1430 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
8dae419f 1431 goto out;
1ac5a404 1432 }
8dae419f 1433 rdev->gsi_ctx.gsi_sqp = sqp;
1ac5a404 1434
8dae419f
DS		 1435 sqp->rcq = qp->rcq;
1436 sqp->scq = qp->scq;
1437 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1438 &qp->qplib_qp);
1439 if (!sah) {
1440 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1441 &sqp->qplib_qp);
1442 rc = -ENODEV;
6ccad848
DS		 1443 ibdev_err(&rdev->ibdev,
1444 "Failed to create AH entry for ShadowQP");
8dae419f
DS		 1445 goto out;
1446 }
1447 rdev->gsi_ctx.gsi_sah = sah;
1ac5a404 1448
8dae419f
DS		 1449 return 0;
1450out:
1451 kfree(sqp_tbl);
1452 return rc;
1453}
1ac5a404 1454
8dae419f
DS		 1455static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1456 struct ib_qp_init_attr *init_attr)
1457{
8dae419f
DS		 1458 struct bnxt_re_dev *rdev;
1459 struct bnxt_qplib_qp *qplqp;
14611b9b 1460 int rc;
9152e0b7 1461
8dae419f
DS		 1462 rdev = qp->rdev;
1463 qplqp = &qp->qplib_qp;
9152e0b7 1464
8dae419f
DS		 1465 qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1466 qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
9152e0b7 1467
8dae419f
DS		 1468 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1469 if (rc) {
6ccad848 1470 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
8dae419f
DS		 1471 goto out;
1472 }
1473
1474 rc = bnxt_re_create_shadow_gsi(qp, pd);
1475out:
1476 return rc;
1477}
1478
1479static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1480 struct ib_qp_init_attr *init_attr,
1481 struct bnxt_qplib_dev_attr *dev_attr)
1482{
1483 bool rc = true;
1ac5a404 1484
8dae419f
DS		 1485 if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1486 init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1487 init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1488 init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1489 init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
6ccad848
DS		 1490 ibdev_err(&rdev->ibdev,
1491 "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1492 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1493 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1494 init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1495 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1496 init_attr->cap.max_inline_data,
1497 dev_attr->max_inline_data);
8dae419f
DS		 1498 rc = false;
1499 }
1500 return rc;
1501}
1502
514aee66
LR		 1503int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1504 struct ib_udata *udata)
8dae419f 1505{
514aee66 1506 struct ib_pd *ib_pd = ib_qp->pd;
8dae419f
DS		 1507 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1508 struct bnxt_re_dev *rdev = pd->rdev;
1509 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
514aee66 1510 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
cb95709e 1511 u32 active_qps;
8dae419f
DS		 1512 int rc;
1513
1514 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1515 if (!rc) {
1516 rc = -EINVAL;
514aee66 1517 goto fail;
8dae419f
DS		 1518 }
1519
8dae419f
DS		 1520 qp->rdev = rdev;
1521 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
1522 if (rc)
1523 goto fail;
1524
1525 if (qp_init_attr->qp_type == IB_QPT_GSI &&
0cfb329d 1526 !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) {
8dae419f
DS		 1527 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1528 if (rc == -ENODEV)
1529 goto qp_destroy;
1530 if (rc)
1531 goto fail;
1532 } else {
1ac5a404
SX		 1533 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1534 if (rc) {
6ccad848 1535 ibdev_err(&rdev->ibdev, "Failed to create HW QP");
6b4521f5 1536 goto free_umem;
1ac5a404 1537 }
8dae419f
DS		 1538 if (udata) {
1539 struct bnxt_re_qp_resp resp;
1540
1541 resp.qpid = qp->qplib_qp.id;
1542 resp.rsvd = 0;
1543 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1544 if (rc) {
6ccad848 1545 ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
8dae419f
DS		 1546 goto qp_destroy;
1547 }
1548 }
1ac5a404
SX		 1549 }
1550
1551 qp->ib_qp.qp_num = qp->qplib_qp.id;
8dae419f
DS		 1552 if (qp_init_attr->qp_type == IB_QPT_GSI)
1553 rdev->gsi_ctx.gsi_qp = qp;
1ac5a404 1554 spin_lock_init(&qp->sq_lock);
018cf599 1555 spin_lock_init(&qp->rq_lock);
1ac5a404
SX		 1556 INIT_LIST_HEAD(&qp->list);
1557 mutex_lock(&rdev->qp_lock);
1558 list_add_tail(&qp->list, &rdev->qp_list);
1ac5a404 1559 mutex_unlock(&rdev->qp_lock);
cb95709e
CA		 1560 active_qps = atomic_inc_return(&rdev->stats.res.qp_count);
1561 if (active_qps > rdev->stats.res.qp_watermark)
1562 rdev->stats.res.qp_watermark = active_qps;
1563 if (qp_init_attr->qp_type == IB_QPT_RC) {
1564 active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count);
1565 if (active_qps > rdev->stats.res.rc_qp_watermark)
1566 rdev->stats.res.rc_qp_watermark = active_qps;
1567 } else if (qp_init_attr->qp_type == IB_QPT_UD) {
1568 active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count);
1569 if (active_qps > rdev->stats.res.ud_qp_watermark)
1570 rdev->stats.res.ud_qp_watermark = active_qps;
1571 }
1ac5a404 1572
514aee66 1573 return 0;
1ac5a404
SX		 1574qp_destroy:
1575 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
6b4521f5 1576free_umem:
836a0fbb
LR		 1577 ib_umem_release(qp->rumem);
1578 ib_umem_release(qp->sumem);
1ac5a404 1579fail:
514aee66 1580 return rc;
1ac5a404
SX		 1581}
1582
1583static u8 __from_ib_qp_state(enum ib_qp_state state)
1584{
1585 switch (state) {
1586 case IB_QPS_RESET:
1587 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1588 case IB_QPS_INIT:
1589 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1590 case IB_QPS_RTR:
1591 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1592 case IB_QPS_RTS:
1593 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1594 case IB_QPS_SQD:
1595 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1596 case IB_QPS_SQE:
1597 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1598 case IB_QPS_ERR:
1599 default:
1600 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1601 }
1602}
1603
1604static enum ib_qp_state __to_ib_qp_state(u8 state)
1605{
1606 switch (state) {
1607 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1608 return IB_QPS_RESET;
1609 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1610 return IB_QPS_INIT;
1611 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1612 return IB_QPS_RTR;
1613 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1614 return IB_QPS_RTS;
1615 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1616 return IB_QPS_SQD;
1617 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1618 return IB_QPS_SQE;
1619 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1620 default:
1621 return IB_QPS_ERR;
1622 }
1623}
1624
1625static u32 __from_ib_mtu(enum ib_mtu mtu)
1626{
1627 switch (mtu) {
1628 case IB_MTU_256:
1629 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1630 case IB_MTU_512:
1631 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1632 case IB_MTU_1024:
1633 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1634 case IB_MTU_2048:
1635 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1636 case IB_MTU_4096:
1637 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1638 default:
1639 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1640 }
1641}
1642
1643static enum ib_mtu __to_ib_mtu(u32 mtu)
1644{
1645 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1646 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1647 return IB_MTU_256;
1648 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1649 return IB_MTU_512;
1650 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1651 return IB_MTU_1024;
1652 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1653 return IB_MTU_2048;
1654 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1655 return IB_MTU_4096;
1656 default:
1657 return IB_MTU_2048;
1658 }
1659}
1660
37cb11ac 1661/* Shared Receive Queues */
119181d1 1662int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
37cb11ac
DS		 1663{
1664 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1665 ib_srq);
1666 struct bnxt_re_dev *rdev = srq->rdev;
1667 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1668 struct bnxt_qplib_nq *nq = NULL;
37cb11ac
DS		 1669
1670 if (qplib_srq->cq)
1671 nq = qplib_srq->cq->nq;
68e326de 1672 bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
836a0fbb 1673 ib_umem_release(srq->umem);
063975fe 1674 atomic_dec(&rdev->stats.res.srq_count);
37cb11ac
DS		 1675 if (nq)
1676 nq->budget--;
119181d1 1677 return 0;
37cb11ac
DS		 1678}
1679
1680static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1681 struct bnxt_re_pd *pd,
1682 struct bnxt_re_srq *srq,
1683 struct ib_udata *udata)
1684{
1685 struct bnxt_re_srq_req ureq;
1686 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1687 struct ib_umem *umem;
1688 int bytes = 0;
89944450
SR		 1689 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1690 udata, struct bnxt_re_ucontext, ib_uctx);
1691
37cb11ac
DS		 1692 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1693 return -EFAULT;
1694
fddcbbb0 1695 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
37cb11ac 1696 bytes = PAGE_ALIGN(bytes);
c320e527
MS		 1697 umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1698 IB_ACCESS_LOCAL_WRITE);
37cb11ac
DS		 1699 if (IS_ERR(umem))
1700 return PTR_ERR(umem);
1701
1702 srq->umem = umem;
6ef999f5 1703 qplib_srq->sg_info.umem = umem;
0c4dcd60
DS		 1704 qplib_srq->sg_info.pgsize = PAGE_SIZE;
1705 qplib_srq->sg_info.pgshft = PAGE_SHIFT;
37cb11ac
DS		 1706 qplib_srq->srq_handle = ureq.srq_handle;
1707 qplib_srq->dpi = &cntx->dpi;
1708
1709 return 0;
1710}
1711
68e326de
LR		 1712int bnxt_re_create_srq(struct ib_srq *ib_srq,
1713 struct ib_srq_init_attr *srq_init_attr,
1714 struct ib_udata *udata)
37cb11ac 1715{
fddcbbb0 1716 struct bnxt_qplib_dev_attr *dev_attr;
37cb11ac 1717 struct bnxt_qplib_nq *nq = NULL;
fddcbbb0
DS		 1718 struct bnxt_re_dev *rdev;
1719 struct bnxt_re_srq *srq;
1720 struct bnxt_re_pd *pd;
1721 struct ib_pd *ib_pd;
cb95709e 1722 u32 active_srqs;
37cb11ac
DS		 1723 int rc, entries;
1724
fddcbbb0
DS		 1725 ib_pd = ib_srq->pd;
1726 pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1727 rdev = pd->rdev;
1728 dev_attr = &rdev->dev_attr;
1729 srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1730
37cb11ac 1731 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
6ccad848 1732 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
37cb11ac
DS		 1733 rc = -EINVAL;
1734 goto exit;
1735 }
1736
1737 if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
8380b74e 1738 rc = -EOPNOTSUPP;
37cb11ac
DS		 1739 goto exit;
1740 }
1741
37cb11ac
DS		 1742 srq->rdev = rdev;
1743 srq->qplib_srq.pd = &pd->qplib_pd;
1744 srq->qplib_srq.dpi = &rdev->dpi_privileged;
1745 /* Allocate 1 more than what's provided so posting max doesn't
1746 * mean empty
1747 */
1748 entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
1749 if (entries > dev_attr->max_srq_wqes + 1)
1750 entries = dev_attr->max_srq_wqes + 1;
37cb11ac 1751 srq->qplib_srq.max_wqe = entries;
fddcbbb0 1752
37cb11ac 1753 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
b898d5c5
SX		 1754 /* 128 byte wqe size for SRQ . So use max sges */
1755 srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
37cb11ac
DS		 1756 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1757 srq->srq_limit = srq_init_attr->attr.srq_limit;
1758 srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
1759 nq = &rdev->nq[0];
1760
1761 if (udata) {
1762 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1763 if (rc)
1764 goto fail;
1765 }
1766
1767 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1768 if (rc) {
6ccad848 1769 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
37cb11ac
DS		 1770 goto fail;
1771 }
1772
1773 if (udata) {
1774 struct bnxt_re_srq_resp resp;
1775
1776 resp.srqid = srq->qplib_srq.id;
1777 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1778 if (rc) {
6ccad848 1779 ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
37cb11ac
DS		 1780 bnxt_qplib_destroy_srq(&rdev->qplib_res,
1781 &srq->qplib_srq);
4a9d46a9 1782 goto fail;
37cb11ac
DS		 1783 }
1784 }
1785 if (nq)
1786 nq->budget++;
cb95709e
CA		 1787 active_srqs = atomic_inc_return(&rdev->stats.res.srq_count);
1788 if (active_srqs > rdev->stats.res.srq_watermark)
1789 rdev->stats.res.srq_watermark = active_srqs;
17f2569d 1790 spin_lock_init(&srq->lock);
37cb11ac 1791
68e326de 1792 return 0;
37cb11ac
DS		 1793
1794fail:
836a0fbb 1795 ib_umem_release(srq->umem);
37cb11ac 1796exit:
68e326de 1797 return rc;
37cb11ac
DS		 1798}
1799
1800int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1801 enum ib_srq_attr_mask srq_attr_mask,
1802 struct ib_udata *udata)
1803{
1804 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1805 ib_srq);
1806 struct bnxt_re_dev *rdev = srq->rdev;
1807 int rc;
1808
1809 switch (srq_attr_mask) {
1810 case IB_SRQ_MAX_WR:
1811 /* SRQ resize is not supported */
1812 break;
1813 case IB_SRQ_LIMIT:
1814 /* Change the SRQ threshold */
1815 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1816 return -EINVAL;
1817
1818 srq->qplib_srq.threshold = srq_attr->srq_limit;
1819 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1820 if (rc) {
6ccad848 1821 ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!");
37cb11ac
DS		 1822 return rc;
1823 }
1824 /* On success, update the shadow */
1825 srq->srq_limit = srq_attr->srq_limit;
1826 /* No need to Build and send response back to udata */
1827 break;
1828 default:
6ccad848
DS		 1829 ibdev_err(&rdev->ibdev,
1830 "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
37cb11ac
DS		 1831 return -EINVAL;
1832 }
1833 return 0;
1834}
1835
1836int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1837{
1838 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1839 ib_srq);
1840 struct bnxt_re_srq tsrq;
1841 struct bnxt_re_dev *rdev = srq->rdev;
1842 int rc;
1843
1844 /* Get live SRQ attr */
1845 tsrq.qplib_srq.id = srq->qplib_srq.id;
1846 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
1847 if (rc) {
6ccad848 1848 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
37cb11ac
DS		 1849 return rc;
1850 }
1851 srq_attr->max_wr = srq->qplib_srq.max_wqe;
1852 srq_attr->max_sge = srq->qplib_srq.max_sge;
1853 srq_attr->srq_limit = tsrq.qplib_srq.threshold;
1854
1855 return 0;
1856}
1857
d34ac5cd
BVA		 1858int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1859 const struct ib_recv_wr **bad_wr)
37cb11ac
DS		 1860{
1861 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1862 ib_srq);
1863 struct bnxt_qplib_swqe wqe;
1864 unsigned long flags;
036ef0a1 1865 int rc = 0;
37cb11ac
DS		 1866
1867 spin_lock_irqsave(&srq->lock, flags);
1868 while (wr) {
1869 /* Transcribe each ib_recv_wr to qplib_swqe */
1870 wqe.num_sge = wr->num_sge;
036ef0a1 1871 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
37cb11ac
DS		 1872 wqe.wr_id = wr->wr_id;
1873 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1874
1875 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1876 if (rc) {
1877 *bad_wr = wr;
1878 break;
1879 }
1880 wr = wr->next;
1881 }
1882 spin_unlock_irqrestore(&srq->lock, flags);
1883
1884 return rc;
1885}
1ac5a404
SX		 1886static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1887 struct bnxt_re_qp *qp1_qp,
1888 int qp_attr_mask)
1889{
8dae419f 1890 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
14611b9b 1891 int rc;
1ac5a404
SX		 1892
1893 if (qp_attr_mask & IB_QP_STATE) {
1894 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1895 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1896 }
1897 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1898 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1899 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1900 }
1901
1902 if (qp_attr_mask & IB_QP_QKEY) {
1903 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1904 /* Using a Random QKEY */
1905 qp->qplib_qp.qkey = 0x81818181;
1906 }
1907 if (qp_attr_mask & IB_QP_SQ_PSN) {
1908 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1909 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1910 }
1911
1912 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1913 if (rc)
6ccad848 1914 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
1ac5a404
SX		 1915 return rc;
1916}
1917
1918int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1919 int qp_attr_mask, struct ib_udata *udata)
1920{
1921 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1922 struct bnxt_re_dev *rdev = qp->rdev;
1923 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1924 enum ib_qp_state curr_qp_state, new_qp_state;
1925 int rc, entries;
942c9b6c 1926 unsigned int flags;
1ac5a404
SX		 1927 u8 nw_type;
1928
26e990ba
JG		 1929 if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1930 return -EOPNOTSUPP;
1931
1ac5a404
SX		 1932 qp->qplib_qp.modify_flags = 0;
1933 if (qp_attr_mask & IB_QP_STATE) {
1934 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1935 new_qp_state = qp_attr->qp_state;
1936 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
d31131bb 1937 ib_qp->qp_type, qp_attr_mask)) {
6ccad848
DS		 1938 ibdev_err(&rdev->ibdev,
1939 "Invalid attribute mask: %#x specified ",
1940 qp_attr_mask);
1941 ibdev_err(&rdev->ibdev,
1942 "for qpn: %#x type: %#x",
1943 ib_qp->qp_num, ib_qp->qp_type);
1944 ibdev_err(&rdev->ibdev,
1945 "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1946 curr_qp_state, new_qp_state);
1ac5a404
SX		 1947 return -EINVAL;
1948 }
1949 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1950 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
f218d67e
SX		 1951
1952 if (!qp->sumem &&
1953 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
6ccad848
DS		 1954 ibdev_dbg(&rdev->ibdev,
1955 "Move QP = %p to flush list\n", qp);
942c9b6c 1956 flags = bnxt_re_lock_cqs(qp);
f218d67e 1957 bnxt_qplib_add_flush_qp(&qp->qplib_qp);
942c9b6c 1958 bnxt_re_unlock_cqs(qp, flags);
f218d67e
SX		 1959 }
1960 if (!qp->sumem &&
1961 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
6ccad848
DS		 1962 ibdev_dbg(&rdev->ibdev,
1963 "Move QP = %p out of flush list\n", qp);
942c9b6c 1964 flags = bnxt_re_lock_cqs(qp);
3b921e3b 1965 bnxt_qplib_clean_qp(&qp->qplib_qp);
942c9b6c 1966 bnxt_re_unlock_cqs(qp, flags);
f218d67e 1967 }
1ac5a404
SX		 1968 }
1969 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1970 qp->qplib_qp.modify_flags |=
1971 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1972 qp->qplib_qp.en_sqd_async_notify = true;
1973 }
1974 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1975 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1976 qp->qplib_qp.access =
1977 __from_ib_access_flags(qp_attr->qp_access_flags);
1978 /* LOCAL_WRITE access must be set to allow RC receive */
1979 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
37f91cff
DS		 1980 /* Temp: Set all params on QP as of now */
1981 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
1982 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
1ac5a404
SX		 1983 }
1984 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1985 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1986 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1987 }
1988 if (qp_attr_mask & IB_QP_QKEY) {
1989 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1990 qp->qplib_qp.qkey = qp_attr->qkey;
1991 }
1992 if (qp_attr_mask & IB_QP_AV) {
d8966fcd
DC		 1993 const struct ib_global_route *grh =
1994 rdma_ah_read_grh(&qp_attr->ah_attr);
47ec3866 1995 const struct ib_gid_attr *sgid_attr;
0a01623b 1996 struct bnxt_re_gid_ctx *ctx;
d8966fcd 1997
1ac5a404
SX		 1998 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1999 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
2000 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
2001 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
2002 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
2003 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
2004 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
d8966fcd 2005 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1ac5a404 2006 sizeof(qp->qplib_qp.ah.dgid.data));
d8966fcd 2007 qp->qplib_qp.ah.flow_label = grh->flow_label;
0a01623b
SX		 2008 sgid_attr = grh->sgid_attr;
2009 /* Get the HW context of the GID. The reference
2010 * of GID table entry is already taken by the caller.
1ac5a404 2011 */
0a01623b
SX		 2012 ctx = rdma_read_gid_hw_context(sgid_attr);
2013 qp->qplib_qp.ah.sgid_index = ctx->idx;
d8966fcd
DC		 2014 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
2015 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
2016 qp->qplib_qp.ah.traffic_class = grh->traffic_class;
2017 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
44c58487
DC		 2018 ether_addr_copy(qp->qplib_qp.ah.dmac,
2019 qp_attr->ah_attr.roce.dmac);
1ac5a404 2020
a70c0739
PP		 2021 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
2022 &qp->qplib_qp.smac[0]);
2023 if (rc)
2024 return rc;
2025
47ec3866
PP		 2026 nw_type = rdma_gid_attr_network_type(sgid_attr);
2027 switch (nw_type) {
2028 case RDMA_NETWORK_IPV4:
2029 qp->qplib_qp.nw_type =
2030 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
2031 break;
2032 case RDMA_NETWORK_IPV6:
2033 qp->qplib_qp.nw_type =
2034 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
2035 break;
2036 default:
2037 qp->qplib_qp.nw_type =
2038 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
2039 break;
1ac5a404
SX		 2040 }
2041 }
2042
2043 if (qp_attr_mask & IB_QP_PATH_MTU) {
2044 qp->qplib_qp.modify_flags |=
2045 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
2046 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
f9b941ba 2047 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
1ac5a404
SX		 2048 } else if (qp_attr->qp_state == IB_QPS_RTR) {
2049 qp->qplib_qp.modify_flags |=
2050 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
2051 qp->qplib_qp.path_mtu =
2052 __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
f9b941ba
DS		 2053 qp->qplib_qp.mtu =
2054 ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1ac5a404
SX		 2055 }
2056
2057 if (qp_attr_mask & IB_QP_TIMEOUT) {
2058 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
2059 qp->qplib_qp.timeout = qp_attr->timeout;
2060 }
2061 if (qp_attr_mask & IB_QP_RETRY_CNT) {
2062 qp->qplib_qp.modify_flags |=
2063 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
2064 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
2065 }
2066 if (qp_attr_mask & IB_QP_RNR_RETRY) {
2067 qp->qplib_qp.modify_flags |=
2068 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
2069 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
2070 }
2071 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
2072 qp->qplib_qp.modify_flags |=
2073 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
2074 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
2075 }
2076 if (qp_attr_mask & IB_QP_RQ_PSN) {
2077 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
2078 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
2079 }
2080 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2081 qp->qplib_qp.modify_flags |=
2082 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
a25d112f
EW		 2083 /* Cap the max_rd_atomic to device max */
2084 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
2085 dev_attr->max_qp_rd_atom);
1ac5a404
SX		 2086 }
2087 if (qp_attr_mask & IB_QP_SQ_PSN) {
2088 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2089 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
2090 }
2091 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
a25d112f
EW		 2092 if (qp_attr->max_dest_rd_atomic >
2093 dev_attr->max_qp_init_rd_atom) {
6ccad848
DS		 2094 ibdev_err(&rdev->ibdev,
2095 "max_dest_rd_atomic requested%d is > dev_max%d",
2096 qp_attr->max_dest_rd_atomic,
2097 dev_attr->max_qp_init_rd_atom);
a25d112f
EW		 2098 return -EINVAL;
2099 }
2100
1ac5a404
SX		 2101 qp->qplib_qp.modify_flags |=
2102 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
2103 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
2104 }
2105 if (qp_attr_mask & IB_QP_CAP) {
2106 qp->qplib_qp.modify_flags |=
2107 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
2108 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
2109 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
2110 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
2111 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
2112 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
2113 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
2114 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
2115 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2116 (qp_attr->cap.max_inline_data >=
2117 dev_attr->max_inline_data)) {
6ccad848
DS		 2118 ibdev_err(&rdev->ibdev,
2119 "Create QP failed - max exceeded");
1ac5a404
SX		 2120 return -EINVAL;
2121 }
2122 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
2123 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2124 dev_attr->max_qp_wqes + 1);
9152e0b7
EW		 2125 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2126 qp_attr->cap.max_send_wr;
2127 /*
2128 * Reserving one slot for Phantom WQE. Some application can
2129 * post one extra entry in this case. Allowing this to avoid
2130 * unexpected Queue full condition
2131 */
2132 qp->qplib_qp.sq.q_full_delta -= 1;
1ac5a404
SX		 2133 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2134 if (qp->qplib_qp.rq.max_wqe) {
2135 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
2136 qp->qplib_qp.rq.max_wqe =
2137 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
9152e0b7
EW		 2138 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2139 qp_attr->cap.max_recv_wr;
1ac5a404
SX		 2140 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2141 } else {
2142 /* SRQ was used prior, just ignore the RQ caps */
2143 }
2144 }
2145 if (qp_attr_mask & IB_QP_DEST_QPN) {
2146 qp->qplib_qp.modify_flags |=
2147 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2148 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2149 }
2150 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2151 if (rc) {
6ccad848 2152 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
1ac5a404
SX		 2153 return rc;
2154 }
8dae419f 2155 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
1ac5a404
SX		 2156 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2157 return rc;
2158}
2159
2160int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2161 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2162{
2163 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2164 struct bnxt_re_dev *rdev = qp->rdev;
e13547bc 2165 struct bnxt_qplib_qp *qplib_qp;
1ac5a404
SX		 2166 int rc;
2167
e13547bc
LR		 2168 qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2169 if (!qplib_qp)
2170 return -ENOMEM;
2171
2172 qplib_qp->id = qp->qplib_qp.id;
2173 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
1ac5a404 2174
e13547bc 2175 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
1ac5a404 2176 if (rc) {
6ccad848 2177 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
e13547bc 2178 goto out;
1ac5a404 2179 }
e13547bc 2180 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
53839b51 2181 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
e13547bc
LR		 2182 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2183 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
2184 qp_attr->pkey_index = qplib_qp->pkey_index;
2185 qp_attr->qkey = qplib_qp->qkey;
44c58487 2186 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
e13547bc
LR		 2187 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
2188 qplib_qp->ah.host_sgid_index,
2189 qplib_qp->ah.hop_limit,
2190 qplib_qp->ah.traffic_class);
2191 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2192 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2193 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2194 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2195 qp_attr->timeout = qplib_qp->timeout;
2196 qp_attr->retry_cnt = qplib_qp->retry_cnt;
2197 qp_attr->rnr_retry = qplib_qp->rnr_retry;
2198 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2199 qp_attr->rq_psn = qplib_qp->rq.psn;
2200 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2201 qp_attr->sq_psn = qplib_qp->sq.psn;
2202 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2203 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2204 IB_SIGNAL_REQ_WR;
2205 qp_attr->dest_qp_num = qplib_qp->dest_qpn;
1ac5a404
SX		 2206
2207 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2208 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2209 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2210 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2211 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2212 qp_init_attr->cap = qp_attr->cap;
2213
e13547bc
LR		 2214out:
2215 kfree(qplib_qp);
2216 return rc;
1ac5a404
SX		 2217}
2218
2219/* Routine for sending QP1 packets for RoCE V1 an V2
2220 */
2221static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
f696bf6d 2222 const struct ib_send_wr *wr,
1ac5a404
SX		 2223 struct bnxt_qplib_swqe *wqe,
2224 int payload_size)
2225{
1ac5a404
SX		 2226 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2227 ib_ah);
2228 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
89af969a 2229 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
1ac5a404 2230 struct bnxt_qplib_sge sge;
1ac5a404
SX		 2231 u8 nw_type;
2232 u16 ether_type;
1ac5a404
SX		 2233 union ib_gid dgid;
2234 bool is_eth = false;
2235 bool is_vlan = false;
2236 bool is_grh = false;
2237 bool is_udp = false;
2238 u8 ip_version = 0;
2239 u16 vlan_id = 0xFFFF;
2240 void *buf;
14611b9b 2241 int i, rc;
1ac5a404
SX		 2242
2243 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2244
a70c0739
PP		 2245 rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2246 if (rc)
2247 return rc;
2248
1ac5a404 2249 /* Get network header type for this GID */
89af969a 2250 nw_type = rdma_gid_attr_network_type(sgid_attr);
1ac5a404
SX		 2251 switch (nw_type) {
2252 case RDMA_NETWORK_IPV4:
2253 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2254 break;
2255 case RDMA_NETWORK_IPV6:
2256 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2257 break;
2258 default:
2259 nw_type = BNXT_RE_ROCE_V1_PACKET;
2260 break;
2261 }
2262 memcpy(&dgid.raw, &qplib_ah->dgid, 16);
89af969a 2263 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
1ac5a404 2264 if (is_udp) {
89af969a 2265 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
1ac5a404
SX		 2266 ip_version = 4;
2267 ether_type = ETH_P_IP;
2268 } else {
2269 ip_version = 6;
2270 ether_type = ETH_P_IPV6;
2271 }
2272 is_grh = false;
2273 } else {
2274 ether_type = ETH_P_IBOE;
2275 is_grh = true;
2276 }
2277
2278 is_eth = true;
272bba19 2279 is_vlan = vlan_id && (vlan_id < 0x1000);
1ac5a404
SX		 2280
2281 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2282 ip_version, is_udp, 0, &qp->qp1_hdr);
2283
2284 /* ETH */
2285 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2286 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2287
2288 /* For vlan, check the sgid for vlan existence */
2289
2290 if (!is_vlan) {
2291 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2292 } else {
2293 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2294 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2295 }
2296
2297 if (is_grh || (ip_version == 6)) {
89af969a
PP		 2298 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2299 sizeof(sgid_attr->gid));
1ac5a404 2300 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
89af969a 2301 sizeof(sgid_attr->gid));
1ac5a404
SX		 2302 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit;
2303 }
2304
2305 if (ip_version == 4) {
2306 qp->qp1_hdr.ip4.tos = 0;
2307 qp->qp1_hdr.ip4.id = 0;
2308 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2309 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2310
89af969a 2311 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
1ac5a404
SX		 2312 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2313 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2314 }
2315
2316 if (is_udp) {
2317 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2318 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2319 qp->qp1_hdr.udp.csum = 0;
2320 }
2321
2322 /* BTH */
2323 if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2324 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2325 qp->qp1_hdr.immediate_present = 1;
2326 } else {
2327 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2328 }
2329 if (wr->send_flags & IB_SEND_SOLICITED)
2330 qp->qp1_hdr.bth.solicited_event = 1;
2331 /* pad_count */
2332 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2333
2334 /* P_key for QP1 is for all members */
2335 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2336 qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2337 qp->qp1_hdr.bth.ack_req = 0;
2338 qp->send_psn++;
2339 qp->send_psn &= BTH_PSN_MASK;
2340 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2341 /* DETH */
2342 /* Use the priviledged Q_Key for QP1 */
2343 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2344 qp->qp1_hdr.deth.source_qpn = IB_QP1;
2345
2346 /* Pack the QP1 to the transmit buffer */
2347 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2348 if (buf) {
6dfa8ae6 2349 ib_ud_header_pack(&qp->qp1_hdr, buf);
1ac5a404
SX		 2350 for (i = wqe->num_sge; i; i--) {
2351 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2352 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2353 wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2354 }
2355
2356 /*
2357 * Max Header buf size for IPV6 RoCE V2 is 86,
2358 * which is same as the QP1 SQ header buffer.
2359 * Header buf size for IPV4 RoCE V2 can be 66.
2360 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2361 * Subtract 20 bytes from QP1 SQ header buf size
2362 */
2363 if (is_udp && ip_version == 4)
2364 sge.size -= 20;
2365 /*
2366 * Max Header buf size for RoCE V1 is 78.
2367 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2368 * Subtract 8 bytes from QP1 SQ header buf size
2369 */
2370 if (!is_udp)
2371 sge.size -= 8;
2372
2373 /* Subtract 4 bytes for non vlan packets */
2374 if (!is_vlan)
2375 sge.size -= 4;
2376
2377 wqe->sg_list[0].addr = sge.addr;
2378 wqe->sg_list[0].lkey = sge.lkey;
2379 wqe->sg_list[0].size = sge.size;
2380 wqe->num_sge++;
2381
2382 } else {
6ccad848 2383 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
1ac5a404
SX		 2384 rc = -ENOMEM;
2385 }
2386 return rc;
2387}
2388
2389/* For the MAD layer, it only provides the recv SGE the size of
2390 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
2391 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
2392 * receive packet (334 bytes) with no VLAN and then copy the GRH
2393 * and the MAD datagram out to the provided SGE.
2394 */
2395static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
d34ac5cd 2396 const struct ib_recv_wr *wr,
1ac5a404
SX		 2397 struct bnxt_qplib_swqe *wqe,
2398 int payload_size)
2399{
8dae419f 2400 struct bnxt_re_sqp_entries *sqp_entry;
1ac5a404 2401 struct bnxt_qplib_sge ref, sge;
8dae419f 2402 struct bnxt_re_dev *rdev;
1ac5a404 2403 u32 rq_prod_index;
8dae419f
DS		 2404
2405 rdev = qp->rdev;
1ac5a404
SX		 2406
2407 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2408
2409 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2410 return -ENOMEM;
2411
2412 /* Create 1 SGE to receive the entire
2413 * ethernet packet
2414 */
2415 /* Save the reference from ULP */
2416 ref.addr = wqe->sg_list[0].addr;
2417 ref.lkey = wqe->sg_list[0].lkey;
2418 ref.size = wqe->sg_list[0].size;
2419
8dae419f 2420 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
1ac5a404
SX		 2421
2422 /* SGE 1 */
2423 wqe->sg_list[0].addr = sge.addr;
2424 wqe->sg_list[0].lkey = sge.lkey;
2425 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2426 sge.size -= wqe->sg_list[0].size;
2427
2428 sqp_entry->sge.addr = ref.addr;
2429 sqp_entry->sge.lkey = ref.lkey;
2430 sqp_entry->sge.size = ref.size;
2431 /* Store the wrid for reporting completion */
2432 sqp_entry->wrid = wqe->wr_id;
2433 /* change the wqe->wrid to table index */
2434 wqe->wr_id = rq_prod_index;
2435 return 0;
2436}
2437
2438static int is_ud_qp(struct bnxt_re_qp *qp)
2439{
374c5285
DS		 2440 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2441 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
1ac5a404
SX		 2442}
2443
2444static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
f696bf6d 2445 const struct ib_send_wr *wr,
1ac5a404
SX		 2446 struct bnxt_qplib_swqe *wqe)
2447{
2448 struct bnxt_re_ah *ah = NULL;
2449
2450 if (is_ud_qp(qp)) {
2451 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2452 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2453 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2454 wqe->send.avid = ah->qplib_ah.id;
2455 }
2456 switch (wr->opcode) {
2457 case IB_WR_SEND:
2458 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2459 break;
2460 case IB_WR_SEND_WITH_IMM:
2461 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2462 wqe->send.imm_data = wr->ex.imm_data;
2463 break;
2464 case IB_WR_SEND_WITH_INV:
2465 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2466 wqe->send.inv_key = wr->ex.invalidate_rkey;
2467 break;
2468 default:
2469 return -EINVAL;
2470 }
2471 if (wr->send_flags & IB_SEND_SIGNALED)
2472 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2473 if (wr->send_flags & IB_SEND_FENCE)
2474 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2475 if (wr->send_flags & IB_SEND_SOLICITED)
2476 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2477 if (wr->send_flags & IB_SEND_INLINE)
2478 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2479
2480 return 0;
2481}
2482
f696bf6d 2483static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
1ac5a404
SX		 2484 struct bnxt_qplib_swqe *wqe)
2485{
2486 switch (wr->opcode) {
2487 case IB_WR_RDMA_WRITE:
2488 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2489 break;
2490 case IB_WR_RDMA_WRITE_WITH_IMM:
2491 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2492 wqe->rdma.imm_data = wr->ex.imm_data;
2493 break;
2494 case IB_WR_RDMA_READ:
2495 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2496 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2497 break;
2498 default:
2499 return -EINVAL;
2500 }
2501 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2502 wqe->rdma.r_key = rdma_wr(wr)->rkey;
2503 if (wr->send_flags & IB_SEND_SIGNALED)
2504 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2505 if (wr->send_flags & IB_SEND_FENCE)
2506 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2507 if (wr->send_flags & IB_SEND_SOLICITED)
2508 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2509 if (wr->send_flags & IB_SEND_INLINE)
2510 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2511
2512 return 0;
2513}
2514
f696bf6d 2515static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
1ac5a404
SX		 2516 struct bnxt_qplib_swqe *wqe)
2517{
2518 switch (wr->opcode) {
2519 case IB_WR_ATOMIC_CMP_AND_SWP:
2520 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
55311d05 2521 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
1ac5a404
SX		 2522 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2523 break;
2524 case IB_WR_ATOMIC_FETCH_AND_ADD:
2525 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2526 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2527 break;
2528 default:
2529 return -EINVAL;
2530 }
2531 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2532 wqe->atomic.r_key = atomic_wr(wr)->rkey;
2533 if (wr->send_flags & IB_SEND_SIGNALED)
2534 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2535 if (wr->send_flags & IB_SEND_FENCE)
2536 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2537 if (wr->send_flags & IB_SEND_SOLICITED)
2538 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2539 return 0;
2540}
2541
f696bf6d 2542static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
1ac5a404
SX		 2543 struct bnxt_qplib_swqe *wqe)
2544{
2545 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2546 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2547
a45bc17b
DS		 2548 /* Need unconditional fence for local invalidate
2549 * opcode to work as expected.
2550 */
2551 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2552
1ac5a404
SX		 2553 if (wr->send_flags & IB_SEND_SIGNALED)
2554 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1ac5a404
SX		 2555 if (wr->send_flags & IB_SEND_SOLICITED)
2556 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2557
2558 return 0;
2559}
2560
f696bf6d 2561static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
1ac5a404
SX		 2562 struct bnxt_qplib_swqe *wqe)
2563{
2564 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2565 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2566 int access = wr->access;
2567
2568 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2569 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2570 wqe->frmr.page_list = mr->pages;
2571 wqe->frmr.page_list_len = mr->npages;
0c4dcd60 2572 wqe->frmr.levels = qplib_frpl->hwq.level;
1ac5a404
SX		 2573 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2574
a45bc17b
DS		 2575 /* Need unconditional fence for reg_mr
2576 * opcode to function as expected.
2577 */
2578
2579 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2580
1ac5a404
SX		 2581 if (wr->wr.send_flags & IB_SEND_SIGNALED)
2582 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2583
2584 if (access & IB_ACCESS_LOCAL_WRITE)
2585 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2586 if (access & IB_ACCESS_REMOTE_READ)
2587 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2588 if (access & IB_ACCESS_REMOTE_WRITE)
2589 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2590 if (access & IB_ACCESS_REMOTE_ATOMIC)
2591 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2592 if (access & IB_ACCESS_MW_BIND)
2593 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2594
2595 wqe->frmr.l_key = wr->key;
2596 wqe->frmr.length = wr->mr->length;
7a3c3a12
SX		 2597 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
2598 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
1ac5a404
SX		 2599 wqe->frmr.va = wr->mr->iova;
2600 return 0;
2601}
2602
2603static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
f696bf6d 2604 const struct ib_send_wr *wr,
1ac5a404
SX		 2605 struct bnxt_qplib_swqe *wqe)
2606{
2607 /* Copy the inline data to the data field */
2608 u8 *in_data;
2609 u32 i, sge_len;
2610 void *sge_addr;
2611
2612 in_data = wqe->inline_data;
2613 for (i = 0; i < wr->num_sge; i++) {
2614 sge_addr = (void *)(unsigned long)
2615 wr->sg_list[i].addr;
2616 sge_len = wr->sg_list[i].length;
2617
2618 if ((sge_len + wqe->inline_len) >
2619 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
6ccad848
DS		 2620 ibdev_err(&rdev->ibdev,
2621 "Inline data size requested > supported value");
1ac5a404
SX		 2622 return -EINVAL;
2623 }
2624 sge_len = wr->sg_list[i].length;
2625
2626 memcpy(in_data, sge_addr, sge_len);
2627 in_data += wr->sg_list[i].length;
2628 wqe->inline_len += wr->sg_list[i].length;
2629 }
2630 return wqe->inline_len;
2631}
2632
2633static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
f696bf6d 2634 const struct ib_send_wr *wr,
1ac5a404
SX		 2635 struct bnxt_qplib_swqe *wqe)
2636{
2637 int payload_sz = 0;
2638
2639 if (wr->send_flags & IB_SEND_INLINE)
2640 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2641 else
2642 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2643 wqe->num_sge);
2644
2645 return payload_sz;
2646}
2647
3fb755b3
SK		 2648static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2649{
2650 if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2651 qp->ib_qp.qp_type == IB_QPT_GSI ||
2652 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2653 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2654 int qp_attr_mask;
2655 struct ib_qp_attr qp_attr;
2656
2657 qp_attr_mask = IB_QP_STATE;
2658 qp_attr.qp_state = IB_QPS_RTS;
2659 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2660 qp->qplib_qp.wqe_cnt = 0;
2661 }
2662}
2663
1ac5a404
SX		 2664static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2665 struct bnxt_re_qp *qp,
f696bf6d 2666 const struct ib_send_wr *wr)
1ac5a404 2667{
1ac5a404
SX		 2668 int rc = 0, payload_sz = 0;
2669 unsigned long flags;
2670
2671 spin_lock_irqsave(&qp->sq_lock, flags);
1ac5a404 2672 while (wr) {
24a5b0ce 2673 struct bnxt_qplib_swqe wqe = {};
1ac5a404
SX		 2674
2675 /* Common */
2676 wqe.num_sge = wr->num_sge;
2677 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
6ccad848
DS		 2678 ibdev_err(&rdev->ibdev,
2679 "Limit exceeded for Send SGEs");
1ac5a404
SX		 2680 rc = -EINVAL;
2681 goto bad;
2682 }
2683
2684 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2685 if (payload_sz < 0) {
2686 rc = -EINVAL;
2687 goto bad;
2688 }
2689 wqe.wr_id = wr->wr_id;
2690
2691 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2692
2693 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2694 if (!rc)
2695 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2696bad:
2697 if (rc) {
6ccad848
DS		 2698 ibdev_err(&rdev->ibdev,
2699 "Post send failed opcode = %#x rc = %d",
2700 wr->opcode, rc);
1ac5a404
SX		 2701 break;
2702 }
2703 wr = wr->next;
2704 }
2705 bnxt_qplib_post_send_db(&qp->qplib_qp);
3fb755b3 2706 bnxt_ud_qp_hw_stall_workaround(qp);
1ac5a404
SX		 2707 spin_unlock_irqrestore(&qp->sq_lock, flags);
2708 return rc;
2709}
2710
d34ac5cd
BVA		 2711int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2712 const struct ib_send_wr **bad_wr)
1ac5a404
SX		 2713{
2714 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2715 struct bnxt_qplib_swqe wqe;
2716 int rc = 0, payload_sz = 0;
2717 unsigned long flags;
2718
2719 spin_lock_irqsave(&qp->sq_lock, flags);
2720 while (wr) {
2721 /* House keeping */
2722 memset(&wqe, 0, sizeof(wqe));
2723
2724 /* Common */
2725 wqe.num_sge = wr->num_sge;
2726 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
6ccad848
DS		 2727 ibdev_err(&qp->rdev->ibdev,
2728 "Limit exceeded for Send SGEs");
1ac5a404
SX		 2729 rc = -EINVAL;
2730 goto bad;
2731 }
2732
2733 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2734 if (payload_sz < 0) {
2735 rc = -EINVAL;
2736 goto bad;
2737 }
2738 wqe.wr_id = wr->wr_id;
2739
2740 switch (wr->opcode) {
2741 case IB_WR_SEND:
2742 case IB_WR_SEND_WITH_IMM:
374c5285 2743 if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
1ac5a404
SX		 2744 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2745 payload_sz);
2746 if (rc)
2747 goto bad;
2748 wqe.rawqp1.lflags |=
2749 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2750 }
2751 switch (wr->send_flags) {
2752 case IB_SEND_IP_CSUM:
2753 wqe.rawqp1.lflags |=
2754 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2755 break;
2756 default:
2757 break;
2758 }
df561f66 2759 fallthrough;
1ac5a404
SX		 2760 case IB_WR_SEND_WITH_INV:
2761 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2762 break;
2763 case IB_WR_RDMA_WRITE:
2764 case IB_WR_RDMA_WRITE_WITH_IMM:
2765 case IB_WR_RDMA_READ:
2766 rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2767 break;
2768 case IB_WR_ATOMIC_CMP_AND_SWP:
2769 case IB_WR_ATOMIC_FETCH_AND_ADD:
2770 rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2771 break;
2772 case IB_WR_RDMA_READ_WITH_INV:
6ccad848
DS		 2773 ibdev_err(&qp->rdev->ibdev,
2774 "RDMA Read with Invalidate is not supported");
1ac5a404
SX		 2775 rc = -EINVAL;
2776 goto bad;
2777 case IB_WR_LOCAL_INV:
2778 rc = bnxt_re_build_inv_wqe(wr, &wqe);
2779 break;
2780 case IB_WR_REG_MR:
2781 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2782 break;
2783 default:
2784 /* Unsupported WRs */
6ccad848
DS		 2785 ibdev_err(&qp->rdev->ibdev,
2786 "WR (%#x) is not supported", wr->opcode);
1ac5a404
SX		 2787 rc = -EINVAL;
2788 goto bad;
2789 }
2790 if (!rc)
2791 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2792bad:
2793 if (rc) {
6ccad848
DS		 2794 ibdev_err(&qp->rdev->ibdev,
2795 "post_send failed op:%#x qps = %#x rc = %d\n",
2796 wr->opcode, qp->qplib_qp.state, rc);
1ac5a404
SX		 2797 *bad_wr = wr;
2798 break;
2799 }
2800 wr = wr->next;
2801 }
2802 bnxt_qplib_post_send_db(&qp->qplib_qp);
3fb755b3 2803 bnxt_ud_qp_hw_stall_workaround(qp);
1ac5a404
SX		 2804 spin_unlock_irqrestore(&qp->sq_lock, flags);
2805
2806 return rc;
2807}
2808
2809static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2810 struct bnxt_re_qp *qp,
d34ac5cd 2811 const struct ib_recv_wr *wr)
1ac5a404
SX		 2812{
2813 struct bnxt_qplib_swqe wqe;
6dfa8ae6 2814 int rc = 0;
1ac5a404 2815
1ac5a404
SX		 2816 while (wr) {
2817 /* House keeping */
2818 memset(&wqe, 0, sizeof(wqe));
2819
2820 /* Common */
2821 wqe.num_sge = wr->num_sge;
2822 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
6ccad848
DS		 2823 ibdev_err(&rdev->ibdev,
2824 "Limit exceeded for Receive SGEs");
1ac5a404
SX		 2825 rc = -EINVAL;
2826 break;
2827 }
6dfa8ae6 2828 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
1ac5a404
SX		 2829 wqe.wr_id = wr->wr_id;
2830 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2831
2832 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2833 if (rc)
2834 break;
2835
2836 wr = wr->next;
2837 }
2838 if (!rc)
2839 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2840 return rc;
2841}
2842
d34ac5cd
BVA		 2843int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
2844 const struct ib_recv_wr **bad_wr)
1ac5a404
SX		 2845{
2846 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2847 struct bnxt_qplib_swqe wqe;
2848 int rc = 0, payload_sz = 0;
018cf599
DS		 2849 unsigned long flags;
2850 u32 count = 0;
1ac5a404 2851
018cf599 2852 spin_lock_irqsave(&qp->rq_lock, flags);
1ac5a404
SX		 2853 while (wr) {
2854 /* House keeping */
2855 memset(&wqe, 0, sizeof(wqe));
2856
2857 /* Common */
2858 wqe.num_sge = wr->num_sge;
2859 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
6ccad848
DS		 2860 ibdev_err(&qp->rdev->ibdev,
2861 "Limit exceeded for Receive SGEs");
1ac5a404
SX		 2862 rc = -EINVAL;
2863 *bad_wr = wr;
2864 break;
2865 }
2866
2867 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2868 wr->num_sge);
2869 wqe.wr_id = wr->wr_id;
2870 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2871
374c5285
DS		 2872 if (ib_qp->qp_type == IB_QPT_GSI &&
2873 qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
1ac5a404
SX		 2874 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2875 payload_sz);
2876 if (!rc)
2877 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2878 if (rc) {
2879 *bad_wr = wr;
2880 break;
2881 }
018cf599
DS		 2882
2883 /* Ring DB if the RQEs posted reaches a threshold value */
2884 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
2885 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2886 count = 0;
2887 }
2888
1ac5a404
SX		 2889 wr = wr->next;
2890 }
018cf599
DS		 2891
2892 if (count)
2893 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2894
2895 spin_unlock_irqrestore(&qp->rq_lock, flags);
2896
1ac5a404
SX		 2897 return rc;
2898}
2899
2900/* Completion Queues */
43d781b9 2901int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
1ac5a404 2902{
ccd9d0d3
SX		 2903 struct bnxt_re_cq *cq;
2904 struct bnxt_qplib_nq *nq;
2905 struct bnxt_re_dev *rdev;
2906
2907 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2908 rdev = cq->rdev;
2909 nq = cq->qplib_cq.nq;
1ac5a404 2910
a52c8e24 2911 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
836a0fbb 2912 ib_umem_release(cq->umem);
1ac5a404 2913
063975fe 2914 atomic_dec(&rdev->stats.res.cq_count);
6a5df91b 2915 nq->budget--;
ccd9d0d3 2916 kfree(cq->cql);
43d781b9 2917 return 0;
1ac5a404
SX		 2918}
2919
e39afe3d
LR		 2920int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
2921 struct ib_udata *udata)
1ac5a404 2922{
e39afe3d 2923 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
1ac5a404 2924 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
e39afe3d 2925 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
1ac5a404
SX		 2926 int rc, entries;
2927 int cqe = attr->cqe;
6a5df91b
SX		 2928 struct bnxt_qplib_nq *nq = NULL;
2929 unsigned int nq_alloc_cnt;
cb95709e 2930 u32 active_cqs;
1ac5a404 2931
1c407cb5
JG		 2932 if (attr->flags)
2933 return -EOPNOTSUPP;
2934
1ac5a404
SX		 2935 /* Validate CQ fields */
2936 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
6ccad848 2937 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
e39afe3d 2938 return -EINVAL;
1ac5a404 2939 }
1ac5a404
SX		 2940
2941 cq->rdev = rdev;
2942 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2943
2944 entries = roundup_pow_of_two(cqe + 1);
2945 if (entries > dev_attr->max_cq_wqes + 1)
2946 entries = dev_attr->max_cq_wqes + 1;
2947
0c4dcd60
DS		 2948 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
2949 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
ff23dfa1 2950 if (udata) {
1ac5a404 2951 struct bnxt_re_cq_req req;
ff23dfa1
SR		 2952 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
2953 udata, struct bnxt_re_ucontext, ib_uctx);
1ac5a404
SX		 2954 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2955 rc = -EFAULT;
2956 goto fail;
2957 }
2958
c320e527 2959 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
1ac5a404 2960 entries * sizeof(struct cq_base),
72b894b0 2961 IB_ACCESS_LOCAL_WRITE);
1ac5a404
SX		 2962 if (IS_ERR(cq->umem)) {
2963 rc = PTR_ERR(cq->umem);
2964 goto fail;
2965 }
6ef999f5 2966 cq->qplib_cq.sg_info.umem = cq->umem;
b3b2c7c5 2967 cq->qplib_cq.dpi = &uctx->dpi;
1ac5a404
SX		 2968 } else {
2969 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2970 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2971 GFP_KERNEL);
2972 if (!cq->cql) {
2973 rc = -ENOMEM;
2974 goto fail;
2975 }
2976
2977 cq->qplib_cq.dpi = &rdev->dpi_privileged;
1ac5a404 2978 }
6a5df91b
SX		 2979 /*
2980 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
2981 * used for getting the NQ index.
2982 */
2983 nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
2984 nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
1ac5a404 2985 cq->qplib_cq.max_wqe = entries;
6a5df91b
SX		 2986 cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
2987 cq->qplib_cq.nq = nq;
1ac5a404
SX		 2988
2989 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2990 if (rc) {
6ccad848 2991 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
1ac5a404
SX		 2992 goto fail;
2993 }
2994
2995 cq->ib_cq.cqe = entries;
2996 cq->cq_period = cq->qplib_cq.period;
6a5df91b 2997 nq->budget++;
1ac5a404 2998
cb95709e
CA		 2999 active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
3000 if (active_cqs > rdev->stats.res.cq_watermark)
3001 rdev->stats.res.cq_watermark = active_cqs;
5a23e0b1 3002 spin_lock_init(&cq->cq_lock);
1ac5a404 3003
ff23dfa1 3004 if (udata) {
1ac5a404
SX		 3005 struct bnxt_re_cq_resp resp;
3006
3007 resp.cqid = cq->qplib_cq.id;
3008 resp.tail = cq->qplib_cq.hwq.cons;
3009 resp.phase = cq->qplib_cq.period;
3010 resp.rsvd = 0;
3011 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
3012 if (rc) {
6ccad848 3013 ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
1ac5a404
SX		 3014 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3015 goto c2fail;
3016 }
3017 }
3018
e39afe3d 3019 return 0;
1ac5a404
SX		 3020
3021c2fail:
836a0fbb 3022 ib_umem_release(cq->umem);
1ac5a404
SX		 3023fail:
3024 kfree(cq->cql);
e39afe3d 3025 return rc;
1ac5a404
SX		 3026}
3027
d54bd5ab
SX		 3028static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
3029{
3030 struct bnxt_re_dev *rdev = cq->rdev;
3031
3032 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
3033
3034 cq->qplib_cq.max_wqe = cq->resize_cqe;
3035 if (cq->resize_umem) {
3036 ib_umem_release(cq->umem);
3037 cq->umem = cq->resize_umem;
3038 cq->resize_umem = NULL;
3039 cq->resize_cqe = 0;
3040 }
3041}
3042
3043int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
3044{
3045 struct bnxt_qplib_sg_info sg_info = {};
3046 struct bnxt_qplib_dpi *orig_dpi = NULL;
3047 struct bnxt_qplib_dev_attr *dev_attr;
3048 struct bnxt_re_ucontext *uctx = NULL;
3049 struct bnxt_re_resize_cq_req req;
3050 struct bnxt_re_dev *rdev;
3051 struct bnxt_re_cq *cq;
3052 int rc, entries;
3053
3054 cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3055 rdev = cq->rdev;
3056 dev_attr = &rdev->dev_attr;
3057 if (!ibcq->uobject) {
3058 ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported");
3059 return -EOPNOTSUPP;
3060 }
3061
3062 if (cq->resize_umem) {
3063 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy",
3064 cq->qplib_cq.id);
3065 return -EBUSY;
3066 }
3067
3068 /* Check the requested cq depth out of supported depth */
3069 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3070 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d",
3071 cq->qplib_cq.id, cqe);
3072 return -EINVAL;
3073 }
3074
3075 entries = roundup_pow_of_two(cqe + 1);
3076 if (entries > dev_attr->max_cq_wqes + 1)
3077 entries = dev_attr->max_cq_wqes + 1;
3078
3079 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
3080 ib_uctx);
3081 /* uverbs consumer */
3082 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
3083 rc = -EFAULT;
3084 goto fail;
3085 }
3086
3087 cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3088 entries * sizeof(struct cq_base),
3089 IB_ACCESS_LOCAL_WRITE);
3090 if (IS_ERR(cq->resize_umem)) {
3091 rc = PTR_ERR(cq->resize_umem);
3092 cq->resize_umem = NULL;
3093 ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %d\n",
3094 __func__, rc);
3095 goto fail;
3096 }
3097 cq->resize_cqe = entries;
3098 memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info));
3099 orig_dpi = cq->qplib_cq.dpi;
3100
3101 cq->qplib_cq.sg_info.umem = cq->resize_umem;
3102 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3103 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3104 cq->qplib_cq.dpi = &uctx->dpi;
3105
3106 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
3107 if (rc) {
3108 ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!",
3109 cq->qplib_cq.id);
3110 goto fail;
3111 }
3112
3113 cq->ib_cq.cqe = cq->resize_cqe;
cb95709e 3114 atomic_inc(&rdev->stats.res.resize_count);
d54bd5ab
SX		 3115
3116 return 0;
3117
3118fail:
3119 if (cq->resize_umem) {
3120 ib_umem_release(cq->resize_umem);
3121 cq->resize_umem = NULL;
3122 cq->resize_cqe = 0;
3123 memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info));
3124 cq->qplib_cq.dpi = orig_dpi;
3125 }
3126 return rc;
3127}
3128
1ac5a404
SX		 3129static u8 __req_to_ib_wc_status(u8 qstatus)
3130{
3131 switch (qstatus) {
3132 case CQ_REQ_STATUS_OK:
3133 return IB_WC_SUCCESS;
3134 case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
3135 return IB_WC_BAD_RESP_ERR;
3136 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
3137 return IB_WC_LOC_LEN_ERR;
3138 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
3139 return IB_WC_LOC_QP_OP_ERR;
3140 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
3141 return IB_WC_LOC_PROT_ERR;
3142 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
3143 return IB_WC_GENERAL_ERR;
3144 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
3145 return IB_WC_REM_INV_REQ_ERR;
3146 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
3147 return IB_WC_REM_ACCESS_ERR;
3148 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
3149 return IB_WC_REM_OP_ERR;
3150 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
3151 return IB_WC_RNR_RETRY_EXC_ERR;
3152 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
3153 return IB_WC_RETRY_EXC_ERR;
3154 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
3155 return IB_WC_WR_FLUSH_ERR;
3156 default:
3157 return IB_WC_GENERAL_ERR;
3158 }
3159 return 0;
3160}
3161
3162static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
3163{
3164 switch (qstatus) {
3165 case CQ_RES_RAWETH_QP1_STATUS_OK:
3166 return IB_WC_SUCCESS;
3167 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
3168 return IB_WC_LOC_ACCESS_ERR;
3169 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
3170 return IB_WC_LOC_LEN_ERR;
3171 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
3172 return IB_WC_LOC_PROT_ERR;
3173 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
3174 return IB_WC_LOC_QP_OP_ERR;
3175 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
3176 return IB_WC_GENERAL_ERR;
3177 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
3178 return IB_WC_WR_FLUSH_ERR;
3179 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
3180 return IB_WC_WR_FLUSH_ERR;
3181 default:
3182 return IB_WC_GENERAL_ERR;
3183 }
3184}
3185
3186static u8 __rc_to_ib_wc_status(u8 qstatus)
3187{
3188 switch (qstatus) {
3189 case CQ_RES_RC_STATUS_OK:
3190 return IB_WC_SUCCESS;
3191 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
3192 return IB_WC_LOC_ACCESS_ERR;
3193 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
3194 return IB_WC_LOC_LEN_ERR;
3195 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
3196 return IB_WC_LOC_PROT_ERR;
3197 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
3198 return IB_WC_LOC_QP_OP_ERR;
3199 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
3200 return IB_WC_GENERAL_ERR;
3201 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
3202 return IB_WC_REM_INV_REQ_ERR;
3203 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
3204 return IB_WC_WR_FLUSH_ERR;
3205 case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
3206 return IB_WC_WR_FLUSH_ERR;
3207 default:
3208 return IB_WC_GENERAL_ERR;
3209 }
3210}
3211
3212static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
3213{
3214 switch (cqe->type) {
3215 case BNXT_QPLIB_SWQE_TYPE_SEND:
3216 wc->opcode = IB_WC_SEND;
3217 break;
3218 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
3219 wc->opcode = IB_WC_SEND;
3220 wc->wc_flags |= IB_WC_WITH_IMM;
3221 break;
3222 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3223 wc->opcode = IB_WC_SEND;
3224 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3225 break;
3226 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3227 wc->opcode = IB_WC_RDMA_WRITE;
3228 break;
3229 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3230 wc->opcode = IB_WC_RDMA_WRITE;
3231 wc->wc_flags |= IB_WC_WITH_IMM;
3232 break;
3233 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3234 wc->opcode = IB_WC_RDMA_READ;
3235 break;
3236 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3237 wc->opcode = IB_WC_COMP_SWAP;
3238 break;
3239 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3240 wc->opcode = IB_WC_FETCH_ADD;
3241 break;
3242 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3243 wc->opcode = IB_WC_LOCAL_INV;
3244 break;
3245 case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3246 wc->opcode = IB_WC_REG_MR;
3247 break;
3248 default:
3249 wc->opcode = IB_WC_SEND;
3250 break;
3251 }
3252
3253 wc->status = __req_to_ib_wc_status(cqe->status);
3254}
3255
3256static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3257 u16 raweth_qp1_flags2)
3258{
6dfa8ae6 3259 bool is_ipv6 = false, is_ipv4 = false;
1ac5a404
SX		 3260
3261 /* raweth_qp1_flags Bit 9-6 indicates itype */
3262 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3263 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3264 return -1;
3265
3266 if (raweth_qp1_flags2 &
3267 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3268 raweth_qp1_flags2 &
3269 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
1ac5a404
SX		 3270 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3271 (raweth_qp1_flags2 &
3272 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3273 (is_ipv6 = true) : (is_ipv4 = true);
3274 return ((is_ipv6) ?
3275 BNXT_RE_ROCEV2_IPV6_PACKET :
3276 BNXT_RE_ROCEV2_IPV4_PACKET);
3277 } else {
3278 return BNXT_RE_ROCE_V1_PACKET;
3279 }
3280}
3281
3282static int bnxt_re_to_ib_nw_type(int nw_type)
3283{
3284 u8 nw_hdr_type = 0xFF;
3285
3286 switch (nw_type) {
3287 case BNXT_RE_ROCE_V1_PACKET:
3288 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3289 break;
3290 case BNXT_RE_ROCEV2_IPV4_PACKET:
3291 nw_hdr_type = RDMA_NETWORK_IPV4;
3292 break;
3293 case BNXT_RE_ROCEV2_IPV6_PACKET:
3294 nw_hdr_type = RDMA_NETWORK_IPV6;
3295 break;
3296 }
3297 return nw_hdr_type;
3298}
3299
3300static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3301 void *rq_hdr_buf)
3302{
3303 u8 *tmp_buf = NULL;
3304 struct ethhdr *eth_hdr;
3305 u16 eth_type;
3306 bool rc = false;
3307
3308 tmp_buf = (u8 *)rq_hdr_buf;
3309 /*
3310 * If dest mac is not same as I/F mac, this could be a
3311 * loopback address or multicast address, check whether
3312 * it is a loopback packet
3313 */
3314 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3315 tmp_buf += 4;
3316 /* Check the ether type */
3317 eth_hdr = (struct ethhdr *)tmp_buf;
3318 eth_type = ntohs(eth_hdr->h_proto);
3319 switch (eth_type) {
3320 case ETH_P_IBOE:
3321 rc = true;
3322 break;
3323 case ETH_P_IP:
3324 case ETH_P_IPV6: {
3325 u32 len;
3326 struct udphdr *udp_hdr;
3327
3328 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3329 sizeof(struct ipv6hdr));
3330 tmp_buf += sizeof(struct ethhdr) + len;
3331 udp_hdr = (struct udphdr *)tmp_buf;
3332 if (ntohs(udp_hdr->dest) ==
3333 ROCE_V2_UDP_DPORT)
3334 rc = true;
3335 break;
3336 }
3337 default:
3338 break;
3339 }
3340 }
3341
3342 return rc;
3343}
3344
8dae419f 3345static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
1ac5a404
SX		 3346 struct bnxt_qplib_cqe *cqe)
3347{
8dae419f 3348 struct bnxt_re_dev *rdev = gsi_qp->rdev;
1ac5a404 3349 struct bnxt_re_sqp_entries *sqp_entry = NULL;
8dae419f 3350 struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
e59a5cec
KA		 3351 dma_addr_t shrq_hdr_buf_map;
3352 struct ib_sge s_sge[2] = {};
3353 struct ib_sge r_sge[2] = {};
8dae419f 3354 struct bnxt_re_ah *gsi_sah;
e59a5cec
KA		 3355 struct ib_recv_wr rwr = {};
3356 dma_addr_t rq_hdr_buf_map;
3357 struct ib_ud_wr udwr = {};
1ac5a404 3358 struct ib_send_wr *swr;
e59a5cec 3359 u32 skip_bytes = 0;
1ac5a404 3360 int pkt_type = 0;
1ac5a404 3361 void *rq_hdr_buf;
1ac5a404 3362 u32 offset = 0;
e59a5cec 3363 u32 tbl_idx;
1ac5a404
SX		 3364 int rc;
3365
1ac5a404
SX		 3366 swr = &udwr.wr;
3367 tbl_idx = cqe->wr_id;
3368
8dae419f
DS		 3369 rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3370 (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3371 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
1ac5a404
SX		 3372 tbl_idx);
3373
3374 /* Shadow QP header buffer */
8dae419f 3375 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
1ac5a404 3376 tbl_idx);
8dae419f 3377 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
1ac5a404
SX		 3378
3379 /* Store this cqe */
3380 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
8dae419f 3381 sqp_entry->qp1_qp = gsi_qp;
1ac5a404
SX		 3382
3383 /* Find packet type from the cqe */
3384
3385 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3386 cqe->raweth_qp1_flags2);
3387 if (pkt_type < 0) {
6ccad848 3388 ibdev_err(&rdev->ibdev, "Invalid packet\n");
1ac5a404
SX		 3389 return -EINVAL;
3390 }
3391
3392 /* Adjust the offset for the user buffer and post in the rq */
3393
3394 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3395 offset = 20;
3396
3397 /*
3398 * QP1 loopback packet has 4 bytes of internal header before
3399 * ether header. Skip these four bytes.
3400 */
3401 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3402 skip_bytes = 4;
3403
3404 /* First send SGE . Skip the ether header*/
3405 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3406 + skip_bytes;
3407 s_sge[0].lkey = 0xFFFFFFFF;
3408 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3409 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3410
3411 /* Second Send SGE */
3412 s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3413 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3414 if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3415 s_sge[1].addr += 8;
3416 s_sge[1].lkey = 0xFFFFFFFF;
3417 s_sge[1].length = 256;
3418
3419 /* First recv SGE */
3420
3421 r_sge[0].addr = shrq_hdr_buf_map;
3422 r_sge[0].lkey = 0xFFFFFFFF;
3423 r_sge[0].length = 40;
3424
3425 r_sge[1].addr = sqp_entry->sge.addr + offset;
3426 r_sge[1].lkey = sqp_entry->sge.lkey;
3427 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3428
3429 /* Create receive work request */
3430 rwr.num_sge = 2;
3431 rwr.sg_list = r_sge;
3432 rwr.wr_id = tbl_idx;
3433 rwr.next = NULL;
3434
8dae419f 3435 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
1ac5a404 3436 if (rc) {
6ccad848
DS		 3437 ibdev_err(&rdev->ibdev,
3438 "Failed to post Rx buffers to shadow QP");
1ac5a404
SX		 3439 return -ENOMEM;
3440 }
3441
3442 swr->num_sge = 2;
3443 swr->sg_list = s_sge;
3444 swr->wr_id = tbl_idx;
3445 swr->opcode = IB_WR_SEND;
3446 swr->next = NULL;
8dae419f
DS		 3447 gsi_sah = rdev->gsi_ctx.gsi_sah;
3448 udwr.ah = &gsi_sah->ib_ah;
3449 udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3450 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
1ac5a404
SX		 3451
3452 /* post data received in the send queue */
0fa0d520 3453 return bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
1ac5a404
SX		 3454}
3455
3456static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3457 struct bnxt_qplib_cqe *cqe)
3458{
3459 wc->opcode = IB_WC_RECV;
3460 wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3461 wc->wc_flags |= IB_WC_GRH;
3462}
3463
2d0e60ee
SX		 3464static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3465 u16 vlan_id)
3466{
3467 /*
3468 * Check if the vlan is configured in the host. If not configured, it
3469 * can be a transparent VLAN. So dont report the vlan id.
3470 */
3471 if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3472 htons(ETH_P_8021Q), vlan_id))
3473 return false;
3474 return true;
3475}
3476
84511455
DS		 3477static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3478 u16 *vid, u8 *sl)
3479{
3480 bool ret = false;
3481 u32 metadata;
3482 u16 tpid;
3483
3484 metadata = orig_cqe->raweth_qp1_metadata;
3485 if (orig_cqe->raweth_qp1_flags2 &
3486 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3487 tpid = ((metadata &
3488 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3489 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3490 if (tpid == ETH_P_8021Q) {
3491 *vid = metadata &
3492 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3493 *sl = (metadata &
3494 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3495 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3496 ret = true;
3497 }
3498 }
3499
3500 return ret;
3501}
3502
1ac5a404
SX		 3503static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3504 struct bnxt_qplib_cqe *cqe)
3505{
3506 wc->opcode = IB_WC_RECV;
3507 wc->status = __rc_to_ib_wc_status(cqe->status);
3508
3509 if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3510 wc->wc_flags |= IB_WC_WITH_IMM;
3511 if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3512 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3513 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3514 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3515 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3516}
3517
8dae419f 3518static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
1ac5a404
SX		 3519 struct ib_wc *wc,
3520 struct bnxt_qplib_cqe *cqe)
3521{
8dae419f
DS		 3522 struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3523 struct bnxt_re_qp *gsi_qp = NULL;
1ac5a404
SX		 3524 struct bnxt_qplib_cqe *orig_cqe = NULL;
3525 struct bnxt_re_sqp_entries *sqp_entry = NULL;
3526 int nw_type;
84511455
DS		 3527 u32 tbl_idx;
3528 u16 vlan_id;
3529 u8 sl;
1ac5a404
SX		 3530
3531 tbl_idx = cqe->wr_id;
3532
8dae419f
DS		 3533 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3534 gsi_qp = sqp_entry->qp1_qp;
1ac5a404
SX		 3535 orig_cqe = &sqp_entry->cqe;
3536
3537 wc->wr_id = sqp_entry->wrid;
3538 wc->byte_len = orig_cqe->length;
8dae419f 3539 wc->qp = &gsi_qp->ib_qp;
1ac5a404
SX		 3540
3541 wc->ex.imm_data = orig_cqe->immdata;
3542 wc->src_qp = orig_cqe->src_qp;
3543 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
84511455 3544 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
2d0e60ee
SX		 3545 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3546 wc->vlan_id = vlan_id;
3547 wc->sl = sl;
3548 wc->wc_flags |= IB_WC_WITH_VLAN;
3549 }
84511455 3550 }
1ac5a404
SX		 3551 wc->port_num = 1;
3552 wc->vendor_err = orig_cqe->status;
3553
3554 wc->opcode = IB_WC_RECV;
3555 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
3556 wc->wc_flags |= IB_WC_GRH;
3557
3558 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
3559 orig_cqe->raweth_qp1_flags2);
3560 if (nw_type >= 0) {
3561 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3562 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3563 }
3564}
3565
374c5285
DS		 3566static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3567 struct ib_wc *wc,
1ac5a404
SX		 3568 struct bnxt_qplib_cqe *cqe)
3569{
6bda3914
SX		 3570 struct bnxt_re_dev *rdev;
3571 u16 vlan_id = 0;
374c5285
DS		 3572 u8 nw_type;
3573
6bda3914 3574 rdev = qp->rdev;
1ac5a404
SX		 3575 wc->opcode = IB_WC_RECV;
3576 wc->status = __rc_to_ib_wc_status(cqe->status);
3577
374c5285 3578 if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
1ac5a404 3579 wc->wc_flags |= IB_WC_WITH_IMM;
374c5285
DS		 3580 /* report only on GSI QP for Thor */
3581 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3582 wc->wc_flags |= IB_WC_GRH;
3583 memcpy(wc->smac, cqe->smac, ETH_ALEN);
3584 wc->wc_flags |= IB_WC_WITH_SMAC;
3585 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
6bda3914
SX		 3586 vlan_id = (cqe->cfa_meta & 0xFFF);
3587 }
3588 /* Mark only if vlan_id is non zero */
3589 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3590 wc->vlan_id = vlan_id;
3591 wc->wc_flags |= IB_WC_WITH_VLAN;
374c5285
DS		 3592 }
3593 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3594 CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3595 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3596 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3597 }
3598
1ac5a404
SX		 3599}
3600
9152e0b7
EW		 3601static int send_phantom_wqe(struct bnxt_re_qp *qp)
3602{
3603 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3604 unsigned long flags;
14611b9b 3605 int rc;
9152e0b7
EW		 3606
3607 spin_lock_irqsave(&qp->sq_lock, flags);
3608
3609 rc = bnxt_re_bind_fence_mw(lib_qp);
3610 if (!rc) {
3611 lib_qp->sq.phantom_wqe_cnt++;
6ccad848
DS		 3612 ibdev_dbg(&qp->rdev->ibdev,
3613 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3614 lib_qp->id, lib_qp->sq.hwq.prod,
3615 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3616 lib_qp->sq.phantom_wqe_cnt);
9152e0b7
EW		 3617 }
3618
3619 spin_unlock_irqrestore(&qp->sq_lock, flags);
3620 return rc;
3621}
3622
1ac5a404
SX		 3623int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3624{
3625 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
8dae419f 3626 struct bnxt_re_qp *qp, *sh_qp;
1ac5a404
SX		 3627 struct bnxt_qplib_cqe *cqe;
3628 int i, ncqe, budget;
9152e0b7
EW		 3629 struct bnxt_qplib_q *sq;
3630 struct bnxt_qplib_qp *lib_qp;
1ac5a404
SX		 3631 u32 tbl_idx;
3632 struct bnxt_re_sqp_entries *sqp_entry = NULL;
3633 unsigned long flags;
3634
d54bd5ab
SX		 3635 /* User CQ; the only processing we do is to
3636 * complete any pending CQ resize operation.
3637 */
3638 if (cq->umem) {
3639 if (cq->resize_umem)
3640 bnxt_re_resize_cq_complete(cq);
3641 return 0;
3642 }
3643
1ac5a404
SX		 3644 spin_lock_irqsave(&cq->cq_lock, flags);
3645 budget = min_t(u32, num_entries, cq->max_cql);
10d1dedf 3646 num_entries = budget;
1ac5a404 3647 if (!cq->cql) {
6ccad848 3648 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
1ac5a404
SX		 3649 goto exit;
3650 }
3651 cqe = &cq->cql[0];
3652 while (budget) {
9152e0b7
EW		 3653 lib_qp = NULL;
3654 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
3655 if (lib_qp) {
3656 sq = &lib_qp->sq;
3657 if (sq->send_phantom) {
3658 qp = container_of(lib_qp,
3659 struct bnxt_re_qp, qplib_qp);
3660 if (send_phantom_wqe(qp) == -ENOMEM)
6ccad848
DS		 3661 ibdev_err(&cq->rdev->ibdev,
3662 "Phantom failed! Scheduled to send again\n");
9152e0b7
EW		 3663 else
3664 sq->send_phantom = false;
3665 }
3666 }
f218d67e
SX		 3667 if (ncqe < budget)
3668 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
3669 cqe + ncqe,
3670 budget - ncqe);
9152e0b7 3671
1ac5a404
SX		 3672 if (!ncqe)
3673 break;
3674
3675 for (i = 0; i < ncqe; i++, cqe++) {
3676 /* Transcribe each qplib_wqe back to ib_wc */
3677 memset(wc, 0, sizeof(*wc));
3678
3679 wc->wr_id = cqe->wr_id;
3680 wc->byte_len = cqe->length;
3681 qp = container_of
3682 ((struct bnxt_qplib_qp *)
3683 (unsigned long)(cqe->qp_handle),
3684 struct bnxt_re_qp, qplib_qp);
1ac5a404
SX		 3685 wc->qp = &qp->ib_qp;
3686 wc->ex.imm_data = cqe->immdata;
3687 wc->src_qp = cqe->src_qp;
3688 memcpy(wc->smac, cqe->smac, ETH_ALEN);
3689 wc->port_num = 1;
3690 wc->vendor_err = cqe->status;
3691
3692 switch (cqe->opcode) {
3693 case CQ_BASE_CQE_TYPE_REQ:
8dae419f
DS		 3694 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3695 if (sh_qp &&
3696 qp->qplib_qp.id == sh_qp->qplib_qp.id) {
1ac5a404
SX		 3697 /* Handle this completion with
3698 * the stored completion
3699 */
3700 memset(wc, 0, sizeof(*wc));
3701 continue;
3702 }
3703 bnxt_re_process_req_wc(wc, cqe);
3704 break;
3705 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3706 if (!cqe->status) {
3707 int rc = 0;
3708
3709 rc = bnxt_re_process_raw_qp_pkt_rx
3710 (qp, cqe);
3711 if (!rc) {
3712 memset(wc, 0, sizeof(*wc));
3713 continue;
3714 }
3715 cqe->status = -1;
3716 }
3717 /* Errors need not be looped back.
3718 * But change the wr_id to the one
3719 * stored in the table
3720 */
3721 tbl_idx = cqe->wr_id;
8dae419f 3722 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
1ac5a404
SX		 3723 wc->wr_id = sqp_entry->wrid;
3724 bnxt_re_process_res_rawqp1_wc(wc, cqe);
3725 break;
3726 case CQ_BASE_CQE_TYPE_RES_RC:
3727 bnxt_re_process_res_rc_wc(wc, cqe);
3728 break;
3729 case CQ_BASE_CQE_TYPE_RES_UD:
8dae419f
DS		 3730 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3731 if (sh_qp &&
3732 qp->qplib_qp.id == sh_qp->qplib_qp.id) {
1ac5a404
SX		 3733 /* Handle this completion with
3734 * the stored completion
3735 */
3736 if (cqe->status) {
3737 continue;
3738 } else {
3739 bnxt_re_process_res_shadow_qp_wc
3740 (qp, wc, cqe);
3741 break;
3742 }
3743 }
374c5285 3744 bnxt_re_process_res_ud_wc(qp, wc, cqe);
1ac5a404
SX		 3745 break;
3746 default:
6ccad848
DS		 3747 ibdev_err(&cq->rdev->ibdev,
3748 "POLL CQ : type 0x%x not handled",
3749 cqe->opcode);
1ac5a404
SX		 3750 continue;
3751 }
3752 wc++;
3753 budget--;
3754 }
3755 }
3756exit:
3757 spin_unlock_irqrestore(&cq->cq_lock, flags);
3758 return num_entries - budget;
3759}
3760
3761int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3762 enum ib_cq_notify_flags ib_cqn_flags)
3763{
3764 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
05127665
SX		 3765 int type = 0, rc = 0;
3766 unsigned long flags;
1ac5a404 3767
05127665 3768 spin_lock_irqsave(&cq->cq_lock, flags);
1ac5a404
SX		 3769 /* Trigger on the very next completion */
3770 if (ib_cqn_flags & IB_CQ_NEXT_COMP)
b353ce55 3771 type = DBC_DBC_TYPE_CQ_ARMALL;
1ac5a404
SX		 3772 /* Trigger on the next solicited completion */
3773 else if (ib_cqn_flags & IB_CQ_SOLICITED)
b353ce55 3774 type = DBC_DBC_TYPE_CQ_ARMSE;
1ac5a404 3775
499e4569
SX		 3776 /* Poll to see if there are missed events */
3777 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
05127665
SX		 3778 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) {
3779 rc = 1;
3780 goto exit;
3781 }
1ac5a404
SX		 3782 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
3783
05127665
SX		 3784exit:
3785 spin_unlock_irqrestore(&cq->cq_lock, flags);
3786 return rc;
1ac5a404
SX		 3787}
3788
3789/* Memory Regions */
3790struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3791{
3792 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3793 struct bnxt_re_dev *rdev = pd->rdev;
3794 struct bnxt_re_mr *mr;
cb95709e 3795 u32 active_mrs;
1ac5a404
SX		 3796 int rc;
3797
3798 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3799 if (!mr)
3800 return ERR_PTR(-ENOMEM);
3801
3802 mr->rdev = rdev;
3803 mr->qplib_mr.pd = &pd->qplib_pd;
3804 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3805 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3806
3807 /* Allocate and register 0 as the address */
3808 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3809 if (rc)
3810 goto fail;
3811
3812 mr->qplib_mr.hwq.level = PBL_LVL_MAX;
3813 mr->qplib_mr.total_size = -1; /* Infinte length */
f6919d56 3814 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
872f3578 3815 PAGE_SIZE);
1ac5a404
SX		 3816 if (rc)
3817 goto fail_mr;
3818
3819 mr->ib_mr.lkey = mr->qplib_mr.lkey;
3820 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
3821 IB_ACCESS_REMOTE_ATOMIC))
3822 mr->ib_mr.rkey = mr->ib_mr.lkey;
cb95709e
CA		 3823 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
3824 if (active_mrs > rdev->stats.res.mr_watermark)
3825 rdev->stats.res.mr_watermark = active_mrs;
1ac5a404
SX		 3826
3827 return &mr->ib_mr;
3828
3829fail_mr:
3830 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3831fail:
3832 kfree(mr);
3833 return ERR_PTR(rc);
3834}
3835
c4367a26 3836int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
1ac5a404
SX		 3837{
3838 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3839 struct bnxt_re_dev *rdev = mr->rdev;
ebbd1dfb 3840 int rc;
1ac5a404 3841
1c980b01 3842 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
9a4467a6 3843 if (rc) {
6ccad848 3844 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
9a4467a6
SX		 3845 return rc;
3846 }
1c980b01 3847
1993519b 3848 if (mr->pages) {
1ac5a404
SX		 3849 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
3850 &mr->qplib_frpl);
3851 kfree(mr->pages);
3852 mr->npages = 0;
3853 mr->pages = NULL;
3854 }
836a0fbb 3855 ib_umem_release(mr->ib_umem);
1ac5a404
SX		 3856
3857 kfree(mr);
063975fe 3858 atomic_dec(&rdev->stats.res.mr_count);
1ac5a404
SX		 3859 return rc;
3860}
3861
3862static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
3863{
3864 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3865
3866 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
3867 return -ENOMEM;
3868
3869 mr->pages[mr->npages++] = addr;
3870 return 0;
3871}
3872
3873int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
3874 unsigned int *sg_offset)
3875{
3876 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3877
3878 mr->npages = 0;
3879 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
3880}
3881
3882struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
42a3b153 3883 u32 max_num_sg)
1ac5a404
SX		 3884{
3885 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3886 struct bnxt_re_dev *rdev = pd->rdev;
3887 struct bnxt_re_mr *mr = NULL;
cb95709e 3888 u32 active_mrs;
1ac5a404
SX		 3889 int rc;
3890
3891 if (type != IB_MR_TYPE_MEM_REG) {
6ccad848 3892 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
1ac5a404
SX		 3893 return ERR_PTR(-EINVAL);
3894 }
3895 if (max_num_sg > MAX_PBL_LVL_1_PGS)
3896 return ERR_PTR(-EINVAL);
3897
3898 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3899 if (!mr)
3900 return ERR_PTR(-ENOMEM);
3901
3902 mr->rdev = rdev;
3903 mr->qplib_mr.pd = &pd->qplib_pd;
3904 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
3905 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3906
3907 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3908 if (rc)
872f3578 3909 goto bail;
1ac5a404
SX		 3910
3911 mr->ib_mr.lkey = mr->qplib_mr.lkey;
3912 mr->ib_mr.rkey = mr->ib_mr.lkey;
3913
3914 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
3915 if (!mr->pages) {
3916 rc = -ENOMEM;
3917 goto fail;
3918 }
3919 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
3920 &mr->qplib_frpl, max_num_sg);
3921 if (rc) {
6ccad848
DS		 3922 ibdev_err(&rdev->ibdev,
3923 "Failed to allocate HW FR page list");
1ac5a404
SX		 3924 goto fail_mr;
3925 }
3926
cb95709e
CA		 3927 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
3928 if (active_mrs > rdev->stats.res.mr_watermark)
3929 rdev->stats.res.mr_watermark = active_mrs;
1ac5a404
SX		 3930 return &mr->ib_mr;
3931
3932fail_mr:
1ac5a404 3933 kfree(mr->pages);
872f3578
SK		 3934fail:
3935 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3936bail:
1ac5a404
SX		 3937 kfree(mr);
3938 return ERR_PTR(rc);
3939}
3940
9152e0b7
EW		 3941struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
3942 struct ib_udata *udata)
3943{
3944 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3945 struct bnxt_re_dev *rdev = pd->rdev;
3946 struct bnxt_re_mw *mw;
cb95709e 3947 u32 active_mws;
9152e0b7
EW		 3948 int rc;
3949
3950 mw = kzalloc(sizeof(*mw), GFP_KERNEL);
3951 if (!mw)
3952 return ERR_PTR(-ENOMEM);
3953 mw->rdev = rdev;
3954 mw->qplib_mw.pd = &pd->qplib_pd;
3955
3956 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
3957 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
3958 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
3959 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
3960 if (rc) {
6ccad848 3961 ibdev_err(&rdev->ibdev, "Allocate MW failed!");
9152e0b7
EW		 3962 goto fail;
3963 }
3964 mw->ib_mw.rkey = mw->qplib_mw.rkey;
3965
cb95709e
CA		 3966 active_mws = atomic_inc_return(&rdev->stats.res.mw_count);
3967 if (active_mws > rdev->stats.res.mw_watermark)
3968 rdev->stats.res.mw_watermark = active_mws;
9152e0b7
EW		 3969 return &mw->ib_mw;
3970
3971fail:
3972 kfree(mw);
3973 return ERR_PTR(rc);
3974}
3975
3976int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
3977{
3978 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
3979 struct bnxt_re_dev *rdev = mw->rdev;
3980 int rc;
3981
3982 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
3983 if (rc) {
6ccad848 3984 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc);
9152e0b7
EW		 3985 return rc;
3986 }
3987
3988 kfree(mw);
063975fe 3989 atomic_dec(&rdev->stats.res.mw_count);
9152e0b7
EW		 3990 return rc;
3991}
3992
0a30e59f
SV		 3993static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr,
3994 int mr_access_flags, struct ib_umem *umem)
1ac5a404
SX		 3995{
3996 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3997 struct bnxt_re_dev *rdev = pd->rdev;
84e71b4d 3998 unsigned long page_size;
0a30e59f 3999 struct bnxt_re_mr *mr;
84e71b4d 4000 int umem_pgs, rc;
cb95709e 4001 u32 active_mrs;
1ac5a404 4002
58d4a671 4003 if (length > BNXT_RE_MAX_MR_SIZE) {
6ccad848
DS		 4004 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
4005 length, BNXT_RE_MAX_MR_SIZE);
58d4a671
SX		 4006 return ERR_PTR(-ENOMEM);
4007 }
4008
0a30e59f
SV		 4009 page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr);
4010 if (!page_size) {
4011 ibdev_err(&rdev->ibdev, "umem page size unsupported!");
4012 return ERR_PTR(-EINVAL);
4013 }
4014
1ac5a404
SX		 4015 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4016 if (!mr)
4017 return ERR_PTR(-ENOMEM);
4018
4019 mr->rdev = rdev;
4020 mr->qplib_mr.pd = &pd->qplib_pd;
4021 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
4022 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
4023
872f3578
SK		 4024 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4025 if (rc) {
0a30e59f
SV		 4026 ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d", rc);
4027 rc = -EIO;
872f3578
SK		 4028 goto free_mr;
4029 }
4030 /* The fixed portion of the rkey is the same as the lkey */
4031 mr->ib_mr.rkey = mr->qplib_mr.rkey;
1ac5a404 4032 mr->ib_umem = umem;
1ac5a404 4033 mr->qplib_mr.va = virt_addr;
84e71b4d 4034 mr->qplib_mr.total_size = length;
3e7e1193 4035
84e71b4d 4036 umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
f6919d56
SX		 4037 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
4038 umem_pgs, page_size);
1ac5a404 4039 if (rc) {
0a30e59f
SV		 4040 ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc);
4041 rc = -EIO;
4042 goto free_mrw;
1ac5a404
SX		 4043 }
4044
1ac5a404
SX		 4045 mr->ib_mr.lkey = mr->qplib_mr.lkey;
4046 mr->ib_mr.rkey = mr->qplib_mr.lkey;
cb95709e
CA		 4047 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4048 if (active_mrs > rdev->stats.res.mr_watermark)
4049 rdev->stats.res.mr_watermark = active_mrs;
1ac5a404
SX		 4050
4051 return &mr->ib_mr;
0a30e59f 4052
1ac5a404
SX		 4053free_mrw:
4054 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
1ac5a404
SX		 4055free_mr:
4056 kfree(mr);
4057 return ERR_PTR(rc);
4058}
4059
0a30e59f
SV		 4060struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
4061 u64 virt_addr, int mr_access_flags,
4062 struct ib_udata *udata)
4063{
4064 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4065 struct bnxt_re_dev *rdev = pd->rdev;
4066 struct ib_umem *umem;
4067 struct ib_mr *ib_mr;
4068
4069 umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
4070 if (IS_ERR(umem))
4071 return ERR_CAST(umem);
4072
4073 ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4074 if (IS_ERR(ib_mr))
4075 ib_umem_release(umem);
4076 return ib_mr;
4077}
4078
4079struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
4080 u64 length, u64 virt_addr, int fd,
4081 int mr_access_flags, struct ib_udata *udata)
4082{
4083 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4084 struct bnxt_re_dev *rdev = pd->rdev;
4085 struct ib_umem_dmabuf *umem_dmabuf;
4086 struct ib_umem *umem;
4087 struct ib_mr *ib_mr;
4088
4089 umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length,
4090 fd, mr_access_flags);
4091 if (IS_ERR(umem_dmabuf))
4092 return ERR_CAST(umem_dmabuf);
4093
4094 umem = &umem_dmabuf->umem;
4095
4096 ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4097 if (IS_ERR(ib_mr))
4098 ib_umem_release(umem);
4099 return ib_mr;
4100}
4101
a2a074ef 4102int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
1ac5a404 4103{
a2a074ef
LR		 4104 struct ib_device *ibdev = ctx->device;
4105 struct bnxt_re_ucontext *uctx =
4106 container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
1ac5a404 4107 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
95b86d1c 4108 struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
24ce9478 4109 struct bnxt_re_user_mmap_entry *entry;
6becfe91 4110 struct bnxt_re_uctx_resp resp = {};
95b86d1c 4111 u32 chip_met_rev_num = 0;
1ac5a404
SX		 4112 int rc;
4113
6ccad848 4114 ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
1ac5a404 4115
72c6ec18 4116 if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
6ccad848
DS		 4117 ibdev_dbg(ibdev, " is different from the device %d ",
4118 BNXT_RE_ABI_VERSION);
a2a074ef 4119 return -EPERM;
1ac5a404
SX		 4120 }
4121
1ac5a404
SX		 4122 uctx->rdev = rdev;
4123
4124 uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
4125 if (!uctx->shpg) {
4126 rc = -ENOMEM;
4127 goto fail;
4128 }
4129 spin_lock_init(&uctx->sh_lock);
4130
a8714595 4131 resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
0cfb329d
DS		 4132 chip_met_rev_num = rdev->chip_ctx->chip_num;
4133 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
95b86d1c 4134 BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
0cfb329d 4135 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
95b86d1c
DS		 4136 BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
4137 resp.chip_id0 = chip_met_rev_num;
a7b36d5f 4138 /*Temp, Use xa_alloc instead */
95b86d1c 4139 resp.dev_id = rdev->en_dev->pdev->devfn;
1ac5a404
SX		 4140 resp.max_qp = rdev->qplib_ctx.qpc_count;
4141 resp.pg_size = PAGE_SIZE;
4142 resp.cqe_sz = sizeof(struct cq_base);
4143 resp.max_cqd = dev_attr->max_cq_wqes;
1ac5a404 4144
87974051
DS		 4145 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
4146 resp.mode = rdev->chip_ctx->modes.wqe_mode;
4147
360da60d
SX		 4148 if (rdev->chip_ctx->modes.db_push)
4149 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED;
4150
24ce9478
SX		 4151 entry = bnxt_re_mmap_entry_insert(uctx, 0, BNXT_RE_MMAP_SH_PAGE, NULL);
4152 if (!entry) {
4153 rc = -ENOMEM;
4154 goto cfail;
4155 }
4156 uctx->shpage_mmap = &entry->rdma_entry;
fa8fad92
CA		 4157 if (rdev->pacing.dbr_pacing)
4158 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;
24ce9478 4159
95b86d1c 4160 rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
1ac5a404 4161 if (rc) {
6ccad848 4162 ibdev_err(ibdev, "Failed to copy user context");
1ac5a404
SX		 4163 rc = -EFAULT;
4164 goto cfail;
4165 }
4166
a2a074ef 4167 return 0;
1ac5a404
SX		 4168cfail:
4169 free_page((unsigned long)uctx->shpg);
4170 uctx->shpg = NULL;
4171fail:
a2a074ef 4172 return rc;
1ac5a404
SX		 4173}
4174
a2a074ef 4175void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
1ac5a404
SX		 4176{
4177 struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4178 struct bnxt_re_ucontext,
4179 ib_uctx);
b3b2c7c5
DS		 4180
4181 struct bnxt_re_dev *rdev = uctx->rdev;
b3b2c7c5 4182
24ce9478
SX		 4183 rdma_user_mmap_entry_remove(uctx->shpage_mmap);
4184 uctx->shpage_mmap = NULL;
1ac5a404
SX		 4185 if (uctx->shpg)
4186 free_page((unsigned long)uctx->shpg);
b3b2c7c5
DS		 4187
4188 if (uctx->dpi.dbr) {
4189 /* Free DPI only if this is the first PD allocated by the
4190 * application and mark the context dpi as NULL
4191 */
0ac20faf 4192 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->dpi);
b3b2c7c5
DS		 4193 uctx->dpi.dbr = NULL;
4194 }
1ac5a404
SX		 4195}
4196
4197/* Helper function to mmap the virtual memory from user app */
4198int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
4199{
4200 struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4201 struct bnxt_re_ucontext,
4202 ib_uctx);
24ce9478
SX		 4203 struct bnxt_re_user_mmap_entry *bnxt_entry;
4204 struct rdma_user_mmap_entry *rdma_entry;
4205 int ret = 0;
1ac5a404
SX		 4206 u64 pfn;
4207
24ce9478
SX		 4208 rdma_entry = rdma_user_mmap_entry_get(&uctx->ib_uctx, vma);
4209 if (!rdma_entry)
1ac5a404
SX		 4210 return -EINVAL;
4211
24ce9478
SX		 4212 bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4213 rdma_entry);
4214
4215 switch (bnxt_entry->mmap_flag) {
360da60d
SX		 4216 case BNXT_RE_MMAP_WC_DB:
4217 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4218 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4219 pgprot_writecombine(vma->vm_page_prot),
4220 rdma_entry);
4221 break;
24ce9478
SX		 4222 case BNXT_RE_MMAP_UC_DB:
4223 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4224 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4225 pgprot_noncached(vma->vm_page_prot),
4226 rdma_entry);
4227 break;
4228 case BNXT_RE_MMAP_SH_PAGE:
4229 ret = vm_insert_page(vma, vma->vm_start, virt_to_page(uctx->shpg));
4230 break;
ea222485
CA		 4231 case BNXT_RE_MMAP_DBR_BAR:
4232 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4233 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4234 pgprot_noncached(vma->vm_page_prot),
4235 rdma_entry);
4236 break;
4237 case BNXT_RE_MMAP_DBR_PAGE:
4238 /* Driver doesn't expect write access for user space */
4239 if (vma->vm_flags & VM_WRITE)
4240 return -EFAULT;
4241 ret = vm_insert_page(vma, vma->vm_start,
4242 virt_to_page((void *)bnxt_entry->mem_offset));
4243 break;
24ce9478
SX		 4244 default:
4245 ret = -EINVAL;
4246 break;
1ac5a404
SX		 4247 }
4248
24ce9478
SX		 4249 rdma_user_mmap_entry_put(rdma_entry);
4250 return ret;
4251}
4252
4253void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
4254{
4255 struct bnxt_re_user_mmap_entry *bnxt_entry;
4256
4257 bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4258 rdma_entry);
4259
4260 kfree(bnxt_entry);
1ac5a404 4261}
360da60d 4262
61a8118f
CA		 4263static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)
4264{
4265 struct bnxt_re_ucontext *uctx;
4266
4267 uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4268 bnxt_re_pacing_alert(uctx->rdev);
4269 return 0;
4270}
4271
360da60d
SX		 4272static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)
4273{
4274 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
4275 enum bnxt_re_alloc_page_type alloc_type;
4276 struct bnxt_re_user_mmap_entry *entry;
4277 enum bnxt_re_mmap_flag mmap_flag;
4278 struct bnxt_qplib_chip_ctx *cctx;
4279 struct bnxt_re_ucontext *uctx;
4280 struct bnxt_re_dev *rdev;
4281 u64 mmap_offset;
4282 u32 length;
4283 u32 dpi;
ea222485 4284 u64 addr;
360da60d
SX		 4285 int err;
4286
4287 uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4288 if (IS_ERR(uctx))
4289 return PTR_ERR(uctx);
4290
4291 err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);
4292 if (err)
4293 return err;
4294
4295 rdev = uctx->rdev;
4296 cctx = rdev->chip_ctx;
4297
4298 switch (alloc_type) {
4299 case BNXT_RE_ALLOC_WC_PAGE:
4300 if (cctx->modes.db_push) {
4301 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi,
4302 uctx, BNXT_QPLIB_DPI_TYPE_WC))
4303 return -ENOMEM;
4304 length = PAGE_SIZE;
4305 dpi = uctx->wcdpi.dpi;
ea222485 4306 addr = (u64)uctx->wcdpi.umdbr;
360da60d
SX		 4307 mmap_flag = BNXT_RE_MMAP_WC_DB;
4308 } else {
4309 return -EINVAL;
4310 }
4311
4312 break;
ea222485
CA		 4313 case BNXT_RE_ALLOC_DBR_BAR_PAGE:
4314 length = PAGE_SIZE;
4315 addr = (u64)rdev->pacing.dbr_bar_addr;
4316 mmap_flag = BNXT_RE_MMAP_DBR_BAR;
4317 break;
4318
4319 case BNXT_RE_ALLOC_DBR_PAGE:
4320 length = PAGE_SIZE;
4321 addr = (u64)rdev->pacing.dbr_page;
4322 mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
4323 break;
360da60d
SX		 4324
4325 default:
4326 return -EOPNOTSUPP;
4327 }
4328
ea222485 4329 entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset);
4251f631
DC		 4330 if (!entry)
4331 return -ENOMEM;
360da60d
SX		 4332
4333 uobj->object = entry;
4334 uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
4335 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4336 &mmap_offset, sizeof(mmap_offset));
4337 if (err)
4338 return err;
4339
4340 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4341 &length, sizeof(length));
4342 if (err)
4343 return err;
4344
4345 err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI,
4346 &dpi, sizeof(length));
4347 if (err)
4348 return err;
4349
4350 return 0;
4351}
4352
4353static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
4354 enum rdma_remove_reason why,
4355 struct uverbs_attr_bundle *attrs)
4356{
4357 struct bnxt_re_user_mmap_entry *entry = uobject->object;
4358 struct bnxt_re_ucontext *uctx = entry->uctx;
4359
4360 switch (entry->mmap_flag) {
4361 case BNXT_RE_MMAP_WC_DB:
4362 if (uctx && uctx->wcdpi.dbr) {
4363 struct bnxt_re_dev *rdev = uctx->rdev;
4364
4365 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
4366 uctx->wcdpi.dbr = NULL;
4367 }
4368 break;
ea222485
CA		 4369 case BNXT_RE_MMAP_DBR_BAR:
4370 case BNXT_RE_MMAP_DBR_PAGE:
4371 break;
360da60d
SX		 4372 default:
4373 goto exit;
4374 }
4375 rdma_user_mmap_entry_remove(&entry->rdma_entry);
4376exit:
4377 return 0;
4378}
4379
4380DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,
4381 UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,
4382 BNXT_RE_OBJECT_ALLOC_PAGE,
4383 UVERBS_ACCESS_NEW,
4384 UA_MANDATORY),
4385 UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,
4386 enum bnxt_re_alloc_page_type,
4387 UA_MANDATORY),
4388 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4389 UVERBS_ATTR_TYPE(u64),
4390 UA_MANDATORY),
4391 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4392 UVERBS_ATTR_TYPE(u32),
4393 UA_MANDATORY),
4394 UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,
4395 UVERBS_ATTR_TYPE(u32),
4396 UA_MANDATORY));
4397
4398DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,
4399 UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,
4400 BNXT_RE_OBJECT_ALLOC_PAGE,
4401 UVERBS_ACCESS_DESTROY,
4402 UA_MANDATORY));
4403
4404DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,
4405 UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),
4406 &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),
4407 &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));
4408
61a8118f
CA		 4409DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);
4410
4411DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,
4412 &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));
4413
360da60d
SX		 4414const struct uapi_definition bnxt_re_uapi_defs[] = {
4415 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
61a8118f 4416 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
360da60d
SX		 4417 {}
4418};
Linux 6.x block layer and io_uring tree(s)