projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/mlx5: Issue FW command to destroy SRQ on reentry
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
ff23dfa1 45#include <rdma/uverbs_ioctl.h>
ac1b36e5 46
be086e7c
MY		 47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
99d195cc 53#include "qedr_roce_cm.h"
82af6d19 54#include "qedr_iw_cm.h"
ac1b36e5 55
3491c9e7
YB		 56#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57#define RDMA_MAX_SGE_PER_SRQ (4)
58#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
a7efd777
RA		 60#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
4c6bb02d
MK		 62enum {
63 QEDR_USER_MMAP_IO_WC = 0,
97f61250 64 QEDR_USER_MMAP_PHYS_PAGE,
4c6bb02d
MK		 65};
66
c75d3ec8
AR		 67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 size_t len)
69{
70 size_t min_len = min_t(size_t, len, udata->outlen);
71
72 return ib_copy_to_udata(udata, src, min_len);
73}
74
a7efd777
RA		 75int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
76{
dbe30dae 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 78 return -EINVAL;
79
80 *pkey = QEDR_ROCE_PKEY_DEFAULT;
81 return 0;
82}
83
e6a38c54
KM		 84int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
85 int index, union ib_gid *sgid)
86{
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89 memset(sgid->raw, 0, sizeof(sgid->raw));
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 sgid->global.interface_id, sgid->global.subnet_prefix);
94
95 return 0;
96}
97
3491c9e7
YB		 98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99{
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
102 struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104 srq_attr->srq_limit = srq->srq_limit;
105 srq_attr->max_wr = qattr->max_srq_wr;
106 srq_attr->max_sge = qattr->max_sge;
107
108 return 0;
109}
110
ac1b36e5
RA		 111int qedr_query_device(struct ib_device *ibdev,
112 struct ib_device_attr *attr, struct ib_udata *udata)
113{
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
116
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
119 "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 dev->rdma_ctx);
121 return -EINVAL;
122 }
123
124 memset(attr, 0, sizeof(*attr));
125
126 attr->fw_ver = qattr->fw_ver;
127 attr->sys_image_guid = qattr->sys_image_guid;
128 attr->max_mr_size = qattr->max_mr_size;
129 attr->page_size_cap = qattr->page_size_caps;
130 attr->vendor_id = qattr->vendor_id;
131 attr->vendor_part_id = qattr->vendor_part_id;
132 attr->hw_ver = qattr->hw_ver;
133 attr->max_qp = qattr->max_qp;
134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 IB_DEVICE_RC_RNR_NAK_GEN |
137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
138
33023fb8
SW		 139 attr->max_send_sge = qattr->max_sge;
140 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 141 attr->max_sge_rd = qattr->max_sge;
142 attr->max_cq = qattr->max_cq;
143 attr->max_cqe = qattr->max_cqe;
144 attr->max_mr = qattr->max_mr;
145 attr->max_mw = qattr->max_mw;
146 attr->max_pd = qattr->max_pd;
147 attr->atomic_cap = dev->atomic_cap;
ac1b36e5
RA		 148 attr->max_qp_init_rd_atom =
149 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
150 attr->max_qp_rd_atom =
151 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
152 attr->max_qp_init_rd_atom);
153
154 attr->max_srq = qattr->max_srq;
155 attr->max_srq_sge = qattr->max_srq_sge;
156 attr->max_srq_wr = qattr->max_srq_wr;
157
158 attr->local_ca_ack_delay = qattr->dev_ack_delay;
159 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
7d11b478 160 attr->max_pkeys = qattr->max_pkey;
ac1b36e5
RA		 161 attr->max_ah = qattr->max_ah;
162
163 return 0;
164}
165
ac1b36e5
RA		 166static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
167 u8 *ib_width)
168{
169 switch (speed) {
170 case 1000:
69054666 171 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 172 *ib_width = IB_WIDTH_1X;
173 break;
174 case 10000:
69054666 175 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 176 *ib_width = IB_WIDTH_1X;
177 break;
178
179 case 20000:
69054666 180 *ib_speed = IB_SPEED_DDR;
ac1b36e5
RA		 181 *ib_width = IB_WIDTH_4X;
182 break;
183
184 case 25000:
69054666 185 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 186 *ib_width = IB_WIDTH_1X;
187 break;
188
189 case 40000:
69054666 190 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 191 *ib_width = IB_WIDTH_4X;
192 break;
193
194 case 50000:
69054666
SO		 195 *ib_speed = IB_SPEED_HDR;
196 *ib_width = IB_WIDTH_1X;
ac1b36e5
RA		 197 break;
198
199 case 100000:
69054666 200 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 201 *ib_width = IB_WIDTH_4X;
202 break;
203
204 default:
205 /* Unsupported */
69054666 206 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 207 *ib_width = IB_WIDTH_1X;
208 }
209}
210
211int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
212{
213 struct qedr_dev *dev;
214 struct qed_rdma_port *rdma_port;
215
216 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 217
218 if (!dev->rdma_ctx) {
219 DP_ERR(dev, "rdma_ctx is NULL\n");
220 return -EINVAL;
221 }
222
223 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 224
c4550c63 225 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 226 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
227 attr->state = IB_PORT_ACTIVE;
72a7720f 228 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
ac1b36e5
RA		 229 } else {
230 attr->state = IB_PORT_DOWN;
72a7720f 231 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
ac1b36e5
RA		 232 }
233 attr->max_mtu = IB_MTU_4096;
234 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
235 attr->lid = 0;
236 attr->lmc = 0;
237 attr->sm_lid = 0;
238 attr->sm_sl = 0;
2f944c0f 239 attr->ip_gids = true;
f5b1b177
KM		 240 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
241 attr->gid_tbl_len = 1;
f5b1b177
KM		 242 } else {
243 attr->gid_tbl_len = QEDR_MAX_SGID;
244 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
245 }
ac1b36e5
RA		 246 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
247 attr->qkey_viol_cntr = 0;
248 get_link_speed_and_width(rdma_port->link_speed,
249 &attr->active_speed, &attr->active_width);
250 attr->max_msg_sz = rdma_port->max_msg_size;
251 attr->max_vl_num = 4;
252
253 return 0;
254}
255
a2a074ef 256int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 257{
a2a074ef 258 struct ib_device *ibdev = uctx->device;
ac1b36e5 259 int rc;
a2a074ef
LR		 260 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
261 struct qedr_alloc_ucontext_resp uresp = {};
97f61250 262 struct qedr_alloc_ucontext_req ureq = {};
ac1b36e5
RA		 263 struct qedr_dev *dev = get_qedr_dev(ibdev);
264 struct qed_rdma_add_user_out_params oparams;
4c6bb02d 265 struct qedr_user_mmap_entry *entry;
ac1b36e5
RA		 266
267 if (!udata)
a2a074ef 268 return -EFAULT;
ac1b36e5 269
97f61250
MK		 270 if (udata->inlen) {
271 rc = ib_copy_from_udata(&ureq, udata,
272 min(sizeof(ureq), udata->inlen));
273 if (rc) {
274 DP_ERR(dev, "Problem copying data from user space\n");
275 return -EFAULT;
276 }
bbe4f424
MK		 277 ctx->edpm_mode = !!(ureq.context_flags &
278 QEDR_ALLOC_UCTX_EDPM_MODE);
97f61250
MK		 279 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
280 }
281
ac1b36e5
RA		 282 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
283 if (rc) {
284 DP_ERR(dev,
285 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
286 rc);
a2a074ef 287 return rc;
ac1b36e5
RA		 288 }
289
290 ctx->dpi = oparams.dpi;
291 ctx->dpi_addr = oparams.dpi_addr;
292 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
293 ctx->dpi_size = oparams.dpi_size;
4c6bb02d
MK		 294 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
295 if (!entry) {
296 rc = -ENOMEM;
297 goto err;
298 }
299
300 entry->io_address = ctx->dpi_phys_addr;
301 entry->length = ctx->dpi_size;
302 entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
303 entry->dpi = ctx->dpi;
304 entry->dev = dev;
305 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
306 ctx->dpi_size);
307 if (rc) {
308 kfree(entry);
309 goto err;
310 }
311 ctx->db_mmap_entry = &entry->rdma_entry;
ac1b36e5 312
93a3d05f
MK		 313 if (!dev->user_dpm_enabled)
314 uresp.dpm_flags = 0;
315 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
316 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
317 else
318 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
bbe4f424
MK		 319 QEDR_DPM_TYPE_ROCE_LEGACY |
320 QEDR_DPM_TYPE_ROCE_EDPM_MODE;
93a3d05f 321
eb7f84e3
MK		 322 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
323 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
324 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
325 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
326 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
327 }
93a3d05f 328
67cbe353
AR		 329 uresp.wids_enabled = 1;
330 uresp.wid_count = oparams.wid_count;
4c6bb02d 331 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
ac1b36e5
RA		 332 uresp.db_size = ctx->dpi_size;
333 uresp.max_send_wr = dev->attr.max_sqe;
334 uresp.max_recv_wr = dev->attr.max_rqe;
335 uresp.max_srq_wr = dev->attr.max_srq_wr;
336 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
337 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
338 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
339 uresp.max_cqes = QEDR_MAX_CQES;
340
c75d3ec8 341 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 342 if (rc)
4c6bb02d 343 goto err;
ac1b36e5
RA		 344
345 ctx->dev = dev;
346
ac1b36e5
RA		 347 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
348 &ctx->ibucontext);
a2a074ef 349 return 0;
4c6bb02d
MK		 350
351err:
352 if (!ctx->db_mmap_entry)
353 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
354 else
355 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
356
357 return rc;
ac1b36e5
RA		 358}
359
a2a074ef 360void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 361{
362 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
ac1b36e5
RA		 363
364 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
365 uctx);
4c6bb02d
MK		 366
367 rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
ac1b36e5
RA		 368}
369
4c6bb02d 370void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
ac1b36e5 371{
4c6bb02d
MK		 372 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
373 struct qedr_dev *dev = entry->dev;
ac1b36e5 374
97f61250
MK		 375 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
376 free_page((unsigned long)entry->address);
377 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
4c6bb02d 378 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
30bf066c 379
4c6bb02d
MK		 380 kfree(entry);
381}
ac1b36e5 382
4c6bb02d
MK		 383int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
384{
385 struct ib_device *dev = ucontext->device;
386 size_t length = vma->vm_end - vma->vm_start;
387 struct rdma_user_mmap_entry *rdma_entry;
388 struct qedr_user_mmap_entry *entry;
389 int rc = 0;
390 u64 pfn;
391
392 ibdev_dbg(dev,
393 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
394 vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
395
396 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
397 if (!rdma_entry) {
398 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
399 vma->vm_pgoff);
ac1b36e5
RA		 400 return -EINVAL;
401 }
4c6bb02d
MK		 402 entry = get_qedr_mmap_entry(rdma_entry);
403 ibdev_dbg(dev,
404 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
405 entry->io_address, length, entry->mmap_flag);
406
407 switch (entry->mmap_flag) {
408 case QEDR_USER_MMAP_IO_WC:
409 pfn = entry->io_address >> PAGE_SHIFT;
410 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
411 pgprot_writecombine(vma->vm_page_prot),
412 rdma_entry);
413 break;
97f61250
MK		 414 case QEDR_USER_MMAP_PHYS_PAGE:
415 rc = vm_insert_page(vma, vma->vm_start,
416 virt_to_page(entry->address));
417 break;
4c6bb02d
MK		 418 default:
419 rc = -EINVAL;
30bf066c 420 }
ac1b36e5 421
4c6bb02d
MK		 422 if (rc)
423 ibdev_dbg(dev,
424 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
425 entry->io_address, length, entry->mmap_flag, rc);
30bf066c 426
4c6bb02d
MK		 427 rdma_user_mmap_entry_put(rdma_entry);
428 return rc;
ac1b36e5 429}
a7efd777 430
ff23dfa1 431int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777 432{
21a428a0 433 struct ib_device *ibdev = ibpd->device;
a7efd777 434 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 435 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 436 u16 pd_id;
437 int rc;
438
439 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
ff23dfa1 440 udata ? "User Lib" : "Kernel");
a7efd777
RA		 441
442 if (!dev->rdma_ctx) {
847cb1a3 443 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 444 return -EINVAL;
a7efd777
RA		 445 }
446
9c1e0228
RA		 447 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
448 if (rc)
21a428a0 449 return rc;
a7efd777 450
a7efd777
RA		 451 pd->pd_id = pd_id;
452
ff23dfa1 453 if (udata) {
57939021
JG		 454 struct qedr_alloc_pd_uresp uresp = {
455 .pd_id = pd_id,
456 };
ff23dfa1
SR		 457 struct qedr_ucontext *context = rdma_udata_to_drv_context(
458 udata, struct qedr_ucontext, ibucontext);
9c1e0228 459
c75d3ec8 460 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 461 if (rc) {
a7efd777 462 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 463 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 464 return rc;
9c1e0228
RA		 465 }
466
ff23dfa1 467 pd->uctx = context;
9c1e0228 468 pd->uctx->pd = pd;
a7efd777
RA		 469 }
470
21a428a0 471 return 0;
a7efd777
RA		 472}
473
91a7c58f 474int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 475{
476 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
477 struct qedr_pd *pd = get_qedr_pd(ibpd);
478
a7efd777
RA		 479 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
480 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
91a7c58f 481 return 0;
a7efd777
RA		 482}
483
484static void qedr_free_pbl(struct qedr_dev *dev,
485 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
486{
487 struct pci_dev *pdev = dev->pdev;
488 int i;
489
490 for (i = 0; i < pbl_info->num_pbls; i++) {
491 if (!pbl[i].va)
492 continue;
493 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
494 pbl[i].va, pbl[i].pa);
495 }
496
497 kfree(pbl);
498}
499
500#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
501#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
502
503#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
504#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
505#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
506
507static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
508 struct qedr_pbl_info *pbl_info,
509 gfp_t flags)
510{
511 struct pci_dev *pdev = dev->pdev;
512 struct qedr_pbl *pbl_table;
513 dma_addr_t *pbl_main_tbl;
514 dma_addr_t pa;
515 void *va;
516 int i;
517
518 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
519 if (!pbl_table)
520 return ERR_PTR(-ENOMEM);
521
522 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 523 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
524 flags);
a7efd777
RA		 525 if (!va)
526 goto err;
527
a7efd777
RA		 528 pbl_table[i].va = va;
529 pbl_table[i].pa = pa;
530 }
531
532 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
533 * the first one with physical pointers to all of the rest
534 */
535 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
536 for (i = 0; i < pbl_info->num_pbls - 1; i++)
537 pbl_main_tbl[i] = pbl_table[i + 1].pa;
538
539 return pbl_table;
540
541err:
542 for (i--; i >= 0; i--)
543 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
544 pbl_table[i].va, pbl_table[i].pa);
545
546 qedr_free_pbl(dev, pbl_info, pbl_table);
547
548 return ERR_PTR(-ENOMEM);
549}
550
551static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
552 struct qedr_pbl_info *pbl_info,
553 u32 num_pbes, int two_layer_capable)
554{
555 u32 pbl_capacity;
556 u32 pbl_size;
557 u32 num_pbls;
558
559 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
560 if (num_pbes > MAX_PBES_TWO_LAYER) {
561 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
562 num_pbes);
563 return -EINVAL;
564 }
565
566 /* calculate required pbl page size */
567 pbl_size = MIN_FW_PBL_PAGE_SIZE;
568 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
569 NUM_PBES_ON_PAGE(pbl_size);
570
571 while (pbl_capacity < num_pbes) {
572 pbl_size *= 2;
573 pbl_capacity = pbl_size / sizeof(u64);
574 pbl_capacity = pbl_capacity * pbl_capacity;
575 }
576
577 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
578 num_pbls++; /* One for the layer0 ( points to the pbls) */
579 pbl_info->two_layered = true;
580 } else {
581 /* One layered PBL */
582 num_pbls = 1;
583 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
584 roundup_pow_of_two((num_pbes * sizeof(u64))));
585 pbl_info->two_layered = false;
586 }
587
588 pbl_info->num_pbls = num_pbls;
589 pbl_info->pbl_size = pbl_size;
590 pbl_info->num_pbes = num_pbes;
591
592 DP_DEBUG(dev, QEDR_MSG_MR,
593 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
594 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
595
596 return 0;
597}
598
599static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
600 struct qedr_pbl *pbl,
e57bb6be 601 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 602{
95ad233f 603 int pbe_cnt, total_num_pbes = 0;
e57bb6be 604 u32 fw_pg_cnt, fw_pg_per_umem_pg;
a7efd777 605 struct qedr_pbl *pbl_tbl;
95ad233f 606 struct sg_dma_page_iter sg_iter;
a7efd777 607 struct regpair *pbe;
e57bb6be 608 u64 pg_addr;
a7efd777
RA		 609
610 if (!pbl_info->num_pbes)
611 return;
612
613 /* If we have a two layered pbl, the first pbl points to the rest
614 * of the pbls and the first entry lays on the second pbl in the table
615 */
616 if (pbl_info->two_layered)
617 pbl_tbl = &pbl[1];
618 else
619 pbl_tbl = pbl;
620
621 pbe = (struct regpair *)pbl_tbl->va;
622 if (!pbe) {
623 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
624 return;
625 }
626
627 pbe_cnt = 0;
628
95ad233f
SS		 629 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
630
631 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
632 pg_addr = sg_page_iter_dma_address(&sg_iter);
633 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
634 pbe->lo = cpu_to_le32(pg_addr);
635 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
636
637 pg_addr += BIT(pg_shift);
638 pbe_cnt++;
639 total_num_pbes++;
640 pbe++;
e57bb6be 641
95ad233f
SS		 642 if (total_num_pbes == pbl_info->num_pbes)
643 return;
644
645 /* If the given pbl is full storing the pbes,
646 * move to next pbl.
647 */
648 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
649 pbl_tbl++;
650 pbe = (struct regpair *)pbl_tbl->va;
651 pbe_cnt = 0;
a7efd777 652 }
95ad233f
SS		 653
654 fw_pg_cnt++;
a7efd777
RA		 655 }
656 }
657}
658
97f61250
MK		 659static int qedr_db_recovery_add(struct qedr_dev *dev,
660 void __iomem *db_addr,
661 void *db_data,
662 enum qed_db_rec_width db_width,
663 enum qed_db_rec_space db_space)
664{
665 if (!db_data) {
666 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
667 return 0;
668 }
669
670 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
671 db_width, db_space);
672}
673
674static void qedr_db_recovery_del(struct qedr_dev *dev,
675 void __iomem *db_addr,
676 void *db_data)
677{
678 if (!db_data) {
679 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
680 return;
681 }
682
683 /* Ignore return code as there is not much we can do about it. Error
684 * log will be printed inside.
685 */
686 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
687}
688
a7efd777 689static int qedr_copy_cq_uresp(struct qedr_dev *dev,
97f61250
MK		 690 struct qedr_cq *cq, struct ib_udata *udata,
691 u32 db_offset)
a7efd777
RA		 692{
693 struct qedr_create_cq_uresp uresp;
694 int rc;
695
696 memset(&uresp, 0, sizeof(uresp));
697
97f61250 698 uresp.db_offset = db_offset;
a7efd777 699 uresp.icid = cq->icid;
a25984f3
MK		 700 if (cq->q.db_mmap_entry)
701 uresp.db_rec_addr =
702 rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
a7efd777 703
c75d3ec8 704 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 705 if (rc)
706 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
707
708 return rc;
709}
710
711static void consume_cqe(struct qedr_cq *cq)
712{
713 if (cq->latest_cqe == cq->toggle_cqe)
714 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
715
716 cq->latest_cqe = qed_chain_consume(&cq->pbl);
717}
718
719static inline int qedr_align_cq_entries(int entries)
720{
721 u64 size, aligned_size;
722
723 /* We allocate an extra entry that we don't report to the FW. */
724 size = (entries + 1) * QEDR_CQE_SIZE;
725 aligned_size = ALIGN(size, PAGE_SIZE);
726
727 return aligned_size / QEDR_CQE_SIZE;
728}
729
97f61250
MK		 730static int qedr_init_user_db_rec(struct ib_udata *udata,
731 struct qedr_dev *dev, struct qedr_userq *q,
732 bool requires_db_rec)
733{
734 struct qedr_ucontext *uctx =
735 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
736 ibucontext);
737 struct qedr_user_mmap_entry *entry;
738 int rc;
739
740 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
741 if (requires_db_rec == 0 || !uctx->db_rec)
742 return 0;
743
744 /* Allocate a page for doorbell recovery, add to mmap */
745 q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
746 if (!q->db_rec_data) {
747 DP_ERR(dev, "get_zeroed_page failed\n");
748 return -ENOMEM;
749 }
750
751 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
752 if (!entry)
753 goto err_free_db_data;
754
755 entry->address = q->db_rec_data;
756 entry->length = PAGE_SIZE;
757 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
758 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
759 &entry->rdma_entry,
760 PAGE_SIZE);
761 if (rc)
762 goto err_free_entry;
763
764 q->db_mmap_entry = &entry->rdma_entry;
765
766 return 0;
767
768err_free_entry:
769 kfree(entry);
770
771err_free_db_data:
772 free_page((unsigned long)q->db_rec_data);
773 q->db_rec_data = NULL;
774 return -ENOMEM;
775}
776
b0ea0fa5 777static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 778 struct qedr_dev *dev,
b0ea0fa5 779 struct qedr_userq *q, u64 buf_addr,
97f61250 780 size_t buf_len, bool requires_db_rec,
72b894b0 781 int access,
69ad0e7f 782 int alloc_and_init)
a7efd777 783{
e57bb6be 784 u32 fw_pages;
a7efd777
RA		 785 int rc;
786
787 q->buf_addr = buf_addr;
788 q->buf_len = buf_len;
c320e527 789 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
a7efd777
RA		 790 if (IS_ERR(q->umem)) {
791 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
792 PTR_ERR(q->umem));
793 return PTR_ERR(q->umem);
794 }
795
e57bb6be 796 fw_pages = ib_umem_page_count(q->umem) <<
95ad233f 797 (PAGE_SHIFT - FW_PAGE_SHIFT);
e57bb6be
RA		 798
799 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 800 if (rc)
801 goto err0;
802
69ad0e7f
KM		 803 if (alloc_and_init) {
804 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
805 if (IS_ERR(q->pbl_tbl)) {
806 rc = PTR_ERR(q->pbl_tbl);
807 goto err0;
808 }
e57bb6be
RA		 809 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
810 FW_PAGE_SHIFT);
69ad0e7f
KM		 811 } else {
812 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 813 if (!q->pbl_tbl) {
814 rc = -ENOMEM;
69ad0e7f 815 goto err0;
89fd2576 816 }
69ad0e7f 817 }
a7efd777 818
97f61250
MK		 819 /* mmap the user address used to store doorbell data for recovery */
820 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
a7efd777
RA		 821
822err0:
823 ib_umem_release(q->umem);
69ad0e7f 824 q->umem = NULL;
a7efd777
RA		 825
826 return rc;
827}
828
829static inline void qedr_init_cq_params(struct qedr_cq *cq,
830 struct qedr_ucontext *ctx,
831 struct qedr_dev *dev, int vector,
832 int chain_entries, int page_cnt,
833 u64 pbl_ptr,
834 struct qed_rdma_create_cq_in_params
835 *params)
836{
837 memset(params, 0, sizeof(*params));
838 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
839 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
840 params->cnq_id = vector;
841 params->cq_size = chain_entries - 1;
842 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
843 params->pbl_num_pages = page_cnt;
844 params->pbl_ptr = pbl_ptr;
845 params->pbl_two_level = 0;
846}
847
848static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
849{
a7efd777
RA		 850 cq->db.data.agg_flags = flags;
851 cq->db.data.value = cpu_to_le32(cons);
852 writeq(cq->db.raw, cq->db_addr);
a7efd777
RA		 853}
854
855int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
856{
857 struct qedr_cq *cq = get_qedr_cq(ibcq);
858 unsigned long sflags;
4dd72636
AR		 859 struct qedr_dev *dev;
860
861 dev = get_qedr_dev(ibcq->device);
862
863 if (cq->destroyed) {
864 DP_ERR(dev,
865 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
866 cq, cq->icid);
867 return -EINVAL;
868 }
869
a7efd777
RA		 870
871 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
872 return 0;
873
874 spin_lock_irqsave(&cq->cq_lock, sflags);
875
876 cq->arm_flags = 0;
877
878 if (flags & IB_CQ_SOLICITED)
879 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
880
881 if (flags & IB_CQ_NEXT_COMP)
882 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
883
884 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
885
886 spin_unlock_irqrestore(&cq->cq_lock, sflags);
887
888 return 0;
889}
890
e39afe3d
LR		 891int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
892 struct ib_udata *udata)
a7efd777 893{
e39afe3d 894 struct ib_device *ibdev = ibcq->device;
ff23dfa1
SR		 895 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
896 udata, struct qedr_ucontext, ibucontext);
a7efd777
RA		 897 struct qed_rdma_destroy_cq_out_params destroy_oparams;
898 struct qed_rdma_destroy_cq_in_params destroy_iparams;
b6db3f71
AL		 899 struct qed_chain_init_params chain_params = {
900 .mode = QED_CHAIN_MODE_PBL,
901 .intended_use = QED_CHAIN_USE_TO_CONSUME,
902 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
903 .elem_size = sizeof(union rdma_cqe),
904 };
a7efd777
RA		 905 struct qedr_dev *dev = get_qedr_dev(ibdev);
906 struct qed_rdma_create_cq_in_params params;
e39afe3d 907 struct qedr_create_cq_ureq ureq = {};
a7efd777
RA		 908 int vector = attr->comp_vector;
909 int entries = attr->cqe;
e39afe3d 910 struct qedr_cq *cq = get_qedr_cq(ibcq);
a7efd777 911 int chain_entries;
97f61250 912 u32 db_offset;
a7efd777
RA		 913 int page_cnt;
914 u64 pbl_ptr;
915 u16 icid;
916 int rc;
917
918 DP_DEBUG(dev, QEDR_MSG_INIT,
919 "create_cq: called from %s. entries=%d, vector=%d\n",
920 udata ? "User Lib" : "Kernel", entries, vector);
921
922 if (entries > QEDR_MAX_CQES) {
923 DP_ERR(dev,
924 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
925 entries, QEDR_MAX_CQES);
e39afe3d 926 return -EINVAL;
a7efd777
RA		 927 }
928
929 chain_entries = qedr_align_cq_entries(entries);
930 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
b6db3f71 931 chain_params.num_elems = chain_entries;
a7efd777 932
97f61250
MK		 933 /* calc db offset. user will add DPI base, kernel will add db addr */
934 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
935
a7efd777 936 if (udata) {
97f61250
MK		 937 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
938 udata->inlen))) {
a7efd777
RA		 939 DP_ERR(dev,
940 "create cq: problem copying data from user space\n");
941 goto err0;
942 }
943
944 if (!ureq.len) {
945 DP_ERR(dev,
946 "create cq: cannot create a cq with 0 entries\n");
947 goto err0;
948 }
949
950 cq->cq_type = QEDR_CQ_TYPE_USER;
951
b0ea0fa5 952 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
72b894b0
CH		 953 ureq.len, true, IB_ACCESS_LOCAL_WRITE,
954 1);
a7efd777
RA		 955 if (rc)
956 goto err0;
957
958 pbl_ptr = cq->q.pbl_tbl->pa;
959 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 960
961 cq->ibcq.cqe = chain_entries;
97f61250 962 cq->q.db_addr = ctx->dpi_addr + db_offset;
a7efd777
RA		 963 } else {
964 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
965
b6db3f71
AL		 966 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
967 &chain_params);
a7efd777 968 if (rc)
97f61250 969 goto err0;
a7efd777
RA		 970
971 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
972 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 973 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 974 }
975
976 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
977 pbl_ptr, &params);
978
979 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
980 if (rc)
97f61250 981 goto err1;
a7efd777
RA		 982
983 cq->icid = icid;
984 cq->sig = QEDR_CQ_MAGIC_NUMBER;
985 spin_lock_init(&cq->cq_lock);
986
ff23dfa1 987 if (udata) {
97f61250
MK		 988 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
989 if (rc)
990 goto err2;
991
992 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
993 &cq->q.db_rec_data->db_data,
994 DB_REC_WIDTH_64B,
995 DB_REC_USER);
a7efd777 996 if (rc)
97f61250
MK		 997 goto err2;
998
a7efd777
RA		 999 } else {
1000 /* Generate doorbell address. */
a7efd777 1001 cq->db.data.icid = cq->icid;
97f61250 1002 cq->db_addr = dev->db_addr + db_offset;
a7efd777
RA		 1003 cq->db.data.params = DB_AGG_CMD_SET <<
1004 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1005
1006 /* point to the very last element, passing it we will toggle */
1007 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1008 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1009 cq->latest_cqe = NULL;
1010 consume_cqe(cq);
1011 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
97f61250
MK		 1012
1013 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1014 DB_REC_WIDTH_64B, DB_REC_KERNEL);
1015 if (rc)
1016 goto err2;
a7efd777
RA		 1017 }
1018
1019 DP_DEBUG(dev, QEDR_MSG_CQ,
1020 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1021 cq->icid, cq, params.cq_size);
1022
e39afe3d 1023 return 0;
a7efd777 1024
97f61250 1025err2:
a7efd777
RA		 1026 destroy_iparams.icid = cq->icid;
1027 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1028 &destroy_oparams);
a7efd777 1029err1:
97f61250
MK		 1030 if (udata) {
1031 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
a7efd777 1032 ib_umem_release(cq->q.umem);
a25984f3 1033 if (cq->q.db_mmap_entry)
97f61250
MK		 1034 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1035 } else {
1036 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1037 }
a7efd777 1038err0:
e39afe3d 1039 return -EINVAL;
a7efd777
RA		 1040}
1041
1042int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1043{
1044 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1045 struct qedr_cq *cq = get_qedr_cq(ibcq);
1046
1047 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1048
1049 return 0;
1050}
1051
4dd72636
AR		 1052#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1053#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1054
a52c8e24 1055void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 1056{
1057 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1058 struct qed_rdma_destroy_cq_out_params oparams;
1059 struct qed_rdma_destroy_cq_in_params iparams;
1060 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 1061 int iter;
a7efd777 1062
942b3b2c 1063 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 1064
4dd72636
AR		 1065 cq->destroyed = 1;
1066
a7efd777 1067 /* GSIs CQs are handled by driver, so they don't exist in the FW */
97f61250
MK		 1068 if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1069 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
e39afe3d 1070 return;
97f61250 1071 }
a1211359 1072
942b3b2c 1073 iparams.icid = cq->icid;
a52c8e24 1074 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
942b3b2c 1075 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777 1076
bdeacabd 1077 if (udata) {
a7efd777
RA		 1078 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1079 ib_umem_release(cq->q.umem);
97f61250
MK		 1080
1081 if (cq->q.db_rec_data) {
1082 qedr_db_recovery_del(dev, cq->q.db_addr,
1083 &cq->q.db_rec_data->db_data);
1084 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1085 }
1086 } else {
1087 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
a7efd777
RA		 1088 }
1089
4dd72636
AR		 1090 /* We don't want the IRQ handler to handle a non-existing CQ so we
1091 * wait until all CNQ interrupts, if any, are received. This will always
1092 * happen and will always happen very fast. If not, then a serious error
1093 * has occured. That is why we can use a long delay.
1094 * We spin for a short time so we don’t lose time on context switching
1095 * in case all the completions are handled in that span. Otherwise
1096 * we sleep for a while and check again. Since the CNQ may be
1097 * associated with (only) the current CPU we use msleep to allow the
1098 * current CPU to be freed.
1099 * The CNQ notification is increased in qedr_irq_handler().
1100 */
1101 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1102 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1103 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1104 iter--;
1105 }
1106
1107 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1108 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1109 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1110 iter--;
1111 }
1112
4dd72636
AR		 1113 /* Note that we don't need to have explicit code to wait for the
1114 * completion of the event handler because it is invoked from the EQ.
1115 * Since the destroy CQ ramrod has also been received on the EQ we can
1116 * be certain that there's no event handler in process.
1117 */
a7efd777 1118}
cecbcddf
RA		 1119
1120static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1121 struct ib_qp_attr *attr,
1122 int attr_mask,
1123 struct qed_rdma_modify_qp_in_params
1124 *qp_params)
1125{
47ec3866 1126 const struct ib_gid_attr *gid_attr;
cecbcddf 1127 enum rdma_network_type nw_type;
d8966fcd 1128 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1129 u32 ipv4_addr;
a70c0739 1130 int ret;
cecbcddf
RA		 1131 int i;
1132
47ec3866 1133 gid_attr = grh->sgid_attr;
a70c0739
PP		 1134 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1135 if (ret)
1136 return ret;
3e44e0ee 1137
47ec3866 1138 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1139 switch (nw_type) {
1140 case RDMA_NETWORK_IPV6:
47ec3866 1141 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1142 sizeof(qp_params->sgid));
1143 memcpy(&qp_params->dgid.bytes[0],
1144 &grh->dgid,
1145 sizeof(qp_params->dgid));
1146 qp_params->roce_mode = ROCE_V2_IPV6;
1147 SET_FIELD(qp_params->modify_flags,
1148 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1149 break;
1150 case RDMA_NETWORK_IB:
47ec3866 1151 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1152 sizeof(qp_params->sgid));
1153 memcpy(&qp_params->dgid.bytes[0],
1154 &grh->dgid,
1155 sizeof(qp_params->dgid));
1156 qp_params->roce_mode = ROCE_V1;
1157 break;
1158 case RDMA_NETWORK_IPV4:
1159 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1160 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1161 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1162 qp_params->sgid.ipv4_addr = ipv4_addr;
1163 ipv4_addr =
1164 qedr_get_ipv4_from_gid(grh->dgid.raw);
1165 qp_params->dgid.ipv4_addr = ipv4_addr;
1166 SET_FIELD(qp_params->modify_flags,
1167 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1168 qp_params->roce_mode = ROCE_V2_IPV4;
1169 break;
cecbcddf
RA		 1170 }
1171
1172 for (i = 0; i < 4; i++) {
1173 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1174 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1175 }
1176
1177 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1178 qp_params->vlan_id = 0;
1179
1180 return 0;
1181}
1182
cecbcddf 1183static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1184 struct ib_qp_init_attr *attrs,
1185 struct ib_udata *udata)
cecbcddf
RA		 1186{
1187 struct qedr_device_attr *qattr = &dev->attr;
1188
1189 /* QP0... attrs->qp_type == IB_QPT_GSI */
1190 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1191 DP_DEBUG(dev, QEDR_MSG_QP,
1192 "create qp: unsupported qp type=0x%x requested\n",
1193 attrs->qp_type);
bb8865f4 1194 return -EOPNOTSUPP;
cecbcddf
RA		 1195 }
1196
1197 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1198 DP_ERR(dev,
1199 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1200 attrs->cap.max_send_wr, qattr->max_sqe);
1201 return -EINVAL;
1202 }
1203
1204 if (attrs->cap.max_inline_data > qattr->max_inline) {
1205 DP_ERR(dev,
1206 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1207 attrs->cap.max_inline_data, qattr->max_inline);
1208 return -EINVAL;
1209 }
1210
1211 if (attrs->cap.max_send_sge > qattr->max_sge) {
1212 DP_ERR(dev,
1213 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1214 attrs->cap.max_send_sge, qattr->max_sge);
1215 return -EINVAL;
1216 }
1217
1218 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1219 DP_ERR(dev,
1220 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1221 attrs->cap.max_recv_sge, qattr->max_sge);
1222 return -EINVAL;
1223 }
1224
1225 /* Unprivileged user space cannot create special QP */
e00b64f7 1226 if (udata && attrs->qp_type == IB_QPT_GSI) {
cecbcddf
RA		 1227 DP_ERR(dev,
1228 "create qp: userspace can't create special QPs of type=0x%x\n",
1229 attrs->qp_type);
1230 return -EINVAL;
1231 }
1232
1233 return 0;
1234}
1235
40b173dd
YB		 1236static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1237 struct qedr_srq *srq, struct ib_udata *udata)
1238{
1239 struct qedr_create_srq_uresp uresp = {};
1240 int rc;
1241
1242 uresp.srq_id = srq->srq_id;
1243
1244 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1245 if (rc)
1246 DP_ERR(dev, "create srq: problem copying data to user space\n");
1247
1248 return rc;
1249}
1250
69ad0e7f 1251static void qedr_copy_rq_uresp(struct qedr_dev *dev,
97f61250
MK		 1252 struct qedr_create_qp_uresp *uresp,
1253 struct qedr_qp *qp)
cecbcddf 1254{
69ad0e7f
KM		 1255 /* iWARP requires two doorbells per RQ. */
1256 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1257 uresp->rq_db_offset =
1258 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1259 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1260 } else {
1261 uresp->rq_db_offset =
1262 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1263 }
1264
cecbcddf 1265 uresp->rq_icid = qp->icid;
a25984f3
MK		 1266 if (qp->urq.db_mmap_entry)
1267 uresp->rq_db_rec_addr =
1268 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
cecbcddf
RA		 1269}
1270
69ad0e7f
KM		 1271static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1272 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1273 struct qedr_qp *qp)
1274{
1275 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1276
1277 /* iWARP uses the same cid for rq and sq */
1278 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1279 uresp->sq_icid = qp->icid;
1280 else
1281 uresp->sq_icid = qp->icid + 1;
97f61250 1282
a25984f3
MK		 1283 if (qp->usq.db_mmap_entry)
1284 uresp->sq_db_rec_addr =
1285 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
cecbcddf
RA		 1286}
1287
1288static int qedr_copy_qp_uresp(struct qedr_dev *dev,
97f61250
MK		 1289 struct qedr_qp *qp, struct ib_udata *udata,
1290 struct qedr_create_qp_uresp *uresp)
cecbcddf 1291{
cecbcddf
RA		 1292 int rc;
1293
97f61250
MK		 1294 memset(uresp, 0, sizeof(*uresp));
1295 qedr_copy_sq_uresp(dev, uresp, qp);
1296 qedr_copy_rq_uresp(dev, uresp, qp);
cecbcddf 1297
97f61250
MK		 1298 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1299 uresp->qp_id = qp->qp_id;
cecbcddf 1300
97f61250 1301 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
cecbcddf
RA		 1302 if (rc)
1303 DP_ERR(dev,
1304 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1305 qp->icid);
1306
1307 return rc;
1308}
1309
df158561
AR		 1310static void qedr_set_common_qp_params(struct qedr_dev *dev,
1311 struct qedr_qp *qp,
1312 struct qedr_pd *pd,
1313 struct ib_qp_init_attr *attrs)
cecbcddf 1314{
cecbcddf 1315 spin_lock_init(&qp->q_lock);
82af6d19
MK		 1316 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1317 kref_init(&qp->refcnt);
1318 init_completion(&qp->iwarp_cm_comp);
1319 }
df158561 1320 qp->pd = pd;
cecbcddf
RA		 1321 qp->qp_type = attrs->qp_type;
1322 qp->max_inline_data = attrs->cap.max_inline_data;
1323 qp->sq.max_sges = attrs->cap.max_send_sge;
1324 qp->state = QED_ROCE_QP_STATE_RESET;
1325 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1326 qp->sq_cq = get_qedr_cq(attrs->send_cq);
cecbcddf
RA		 1327 qp->dev = dev;
1328
3491c9e7
YB		 1329 if (attrs->srq) {
1330 qp->srq = get_qedr_srq(attrs->srq);
1331 } else {
1332 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1333 qp->rq.max_sges = attrs->cap.max_recv_sge;
1334 DP_DEBUG(dev, QEDR_MSG_QP,
1335 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1336 qp->rq.max_sges, qp->rq_cq->icid);
1337 }
1338
cecbcddf
RA		 1339 DP_DEBUG(dev, QEDR_MSG_QP,
1340 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1341 pd->pd_id, qp->qp_type, qp->max_inline_data,
1342 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1343 DP_DEBUG(dev, QEDR_MSG_QP,
1344 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1345 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1346}
1347
97f61250 1348static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1349{
97f61250
MK		 1350 int rc;
1351
cecbcddf
RA		 1352 qp->sq.db = dev->db_addr +
1353 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1354 qp->sq.db_data.data.icid = qp->icid + 1;
97f61250
MK		 1355 rc = qedr_db_recovery_add(dev, qp->sq.db,
1356 &qp->sq.db_data,
1357 DB_REC_WIDTH_32B,
1358 DB_REC_KERNEL);
1359 if (rc)
1360 return rc;
1361
3491c9e7
YB		 1362 if (!qp->srq) {
1363 qp->rq.db = dev->db_addr +
1364 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1365 qp->rq.db_data.data.icid = qp->icid;
97f61250
MK		 1366
1367 rc = qedr_db_recovery_add(dev, qp->rq.db,
1368 &qp->rq.db_data,
1369 DB_REC_WIDTH_32B,
1370 DB_REC_KERNEL);
1371 if (rc)
1372 qedr_db_recovery_del(dev, qp->sq.db,
1373 &qp->sq.db_data);
3491c9e7 1374 }
97f61250
MK		 1375
1376 return rc;
3491c9e7
YB		 1377}
1378
68e326de 1379static int qedr_check_srq_params(struct qedr_dev *dev,
3491c9e7
YB		 1380 struct ib_srq_init_attr *attrs,
1381 struct ib_udata *udata)
1382{
1383 struct qedr_device_attr *qattr = &dev->attr;
1384
1385 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1386 DP_ERR(dev,
1387 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1388 attrs->attr.max_wr, qattr->max_srq_wr);
1389 return -EINVAL;
1390 }
1391
1392 if (attrs->attr.max_sge > qattr->max_sge) {
1393 DP_ERR(dev,
1394 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1395 attrs->attr.max_sge, qattr->max_sge);
1396 return -EINVAL;
1397 }
1398
1399 return 0;
1400}
1401
40b173dd
YB		 1402static void qedr_free_srq_user_params(struct qedr_srq *srq)
1403{
1404 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1405 ib_umem_release(srq->usrq.umem);
1406 ib_umem_release(srq->prod_umem);
1407}
1408
3491c9e7
YB		 1409static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1410{
1411 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1412 struct qedr_dev *dev = srq->dev;
1413
1414 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1415
1416 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1417 hw_srq->virt_prod_pair_addr,
1418 hw_srq->phy_prod_pair_addr);
1419}
1420
b0ea0fa5 1421static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1422 struct qedr_srq *srq,
1423 struct qedr_create_srq_ureq *ureq,
72b894b0 1424 int access)
40b173dd
YB		 1425{
1426 struct scatterlist *sg;
1427 int rc;
1428
b0ea0fa5 1429 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
72b894b0 1430 ureq->srq_len, false, access, 1);
40b173dd
YB		 1431 if (rc)
1432 return rc;
1433
c320e527
MS		 1434 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1435 sizeof(struct rdma_srq_producers), access);
40b173dd
YB		 1436 if (IS_ERR(srq->prod_umem)) {
1437 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1438 ib_umem_release(srq->usrq.umem);
1439 DP_ERR(srq->dev,
1440 "create srq: failed ib_umem_get for producer, got %ld\n",
1441 PTR_ERR(srq->prod_umem));
1442 return PTR_ERR(srq->prod_umem);
1443 }
1444
1445 sg = srq->prod_umem->sg_head.sgl;
1446 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1447
1448 return 0;
1449}
1450
3491c9e7
YB		 1451static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1452 struct qedr_dev *dev,
1453 struct ib_srq_init_attr *init_attr)
1454{
1455 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
b6db3f71
AL		 1456 struct qed_chain_init_params params = {
1457 .mode = QED_CHAIN_MODE_PBL,
1458 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1459 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1460 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE,
1461 };
3491c9e7
YB		 1462 dma_addr_t phy_prod_pair_addr;
1463 u32 num_elems;
1464 void *va;
1465 int rc;
1466
1467 va = dma_alloc_coherent(&dev->pdev->dev,
1468 sizeof(struct rdma_srq_producers),
1469 &phy_prod_pair_addr, GFP_KERNEL);
1470 if (!va) {
1471 DP_ERR(dev,
1472 "create srq: failed to allocate dma memory for producer\n");
1473 return -ENOMEM;
1474 }
1475
1476 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1477 hw_srq->virt_prod_pair_addr = va;
1478
1479 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
b6db3f71
AL		 1480 params.num_elems = num_elems;
1481
1482 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
3491c9e7
YB		 1483 if (rc)
1484 goto err0;
1485
1486 hw_srq->num_elems = num_elems;
1487
1488 return 0;
1489
1490err0:
1491 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1492 va, phy_prod_pair_addr);
1493 return rc;
1494}
1495
68e326de
LR		 1496int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1497 struct ib_udata *udata)
3491c9e7
YB		 1498{
1499 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1500 struct qed_rdma_create_srq_in_params in_params = {};
68e326de 1501 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
3491c9e7 1502 struct qed_rdma_create_srq_out_params out_params;
68e326de 1503 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
40b173dd 1504 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1505 u64 pbl_base_addr, phy_prod_pair_addr;
1506 struct qedr_srq_hwq_info *hw_srq;
1507 u32 page_cnt, page_size;
68e326de 1508 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3491c9e7
YB		 1509 int rc = 0;
1510
1511 DP_DEBUG(dev, QEDR_MSG_QP,
1512 "create SRQ called from %s (pd %p)\n",
1513 (udata) ? "User lib" : "kernel", pd);
1514
68e326de 1515 rc = qedr_check_srq_params(dev, init_attr, udata);
3491c9e7 1516 if (rc)
68e326de 1517 return -EINVAL;
3491c9e7
YB		 1518
1519 srq->dev = dev;
1520 hw_srq = &srq->hw_srq;
1521 spin_lock_init(&srq->lock);
1522
1523 hw_srq->max_wr = init_attr->attr.max_wr;
1524 hw_srq->max_sges = init_attr->attr.max_sge;
1525
89944450 1526 if (udata) {
97f61250
MK		 1527 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1528 udata->inlen))) {
40b173dd
YB		 1529 DP_ERR(dev,
1530 "create srq: problem copying data from user space\n");
1531 goto err0;
1532 }
1533
72b894b0 1534 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
40b173dd
YB		 1535 if (rc)
1536 goto err0;
1537
1538 page_cnt = srq->usrq.pbl_info.num_pbes;
1539 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1540 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1541 page_size = PAGE_SIZE;
40b173dd
YB		 1542 } else {
1543 struct qed_chain *pbl;
1544
1545 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1546 if (rc)
1547 goto err0;
1548
1549 pbl = &hw_srq->pbl;
1550 page_cnt = qed_chain_get_page_cnt(pbl);
1551 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1552 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1553 page_size = QED_CHAIN_PAGE_SIZE;
1554 }
3491c9e7 1555
3491c9e7
YB		 1556 in_params.pd_id = pd->pd_id;
1557 in_params.pbl_base_addr = pbl_base_addr;
1558 in_params.prod_pair_addr = phy_prod_pair_addr;
1559 in_params.num_pages = page_cnt;
1560 in_params.page_size = page_size;
1561
1562 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1563 if (rc)
1564 goto err1;
1565
1566 srq->srq_id = out_params.srq_id;
1567
40b173dd
YB		 1568 if (udata) {
1569 rc = qedr_copy_srq_uresp(dev, srq, udata);
1570 if (rc)
1571 goto err2;
1572 }
1573
9fd15987 1574 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1575 if (rc)
1576 goto err2;
1577
1578 DP_DEBUG(dev, QEDR_MSG_SRQ,
1579 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
68e326de 1580 return 0;
3491c9e7
YB		 1581
1582err2:
1583 destroy_in_params.srq_id = srq->srq_id;
1584
1585 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1586err1:
40b173dd
YB		 1587 if (udata)
1588 qedr_free_srq_user_params(srq);
1589 else
1590 qedr_free_srq_kernel_params(srq);
3491c9e7 1591err0:
68e326de 1592 return -EFAULT;
3491c9e7
YB		 1593}
1594
68e326de 1595void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1596{
1597 struct qed_rdma_destroy_srq_in_params in_params = {};
1598 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1599 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1600
9fd15987 1601 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7
YB		 1602 in_params.srq_id = srq->srq_id;
1603 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1604
e00b64f7 1605 if (ibsrq->uobject)
40b173dd
YB		 1606 qedr_free_srq_user_params(srq);
1607 else
1608 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1609
1610 DP_DEBUG(dev, QEDR_MSG_SRQ,
1611 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1612 srq->srq_id);
3491c9e7
YB		 1613}
1614
1615int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1616 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1617{
1618 struct qed_rdma_modify_srq_in_params in_params = {};
1619 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1620 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1621 int rc;
1622
1623 if (attr_mask & IB_SRQ_MAX_WR) {
1624 DP_ERR(dev,
1625 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1626 attr_mask, srq);
1627 return -EINVAL;
1628 }
1629
1630 if (attr_mask & IB_SRQ_LIMIT) {
1631 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1632 DP_ERR(dev,
1633 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1634 attr->srq_limit, srq->hw_srq.max_wr);
1635 return -EINVAL;
1636 }
1637
1638 in_params.srq_id = srq->srq_id;
1639 in_params.wqe_limit = attr->srq_limit;
1640 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1641 if (rc)
1642 return rc;
1643 }
1644
1645 srq->srq_limit = attr->srq_limit;
1646
1647 DP_DEBUG(dev, QEDR_MSG_SRQ,
1648 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1649
1650 return 0;
cecbcddf
RA		 1651}
1652
1653static inline void
df158561
AR		 1654qedr_init_common_qp_in_params(struct qedr_dev *dev,
1655 struct qedr_pd *pd,
1656 struct qedr_qp *qp,
1657 struct ib_qp_init_attr *attrs,
1658 bool fmr_and_reserved_lkey,
1659 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1660{
df158561
AR		 1661 /* QP handle to be written in an async event */
1662 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1663 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1664
1665 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1666 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1667 params->pd = pd->pd_id;
1668 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1669 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1670 params->stats_queue = 0;
df158561
AR		 1671 params->srq_id = 0;
1672 params->use_srq = false;
3491c9e7
YB		 1673
1674 if (!qp->srq) {
1675 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1676
1677 } else {
1678 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1679 params->srq_id = qp->srq->srq_id;
1680 params->use_srq = true;
1681 }
cecbcddf
RA		 1682}
1683
df158561 1684static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1685{
df158561
AR		 1686 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1687 "qp=%p. "
1688 "sq_addr=0x%llx, "
1689 "sq_len=%zd, "
1690 "rq_addr=0x%llx, "
1691 "rq_len=%zd"
1692 "\n",
1693 qp,
1694 qp->usq.buf_addr,
1695 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1696}
cecbcddf 1697
69ad0e7f
KM		 1698static inline void
1699qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1700 struct qedr_qp *qp,
1701 struct qed_rdma_create_qp_out_params *out_params)
1702{
1703 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1704 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1705
1706 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1707 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1708 if (!qp->srq) {
1709 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1710 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1711 }
69ad0e7f
KM		 1712
1713 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1714 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1715}
1716
97f61250
MK		 1717static void qedr_cleanup_user(struct qedr_dev *dev,
1718 struct qedr_ucontext *ctx,
1719 struct qedr_qp *qp)
df158561 1720{
836a0fbb 1721 ib_umem_release(qp->usq.umem);
df158561 1722 qp->usq.umem = NULL;
cecbcddf 1723
836a0fbb 1724 ib_umem_release(qp->urq.umem);
df158561 1725 qp->urq.umem = NULL;
24e412c1
MK		 1726
1727 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1728 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1729 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1730 } else {
1731 kfree(qp->usq.pbl_tbl);
1732 kfree(qp->urq.pbl_tbl);
1733 }
97f61250
MK		 1734
1735 if (qp->usq.db_rec_data) {
1736 qedr_db_recovery_del(dev, qp->usq.db_addr,
1737 &qp->usq.db_rec_data->db_data);
1738 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1739 }
1740
1741 if (qp->urq.db_rec_data) {
1742 qedr_db_recovery_del(dev, qp->urq.db_addr,
1743 &qp->urq.db_rec_data->db_data);
1744 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1745 }
b4bc7660
MK		 1746
1747 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1748 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1749 &qp->urq.db_rec_db2_data);
cecbcddf
RA		 1750}
1751
df158561
AR		 1752static int qedr_create_user_qp(struct qedr_dev *dev,
1753 struct qedr_qp *qp,
1754 struct ib_pd *ibpd,
1755 struct ib_udata *udata,
1756 struct ib_qp_init_attr *attrs)
cecbcddf 1757{
df158561
AR		 1758 struct qed_rdma_create_qp_in_params in_params;
1759 struct qed_rdma_create_qp_out_params out_params;
1760 struct qedr_pd *pd = get_qedr_pd(ibpd);
97f61250 1761 struct qedr_create_qp_uresp uresp;
bbe4f424 1762 struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
df158561 1763 struct qedr_create_qp_ureq ureq;
69ad0e7f 1764 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
df158561 1765 int rc = -EINVAL;
cecbcddf 1766
82af6d19 1767 qp->create_type = QEDR_QP_CREATE_USER;
df158561 1768 memset(&ureq, 0, sizeof(ureq));
97f61250 1769 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
df158561
AR		 1770 if (rc) {
1771 DP_ERR(dev, "Problem copying data from user space\n");
1772 return rc;
1773 }
cecbcddf 1774
72b894b0 1775 /* SQ - read access only (0) */
b0ea0fa5 1776 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
72b894b0 1777 ureq.sq_len, true, 0, alloc_and_init);
df158561
AR		 1778 if (rc)
1779 return rc;
cecbcddf 1780
40b173dd 1781 if (!qp->srq) {
72b894b0 1782 /* RQ - read access only (0) */
b0ea0fa5 1783 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
72b894b0 1784 ureq.rq_len, true, 0, alloc_and_init);
40b173dd
YB		 1785 if (rc)
1786 return rc;
1787 }
df158561
AR		 1788
1789 memset(&in_params, 0, sizeof(in_params));
1790 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1791 in_params.qp_handle_lo = ureq.qp_handle_lo;
1792 in_params.qp_handle_hi = ureq.qp_handle_hi;
1793 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1794 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
40b173dd
YB		 1795 if (!qp->srq) {
1796 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1797 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1798 }
df158561 1799
bbe4f424
MK		 1800 if (ctx)
1801 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1802
df158561
AR		 1803 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1804 &in_params, &out_params);
1805
1806 if (!qp->qed_qp) {
1807 rc = -ENOMEM;
1808 goto err1;
1809 }
1810
69ad0e7f
KM		 1811 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1812 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1813
df158561
AR		 1814 qp->qp_id = out_params.qp_id;
1815 qp->icid = out_params.icid;
1816
97f61250
MK		 1817 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1818 if (rc)
1819 goto err;
1820
1821 /* db offset was calculated in copy_qp_uresp, now set in the user q */
1822 ctx = pd->uctx;
1823 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1824 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1825
b4bc7660
MK		 1826 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1827 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1828
1829 /* calculate the db_rec_db2 data since it is constant so no
1830 * need to reflect from user
1831 */
1832 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1833 qp->urq.db_rec_db2_data.data.value =
1834 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1835 }
1836
97f61250
MK		 1837 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1838 &qp->usq.db_rec_data->db_data,
1839 DB_REC_WIDTH_32B,
1840 DB_REC_USER);
1841 if (rc)
1842 goto err;
1843
1844 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1845 &qp->urq.db_rec_data->db_data,
1846 DB_REC_WIDTH_32B,
1847 DB_REC_USER);
df158561
AR		 1848 if (rc)
1849 goto err;
1850
b4bc7660
MK		 1851 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1852 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1853 &qp->urq.db_rec_db2_data,
1854 DB_REC_WIDTH_32B,
1855 DB_REC_USER);
1856 if (rc)
1857 goto err;
1858 }
df158561 1859 qedr_qp_user_print(dev, qp);
cecbcddf 1860
97f61250 1861 return rc;
df158561
AR		 1862err:
1863 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1864 if (rc)
1865 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1866
1867err1:
97f61250 1868 qedr_cleanup_user(dev, ctx, qp);
df158561 1869 return rc;
cecbcddf
RA		 1870}
1871
97f61250 1872static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
f5b1b177 1873{
97f61250
MK		 1874 int rc;
1875
f5b1b177
KM		 1876 qp->sq.db = dev->db_addr +
1877 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1878 qp->sq.db_data.data.icid = qp->icid;
1879
97f61250
MK		 1880 rc = qedr_db_recovery_add(dev, qp->sq.db,
1881 &qp->sq.db_data,
1882 DB_REC_WIDTH_32B,
1883 DB_REC_KERNEL);
1884 if (rc)
1885 return rc;
1886
f5b1b177
KM		 1887 qp->rq.db = dev->db_addr +
1888 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1889 qp->rq.db_data.data.icid = qp->icid;
1890 qp->rq.iwarp_db2 = dev->db_addr +
1891 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1892 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1893 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
97f61250
MK		 1894
1895 rc = qedr_db_recovery_add(dev, qp->rq.db,
1896 &qp->rq.db_data,
1897 DB_REC_WIDTH_32B,
1898 DB_REC_KERNEL);
b4bc7660
MK		 1899 if (rc)
1900 return rc;
1901
1902 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1903 &qp->rq.iwarp_db2_data,
1904 DB_REC_WIDTH_32B,
1905 DB_REC_KERNEL);
97f61250 1906 return rc;
f5b1b177
KM		 1907}
1908
df158561
AR		 1909static int
1910qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1911 struct qedr_qp *qp,
1912 struct qed_rdma_create_qp_in_params *in_params,
1913 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1914{
df158561 1915 struct qed_rdma_create_qp_out_params out_params;
b6db3f71
AL		 1916 struct qed_chain_init_params params = {
1917 .mode = QED_CHAIN_MODE_PBL,
1918 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1919 };
cecbcddf
RA		 1920 int rc;
1921
b6db3f71
AL		 1922 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
1923 params.num_elems = n_sq_elems;
1924 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
cecbcddf 1925
b6db3f71 1926 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
df158561
AR		 1927 if (rc)
1928 return rc;
cecbcddf 1929
df158561
AR		 1930 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1931 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1932
b6db3f71 1933 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
35406697 1934 params.num_elems = n_rq_elems;
b6db3f71
AL		 1935 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
1936
1937 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
df158561
AR		 1938 if (rc)
1939 return rc;
cecbcddf 1940
df158561
AR		 1941 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1942 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1943
df158561
AR		 1944 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1945 in_params, &out_params);
cecbcddf 1946
df158561
AR		 1947 if (!qp->qed_qp)
1948 return -EINVAL;
cecbcddf 1949
df158561
AR		 1950 qp->qp_id = out_params.qp_id;
1951 qp->icid = out_params.icid;
cecbcddf 1952
97f61250 1953 return qedr_set_roce_db_info(dev, qp);
f5b1b177 1954}
cecbcddf 1955
f5b1b177
KM		 1956static int
1957qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1958 struct qedr_qp *qp,
1959 struct qed_rdma_create_qp_in_params *in_params,
1960 u32 n_sq_elems, u32 n_rq_elems)
1961{
1962 struct qed_rdma_create_qp_out_params out_params;
b6db3f71
AL		 1963 struct qed_chain_init_params params = {
1964 .mode = QED_CHAIN_MODE_PBL,
1965 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1966 };
f5b1b177
KM		 1967 int rc;
1968
1969 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1970 QEDR_SQE_ELEMENT_SIZE,
15506586 1971 QED_CHAIN_PAGE_SIZE,
f5b1b177
KM		 1972 QED_CHAIN_MODE_PBL);
1973 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1974 QEDR_RQE_ELEMENT_SIZE,
15506586 1975 QED_CHAIN_PAGE_SIZE,
f5b1b177
KM		 1976 QED_CHAIN_MODE_PBL);
1977
1978 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1979 in_params, &out_params);
1980
1981 if (!qp->qed_qp)
1982 return -EINVAL;
1983
1984 /* Now we allocate the chain */
f5b1b177 1985
b6db3f71
AL		 1986 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
1987 params.num_elems = n_sq_elems;
1988 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
1989 params.ext_pbl_virt = out_params.sq_pbl_virt;
1990 params.ext_pbl_phys = out_params.sq_pbl_phys;
f5b1b177 1991
b6db3f71 1992 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
f5b1b177
KM		 1993 if (rc)
1994 goto err;
1995
b6db3f71
AL		 1996 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1997 params.num_elems = n_rq_elems;
1998 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
1999 params.ext_pbl_virt = out_params.rq_pbl_virt;
2000 params.ext_pbl_phys = out_params.rq_pbl_phys;
f5b1b177 2001
b6db3f71 2002 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
f5b1b177
KM		 2003 if (rc)
2004 goto err;
2005
2006 qp->qp_id = out_params.qp_id;
2007 qp->icid = out_params.icid;
2008
97f61250 2009 return qedr_set_iwarp_db_info(dev, qp);
f5b1b177
KM		 2010
2011err:
2012 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2013
2014 return rc;
cecbcddf
RA		 2015}
2016
df158561 2017static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 2018{
df158561
AR		 2019 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2020 kfree(qp->wqe_wr_id);
cecbcddf 2021
df158561
AR		 2022 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2023 kfree(qp->rqe_wr_id);
97f61250
MK		 2024
2025 /* GSI qp is not registered to db mechanism so no need to delete */
2026 if (qp->qp_type == IB_QPT_GSI)
2027 return;
2028
2029 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2030
b4bc7660 2031 if (!qp->srq) {
97f61250 2032 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
b4bc7660
MK		 2033
2034 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2035 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2036 &qp->rq.iwarp_db2_data);
2037 }
cecbcddf
RA		 2038}
2039
df158561
AR		 2040static int qedr_create_kernel_qp(struct qedr_dev *dev,
2041 struct qedr_qp *qp,
2042 struct ib_pd *ibpd,
2043 struct ib_qp_init_attr *attrs)
cecbcddf 2044{
df158561
AR		 2045 struct qed_rdma_create_qp_in_params in_params;
2046 struct qedr_pd *pd = get_qedr_pd(ibpd);
2047 int rc = -EINVAL;
2048 u32 n_rq_elems;
2049 u32 n_sq_elems;
2050 u32 n_sq_entries;
cecbcddf 2051
df158561 2052 memset(&in_params, 0, sizeof(in_params));
82af6d19 2053 qp->create_type = QEDR_QP_CREATE_KERNEL;
cecbcddf 2054
df158561
AR		 2055 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2056 * the ring. The ring should allow at least a single WR, even if the
2057 * user requested none, due to allocation issues.
2058 * We should add an extra WR since the prod and cons indices of
2059 * wqe_wr_id are managed in such a way that the WQ is considered full
2060 * when (prod+1)%max_wr==cons. We currently don't do that because we
2061 * double the number of entries due an iSER issue that pushes far more
2062 * WRs than indicated. If we decline its ib_post_send() then we get
2063 * error prints in the dmesg we'd like to avoid.
2064 */
2065 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2066 dev->attr.max_sqe);
cecbcddf 2067
6396bb22 2068 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 2069 GFP_KERNEL);
2070 if (!qp->wqe_wr_id) {
2071 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2072 return -ENOMEM;
2073 }
cecbcddf 2074
df158561
AR		 2075 /* QP handle to be written in CQE */
2076 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2077 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 2078
df158561
AR		 2079 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2080 * the ring. There ring should allow at least a single WR, even if the
2081 * user requested none, due to allocation issues.
2082 */
2083 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 2084
df158561 2085 /* Allocate driver internal RQ array */
6396bb22 2086 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 2087 GFP_KERNEL);
2088 if (!qp->rqe_wr_id) {
2089 DP_ERR(dev,
2090 "create qp: failed RQ shadow memory allocation\n");
2091 kfree(qp->wqe_wr_id);
2092 return -ENOMEM;
cecbcddf
RA		 2093 }
2094
df158561 2095 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 2096
df158561
AR		 2097 n_sq_entries = attrs->cap.max_send_wr;
2098 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2099 n_sq_entries = max_t(u32, n_sq_entries, 1);
2100 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 2101
df158561
AR		 2102 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2103
f5b1b177
KM		 2104 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2105 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2106 n_sq_elems, n_rq_elems);
2107 else
2108 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2109 n_sq_elems, n_rq_elems);
df158561
AR		 2110 if (rc)
2111 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 2112
2113 return rc;
2114}
2115
2116struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2117 struct ib_qp_init_attr *attrs,
2118 struct ib_udata *udata)
2119{
2120 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 2121 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 2122 struct qedr_qp *qp;
181d8015 2123 struct ib_qp *ibqp;
cecbcddf
RA		 2124 int rc = 0;
2125
2126 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2127 udata ? "user library" : "kernel", pd);
2128
e00b64f7 2129 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf
RA		 2130 if (rc)
2131 return ERR_PTR(rc);
2132
cecbcddf 2133 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 2134 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2135 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 2136 get_qedr_cq(attrs->send_cq),
2137 get_qedr_cq(attrs->send_cq)->icid,
2138 get_qedr_cq(attrs->recv_cq),
3491c9e7 2139 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 2140
df158561
AR		 2141 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2142 if (!qp) {
2143 DP_ERR(dev, "create qp: failed allocating memory\n");
2144 return ERR_PTR(-ENOMEM);
2145 }
2146
2147 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 2148
04886779 2149 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 2150 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2151 if (IS_ERR(ibqp))
2152 kfree(qp);
2153 return ibqp;
04886779
RA		 2154 }
2155
df158561
AR		 2156 if (udata)
2157 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2158 else
2159 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 2160
df158561
AR		 2161 if (rc)
2162 goto err;
cecbcddf 2163
cecbcddf
RA		 2164 qp->ibqp.qp_num = qp->qp_id;
2165
1212767e 2166 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 2167 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e
YB		 2168 if (rc)
2169 goto err;
2170 }
de0089e6 2171
cecbcddf
RA		 2172 return &qp->ibqp;
2173
df158561 2174err:
cecbcddf
RA		 2175 kfree(qp);
2176
2177 return ERR_PTR(-EFAULT);
2178}
2179
27a4b1a6 2180static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 2181{
2182 switch (qp_state) {
2183 case QED_ROCE_QP_STATE_RESET:
2184 return IB_QPS_RESET;
2185 case QED_ROCE_QP_STATE_INIT:
2186 return IB_QPS_INIT;
2187 case QED_ROCE_QP_STATE_RTR:
2188 return IB_QPS_RTR;
2189 case QED_ROCE_QP_STATE_RTS:
2190 return IB_QPS_RTS;
2191 case QED_ROCE_QP_STATE_SQD:
2192 return IB_QPS_SQD;
2193 case QED_ROCE_QP_STATE_ERR:
2194 return IB_QPS_ERR;
2195 case QED_ROCE_QP_STATE_SQE:
2196 return IB_QPS_SQE;
2197 }
2198 return IB_QPS_ERR;
2199}
2200
27a4b1a6
RA		 2201static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2202 enum ib_qp_state qp_state)
cecbcddf
RA		 2203{
2204 switch (qp_state) {
2205 case IB_QPS_RESET:
2206 return QED_ROCE_QP_STATE_RESET;
2207 case IB_QPS_INIT:
2208 return QED_ROCE_QP_STATE_INIT;
2209 case IB_QPS_RTR:
2210 return QED_ROCE_QP_STATE_RTR;
2211 case IB_QPS_RTS:
2212 return QED_ROCE_QP_STATE_RTS;
2213 case IB_QPS_SQD:
2214 return QED_ROCE_QP_STATE_SQD;
2215 case IB_QPS_ERR:
2216 return QED_ROCE_QP_STATE_ERR;
2217 default:
2218 return QED_ROCE_QP_STATE_ERR;
2219 }
2220}
2221
2222static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2223{
2224 qed_chain_reset(&qph->pbl);
2225 qph->prod = 0;
2226 qph->cons = 0;
2227 qph->wqe_cons = 0;
2228 qph->db_data.data.value = cpu_to_le16(0);
2229}
2230
2231static int qedr_update_qp_state(struct qedr_dev *dev,
2232 struct qedr_qp *qp,
caf61b1b 2233 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 2234 enum qed_roce_qp_state new_state)
2235{
2236 int status = 0;
2237
caf61b1b 2238 if (new_state == cur_state)
865cea40 2239 return 0;
cecbcddf 2240
caf61b1b 2241 switch (cur_state) {
cecbcddf
RA		 2242 case QED_ROCE_QP_STATE_RESET:
2243 switch (new_state) {
2244 case QED_ROCE_QP_STATE_INIT:
2245 qp->prev_wqe_size = 0;
2246 qedr_reset_qp_hwq_info(&qp->sq);
2247 qedr_reset_qp_hwq_info(&qp->rq);
2248 break;
2249 default:
2250 status = -EINVAL;
2251 break;
790b57f6 2252 }
cecbcddf
RA		 2253 break;
2254 case QED_ROCE_QP_STATE_INIT:
2255 switch (new_state) {
2256 case QED_ROCE_QP_STATE_RTR:
2257 /* Update doorbell (in case post_recv was
2258 * done before move to RTR)
2259 */
f5b1b177
KM		 2260
2261 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177 2262 writel(qp->rq.db_data.raw, qp->rq.db);
f5b1b177 2263 }
cecbcddf
RA		 2264 break;
2265 case QED_ROCE_QP_STATE_ERR:
2266 break;
2267 default:
2268 /* Invalid state change. */
2269 status = -EINVAL;
2270 break;
790b57f6 2271 }
cecbcddf
RA		 2272 break;
2273 case QED_ROCE_QP_STATE_RTR:
2274 /* RTR->XXX */
2275 switch (new_state) {
2276 case QED_ROCE_QP_STATE_RTS:
2277 break;
2278 case QED_ROCE_QP_STATE_ERR:
2279 break;
2280 default:
2281 /* Invalid state change. */
2282 status = -EINVAL;
2283 break;
790b57f6 2284 }
cecbcddf
RA		 2285 break;
2286 case QED_ROCE_QP_STATE_RTS:
2287 /* RTS->XXX */
2288 switch (new_state) {
2289 case QED_ROCE_QP_STATE_SQD:
2290 break;
2291 case QED_ROCE_QP_STATE_ERR:
2292 break;
2293 default:
2294 /* Invalid state change. */
2295 status = -EINVAL;
2296 break;
790b57f6 2297 }
cecbcddf
RA		 2298 break;
2299 case QED_ROCE_QP_STATE_SQD:
2300 /* SQD->XXX */
2301 switch (new_state) {
2302 case QED_ROCE_QP_STATE_RTS:
2303 case QED_ROCE_QP_STATE_ERR:
2304 break;
2305 default:
2306 /* Invalid state change. */
2307 status = -EINVAL;
2308 break;
790b57f6 2309 }
cecbcddf
RA		 2310 break;
2311 case QED_ROCE_QP_STATE_ERR:
2312 /* ERR->XXX */
2313 switch (new_state) {
2314 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2315 if ((qp->rq.prod != qp->rq.cons) ||
2316 (qp->sq.prod != qp->sq.cons)) {
2317 DP_NOTICE(dev,
2318 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2319 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2320 qp->sq.cons);
2321 status = -EINVAL;
2322 }
cecbcddf
RA		 2323 break;
2324 default:
2325 status = -EINVAL;
2326 break;
790b57f6 2327 }
cecbcddf
RA		 2328 break;
2329 default:
2330 status = -EINVAL;
2331 break;
790b57f6 2332 }
cecbcddf
RA		 2333
2334 return status;
2335}
2336
2337int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2338 int attr_mask, struct ib_udata *udata)
2339{
2340 struct qedr_qp *qp = get_qedr_qp(ibqp);
2341 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2342 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2343 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2344 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2345 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2346 int rc = 0;
2347
2348 DP_DEBUG(dev, QEDR_MSG_QP,
2349 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2350 attr->qp_state);
2351
2352 old_qp_state = qedr_get_ibqp_state(qp->state);
2353 if (attr_mask & IB_QP_STATE)
2354 new_qp_state = attr->qp_state;
2355 else
2356 new_qp_state = old_qp_state;
2357
f5b1b177
KM		 2358 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2359 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2360 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2361 DP_ERR(dev,
2362 "modify qp: invalid attribute mask=0x%x specified for\n"
2363 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2364 attr_mask, qp->qp_id, ibqp->qp_type,
2365 old_qp_state, new_qp_state);
2366 rc = -EINVAL;
2367 goto err;
2368 }
cecbcddf
RA		 2369 }
2370
2371 /* Translate the masks... */
2372 if (attr_mask & IB_QP_STATE) {
2373 SET_FIELD(qp_params.modify_flags,
2374 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2375 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2376 }
2377
2378 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2379 qp_params.sqd_async = true;
2380
2381 if (attr_mask & IB_QP_PKEY_INDEX) {
2382 SET_FIELD(qp_params.modify_flags,
2383 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2384 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2385 rc = -EINVAL;
2386 goto err;
2387 }
2388
2389 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2390 }
2391
2392 if (attr_mask & IB_QP_QKEY)
2393 qp->qkey = attr->qkey;
2394
2395 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2396 SET_FIELD(qp_params.modify_flags,
2397 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2398 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2399 IB_ACCESS_REMOTE_READ;
2400 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2401 IB_ACCESS_REMOTE_WRITE;
2402 qp_params.incoming_atomic_en = attr->qp_access_flags &
2403 IB_ACCESS_REMOTE_ATOMIC;
2404 }
2405
2406 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2407 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2408 return -EINVAL;
2409
cecbcddf
RA		 2410 if (attr_mask & IB_QP_PATH_MTU) {
2411 if (attr->path_mtu < IB_MTU_256 ||
2412 attr->path_mtu > IB_MTU_4096) {
2413 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2414 rc = -EINVAL;
2415 goto err;
2416 }
2417 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2418 ib_mtu_enum_to_int(iboe_get_mtu
2419 (dev->ndev->mtu)));
2420 }
2421
2422 if (!qp->mtu) {
2423 qp->mtu =
2424 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2425 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2426 }
2427
2428 SET_FIELD(qp_params.modify_flags,
2429 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2430
d8966fcd
DC		 2431 qp_params.traffic_class_tos = grh->traffic_class;
2432 qp_params.flow_label = grh->flow_label;
2433 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2434
d8966fcd 2435 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2436
2437 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2438 if (rc) {
2439 DP_ERR(dev,
2440 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2441 grh->sgid_index, rc);
cecbcddf
RA		 2442 return rc;
2443 }
2444
2445 rc = qedr_get_dmac(dev, &attr->ah_attr,
2446 qp_params.remote_mac_addr);
2447 if (rc)
2448 return rc;
2449
2450 qp_params.use_local_mac = true;
2451 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2452
2453 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2454 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2455 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2456 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2457 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2458 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2459 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2460 qp_params.remote_mac_addr);
cecbcddf
RA		 2461
2462 qp_params.mtu = qp->mtu;
2463 qp_params.lb_indication = false;
2464 }
2465
2466 if (!qp_params.mtu) {
2467 /* Stay with current MTU */
2468 if (qp->mtu)
2469 qp_params.mtu = qp->mtu;
2470 else
2471 qp_params.mtu =
2472 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2473 }
2474
2475 if (attr_mask & IB_QP_TIMEOUT) {
2476 SET_FIELD(qp_params.modify_flags,
2477 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2478
c3594f22
KM		 2479 /* The received timeout value is an exponent used like this:
2480 * "12.7.34 LOCAL ACK TIMEOUT
2481 * Value representing the transport (ACK) timeout for use by
2482 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2483 * The FW expects timeout in msec so we need to divide the usec
2484 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2485 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2486 * The value of zero means infinite so we use a 'max_t' to make
2487 * sure that sub 1 msec values will be configured as 1 msec.
2488 */
2489 if (attr->timeout)
2490 qp_params.ack_timeout =
2491 1 << max_t(int, attr->timeout - 8, 0);
2492 else
cecbcddf 2493 qp_params.ack_timeout = 0;
cecbcddf 2494 }
c3594f22 2495
cecbcddf
RA		 2496 if (attr_mask & IB_QP_RETRY_CNT) {
2497 SET_FIELD(qp_params.modify_flags,
2498 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2499 qp_params.retry_cnt = attr->retry_cnt;
2500 }
2501
2502 if (attr_mask & IB_QP_RNR_RETRY) {
2503 SET_FIELD(qp_params.modify_flags,
2504 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2505 qp_params.rnr_retry_cnt = attr->rnr_retry;
2506 }
2507
2508 if (attr_mask & IB_QP_RQ_PSN) {
2509 SET_FIELD(qp_params.modify_flags,
2510 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2511 qp_params.rq_psn = attr->rq_psn;
2512 qp->rq_psn = attr->rq_psn;
2513 }
2514
2515 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2516 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2517 rc = -EINVAL;
2518 DP_ERR(dev,
2519 "unsupported max_rd_atomic=%d, supported=%d\n",
2520 attr->max_rd_atomic,
2521 dev->attr.max_qp_req_rd_atomic_resc);
2522 goto err;
2523 }
2524
2525 SET_FIELD(qp_params.modify_flags,
2526 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2527 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2528 }
2529
2530 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2531 SET_FIELD(qp_params.modify_flags,
2532 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2533 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2534 }
2535
2536 if (attr_mask & IB_QP_SQ_PSN) {
2537 SET_FIELD(qp_params.modify_flags,
2538 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2539 qp_params.sq_psn = attr->sq_psn;
2540 qp->sq_psn = attr->sq_psn;
2541 }
2542
2543 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2544 if (attr->max_dest_rd_atomic >
2545 dev->attr.max_qp_resp_rd_atomic_resc) {
2546 DP_ERR(dev,
2547 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2548 attr->max_dest_rd_atomic,
2549 dev->attr.max_qp_resp_rd_atomic_resc);
2550
2551 rc = -EINVAL;
2552 goto err;
2553 }
2554
2555 SET_FIELD(qp_params.modify_flags,
2556 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2557 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2558 }
2559
2560 if (attr_mask & IB_QP_DEST_QPN) {
2561 SET_FIELD(qp_params.modify_flags,
2562 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2563
2564 qp_params.dest_qp = attr->dest_qp_num;
2565 qp->dest_qp_num = attr->dest_qp_num;
2566 }
2567
caf61b1b
KM		 2568 cur_state = qp->state;
2569
2570 /* Update the QP state before the actual ramrod to prevent a race with
2571 * fast path. Modifying the QP state to error will cause the device to
2572 * flush the CQEs and while polling the flushed CQEs will considered as
2573 * a potential issue if the QP isn't in error state.
2574 */
2575 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2576 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2577 qp->state = QED_ROCE_QP_STATE_ERR;
2578
cecbcddf
RA		 2579 if (qp->qp_type != IB_QPT_GSI)
2580 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2581 qp->qed_qp, &qp_params);
2582
2583 if (attr_mask & IB_QP_STATE) {
2584 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2585 rc = qedr_update_qp_state(dev, qp, cur_state,
2586 qp_params.new_state);
cecbcddf
RA		 2587 qp->state = qp_params.new_state;
2588 }
2589
2590err:
2591 return rc;
2592}
2593
2594static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2595{
2596 int ib_qp_acc_flags = 0;
2597
2598 if (params->incoming_rdma_write_en)
2599 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2600 if (params->incoming_rdma_read_en)
2601 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2602 if (params->incoming_atomic_en)
2603 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2604 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2605 return ib_qp_acc_flags;
2606}
2607
2608int qedr_query_qp(struct ib_qp *ibqp,
2609 struct ib_qp_attr *qp_attr,
2610 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2611{
2612 struct qed_rdma_query_qp_out_params params;
2613 struct qedr_qp *qp = get_qedr_qp(ibqp);
2614 struct qedr_dev *dev = qp->dev;
2615 int rc = 0;
2616
2617 memset(&params, 0, sizeof(params));
2618
2619 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2620 if (rc)
2621 goto err;
2622
2623 memset(qp_attr, 0, sizeof(*qp_attr));
2624 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2625
2626 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2627 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2628 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2629 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2630 qp_attr->rq_psn = params.rq_psn;
2631 qp_attr->sq_psn = params.sq_psn;
2632 qp_attr->dest_qp_num = params.dest_qp;
2633
2634 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2635
2636 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2637 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2638 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2639 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 2640 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 2641 qp_init_attr->cap = qp_attr->cap;
2642
44c58487 2643 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2644 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2645 params.flow_label, qp->sgid_idx,
2646 params.hop_limit_ttl, params.traffic_class_tos);
2647 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2648 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2649 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2650 qp_attr->timeout = params.timeout;
2651 qp_attr->rnr_retry = params.rnr_retry;
2652 qp_attr->retry_cnt = params.retry_cnt;
2653 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2654 qp_attr->pkey_index = params.pkey_index;
2655 qp_attr->port_num = 1;
d8966fcd
DC		 2656 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2657 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2658 qp_attr->alt_pkey_index = 0;
2659 qp_attr->alt_port_num = 0;
2660 qp_attr->alt_timeout = 0;
2661 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2662
2663 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2664 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2665 qp_attr->max_rd_atomic = params.max_rd_atomic;
2666 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2667
2668 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2669 qp_attr->cap.max_inline_data);
2670
2671err:
2672 return rc;
2673}
2674
bdeacabd
SR		 2675static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2676 struct ib_udata *udata)
df158561 2677{
97f61250
MK		 2678 struct qedr_ucontext *ctx =
2679 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2680 ibucontext);
2681 int rc;
df158561
AR		 2682
2683 if (qp->qp_type != IB_QPT_GSI) {
2684 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2685 if (rc)
2686 return rc;
2687 }
2688
82af6d19 2689 if (qp->create_type == QEDR_QP_CREATE_USER)
97f61250 2690 qedr_cleanup_user(dev, ctx, qp);
df158561
AR		 2691 else
2692 qedr_cleanup_kernel(dev, qp);
2693
2694 return 0;
2695}
2696
c4367a26 2697int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2698{
2699 struct qedr_qp *qp = get_qedr_qp(ibqp);
2700 struct qedr_dev *dev = qp->dev;
2701 struct ib_qp_attr attr;
2702 int attr_mask = 0;
cecbcddf
RA		 2703
2704 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2705 qp, qp->qp_type);
2706
f5b1b177
KM		 2707 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2708 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2709 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2710 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2711
f5b1b177
KM		 2712 attr.qp_state = IB_QPS_ERR;
2713 attr_mask |= IB_QP_STATE;
cecbcddf 2714
f5b1b177
KM		 2715 /* Change the QP state to ERROR */
2716 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2717 }
e411e058 2718 } else {
82af6d19
MK		 2719 /* If connection establishment started the WAIT_FOR_CONNECT
2720 * bit will be on and we need to Wait for the establishment
2721 * to complete before destroying the qp.
2722 */
2723 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2724 &qp->iwarp_cm_flags))
2725 wait_for_completion(&qp->iwarp_cm_comp);
2726
2727 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2728 * bit will be on, and we need to wait for the disconnect to
2729 * complete before continuing. We can use the same completion,
2730 * iwarp_cm_comp, since this is the only place that waits for
2731 * this completion and it is sequential. In addition,
2732 * disconnect can't occur before the connection is fully
2733 * established, therefore if WAIT_FOR_DISCONNECT is on it
2734 * means WAIT_FOR_CONNECT is also on and the completion for
2735 * CONNECT already occurred.
2736 */
2737 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2738 &qp->iwarp_cm_flags))
2739 wait_for_completion(&qp->iwarp_cm_comp);
cecbcddf
RA		 2740 }
2741
df158561 2742 if (qp->qp_type == IB_QPT_GSI)
04886779 2743 qedr_destroy_gsi_qp(dev);
cecbcddf 2744
82af6d19
MK		 2745 /* We need to remove the entry from the xarray before we release the
2746 * qp_id to avoid a race of the qp_id being reallocated and failing
2747 * on xa_insert
2748 */
2749 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2750 xa_erase(&dev->qps, qp->qp_id);
2751
bdeacabd 2752 qedr_free_qp_resources(dev, qp, udata);
cecbcddf 2753
82af6d19
MK		 2754 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2755 qedr_iw_qp_rem_ref(&qp->ibqp);
2756
cf167e5e 2757 return 0;
cecbcddf 2758}
e0290cce 2759
fa5d010c 2760int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
d3456914 2761 struct ib_udata *udata)
04886779 2762{
d3456914 2763 struct qedr_ah *ah = get_qedr_ah(ibah);
04886779 2764
fa5d010c 2765 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
04886779 2766
d3456914 2767 return 0;
04886779
RA		 2768}
2769
9a9ebf8c 2770int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
04886779
RA		 2771{
2772 struct qedr_ah *ah = get_qedr_ah(ibah);
2773
d97099fe 2774 rdma_destroy_ah_attr(&ah->attr);
9a9ebf8c 2775 return 0;
04886779
RA		 2776}
2777
e0290cce
RA		 2778static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2779{
2780 struct qedr_pbl *pbl, *tmp;
2781
2782 if (info->pbl_table)
2783 list_add_tail(&info->pbl_table->list_entry,
2784 &info->free_pbl_list);
2785
2786 if (!list_empty(&info->inuse_pbl_list))
2787 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2788
2789 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2790 list_del(&pbl->list_entry);
2791 qedr_free_pbl(dev, &info->pbl_info, pbl);
2792 }
2793}
2794
2795static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2796 size_t page_list_len, bool two_layered)
2797{
2798 struct qedr_pbl *tmp;
2799 int rc;
2800
2801 INIT_LIST_HEAD(&info->free_pbl_list);
2802 INIT_LIST_HEAD(&info->inuse_pbl_list);
2803
2804 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2805 page_list_len, two_layered);
2806 if (rc)
2807 goto done;
2808
2809 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2810 if (IS_ERR(info->pbl_table)) {
2811 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2812 goto done;
2813 }
2814
2815 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2816 &info->pbl_table->pa);
2817
2818 /* in usual case we use 2 PBLs, so we add one to free
2819 * list and allocating another one
2820 */
2821 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2822 if (IS_ERR(tmp)) {
e0290cce
RA		 2823 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2824 goto done;
2825 }
2826
2827 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2828
2829 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2830
2831done:
2832 if (rc)
2833 free_mr_info(dev, info);
2834
2835 return rc;
2836}
2837
2838struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2839 u64 usr_addr, int acc, struct ib_udata *udata)
2840{
2841 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2842 struct qedr_mr *mr;
2843 struct qedr_pd *pd;
2844 int rc = -ENOMEM;
2845
2846 pd = get_qedr_pd(ibpd);
2847 DP_DEBUG(dev, QEDR_MSG_MR,
2848 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2849 pd->pd_id, start, len, usr_addr, acc);
2850
2851 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2852 return ERR_PTR(-EINVAL);
2853
2854 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2855 if (!mr)
2856 return ERR_PTR(rc);
2857
2858 mr->type = QEDR_MR_USER;
2859
c320e527 2860 mr->umem = ib_umem_get(ibpd->device, start, len, acc);
e0290cce
RA		 2861 if (IS_ERR(mr->umem)) {
2862 rc = -EFAULT;
2863 goto err0;
2864 }
2865
2866 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2867 if (rc)
2868 goto err1;
2869
2870 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2871 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2872
2873 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2874 if (rc) {
2875 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2876 goto err1;
2877 }
2878
2879 /* Index only, 18 bit long, lkey = itid << 8 | key */
2880 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2881 mr->hw_mr.key = 0;
2882 mr->hw_mr.pd = pd->pd_id;
2883 mr->hw_mr.local_read = 1;
2884 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2885 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2886 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2887 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2888 mr->hw_mr.mw_bind = false;
2889 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2890 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2891 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 2892 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 2893 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2894 mr->hw_mr.length = len;
2895 mr->hw_mr.vaddr = usr_addr;
2896 mr->hw_mr.zbva = false;
2897 mr->hw_mr.phy_mr = false;
2898 mr->hw_mr.dma_mr = false;
2899
2900 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2901 if (rc) {
2902 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2903 goto err2;
2904 }
2905
2906 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2907 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2908 mr->hw_mr.remote_atomic)
2909 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2910
2911 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2912 mr->ibmr.lkey);
2913 return &mr->ibmr;
2914
2915err2:
2916 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2917err1:
2918 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2919err0:
2920 kfree(mr);
2921 return ERR_PTR(rc);
2922}
2923
c4367a26 2924int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 2925{
2926 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2927 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2928 int rc = 0;
2929
2930 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2931 if (rc)
2932 return rc;
2933
2934 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2935
24e412c1
MK		 2936 if (mr->type != QEDR_MR_DMA)
2937 free_mr_info(dev, &mr->info);
e0290cce
RA		 2938
2939 /* it could be user registered memory. */
836a0fbb 2940 ib_umem_release(mr->umem);
e0290cce
RA		 2941
2942 kfree(mr);
2943
2944 return rc;
2945}
2946
27a4b1a6
RA		 2947static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2948 int max_page_list_len)
e0290cce
RA		 2949{
2950 struct qedr_pd *pd = get_qedr_pd(ibpd);
2951 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2952 struct qedr_mr *mr;
2953 int rc = -ENOMEM;
2954
2955 DP_DEBUG(dev, QEDR_MSG_MR,
2956 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2957 max_page_list_len);
2958
2959 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2960 if (!mr)
2961 return ERR_PTR(rc);
2962
2963 mr->dev = dev;
2964 mr->type = QEDR_MR_FRMR;
2965
2966 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2967 if (rc)
2968 goto err0;
2969
2970 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2971 if (rc) {
2972 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2973 goto err0;
2974 }
2975
2976 /* Index only, 18 bit long, lkey = itid << 8 | key */
2977 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2978 mr->hw_mr.key = 0;
2979 mr->hw_mr.pd = pd->pd_id;
2980 mr->hw_mr.local_read = 1;
2981 mr->hw_mr.local_write = 0;
2982 mr->hw_mr.remote_read = 0;
2983 mr->hw_mr.remote_write = 0;
2984 mr->hw_mr.remote_atomic = 0;
2985 mr->hw_mr.mw_bind = false;
2986 mr->hw_mr.pbl_ptr = 0;
2987 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2988 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2989 mr->hw_mr.fbo = 0;
2990 mr->hw_mr.length = 0;
2991 mr->hw_mr.vaddr = 0;
2992 mr->hw_mr.zbva = false;
2993 mr->hw_mr.phy_mr = true;
2994 mr->hw_mr.dma_mr = false;
2995
2996 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2997 if (rc) {
2998 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2999 goto err1;
3000 }
3001
3002 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3003 mr->ibmr.rkey = mr->ibmr.lkey;
3004
3005 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3006 return mr;
3007
3008err1:
3009 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3010err0:
3011 kfree(mr);
3012 return ERR_PTR(rc);
3013}
3014
c4367a26 3015struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
42a3b153 3016 u32 max_num_sg)
e0290cce 3017{
e0290cce
RA		 3018 struct qedr_mr *mr;
3019
3020 if (mr_type != IB_MR_TYPE_MEM_REG)
3021 return ERR_PTR(-EINVAL);
3022
3023 mr = __qedr_alloc_mr(ibpd, max_num_sg);
3024
3025 if (IS_ERR(mr))
3026 return ERR_PTR(-EINVAL);
3027
e0290cce
RA		 3028 return &mr->ibmr;
3029}
3030
3031static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3032{
3033 struct qedr_mr *mr = get_qedr_mr(ibmr);
3034 struct qedr_pbl *pbl_table;
3035 struct regpair *pbe;
3036 u32 pbes_in_page;
3037
3038 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 3039 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 3040 return -ENOMEM;
3041 }
3042
3043 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3044 mr->npages, addr);
3045
3046 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3047 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3048 pbe = (struct regpair *)pbl_table->va;
3049 pbe += mr->npages % pbes_in_page;
3050 pbe->lo = cpu_to_le32((u32)addr);
3051 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3052
3053 mr->npages++;
3054
3055 return 0;
3056}
3057
3058static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3059{
3060 int work = info->completed - info->completed_handled - 1;
3061
3062 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3063 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3064 struct qedr_pbl *pbl;
3065
3066 /* Free all the page list that are possible to be freed
3067 * (all the ones that were invalidated), under the assumption
3068 * that if an FMR was completed successfully that means that
3069 * if there was an invalidate operation before it also ended
3070 */
3071 pbl = list_first_entry(&info->inuse_pbl_list,
3072 struct qedr_pbl, list_entry);
aafec388 3073 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 3074 info->completed_handled++;
3075 }
3076}
3077
3078int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3079 int sg_nents, unsigned int *sg_offset)
3080{
3081 struct qedr_mr *mr = get_qedr_mr(ibmr);
3082
3083 mr->npages = 0;
3084
3085 handle_completed_mrs(mr->dev, &mr->info);
3086 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3087}
3088
3089struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3090{
3091 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3092 struct qedr_pd *pd = get_qedr_pd(ibpd);
3093 struct qedr_mr *mr;
3094 int rc;
3095
3096 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3097 if (!mr)
3098 return ERR_PTR(-ENOMEM);
3099
3100 mr->type = QEDR_MR_DMA;
3101
3102 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3103 if (rc) {
3104 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3105 goto err1;
3106 }
3107
3108 /* index only, 18 bit long, lkey = itid << 8 | key */
3109 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3110 mr->hw_mr.pd = pd->pd_id;
3111 mr->hw_mr.local_read = 1;
3112 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3113 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3114 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3115 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3116 mr->hw_mr.dma_mr = true;
3117
3118 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3119 if (rc) {
3120 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3121 goto err2;
3122 }
3123
3124 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3125 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3126 mr->hw_mr.remote_atomic)
3127 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3128
3129 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3130 return &mr->ibmr;
3131
3132err2:
3133 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3134err1:
3135 kfree(mr);
3136 return ERR_PTR(rc);
3137}
afa0e13b
RA		 3138
3139static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3140{
3141 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3142}
3143
3144static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3145{
3146 int i, len = 0;
3147
3148 for (i = 0; i < num_sge; i++)
3149 len += sg_list[i].length;
3150
3151 return len;
3152}
3153
3154static void swap_wqe_data64(u64 *p)
3155{
3156 int i;
3157
3158 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3159 *p = cpu_to_be64(cpu_to_le64(*p));
3160}
3161
3162static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3163 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 3164 const struct ib_send_wr *wr,
3165 const struct ib_send_wr **bad_wr,
3166 u8 *bits, u8 bit)
afa0e13b
RA		 3167{
3168 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3169 char *seg_prt, *wqe;
3170 int i, seg_siz;
3171
3172 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3173 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3174 *bad_wr = wr;
3175 return 0;
3176 }
3177
3178 if (!data_size)
3179 return data_size;
3180
3181 *bits |= bit;
3182
3183 seg_prt = NULL;
3184 wqe = NULL;
3185 seg_siz = 0;
3186
3187 /* Copy data inline */
3188 for (i = 0; i < wr->num_sge; i++) {
3189 u32 len = wr->sg_list[i].length;
3190 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3191
3192 while (len > 0) {
3193 u32 cur;
3194
3195 /* New segment required */
3196 if (!seg_siz) {
3197 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3198 seg_prt = wqe;
3199 seg_siz = sizeof(struct rdma_sq_common_wqe);
3200 (*wqe_size)++;
3201 }
3202
3203 /* Calculate currently allowed length */
3204 cur = min_t(u32, len, seg_siz);
3205 memcpy(seg_prt, src, cur);
3206
3207 /* Update segment variables */
3208 seg_prt += cur;
3209 seg_siz -= cur;
3210
3211 /* Update sge variables */
3212 src += cur;
3213 len -= cur;
3214
3215 /* Swap fully-completed segments */
3216 if (!seg_siz)
3217 swap_wqe_data64((u64 *)wqe);
3218 }
3219 }
3220
3221 /* swap last not completed segment */
3222 if (seg_siz)
3223 swap_wqe_data64((u64 *)wqe);
3224
3225 return data_size;
3226}
3227
3228#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3229 do { \
3230 DMA_REGPAIR_LE(sge->addr, vaddr); \
3231 (sge)->length = cpu_to_le32(vlength); \
3232 (sge)->flags = cpu_to_le32(vflags); \
3233 } while (0)
3234
3235#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3236 do { \
3237 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3238 (hdr)->num_sges = num_sge; \
3239 } while (0)
3240
3241#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3242 do { \
3243 DMA_REGPAIR_LE(sge->addr, vaddr); \
3244 (sge)->length = cpu_to_le32(vlength); \
3245 (sge)->l_key = cpu_to_le32(vlkey); \
3246 } while (0)
3247
3248static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 3249 const struct ib_send_wr *wr)
afa0e13b
RA		 3250{
3251 u32 data_size = 0;
3252 int i;
3253
3254 for (i = 0; i < wr->num_sge; i++) {
3255 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3256
3257 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3258 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3259 sge->length = cpu_to_le32(wr->sg_list[i].length);
3260 data_size += wr->sg_list[i].length;
3261 }
3262
3263 if (wqe_size)
3264 *wqe_size += wr->num_sge;
3265
3266 return data_size;
3267}
3268
3269static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3270 struct qedr_qp *qp,
3271 struct rdma_sq_rdma_wqe_1st *rwqe,
3272 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3273 const struct ib_send_wr *wr,
3274 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3275{
3276 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3277 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3278
8b0cabc6
AR		 3279 if (wr->send_flags & IB_SEND_INLINE &&
3280 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3281 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3282 u8 flags = 0;
3283
3284 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3285 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3286 bad_wr, &rwqe->flags, flags);
3287 }
3288
3289 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3290}
3291
3292static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3293 struct qedr_qp *qp,
3294 struct rdma_sq_send_wqe_1st *swqe,
3295 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3296 const struct ib_send_wr *wr,
3297 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3298{
3299 memset(swqe2, 0, sizeof(*swqe2));
3300 if (wr->send_flags & IB_SEND_INLINE) {
3301 u8 flags = 0;
3302
3303 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3304 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3305 bad_wr, &swqe->flags, flags);
3306 }
3307
3308 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3309}
3310
3311static int qedr_prepare_reg(struct qedr_qp *qp,
3312 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3313 const struct ib_reg_wr *wr)
afa0e13b
RA		 3314{
3315 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3316 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3317
3318 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3319 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3320 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3321 fwqe1->l_key = wr->key;
3322
08c4cf51
AR		 3323 fwqe2->access_ctrl = 0;
3324
afa0e13b
RA		 3325 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3326 !!(wr->access & IB_ACCESS_REMOTE_READ));
3327 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3328 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3329 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3330 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3331 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3332 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3333 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3334 fwqe2->fmr_ctrl = 0;
3335
3336 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3337 ilog2(mr->ibmr.page_size) - 12);
3338
3339 fwqe2->length_hi = 0;
3340 fwqe2->length_lo = mr->ibmr.length;
3341 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3342 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3343
3344 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3345
3346 return 0;
3347}
3348
27a4b1a6 3349static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3350{
3351 switch (opcode) {
3352 case IB_WR_RDMA_WRITE:
3353 case IB_WR_RDMA_WRITE_WITH_IMM:
3354 return IB_WC_RDMA_WRITE;
3355 case IB_WR_SEND_WITH_IMM:
3356 case IB_WR_SEND:
3357 case IB_WR_SEND_WITH_INV:
3358 return IB_WC_SEND;
3359 case IB_WR_RDMA_READ:
fb1a22be 3360 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3361 return IB_WC_RDMA_READ;
3362 case IB_WR_ATOMIC_CMP_AND_SWP:
3363 return IB_WC_COMP_SWAP;
3364 case IB_WR_ATOMIC_FETCH_AND_ADD:
3365 return IB_WC_FETCH_ADD;
3366 case IB_WR_REG_MR:
3367 return IB_WC_REG_MR;
3368 case IB_WR_LOCAL_INV:
3369 return IB_WC_LOCAL_INV;
3370 default:
3371 return IB_WC_SEND;
3372 }
3373}
3374
f696bf6d
BVA		 3375static inline bool qedr_can_post_send(struct qedr_qp *qp,
3376 const struct ib_send_wr *wr)
afa0e13b
RA		 3377{
3378 int wq_is_full, err_wr, pbl_is_full;
3379 struct qedr_dev *dev = qp->dev;
3380
3381 /* prevent SQ overflow and/or processing of a bad WR */
3382 err_wr = wr->num_sge > qp->sq.max_sges;
3383 wq_is_full = qedr_wq_is_full(&qp->sq);
3384 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3385 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3386 if (wq_is_full || err_wr || pbl_is_full) {
3387 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3388 DP_ERR(dev,
3389 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3390 qp);
3391 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3392 }
3393
3394 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3395 DP_ERR(dev,
3396 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3397 qp);
3398 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3399 }
3400
3401 if (pbl_is_full &&
3402 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3403 DP_ERR(dev,
3404 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3405 qp);
3406 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3407 }
3408 return false;
3409 }
3410 return true;
3411}
3412
d34ac5cd
BVA		 3413static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3414 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3415{
3416 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3417 struct qedr_qp *qp = get_qedr_qp(ibqp);
3418 struct rdma_sq_atomic_wqe_1st *awqe1;
3419 struct rdma_sq_atomic_wqe_2nd *awqe2;
3420 struct rdma_sq_atomic_wqe_3rd *awqe3;
3421 struct rdma_sq_send_wqe_2st *swqe2;
3422 struct rdma_sq_local_inv_wqe *iwqe;
3423 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3424 struct rdma_sq_send_wqe_1st *swqe;
3425 struct rdma_sq_rdma_wqe_1st *rwqe;
3426 struct rdma_sq_fmr_wqe_1st *fwqe1;
3427 struct rdma_sq_common_wqe *wqe;
3428 u32 length;
3429 int rc = 0;
3430 bool comp;
3431
3432 if (!qedr_can_post_send(qp, wr)) {
3433 *bad_wr = wr;
3434 return -ENOMEM;
3435 }
3436
3437 wqe = qed_chain_produce(&qp->sq.pbl);
3438 qp->wqe_wr_id[qp->sq.prod].signaled =
3439 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3440
3441 wqe->flags = 0;
3442 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3443 !!(wr->send_flags & IB_SEND_SOLICITED));
3444 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3445 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3446 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3447 !!(wr->send_flags & IB_SEND_FENCE));
3448 wqe->prev_wqe_size = qp->prev_wqe_size;
3449
3450 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3451
3452 switch (wr->opcode) {
3453 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3454 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3455 rc = -EINVAL;
3456 *bad_wr = wr;
3457 break;
3458 }
afa0e13b
RA		 3459 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3460 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3461 swqe->wqe_size = 2;
3462 swqe2 = qed_chain_produce(&qp->sq.pbl);
3463
7bed7ebc 3464 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3465 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3466 wr, bad_wr);
3467 swqe->length = cpu_to_le32(length);
3468 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3469 qp->prev_wqe_size = swqe->wqe_size;
3470 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3471 break;
3472 case IB_WR_SEND:
3473 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3474 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3475
3476 swqe->wqe_size = 2;
3477 swqe2 = qed_chain_produce(&qp->sq.pbl);
3478 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3479 wr, bad_wr);
3480 swqe->length = cpu_to_le32(length);
3481 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3482 qp->prev_wqe_size = swqe->wqe_size;
3483 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3484 break;
3485 case IB_WR_SEND_WITH_INV:
3486 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3487 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3488 swqe2 = qed_chain_produce(&qp->sq.pbl);
3489 swqe->wqe_size = 2;
3490 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3491 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3492 wr, bad_wr);
3493 swqe->length = cpu_to_le32(length);
3494 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3495 qp->prev_wqe_size = swqe->wqe_size;
3496 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3497 break;
3498
3499 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3500 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3501 rc = -EINVAL;
3502 *bad_wr = wr;
3503 break;
3504 }
afa0e13b
RA		 3505 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3506 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3507
3508 rwqe->wqe_size = 2;
3509 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3510 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3511 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3512 wr, bad_wr);
3513 rwqe->length = cpu_to_le32(length);
3514 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3515 qp->prev_wqe_size = rwqe->wqe_size;
3516 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3517 break;
3518 case IB_WR_RDMA_WRITE:
3519 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3520 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3521
3522 rwqe->wqe_size = 2;
3523 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3524 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3525 wr, bad_wr);
3526 rwqe->length = cpu_to_le32(length);
3527 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3528 qp->prev_wqe_size = rwqe->wqe_size;
3529 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3530 break;
3531 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3532 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
df561f66 3533 fallthrough; /* same is identical to RDMA READ */
afa0e13b
RA		 3534
3535 case IB_WR_RDMA_READ:
3536 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3537 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3538
3539 rwqe->wqe_size = 2;
3540 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3541 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3542 wr, bad_wr);
3543 rwqe->length = cpu_to_le32(length);
3544 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3545 qp->prev_wqe_size = rwqe->wqe_size;
3546 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3547 break;
3548
3549 case IB_WR_ATOMIC_CMP_AND_SWP:
3550 case IB_WR_ATOMIC_FETCH_AND_ADD:
3551 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3552 awqe1->wqe_size = 4;
3553
3554 awqe2 = qed_chain_produce(&qp->sq.pbl);
3555 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3556 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3557
3558 awqe3 = qed_chain_produce(&qp->sq.pbl);
3559
3560 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3561 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3562 DMA_REGPAIR_LE(awqe3->swap_data,
3563 atomic_wr(wr)->compare_add);
3564 } else {
3565 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3566 DMA_REGPAIR_LE(awqe3->swap_data,
3567 atomic_wr(wr)->swap);
3568 DMA_REGPAIR_LE(awqe3->cmp_data,
3569 atomic_wr(wr)->compare_add);
3570 }
3571
3572 qedr_prepare_sq_sges(qp, NULL, wr);
3573
3574 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3575 qp->prev_wqe_size = awqe1->wqe_size;
3576 break;
3577
3578 case IB_WR_LOCAL_INV:
3579 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3580 iwqe->wqe_size = 1;
3581
3582 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3583 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3584 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3585 qp->prev_wqe_size = iwqe->wqe_size;
3586 break;
3587 case IB_WR_REG_MR:
3588 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3589 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3590 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3591 fwqe1->wqe_size = 2;
3592
3593 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3594 if (rc) {
3595 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3596 *bad_wr = wr;
3597 break;
3598 }
3599
3600 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3601 qp->prev_wqe_size = fwqe1->wqe_size;
3602 break;
3603 default:
3604 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3605 rc = -EINVAL;
3606 *bad_wr = wr;
3607 break;
3608 }
3609
3610 if (*bad_wr) {
3611 u16 value;
3612
3613 /* Restore prod to its position before
3614 * this WR was processed
3615 */
3616 value = le16_to_cpu(qp->sq.db_data.data.value);
3617 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3618
3619 /* Restore prev_wqe_size */
3620 qp->prev_wqe_size = wqe->prev_wqe_size;
3621 rc = -EINVAL;
3622 DP_ERR(dev, "POST SEND FAILED\n");
3623 }
3624
3625 return rc;
3626}
3627
d34ac5cd
BVA		 3628int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3629 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3630{
3631 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3632 struct qedr_qp *qp = get_qedr_qp(ibqp);
3633 unsigned long flags;
3634 int rc = 0;
3635
3636 *bad_wr = NULL;
3637
04886779
RA		 3638 if (qp->qp_type == IB_QPT_GSI)
3639 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3640
afa0e13b
RA		 3641 spin_lock_irqsave(&qp->q_lock, flags);
3642
f5b1b177
KM		 3643 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3644 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3645 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3646 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3647 spin_unlock_irqrestore(&qp->q_lock, flags);
3648 *bad_wr = wr;
3649 DP_DEBUG(dev, QEDR_MSG_CQ,
3650 "QP in wrong state! QP icid=0x%x state %d\n",
3651 qp->icid, qp->state);
3652 return -EINVAL;
3653 }
afa0e13b
RA		 3654 }
3655
afa0e13b
RA		 3656 while (wr) {
3657 rc = __qedr_post_send(ibqp, wr, bad_wr);
3658 if (rc)
3659 break;
3660
3661 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3662
3663 qedr_inc_sw_prod(&qp->sq);
3664
3665 qp->sq.db_data.data.value++;
3666
3667 wr = wr->next;
3668 }
3669
3670 /* Trigger doorbell
3671 * If there was a failure in the first WR then it will be triggered in
3672 * vane. However this is not harmful (as long as the producer value is
3673 * unchanged). For performance reasons we avoid checking for this
3674 * redundant doorbell.
09c4854f
KM		 3675 *
3676 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3677 * soon as we give the doorbell, we could get a completion
3678 * for this wr, therefore we need to make sure that the
3679 * memory is updated before giving the doorbell.
3680 * During qedr_poll_cq, rmb is called before accessing the
3681 * cqe. This covers for the smp_rmb as well.
afa0e13b 3682 */
09c4854f 3683 smp_wmb();
afa0e13b
RA		 3684 writel(qp->sq.db_data.raw, qp->sq.db);
3685
afa0e13b
RA		 3686 spin_unlock_irqrestore(&qp->q_lock, flags);
3687
3688 return rc;
3689}
3690
3491c9e7
YB		 3691static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3692{
3693 u32 used;
3694
3695 /* Calculate number of elements used based on producer
3696 * count and consumer count and subtract it from max
3697 * work request supported so that we get elements left.
3698 */
acca72e2 3699 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3491c9e7
YB		 3700
3701 return hw_srq->max_wr - used;
3702}
3703
3704int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3705 const struct ib_recv_wr **bad_wr)
3706{
3707 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3708 struct qedr_srq_hwq_info *hw_srq;
3709 struct qedr_dev *dev = srq->dev;
3710 struct qed_chain *pbl;
3711 unsigned long flags;
3712 int status = 0;
3713 u32 num_sge;
3491c9e7
YB		 3714
3715 spin_lock_irqsave(&srq->lock, flags);
3716
3717 hw_srq = &srq->hw_srq;
3718 pbl = &srq->hw_srq.pbl;
3719 while (wr) {
3720 struct rdma_srq_wqe_header *hdr;
3721 int i;
3722
3723 if (!qedr_srq_elem_left(hw_srq) ||
3724 wr->num_sge > srq->hw_srq.max_sges) {
3725 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
acca72e2
YB		 3726 hw_srq->wr_prod_cnt,
3727 atomic_read(&hw_srq->wr_cons_cnt),
3491c9e7
YB		 3728 wr->num_sge, srq->hw_srq.max_sges);
3729 status = -ENOMEM;
3730 *bad_wr = wr;
3731 break;
3732 }
3733
3734 hdr = qed_chain_produce(pbl);
3735 num_sge = wr->num_sge;
3736 /* Set number of sge and work request id in header */
3737 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3738
3739 srq->hw_srq.wr_prod_cnt++;
3740 hw_srq->wqe_prod++;
3741 hw_srq->sge_prod++;
3742
3743 DP_DEBUG(dev, QEDR_MSG_SRQ,
3744 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3745 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3746
3747 for (i = 0; i < wr->num_sge; i++) {
3748 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3749
3750 /* Set SGE length, lkey and address */
3751 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3752 wr->sg_list[i].length, wr->sg_list[i].lkey);
3753
3754 DP_DEBUG(dev, QEDR_MSG_SRQ,
3755 "[%d]: len %d key %x addr %x:%x\n",
3756 i, srq_sge->length, srq_sge->l_key,
3757 srq_sge->addr.hi, srq_sge->addr.lo);
3758 hw_srq->sge_prod++;
3759 }
3760
acca72e2 3761 /* Update WQE and SGE information before
3491c9e7
YB		 3762 * updating producer.
3763 */
acca72e2 3764 dma_wmb();
3491c9e7
YB		 3765
3766 /* SRQ producer is 8 bytes. Need to update SGE producer index
3767 * in first 4 bytes and need to update WQE producer in
3768 * next 4 bytes.
3769 */
acca72e2
YB		 3770 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3771 /* Make sure sge producer is updated first */
3772 dma_wmb();
3773 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3491c9e7 3774
3491c9e7
YB		 3775 wr = wr->next;
3776 }
3777
3778 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3779 qed_chain_get_elem_left(pbl));
3780 spin_unlock_irqrestore(&srq->lock, flags);
3781
3782 return status;
3783}
3784
d34ac5cd
BVA		 3785int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3786 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3787{
3788 struct qedr_qp *qp = get_qedr_qp(ibqp);
3789 struct qedr_dev *dev = qp->dev;
3790 unsigned long flags;
3791 int status = 0;
3792
04886779
RA		 3793 if (qp->qp_type == IB_QPT_GSI)
3794 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3795
afa0e13b
RA		 3796 spin_lock_irqsave(&qp->q_lock, flags);
3797
922d9a40 3798 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3799 spin_unlock_irqrestore(&qp->q_lock, flags);
3800 *bad_wr = wr;
3801 return -EINVAL;
3802 }
3803
3804 while (wr) {
3805 int i;
3806
3807 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3808 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3809 wr->num_sge > qp->rq.max_sges) {
3810 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3811 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3812 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3813 qp->rq.max_sges);
3814 status = -ENOMEM;
3815 *bad_wr = wr;
3816 break;
3817 }
3818 for (i = 0; i < wr->num_sge; i++) {
3819 u32 flags = 0;
3820 struct rdma_rq_sge *rqe =
3821 qed_chain_produce(&qp->rq.pbl);
3822
3823 /* First one must include the number
3824 * of SGE in the list
3825 */
3826 if (!i)
3827 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3828 wr->num_sge);
3829
d52c89f1 3830 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3831 wr->sg_list[i].lkey);
3832
3833 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3834 wr->sg_list[i].length, flags);
3835 }
3836
3837 /* Special case of no sges. FW requires between 1-4 sges...
3838 * in this case we need to post 1 sge with length zero. this is
3839 * because rdma write with immediate consumes an RQ.
3840 */
3841 if (!wr->num_sge) {
3842 u32 flags = 0;
3843 struct rdma_rq_sge *rqe =
3844 qed_chain_produce(&qp->rq.pbl);
3845
3846 /* First one must include the number
3847 * of SGE in the list
3848 */
d52c89f1 3849 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3850 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3851
3852 RQ_SGE_SET(rqe, 0, 0, flags);
3853 i = 1;
3854 }
3855
3856 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3857 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3858
3859 qedr_inc_sw_prod(&qp->rq);
3860
09c4854f
KM		 3861 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3862 * soon as we give the doorbell, we could get a completion
3863 * for this wr, therefore we need to make sure that the
3864 * memory is update before giving the doorbell.
3865 * During qedr_poll_cq, rmb is called before accessing the
3866 * cqe. This covers for the smp_rmb as well.
3867 */
3868 smp_wmb();
afa0e13b
RA		 3869
3870 qp->rq.db_data.data.value++;
3871
3872 writel(qp->rq.db_data.raw, qp->rq.db);
3873
f5b1b177
KM		 3874 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3875 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
f5b1b177
KM		 3876 }
3877
afa0e13b
RA		 3878 wr = wr->next;
3879 }
3880
3881 spin_unlock_irqrestore(&qp->q_lock, flags);
3882
3883 return status;
3884}
3885
3886static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3887{
3888 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3889
3890 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3891 cq->pbl_toggle;
3892}
3893
3894static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3895{
3896 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3897 struct qedr_qp *qp;
3898
3899 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3900 resp_cqe->qp_handle.lo,
3901 u64);
3902 return qp;
3903}
3904
3905static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3906{
3907 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3908
3909 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3910}
3911
3912/* Return latest CQE (needs processing) */
3913static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3914{
3915 return cq->latest_cqe;
3916}
3917
3918/* In fmr we need to increase the number of fmr completed counter for the fmr
3919 * algorithm determining whether we can free a pbl or not.
3920 * we need to perform this whether the work request was signaled or not. for
3921 * this purpose we call this function from the condition that checks if a wr
3922 * should be skipped, to make sure we don't miss it ( possibly this fmr
3923 * operation was not signalted)
3924 */
3925static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3926{
3927 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3928 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3929}
3930
3931static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3932 struct qedr_cq *cq, int num_entries,
3933 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3934 int force)
3935{
3936 u16 cnt = 0;
3937
3938 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3939 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3940 qedr_chk_if_fmr(qp);
3941 /* skip WC */
3942 goto next_cqe;
3943 }
3944
3945 /* fill WC */
3946 wc->status = status;
27035a1b 3947 wc->vendor_err = 0;
afa0e13b
RA		 3948 wc->wc_flags = 0;
3949 wc->src_qp = qp->id;
3950 wc->qp = &qp->ibqp;
3951
3952 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3953 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3954
3955 switch (wc->opcode) {
3956 case IB_WC_RDMA_WRITE:
3957 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3958 break;
3959 case IB_WC_COMP_SWAP:
3960 case IB_WC_FETCH_ADD:
3961 wc->byte_len = 8;
3962 break;
3963 case IB_WC_REG_MR:
3964 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3965 break;
dac27386
MK		 3966 case IB_WC_RDMA_READ:
3967 case IB_WC_SEND:
3968 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3969 break;
afa0e13b
RA		 3970 default:
3971 break;
3972 }
3973
3974 num_entries--;
3975 wc++;
3976 cnt++;
3977next_cqe:
3978 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3979 qed_chain_consume(&qp->sq.pbl);
3980 qedr_inc_sw_cons(&qp->sq);
3981 }
3982
3983 return cnt;
3984}
3985
3986static int qedr_poll_cq_req(struct qedr_dev *dev,
3987 struct qedr_qp *qp, struct qedr_cq *cq,
3988 int num_entries, struct ib_wc *wc,
3989 struct rdma_cqe_requester *req)
3990{
3991 int cnt = 0;
3992
3993 switch (req->status) {
3994 case RDMA_CQE_REQ_STS_OK:
3995 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3996 IB_WC_SUCCESS, 0);
3997 break;
3998 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 3999 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 4000 DP_DEBUG(dev, QEDR_MSG_CQ,
4001 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4002 cq->icid, qp->icid);
afa0e13b 4003 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 4004 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 4005 break;
4006 default:
4007 /* process all WQE before the cosumer */
4008 qp->state = QED_ROCE_QP_STATE_ERR;
4009 cnt = process_req(dev, qp, cq, num_entries, wc,
4010 req->sq_cons - 1, IB_WC_SUCCESS, 0);
4011 wc += cnt;
4012 /* if we have extra WC fill it with actual error info */
4013 if (cnt < num_entries) {
4014 enum ib_wc_status wc_status;
4015
4016 switch (req->status) {
4017 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4018 DP_ERR(dev,
4019 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4020 cq->icid, qp->icid);
4021 wc_status = IB_WC_BAD_RESP_ERR;
4022 break;
4023 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4024 DP_ERR(dev,
4025 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4026 cq->icid, qp->icid);
4027 wc_status = IB_WC_LOC_LEN_ERR;
4028 break;
4029 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4030 DP_ERR(dev,
4031 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4032 cq->icid, qp->icid);
4033 wc_status = IB_WC_LOC_QP_OP_ERR;
4034 break;
4035 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4036 DP_ERR(dev,
4037 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4038 cq->icid, qp->icid);
4039 wc_status = IB_WC_LOC_PROT_ERR;
4040 break;
4041 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4042 DP_ERR(dev,
4043 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4044 cq->icid, qp->icid);
4045 wc_status = IB_WC_MW_BIND_ERR;
4046 break;
4047 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4048 DP_ERR(dev,
4049 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4050 cq->icid, qp->icid);
4051 wc_status = IB_WC_REM_INV_REQ_ERR;
4052 break;
4053 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4054 DP_ERR(dev,
4055 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4056 cq->icid, qp->icid);
4057 wc_status = IB_WC_REM_ACCESS_ERR;
4058 break;
4059 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4060 DP_ERR(dev,
4061 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4062 cq->icid, qp->icid);
4063 wc_status = IB_WC_REM_OP_ERR;
4064 break;
4065 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4066 DP_ERR(dev,
4067 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4068 cq->icid, qp->icid);
4069 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4070 break;
4071 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4072 DP_ERR(dev,
4073 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4074 cq->icid, qp->icid);
4075 wc_status = IB_WC_RETRY_EXC_ERR;
4076 break;
4077 default:
4078 DP_ERR(dev,
4079 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4080 cq->icid, qp->icid);
4081 wc_status = IB_WC_GENERAL_ERR;
4082 }
4083 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4084 wc_status, 1);
4085 }
4086 }
4087
4088 return cnt;
4089}
4090
b6acd71f 4091static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 4092{
b6acd71f 4093 switch (status) {
afa0e13b 4094 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 4095 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 4096 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 4097 return IB_WC_LOC_LEN_ERR;
afa0e13b 4098 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 4099 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 4100 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 4101 return IB_WC_LOC_PROT_ERR;
afa0e13b 4102 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 4103 return IB_WC_MW_BIND_ERR;
afa0e13b 4104 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 4105 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 4106 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 4107 return IB_WC_SUCCESS;
4108 default:
4109 return IB_WC_GENERAL_ERR;
4110 }
4111}
afa0e13b 4112
b6acd71f
AR		 4113static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4114 struct ib_wc *wc)
4115{
4116 wc->status = IB_WC_SUCCESS;
4117 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 4118
b6acd71f 4119 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 4120 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 4121 wc->wc_flags |= IB_WC_WITH_IMM;
4122
4123 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 4124 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4125
b6acd71f
AR		 4126 if (resp->flags & QEDR_RESP_INV)
4127 return -EINVAL;
4128
4129 } else if (resp->flags & QEDR_RESP_INV) {
4130 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4131 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4132
4133 if (resp->flags & QEDR_RESP_RDMA)
4134 return -EINVAL;
4135
4136 } else if (resp->flags & QEDR_RESP_RDMA) {
4137 return -EINVAL;
4138 }
4139
4140 return 0;
4141}
4142
4143static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4144 struct qedr_cq *cq, struct ib_wc *wc,
4145 struct rdma_cqe_responder *resp, u64 wr_id)
4146{
4147 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4148 wc->opcode = IB_WC_RECV;
4149 wc->wc_flags = 0;
4150
4151 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4152 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4153 DP_ERR(dev,
4154 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4155 cq, cq->icid, resp->flags);
4156
4157 } else {
4158 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4159 if (wc->status == IB_WC_GENERAL_ERR)
4160 DP_ERR(dev,
4161 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4162 cq, cq->icid, resp->status);
afa0e13b
RA		 4163 }
4164
b6acd71f 4165 /* Fill the rest of the WC */
27035a1b 4166 wc->vendor_err = 0;
afa0e13b
RA		 4167 wc->src_qp = qp->id;
4168 wc->qp = &qp->ibqp;
4169 wc->wr_id = wr_id;
4170}
4171
3491c9e7
YB		 4172static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4173 struct qedr_cq *cq, struct ib_wc *wc,
4174 struct rdma_cqe_responder *resp)
4175{
4176 struct qedr_srq *srq = qp->srq;
4177 u64 wr_id;
4178
4179 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4180 le32_to_cpu(resp->srq_wr_id.lo), u64);
4181
4182 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4183 wc->status = IB_WC_WR_FLUSH_ERR;
4184 wc->vendor_err = 0;
4185 wc->wr_id = wr_id;
4186 wc->byte_len = 0;
4187 wc->src_qp = qp->id;
4188 wc->qp = &qp->ibqp;
4189 wc->wr_id = wr_id;
4190 } else {
4191 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4192 }
acca72e2 4193 atomic_inc(&srq->hw_srq.wr_cons_cnt);
3491c9e7
YB		 4194
4195 return 1;
4196}
afa0e13b
RA		 4197static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4198 struct qedr_cq *cq, struct ib_wc *wc,
4199 struct rdma_cqe_responder *resp)
4200{
4201 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4202
4203 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4204
4205 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4206 qed_chain_consume(&qp->rq.pbl);
4207 qedr_inc_sw_cons(&qp->rq);
4208
4209 return 1;
4210}
4211
4212static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4213 int num_entries, struct ib_wc *wc, u16 hw_cons)
4214{
4215 u16 cnt = 0;
4216
4217 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4218 /* fill WC */
4219 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 4220 wc->vendor_err = 0;
afa0e13b
RA		 4221 wc->wc_flags = 0;
4222 wc->src_qp = qp->id;
4223 wc->byte_len = 0;
4224 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4225 wc->qp = &qp->ibqp;
4226 num_entries--;
4227 wc++;
4228 cnt++;
4229 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4230 qed_chain_consume(&qp->rq.pbl);
4231 qedr_inc_sw_cons(&qp->rq);
4232 }
4233
4234 return cnt;
4235}
4236
4237static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4238 struct rdma_cqe_responder *resp, int *update)
4239{
50bc60cb 4240 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 4241 consume_cqe(cq);
4242 *update |= 1;
4243 }
4244}
4245
3491c9e7
YB		 4246static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4247 struct qedr_cq *cq, int num_entries,
4248 struct ib_wc *wc,
4249 struct rdma_cqe_responder *resp)
4250{
4251 int cnt;
4252
4253 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4254 consume_cqe(cq);
4255
4256 return cnt;
4257}
4258
afa0e13b
RA		 4259static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4260 struct qedr_cq *cq, int num_entries,
4261 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4262 int *update)
4263{
4264 int cnt;
4265
4266 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4267 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4268 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4269 try_consume_resp_cqe(cq, qp, resp, update);
4270 } else {
4271 cnt = process_resp_one(dev, qp, cq, wc, resp);
4272 consume_cqe(cq);
4273 *update |= 1;
4274 }
4275
4276 return cnt;
4277}
4278
4279static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4280 struct rdma_cqe_requester *req, int *update)
4281{
4282 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4283 consume_cqe(cq);
4284 *update |= 1;
4285 }
4286}
4287
4288int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4289{
4290 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4291 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4292 union rdma_cqe *cqe;
afa0e13b
RA		 4293 u32 old_cons, new_cons;
4294 unsigned long flags;
4295 int update = 0;
4296 int done = 0;
4297
4dd72636
AR		 4298 if (cq->destroyed) {
4299 DP_ERR(dev,
4300 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4301 cq, cq->icid);
4302 return 0;
4303 }
4304
04886779
RA		 4305 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4306 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4307
afa0e13b 4308 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4309 cqe = cq->latest_cqe;
afa0e13b
RA		 4310 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4311 while (num_entries && is_valid_cqe(cq, cqe)) {
4312 struct qedr_qp *qp;
4313 int cnt = 0;
4314
4315 /* prevent speculative reads of any field of CQE */
4316 rmb();
4317
4318 qp = cqe_get_qp(cqe);
4319 if (!qp) {
4320 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4321 break;
4322 }
4323
4324 wc->qp = &qp->ibqp;
4325
4326 switch (cqe_get_type(cqe)) {
4327 case RDMA_CQE_TYPE_REQUESTER:
4328 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4329 &cqe->req);
4330 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4331 break;
4332 case RDMA_CQE_TYPE_RESPONDER_RQ:
4333 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4334 &cqe->resp, &update);
4335 break;
3491c9e7
YB		 4336 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4337 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4338 wc, &cqe->resp);
4339 update = 1;
4340 break;
afa0e13b
RA		 4341 case RDMA_CQE_TYPE_INVALID:
4342 default:
4343 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4344 cqe_get_type(cqe));
4345 }
4346 num_entries -= cnt;
4347 wc += cnt;
4348 done += cnt;
4349
4350 cqe = get_cqe(cq);
4351 }
4352 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4353
4354 cq->cq_cons += new_cons - old_cons;
4355
4356 if (update)
4357 /* doorbell notifies abount latest VALID entry,
4358 * but chain already point to the next INVALID one
4359 */
4360 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4361
4362 spin_unlock_irqrestore(&cq->cq_lock, flags);
4363 return done;
4364}
993d1b52
RA		 4365
4366int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
e26e7b88
LR		 4367 u8 port_num, const struct ib_wc *in_wc,
4368 const struct ib_grh *in_grh, const struct ib_mad *in,
4369 struct ib_mad *out_mad, size_t *out_mad_size,
4370 u16 *out_mad_pkey_index)
993d1b52 4371{
993d1b52
RA		 4372 return IB_MAD_RESULT_SUCCESS;
4373}
Linux 6.x block layer and io_uring tree(s)