projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/core: Remove ib_alloc_mr_user function
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
ff23dfa1 45#include <rdma/uverbs_ioctl.h>
ac1b36e5 46
be086e7c
MY		 47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
99d195cc 53#include "qedr_roce_cm.h"
82af6d19 54#include "qedr_iw_cm.h"
ac1b36e5 55
3491c9e7
YB		 56#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57#define RDMA_MAX_SGE_PER_SRQ (4)
58#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
a7efd777
RA		 60#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
4c6bb02d
MK		 62enum {
63 QEDR_USER_MMAP_IO_WC = 0,
97f61250 64 QEDR_USER_MMAP_PHYS_PAGE,
4c6bb02d
MK		 65};
66
c75d3ec8
AR		 67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 size_t len)
69{
70 size_t min_len = min_t(size_t, len, udata->outlen);
71
72 return ib_copy_to_udata(udata, src, min_len);
73}
74
a7efd777
RA		 75int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
76{
dbe30dae 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 78 return -EINVAL;
79
80 *pkey = QEDR_ROCE_PKEY_DEFAULT;
81 return 0;
82}
83
e6a38c54
KM		 84int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
85 int index, union ib_gid *sgid)
86{
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89 memset(sgid->raw, 0, sizeof(sgid->raw));
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 sgid->global.interface_id, sgid->global.subnet_prefix);
94
95 return 0;
96}
97
3491c9e7
YB		 98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99{
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
102 struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104 srq_attr->srq_limit = srq->srq_limit;
105 srq_attr->max_wr = qattr->max_srq_wr;
106 srq_attr->max_sge = qattr->max_sge;
107
108 return 0;
109}
110
ac1b36e5
RA		 111int qedr_query_device(struct ib_device *ibdev,
112 struct ib_device_attr *attr, struct ib_udata *udata)
113{
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
116
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
119 "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 dev->rdma_ctx);
121 return -EINVAL;
122 }
123
124 memset(attr, 0, sizeof(*attr));
125
126 attr->fw_ver = qattr->fw_ver;
127 attr->sys_image_guid = qattr->sys_image_guid;
128 attr->max_mr_size = qattr->max_mr_size;
129 attr->page_size_cap = qattr->page_size_caps;
130 attr->vendor_id = qattr->vendor_id;
131 attr->vendor_part_id = qattr->vendor_part_id;
132 attr->hw_ver = qattr->hw_ver;
133 attr->max_qp = qattr->max_qp;
134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 IB_DEVICE_RC_RNR_NAK_GEN |
137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
138
33023fb8
SW		 139 attr->max_send_sge = qattr->max_sge;
140 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 141 attr->max_sge_rd = qattr->max_sge;
142 attr->max_cq = qattr->max_cq;
143 attr->max_cqe = qattr->max_cqe;
144 attr->max_mr = qattr->max_mr;
145 attr->max_mw = qattr->max_mw;
146 attr->max_pd = qattr->max_pd;
147 attr->atomic_cap = dev->atomic_cap;
ac1b36e5
RA		 148 attr->max_qp_init_rd_atom =
149 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
150 attr->max_qp_rd_atom =
151 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
152 attr->max_qp_init_rd_atom);
153
154 attr->max_srq = qattr->max_srq;
155 attr->max_srq_sge = qattr->max_srq_sge;
156 attr->max_srq_wr = qattr->max_srq_wr;
157
158 attr->local_ca_ack_delay = qattr->dev_ack_delay;
159 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
160 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
161 attr->max_ah = qattr->max_ah;
162
163 return 0;
164}
165
ac1b36e5
RA		 166static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
167 u8 *ib_width)
168{
169 switch (speed) {
170 case 1000:
69054666 171 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 172 *ib_width = IB_WIDTH_1X;
173 break;
174 case 10000:
69054666 175 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 176 *ib_width = IB_WIDTH_1X;
177 break;
178
179 case 20000:
69054666 180 *ib_speed = IB_SPEED_DDR;
ac1b36e5
RA		 181 *ib_width = IB_WIDTH_4X;
182 break;
183
184 case 25000:
69054666 185 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 186 *ib_width = IB_WIDTH_1X;
187 break;
188
189 case 40000:
69054666 190 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 191 *ib_width = IB_WIDTH_4X;
192 break;
193
194 case 50000:
69054666
SO		 195 *ib_speed = IB_SPEED_HDR;
196 *ib_width = IB_WIDTH_1X;
ac1b36e5
RA		 197 break;
198
199 case 100000:
69054666 200 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 201 *ib_width = IB_WIDTH_4X;
202 break;
203
204 default:
205 /* Unsupported */
69054666 206 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 207 *ib_width = IB_WIDTH_1X;
208 }
209}
210
211int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
212{
213 struct qedr_dev *dev;
214 struct qed_rdma_port *rdma_port;
215
216 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 217
218 if (!dev->rdma_ctx) {
219 DP_ERR(dev, "rdma_ctx is NULL\n");
220 return -EINVAL;
221 }
222
223 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 224
c4550c63 225 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 226 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
227 attr->state = IB_PORT_ACTIVE;
72a7720f 228 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
ac1b36e5
RA		 229 } else {
230 attr->state = IB_PORT_DOWN;
72a7720f 231 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
ac1b36e5
RA		 232 }
233 attr->max_mtu = IB_MTU_4096;
234 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
235 attr->lid = 0;
236 attr->lmc = 0;
237 attr->sm_lid = 0;
238 attr->sm_sl = 0;
2f944c0f 239 attr->ip_gids = true;
f5b1b177
KM		 240 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
241 attr->gid_tbl_len = 1;
242 attr->pkey_tbl_len = 1;
243 } else {
244 attr->gid_tbl_len = QEDR_MAX_SGID;
245 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
246 }
ac1b36e5
RA		 247 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
248 attr->qkey_viol_cntr = 0;
249 get_link_speed_and_width(rdma_port->link_speed,
250 &attr->active_speed, &attr->active_width);
251 attr->max_msg_sz = rdma_port->max_msg_size;
252 attr->max_vl_num = 4;
253
254 return 0;
255}
256
a2a074ef 257int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 258{
a2a074ef 259 struct ib_device *ibdev = uctx->device;
ac1b36e5 260 int rc;
a2a074ef
LR		 261 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
262 struct qedr_alloc_ucontext_resp uresp = {};
97f61250 263 struct qedr_alloc_ucontext_req ureq = {};
ac1b36e5
RA		 264 struct qedr_dev *dev = get_qedr_dev(ibdev);
265 struct qed_rdma_add_user_out_params oparams;
4c6bb02d 266 struct qedr_user_mmap_entry *entry;
ac1b36e5
RA		 267
268 if (!udata)
a2a074ef 269 return -EFAULT;
ac1b36e5 270
97f61250
MK		 271 if (udata->inlen) {
272 rc = ib_copy_from_udata(&ureq, udata,
273 min(sizeof(ureq), udata->inlen));
274 if (rc) {
275 DP_ERR(dev, "Problem copying data from user space\n");
276 return -EFAULT;
277 }
278
279 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
280 }
281
ac1b36e5
RA		 282 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
283 if (rc) {
284 DP_ERR(dev,
285 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
286 rc);
a2a074ef 287 return rc;
ac1b36e5
RA		 288 }
289
290 ctx->dpi = oparams.dpi;
291 ctx->dpi_addr = oparams.dpi_addr;
292 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
293 ctx->dpi_size = oparams.dpi_size;
4c6bb02d
MK		 294 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
295 if (!entry) {
296 rc = -ENOMEM;
297 goto err;
298 }
299
300 entry->io_address = ctx->dpi_phys_addr;
301 entry->length = ctx->dpi_size;
302 entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
303 entry->dpi = ctx->dpi;
304 entry->dev = dev;
305 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
306 ctx->dpi_size);
307 if (rc) {
308 kfree(entry);
309 goto err;
310 }
311 ctx->db_mmap_entry = &entry->rdma_entry;
ac1b36e5 312
93a3d05f
MK		 313 if (!dev->user_dpm_enabled)
314 uresp.dpm_flags = 0;
315 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
316 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
317 else
318 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
319 QEDR_DPM_TYPE_ROCE_LEGACY;
320
321 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
322 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
323 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
324
67cbe353
AR		 325 uresp.wids_enabled = 1;
326 uresp.wid_count = oparams.wid_count;
4c6bb02d 327 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
ac1b36e5
RA		 328 uresp.db_size = ctx->dpi_size;
329 uresp.max_send_wr = dev->attr.max_sqe;
330 uresp.max_recv_wr = dev->attr.max_rqe;
331 uresp.max_srq_wr = dev->attr.max_srq_wr;
332 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
333 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
334 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
335 uresp.max_cqes = QEDR_MAX_CQES;
336
c75d3ec8 337 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 338 if (rc)
4c6bb02d 339 goto err;
ac1b36e5
RA		 340
341 ctx->dev = dev;
342
ac1b36e5
RA		 343 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
344 &ctx->ibucontext);
a2a074ef 345 return 0;
4c6bb02d
MK		 346
347err:
348 if (!ctx->db_mmap_entry)
349 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
350 else
351 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
352
353 return rc;
ac1b36e5
RA		 354}
355
a2a074ef 356void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 357{
358 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
ac1b36e5
RA		 359
360 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
361 uctx);
4c6bb02d
MK		 362
363 rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
ac1b36e5
RA		 364}
365
4c6bb02d 366void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
ac1b36e5 367{
4c6bb02d
MK		 368 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
369 struct qedr_dev *dev = entry->dev;
ac1b36e5 370
97f61250
MK		 371 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
372 free_page((unsigned long)entry->address);
373 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
4c6bb02d 374 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
30bf066c 375
4c6bb02d
MK		 376 kfree(entry);
377}
ac1b36e5 378
4c6bb02d
MK		 379int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
380{
381 struct ib_device *dev = ucontext->device;
382 size_t length = vma->vm_end - vma->vm_start;
383 struct rdma_user_mmap_entry *rdma_entry;
384 struct qedr_user_mmap_entry *entry;
385 int rc = 0;
386 u64 pfn;
387
388 ibdev_dbg(dev,
389 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
390 vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
391
392 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
393 if (!rdma_entry) {
394 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
395 vma->vm_pgoff);
ac1b36e5
RA		 396 return -EINVAL;
397 }
4c6bb02d
MK		 398 entry = get_qedr_mmap_entry(rdma_entry);
399 ibdev_dbg(dev,
400 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
401 entry->io_address, length, entry->mmap_flag);
402
403 switch (entry->mmap_flag) {
404 case QEDR_USER_MMAP_IO_WC:
405 pfn = entry->io_address >> PAGE_SHIFT;
406 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
407 pgprot_writecombine(vma->vm_page_prot),
408 rdma_entry);
409 break;
97f61250
MK		 410 case QEDR_USER_MMAP_PHYS_PAGE:
411 rc = vm_insert_page(vma, vma->vm_start,
412 virt_to_page(entry->address));
413 break;
4c6bb02d
MK		 414 default:
415 rc = -EINVAL;
30bf066c 416 }
ac1b36e5 417
4c6bb02d
MK		 418 if (rc)
419 ibdev_dbg(dev,
420 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
421 entry->io_address, length, entry->mmap_flag, rc);
30bf066c 422
4c6bb02d
MK		 423 rdma_user_mmap_entry_put(rdma_entry);
424 return rc;
ac1b36e5 425}
a7efd777 426
ff23dfa1 427int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777 428{
21a428a0 429 struct ib_device *ibdev = ibpd->device;
a7efd777 430 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 431 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 432 u16 pd_id;
433 int rc;
434
435 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
ff23dfa1 436 udata ? "User Lib" : "Kernel");
a7efd777
RA		 437
438 if (!dev->rdma_ctx) {
847cb1a3 439 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 440 return -EINVAL;
a7efd777
RA		 441 }
442
9c1e0228
RA		 443 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
444 if (rc)
21a428a0 445 return rc;
a7efd777 446
a7efd777
RA		 447 pd->pd_id = pd_id;
448
ff23dfa1 449 if (udata) {
57939021
JG		 450 struct qedr_alloc_pd_uresp uresp = {
451 .pd_id = pd_id,
452 };
ff23dfa1
SR		 453 struct qedr_ucontext *context = rdma_udata_to_drv_context(
454 udata, struct qedr_ucontext, ibucontext);
9c1e0228 455
c75d3ec8 456 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 457 if (rc) {
a7efd777 458 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 459 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 460 return rc;
9c1e0228
RA		 461 }
462
ff23dfa1 463 pd->uctx = context;
9c1e0228 464 pd->uctx->pd = pd;
a7efd777
RA		 465 }
466
21a428a0 467 return 0;
a7efd777
RA		 468}
469
c4367a26 470void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 471{
472 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
473 struct qedr_pd *pd = get_qedr_pd(ibpd);
474
a7efd777
RA		 475 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
476 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
a7efd777
RA		 477}
478
479static void qedr_free_pbl(struct qedr_dev *dev,
480 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
481{
482 struct pci_dev *pdev = dev->pdev;
483 int i;
484
485 for (i = 0; i < pbl_info->num_pbls; i++) {
486 if (!pbl[i].va)
487 continue;
488 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
489 pbl[i].va, pbl[i].pa);
490 }
491
492 kfree(pbl);
493}
494
495#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
496#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
497
498#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
499#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
500#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
501
502static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
503 struct qedr_pbl_info *pbl_info,
504 gfp_t flags)
505{
506 struct pci_dev *pdev = dev->pdev;
507 struct qedr_pbl *pbl_table;
508 dma_addr_t *pbl_main_tbl;
509 dma_addr_t pa;
510 void *va;
511 int i;
512
513 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
514 if (!pbl_table)
515 return ERR_PTR(-ENOMEM);
516
517 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 518 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
519 flags);
a7efd777
RA		 520 if (!va)
521 goto err;
522
a7efd777
RA		 523 pbl_table[i].va = va;
524 pbl_table[i].pa = pa;
525 }
526
527 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
528 * the first one with physical pointers to all of the rest
529 */
530 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
531 for (i = 0; i < pbl_info->num_pbls - 1; i++)
532 pbl_main_tbl[i] = pbl_table[i + 1].pa;
533
534 return pbl_table;
535
536err:
537 for (i--; i >= 0; i--)
538 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
539 pbl_table[i].va, pbl_table[i].pa);
540
541 qedr_free_pbl(dev, pbl_info, pbl_table);
542
543 return ERR_PTR(-ENOMEM);
544}
545
546static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
547 struct qedr_pbl_info *pbl_info,
548 u32 num_pbes, int two_layer_capable)
549{
550 u32 pbl_capacity;
551 u32 pbl_size;
552 u32 num_pbls;
553
554 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
555 if (num_pbes > MAX_PBES_TWO_LAYER) {
556 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
557 num_pbes);
558 return -EINVAL;
559 }
560
561 /* calculate required pbl page size */
562 pbl_size = MIN_FW_PBL_PAGE_SIZE;
563 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
564 NUM_PBES_ON_PAGE(pbl_size);
565
566 while (pbl_capacity < num_pbes) {
567 pbl_size *= 2;
568 pbl_capacity = pbl_size / sizeof(u64);
569 pbl_capacity = pbl_capacity * pbl_capacity;
570 }
571
572 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
573 num_pbls++; /* One for the layer0 ( points to the pbls) */
574 pbl_info->two_layered = true;
575 } else {
576 /* One layered PBL */
577 num_pbls = 1;
578 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
579 roundup_pow_of_two((num_pbes * sizeof(u64))));
580 pbl_info->two_layered = false;
581 }
582
583 pbl_info->num_pbls = num_pbls;
584 pbl_info->pbl_size = pbl_size;
585 pbl_info->num_pbes = num_pbes;
586
587 DP_DEBUG(dev, QEDR_MSG_MR,
588 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
589 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
590
591 return 0;
592}
593
594static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
595 struct qedr_pbl *pbl,
e57bb6be 596 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 597{
95ad233f 598 int pbe_cnt, total_num_pbes = 0;
e57bb6be 599 u32 fw_pg_cnt, fw_pg_per_umem_pg;
a7efd777 600 struct qedr_pbl *pbl_tbl;
95ad233f 601 struct sg_dma_page_iter sg_iter;
a7efd777 602 struct regpair *pbe;
e57bb6be 603 u64 pg_addr;
a7efd777
RA		 604
605 if (!pbl_info->num_pbes)
606 return;
607
608 /* If we have a two layered pbl, the first pbl points to the rest
609 * of the pbls and the first entry lays on the second pbl in the table
610 */
611 if (pbl_info->two_layered)
612 pbl_tbl = &pbl[1];
613 else
614 pbl_tbl = pbl;
615
616 pbe = (struct regpair *)pbl_tbl->va;
617 if (!pbe) {
618 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
619 return;
620 }
621
622 pbe_cnt = 0;
623
95ad233f
SS		 624 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
625
626 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
627 pg_addr = sg_page_iter_dma_address(&sg_iter);
628 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
629 pbe->lo = cpu_to_le32(pg_addr);
630 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
631
632 pg_addr += BIT(pg_shift);
633 pbe_cnt++;
634 total_num_pbes++;
635 pbe++;
e57bb6be 636
95ad233f
SS		 637 if (total_num_pbes == pbl_info->num_pbes)
638 return;
639
640 /* If the given pbl is full storing the pbes,
641 * move to next pbl.
642 */
643 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
644 pbl_tbl++;
645 pbe = (struct regpair *)pbl_tbl->va;
646 pbe_cnt = 0;
a7efd777 647 }
95ad233f
SS		 648
649 fw_pg_cnt++;
a7efd777
RA		 650 }
651 }
652}
653
97f61250
MK		 654static int qedr_db_recovery_add(struct qedr_dev *dev,
655 void __iomem *db_addr,
656 void *db_data,
657 enum qed_db_rec_width db_width,
658 enum qed_db_rec_space db_space)
659{
660 if (!db_data) {
661 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
662 return 0;
663 }
664
665 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
666 db_width, db_space);
667}
668
669static void qedr_db_recovery_del(struct qedr_dev *dev,
670 void __iomem *db_addr,
671 void *db_data)
672{
673 if (!db_data) {
674 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
675 return;
676 }
677
678 /* Ignore return code as there is not much we can do about it. Error
679 * log will be printed inside.
680 */
681 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
682}
683
a7efd777 684static int qedr_copy_cq_uresp(struct qedr_dev *dev,
97f61250
MK		 685 struct qedr_cq *cq, struct ib_udata *udata,
686 u32 db_offset)
a7efd777
RA		 687{
688 struct qedr_create_cq_uresp uresp;
689 int rc;
690
691 memset(&uresp, 0, sizeof(uresp));
692
97f61250 693 uresp.db_offset = db_offset;
a7efd777 694 uresp.icid = cq->icid;
a25984f3
MK		 695 if (cq->q.db_mmap_entry)
696 uresp.db_rec_addr =
697 rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
a7efd777 698
c75d3ec8 699 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 700 if (rc)
701 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
702
703 return rc;
704}
705
706static void consume_cqe(struct qedr_cq *cq)
707{
708 if (cq->latest_cqe == cq->toggle_cqe)
709 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
710
711 cq->latest_cqe = qed_chain_consume(&cq->pbl);
712}
713
714static inline int qedr_align_cq_entries(int entries)
715{
716 u64 size, aligned_size;
717
718 /* We allocate an extra entry that we don't report to the FW. */
719 size = (entries + 1) * QEDR_CQE_SIZE;
720 aligned_size = ALIGN(size, PAGE_SIZE);
721
722 return aligned_size / QEDR_CQE_SIZE;
723}
724
97f61250
MK		 725static int qedr_init_user_db_rec(struct ib_udata *udata,
726 struct qedr_dev *dev, struct qedr_userq *q,
727 bool requires_db_rec)
728{
729 struct qedr_ucontext *uctx =
730 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
731 ibucontext);
732 struct qedr_user_mmap_entry *entry;
733 int rc;
734
735 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
736 if (requires_db_rec == 0 || !uctx->db_rec)
737 return 0;
738
739 /* Allocate a page for doorbell recovery, add to mmap */
740 q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
741 if (!q->db_rec_data) {
742 DP_ERR(dev, "get_zeroed_page failed\n");
743 return -ENOMEM;
744 }
745
746 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
747 if (!entry)
748 goto err_free_db_data;
749
750 entry->address = q->db_rec_data;
751 entry->length = PAGE_SIZE;
752 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
753 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
754 &entry->rdma_entry,
755 PAGE_SIZE);
756 if (rc)
757 goto err_free_entry;
758
759 q->db_mmap_entry = &entry->rdma_entry;
760
761 return 0;
762
763err_free_entry:
764 kfree(entry);
765
766err_free_db_data:
767 free_page((unsigned long)q->db_rec_data);
768 q->db_rec_data = NULL;
769 return -ENOMEM;
770}
771
b0ea0fa5 772static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 773 struct qedr_dev *dev,
b0ea0fa5 774 struct qedr_userq *q, u64 buf_addr,
97f61250 775 size_t buf_len, bool requires_db_rec,
72b894b0 776 int access,
69ad0e7f 777 int alloc_and_init)
a7efd777 778{
e57bb6be 779 u32 fw_pages;
a7efd777
RA		 780 int rc;
781
782 q->buf_addr = buf_addr;
783 q->buf_len = buf_len;
c320e527 784 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
a7efd777
RA		 785 if (IS_ERR(q->umem)) {
786 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
787 PTR_ERR(q->umem));
788 return PTR_ERR(q->umem);
789 }
790
e57bb6be 791 fw_pages = ib_umem_page_count(q->umem) <<
95ad233f 792 (PAGE_SHIFT - FW_PAGE_SHIFT);
e57bb6be
RA		 793
794 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 795 if (rc)
796 goto err0;
797
69ad0e7f
KM		 798 if (alloc_and_init) {
799 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
800 if (IS_ERR(q->pbl_tbl)) {
801 rc = PTR_ERR(q->pbl_tbl);
802 goto err0;
803 }
e57bb6be
RA		 804 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
805 FW_PAGE_SHIFT);
69ad0e7f
KM		 806 } else {
807 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 808 if (!q->pbl_tbl) {
809 rc = -ENOMEM;
69ad0e7f 810 goto err0;
89fd2576 811 }
69ad0e7f 812 }
a7efd777 813
97f61250
MK		 814 /* mmap the user address used to store doorbell data for recovery */
815 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
a7efd777
RA		 816
817err0:
818 ib_umem_release(q->umem);
69ad0e7f 819 q->umem = NULL;
a7efd777
RA		 820
821 return rc;
822}
823
824static inline void qedr_init_cq_params(struct qedr_cq *cq,
825 struct qedr_ucontext *ctx,
826 struct qedr_dev *dev, int vector,
827 int chain_entries, int page_cnt,
828 u64 pbl_ptr,
829 struct qed_rdma_create_cq_in_params
830 *params)
831{
832 memset(params, 0, sizeof(*params));
833 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
834 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
835 params->cnq_id = vector;
836 params->cq_size = chain_entries - 1;
837 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
838 params->pbl_num_pages = page_cnt;
839 params->pbl_ptr = pbl_ptr;
840 params->pbl_two_level = 0;
841}
842
843static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
844{
a7efd777
RA		 845 cq->db.data.agg_flags = flags;
846 cq->db.data.value = cpu_to_le32(cons);
847 writeq(cq->db.raw, cq->db_addr);
a7efd777
RA		 848}
849
850int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
851{
852 struct qedr_cq *cq = get_qedr_cq(ibcq);
853 unsigned long sflags;
4dd72636
AR		 854 struct qedr_dev *dev;
855
856 dev = get_qedr_dev(ibcq->device);
857
858 if (cq->destroyed) {
859 DP_ERR(dev,
860 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
861 cq, cq->icid);
862 return -EINVAL;
863 }
864
a7efd777
RA		 865
866 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
867 return 0;
868
869 spin_lock_irqsave(&cq->cq_lock, sflags);
870
871 cq->arm_flags = 0;
872
873 if (flags & IB_CQ_SOLICITED)
874 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
875
876 if (flags & IB_CQ_NEXT_COMP)
877 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
878
879 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
880
881 spin_unlock_irqrestore(&cq->cq_lock, sflags);
882
883 return 0;
884}
885
e39afe3d
LR		 886int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
887 struct ib_udata *udata)
a7efd777 888{
e39afe3d 889 struct ib_device *ibdev = ibcq->device;
ff23dfa1
SR		 890 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
891 udata, struct qedr_ucontext, ibucontext);
a7efd777
RA		 892 struct qed_rdma_destroy_cq_out_params destroy_oparams;
893 struct qed_rdma_destroy_cq_in_params destroy_iparams;
894 struct qedr_dev *dev = get_qedr_dev(ibdev);
895 struct qed_rdma_create_cq_in_params params;
e39afe3d 896 struct qedr_create_cq_ureq ureq = {};
a7efd777
RA		 897 int vector = attr->comp_vector;
898 int entries = attr->cqe;
e39afe3d 899 struct qedr_cq *cq = get_qedr_cq(ibcq);
a7efd777 900 int chain_entries;
97f61250 901 u32 db_offset;
a7efd777
RA		 902 int page_cnt;
903 u64 pbl_ptr;
904 u16 icid;
905 int rc;
906
907 DP_DEBUG(dev, QEDR_MSG_INIT,
908 "create_cq: called from %s. entries=%d, vector=%d\n",
909 udata ? "User Lib" : "Kernel", entries, vector);
910
911 if (entries > QEDR_MAX_CQES) {
912 DP_ERR(dev,
913 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
914 entries, QEDR_MAX_CQES);
e39afe3d 915 return -EINVAL;
a7efd777
RA		 916 }
917
918 chain_entries = qedr_align_cq_entries(entries);
919 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
920
97f61250
MK		 921 /* calc db offset. user will add DPI base, kernel will add db addr */
922 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
923
a7efd777 924 if (udata) {
97f61250
MK		 925 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
926 udata->inlen))) {
a7efd777
RA		 927 DP_ERR(dev,
928 "create cq: problem copying data from user space\n");
929 goto err0;
930 }
931
932 if (!ureq.len) {
933 DP_ERR(dev,
934 "create cq: cannot create a cq with 0 entries\n");
935 goto err0;
936 }
937
938 cq->cq_type = QEDR_CQ_TYPE_USER;
939
b0ea0fa5 940 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
72b894b0
CH		 941 ureq.len, true, IB_ACCESS_LOCAL_WRITE,
942 1);
a7efd777
RA		 943 if (rc)
944 goto err0;
945
946 pbl_ptr = cq->q.pbl_tbl->pa;
947 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 948
949 cq->ibcq.cqe = chain_entries;
97f61250 950 cq->q.db_addr = ctx->dpi_addr + db_offset;
a7efd777
RA		 951 } else {
952 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
953
954 rc = dev->ops->common->chain_alloc(dev->cdev,
955 QED_CHAIN_USE_TO_CONSUME,
956 QED_CHAIN_MODE_PBL,
957 QED_CHAIN_CNT_TYPE_U32,
958 chain_entries,
959 sizeof(union rdma_cqe),
1a4a6975 960 &cq->pbl, NULL);
a7efd777 961 if (rc)
97f61250 962 goto err0;
a7efd777
RA		 963
964 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
965 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 966 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 967 }
968
969 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
970 pbl_ptr, &params);
971
972 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
973 if (rc)
97f61250 974 goto err1;
a7efd777
RA		 975
976 cq->icid = icid;
977 cq->sig = QEDR_CQ_MAGIC_NUMBER;
978 spin_lock_init(&cq->cq_lock);
979
ff23dfa1 980 if (udata) {
97f61250
MK		 981 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
982 if (rc)
983 goto err2;
984
985 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
986 &cq->q.db_rec_data->db_data,
987 DB_REC_WIDTH_64B,
988 DB_REC_USER);
a7efd777 989 if (rc)
97f61250
MK		 990 goto err2;
991
a7efd777
RA		 992 } else {
993 /* Generate doorbell address. */
a7efd777 994 cq->db.data.icid = cq->icid;
97f61250 995 cq->db_addr = dev->db_addr + db_offset;
a7efd777
RA		 996 cq->db.data.params = DB_AGG_CMD_SET <<
997 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
998
999 /* point to the very last element, passing it we will toggle */
1000 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1001 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1002 cq->latest_cqe = NULL;
1003 consume_cqe(cq);
1004 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
97f61250
MK		 1005
1006 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1007 DB_REC_WIDTH_64B, DB_REC_KERNEL);
1008 if (rc)
1009 goto err2;
a7efd777
RA		 1010 }
1011
1012 DP_DEBUG(dev, QEDR_MSG_CQ,
1013 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1014 cq->icid, cq, params.cq_size);
1015
e39afe3d 1016 return 0;
a7efd777 1017
97f61250 1018err2:
a7efd777
RA		 1019 destroy_iparams.icid = cq->icid;
1020 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1021 &destroy_oparams);
a7efd777 1022err1:
97f61250
MK		 1023 if (udata) {
1024 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
a7efd777 1025 ib_umem_release(cq->q.umem);
a25984f3 1026 if (cq->q.db_mmap_entry)
97f61250
MK		 1027 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1028 } else {
1029 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1030 }
a7efd777 1031err0:
e39afe3d 1032 return -EINVAL;
a7efd777
RA		 1033}
1034
1035int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1036{
1037 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1038 struct qedr_cq *cq = get_qedr_cq(ibcq);
1039
1040 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1041
1042 return 0;
1043}
1044
4dd72636
AR		 1045#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1046#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1047
a52c8e24 1048void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 1049{
1050 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1051 struct qed_rdma_destroy_cq_out_params oparams;
1052 struct qed_rdma_destroy_cq_in_params iparams;
1053 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 1054 int iter;
a7efd777 1055
942b3b2c 1056 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 1057
4dd72636
AR		 1058 cq->destroyed = 1;
1059
a7efd777 1060 /* GSIs CQs are handled by driver, so they don't exist in the FW */
97f61250
MK		 1061 if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1062 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
e39afe3d 1063 return;
97f61250 1064 }
a1211359 1065
942b3b2c 1066 iparams.icid = cq->icid;
a52c8e24 1067 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
942b3b2c 1068 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777 1069
bdeacabd 1070 if (udata) {
a7efd777
RA		 1071 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1072 ib_umem_release(cq->q.umem);
97f61250
MK		 1073
1074 if (cq->q.db_rec_data) {
1075 qedr_db_recovery_del(dev, cq->q.db_addr,
1076 &cq->q.db_rec_data->db_data);
1077 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1078 }
1079 } else {
1080 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
a7efd777
RA		 1081 }
1082
4dd72636
AR		 1083 /* We don't want the IRQ handler to handle a non-existing CQ so we
1084 * wait until all CNQ interrupts, if any, are received. This will always
1085 * happen and will always happen very fast. If not, then a serious error
1086 * has occured. That is why we can use a long delay.
1087 * We spin for a short time so we don’t lose time on context switching
1088 * in case all the completions are handled in that span. Otherwise
1089 * we sleep for a while and check again. Since the CNQ may be
1090 * associated with (only) the current CPU we use msleep to allow the
1091 * current CPU to be freed.
1092 * The CNQ notification is increased in qedr_irq_handler().
1093 */
1094 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1095 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1096 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1097 iter--;
1098 }
1099
1100 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1101 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1102 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1103 iter--;
1104 }
1105
4dd72636
AR		 1106 /* Note that we don't need to have explicit code to wait for the
1107 * completion of the event handler because it is invoked from the EQ.
1108 * Since the destroy CQ ramrod has also been received on the EQ we can
1109 * be certain that there's no event handler in process.
1110 */
a7efd777 1111}
cecbcddf
RA		 1112
1113static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1114 struct ib_qp_attr *attr,
1115 int attr_mask,
1116 struct qed_rdma_modify_qp_in_params
1117 *qp_params)
1118{
47ec3866 1119 const struct ib_gid_attr *gid_attr;
cecbcddf 1120 enum rdma_network_type nw_type;
d8966fcd 1121 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1122 u32 ipv4_addr;
a70c0739 1123 int ret;
cecbcddf
RA		 1124 int i;
1125
47ec3866 1126 gid_attr = grh->sgid_attr;
a70c0739
PP		 1127 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1128 if (ret)
1129 return ret;
3e44e0ee 1130
47ec3866 1131 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1132 switch (nw_type) {
1133 case RDMA_NETWORK_IPV6:
47ec3866 1134 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1135 sizeof(qp_params->sgid));
1136 memcpy(&qp_params->dgid.bytes[0],
1137 &grh->dgid,
1138 sizeof(qp_params->dgid));
1139 qp_params->roce_mode = ROCE_V2_IPV6;
1140 SET_FIELD(qp_params->modify_flags,
1141 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1142 break;
1143 case RDMA_NETWORK_IB:
47ec3866 1144 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1145 sizeof(qp_params->sgid));
1146 memcpy(&qp_params->dgid.bytes[0],
1147 &grh->dgid,
1148 sizeof(qp_params->dgid));
1149 qp_params->roce_mode = ROCE_V1;
1150 break;
1151 case RDMA_NETWORK_IPV4:
1152 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1153 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1154 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1155 qp_params->sgid.ipv4_addr = ipv4_addr;
1156 ipv4_addr =
1157 qedr_get_ipv4_from_gid(grh->dgid.raw);
1158 qp_params->dgid.ipv4_addr = ipv4_addr;
1159 SET_FIELD(qp_params->modify_flags,
1160 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1161 qp_params->roce_mode = ROCE_V2_IPV4;
1162 break;
cecbcddf
RA		 1163 }
1164
1165 for (i = 0; i < 4; i++) {
1166 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1167 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1168 }
1169
1170 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1171 qp_params->vlan_id = 0;
1172
1173 return 0;
1174}
1175
cecbcddf 1176static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1177 struct ib_qp_init_attr *attrs,
1178 struct ib_udata *udata)
cecbcddf
RA		 1179{
1180 struct qedr_device_attr *qattr = &dev->attr;
1181
1182 /* QP0... attrs->qp_type == IB_QPT_GSI */
1183 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1184 DP_DEBUG(dev, QEDR_MSG_QP,
1185 "create qp: unsupported qp type=0x%x requested\n",
1186 attrs->qp_type);
bb8865f4 1187 return -EOPNOTSUPP;
cecbcddf
RA		 1188 }
1189
1190 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1191 DP_ERR(dev,
1192 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1193 attrs->cap.max_send_wr, qattr->max_sqe);
1194 return -EINVAL;
1195 }
1196
1197 if (attrs->cap.max_inline_data > qattr->max_inline) {
1198 DP_ERR(dev,
1199 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1200 attrs->cap.max_inline_data, qattr->max_inline);
1201 return -EINVAL;
1202 }
1203
1204 if (attrs->cap.max_send_sge > qattr->max_sge) {
1205 DP_ERR(dev,
1206 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1207 attrs->cap.max_send_sge, qattr->max_sge);
1208 return -EINVAL;
1209 }
1210
1211 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1212 DP_ERR(dev,
1213 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1214 attrs->cap.max_recv_sge, qattr->max_sge);
1215 return -EINVAL;
1216 }
1217
1218 /* Unprivileged user space cannot create special QP */
e00b64f7 1219 if (udata && attrs->qp_type == IB_QPT_GSI) {
cecbcddf
RA		 1220 DP_ERR(dev,
1221 "create qp: userspace can't create special QPs of type=0x%x\n",
1222 attrs->qp_type);
1223 return -EINVAL;
1224 }
1225
1226 return 0;
1227}
1228
40b173dd
YB		 1229static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1230 struct qedr_srq *srq, struct ib_udata *udata)
1231{
1232 struct qedr_create_srq_uresp uresp = {};
1233 int rc;
1234
1235 uresp.srq_id = srq->srq_id;
1236
1237 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1238 if (rc)
1239 DP_ERR(dev, "create srq: problem copying data to user space\n");
1240
1241 return rc;
1242}
1243
69ad0e7f 1244static void qedr_copy_rq_uresp(struct qedr_dev *dev,
97f61250
MK		 1245 struct qedr_create_qp_uresp *uresp,
1246 struct qedr_qp *qp)
cecbcddf 1247{
69ad0e7f
KM		 1248 /* iWARP requires two doorbells per RQ. */
1249 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1250 uresp->rq_db_offset =
1251 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1252 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1253 } else {
1254 uresp->rq_db_offset =
1255 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1256 }
1257
cecbcddf 1258 uresp->rq_icid = qp->icid;
a25984f3
MK		 1259 if (qp->urq.db_mmap_entry)
1260 uresp->rq_db_rec_addr =
1261 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
cecbcddf
RA		 1262}
1263
69ad0e7f
KM		 1264static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1265 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1266 struct qedr_qp *qp)
1267{
1268 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1269
1270 /* iWARP uses the same cid for rq and sq */
1271 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1272 uresp->sq_icid = qp->icid;
1273 else
1274 uresp->sq_icid = qp->icid + 1;
97f61250 1275
a25984f3
MK		 1276 if (qp->usq.db_mmap_entry)
1277 uresp->sq_db_rec_addr =
1278 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
cecbcddf
RA		 1279}
1280
1281static int qedr_copy_qp_uresp(struct qedr_dev *dev,
97f61250
MK		 1282 struct qedr_qp *qp, struct ib_udata *udata,
1283 struct qedr_create_qp_uresp *uresp)
cecbcddf 1284{
cecbcddf
RA		 1285 int rc;
1286
97f61250
MK		 1287 memset(uresp, 0, sizeof(*uresp));
1288 qedr_copy_sq_uresp(dev, uresp, qp);
1289 qedr_copy_rq_uresp(dev, uresp, qp);
cecbcddf 1290
97f61250
MK		 1291 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1292 uresp->qp_id = qp->qp_id;
cecbcddf 1293
97f61250 1294 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
cecbcddf
RA		 1295 if (rc)
1296 DP_ERR(dev,
1297 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1298 qp->icid);
1299
1300 return rc;
1301}
1302
df158561
AR		 1303static void qedr_set_common_qp_params(struct qedr_dev *dev,
1304 struct qedr_qp *qp,
1305 struct qedr_pd *pd,
1306 struct ib_qp_init_attr *attrs)
cecbcddf 1307{
cecbcddf 1308 spin_lock_init(&qp->q_lock);
82af6d19
MK		 1309 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1310 kref_init(&qp->refcnt);
1311 init_completion(&qp->iwarp_cm_comp);
1312 }
df158561 1313 qp->pd = pd;
cecbcddf
RA		 1314 qp->qp_type = attrs->qp_type;
1315 qp->max_inline_data = attrs->cap.max_inline_data;
1316 qp->sq.max_sges = attrs->cap.max_send_sge;
1317 qp->state = QED_ROCE_QP_STATE_RESET;
1318 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1319 qp->sq_cq = get_qedr_cq(attrs->send_cq);
cecbcddf
RA		 1320 qp->dev = dev;
1321
3491c9e7
YB		 1322 if (attrs->srq) {
1323 qp->srq = get_qedr_srq(attrs->srq);
1324 } else {
1325 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1326 qp->rq.max_sges = attrs->cap.max_recv_sge;
1327 DP_DEBUG(dev, QEDR_MSG_QP,
1328 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1329 qp->rq.max_sges, qp->rq_cq->icid);
1330 }
1331
cecbcddf
RA		 1332 DP_DEBUG(dev, QEDR_MSG_QP,
1333 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1334 pd->pd_id, qp->qp_type, qp->max_inline_data,
1335 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1336 DP_DEBUG(dev, QEDR_MSG_QP,
1337 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1338 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1339}
1340
97f61250 1341static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1342{
97f61250
MK		 1343 int rc;
1344
cecbcddf
RA		 1345 qp->sq.db = dev->db_addr +
1346 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1347 qp->sq.db_data.data.icid = qp->icid + 1;
97f61250
MK		 1348 rc = qedr_db_recovery_add(dev, qp->sq.db,
1349 &qp->sq.db_data,
1350 DB_REC_WIDTH_32B,
1351 DB_REC_KERNEL);
1352 if (rc)
1353 return rc;
1354
3491c9e7
YB		 1355 if (!qp->srq) {
1356 qp->rq.db = dev->db_addr +
1357 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1358 qp->rq.db_data.data.icid = qp->icid;
97f61250
MK		 1359
1360 rc = qedr_db_recovery_add(dev, qp->rq.db,
1361 &qp->rq.db_data,
1362 DB_REC_WIDTH_32B,
1363 DB_REC_KERNEL);
1364 if (rc)
1365 qedr_db_recovery_del(dev, qp->sq.db,
1366 &qp->sq.db_data);
3491c9e7 1367 }
97f61250
MK		 1368
1369 return rc;
3491c9e7
YB		 1370}
1371
68e326de 1372static int qedr_check_srq_params(struct qedr_dev *dev,
3491c9e7
YB		 1373 struct ib_srq_init_attr *attrs,
1374 struct ib_udata *udata)
1375{
1376 struct qedr_device_attr *qattr = &dev->attr;
1377
1378 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1379 DP_ERR(dev,
1380 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1381 attrs->attr.max_wr, qattr->max_srq_wr);
1382 return -EINVAL;
1383 }
1384
1385 if (attrs->attr.max_sge > qattr->max_sge) {
1386 DP_ERR(dev,
1387 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1388 attrs->attr.max_sge, qattr->max_sge);
1389 return -EINVAL;
1390 }
1391
1392 return 0;
1393}
1394
40b173dd
YB		 1395static void qedr_free_srq_user_params(struct qedr_srq *srq)
1396{
1397 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1398 ib_umem_release(srq->usrq.umem);
1399 ib_umem_release(srq->prod_umem);
1400}
1401
3491c9e7
YB		 1402static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1403{
1404 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1405 struct qedr_dev *dev = srq->dev;
1406
1407 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1408
1409 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1410 hw_srq->virt_prod_pair_addr,
1411 hw_srq->phy_prod_pair_addr);
1412}
1413
b0ea0fa5 1414static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1415 struct qedr_srq *srq,
1416 struct qedr_create_srq_ureq *ureq,
72b894b0 1417 int access)
40b173dd
YB		 1418{
1419 struct scatterlist *sg;
1420 int rc;
1421
b0ea0fa5 1422 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
72b894b0 1423 ureq->srq_len, false, access, 1);
40b173dd
YB		 1424 if (rc)
1425 return rc;
1426
c320e527
MS		 1427 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1428 sizeof(struct rdma_srq_producers), access);
40b173dd
YB		 1429 if (IS_ERR(srq->prod_umem)) {
1430 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1431 ib_umem_release(srq->usrq.umem);
1432 DP_ERR(srq->dev,
1433 "create srq: failed ib_umem_get for producer, got %ld\n",
1434 PTR_ERR(srq->prod_umem));
1435 return PTR_ERR(srq->prod_umem);
1436 }
1437
1438 sg = srq->prod_umem->sg_head.sgl;
1439 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1440
1441 return 0;
1442}
1443
3491c9e7
YB		 1444static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1445 struct qedr_dev *dev,
1446 struct ib_srq_init_attr *init_attr)
1447{
1448 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1449 dma_addr_t phy_prod_pair_addr;
1450 u32 num_elems;
1451 void *va;
1452 int rc;
1453
1454 va = dma_alloc_coherent(&dev->pdev->dev,
1455 sizeof(struct rdma_srq_producers),
1456 &phy_prod_pair_addr, GFP_KERNEL);
1457 if (!va) {
1458 DP_ERR(dev,
1459 "create srq: failed to allocate dma memory for producer\n");
1460 return -ENOMEM;
1461 }
1462
1463 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1464 hw_srq->virt_prod_pair_addr = va;
1465
1466 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1467 rc = dev->ops->common->chain_alloc(dev->cdev,
1468 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1469 QED_CHAIN_MODE_PBL,
1470 QED_CHAIN_CNT_TYPE_U32,
1471 num_elems,
1472 QEDR_SRQ_WQE_ELEM_SIZE,
1473 &hw_srq->pbl, NULL);
1474 if (rc)
1475 goto err0;
1476
1477 hw_srq->num_elems = num_elems;
1478
1479 return 0;
1480
1481err0:
1482 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1483 va, phy_prod_pair_addr);
1484 return rc;
1485}
1486
68e326de
LR		 1487int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1488 struct ib_udata *udata)
3491c9e7
YB		 1489{
1490 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1491 struct qed_rdma_create_srq_in_params in_params = {};
68e326de 1492 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
3491c9e7 1493 struct qed_rdma_create_srq_out_params out_params;
68e326de 1494 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
40b173dd 1495 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1496 u64 pbl_base_addr, phy_prod_pair_addr;
1497 struct qedr_srq_hwq_info *hw_srq;
1498 u32 page_cnt, page_size;
68e326de 1499 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3491c9e7
YB		 1500 int rc = 0;
1501
1502 DP_DEBUG(dev, QEDR_MSG_QP,
1503 "create SRQ called from %s (pd %p)\n",
1504 (udata) ? "User lib" : "kernel", pd);
1505
68e326de 1506 rc = qedr_check_srq_params(dev, init_attr, udata);
3491c9e7 1507 if (rc)
68e326de 1508 return -EINVAL;
3491c9e7
YB		 1509
1510 srq->dev = dev;
1511 hw_srq = &srq->hw_srq;
1512 spin_lock_init(&srq->lock);
1513
1514 hw_srq->max_wr = init_attr->attr.max_wr;
1515 hw_srq->max_sges = init_attr->attr.max_sge;
1516
89944450 1517 if (udata) {
97f61250
MK		 1518 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1519 udata->inlen))) {
40b173dd
YB		 1520 DP_ERR(dev,
1521 "create srq: problem copying data from user space\n");
1522 goto err0;
1523 }
1524
72b894b0 1525 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
40b173dd
YB		 1526 if (rc)
1527 goto err0;
1528
1529 page_cnt = srq->usrq.pbl_info.num_pbes;
1530 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1531 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1532 page_size = PAGE_SIZE;
40b173dd
YB		 1533 } else {
1534 struct qed_chain *pbl;
1535
1536 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1537 if (rc)
1538 goto err0;
1539
1540 pbl = &hw_srq->pbl;
1541 page_cnt = qed_chain_get_page_cnt(pbl);
1542 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1543 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1544 page_size = QED_CHAIN_PAGE_SIZE;
1545 }
3491c9e7 1546
3491c9e7
YB		 1547 in_params.pd_id = pd->pd_id;
1548 in_params.pbl_base_addr = pbl_base_addr;
1549 in_params.prod_pair_addr = phy_prod_pair_addr;
1550 in_params.num_pages = page_cnt;
1551 in_params.page_size = page_size;
1552
1553 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1554 if (rc)
1555 goto err1;
1556
1557 srq->srq_id = out_params.srq_id;
1558
40b173dd
YB		 1559 if (udata) {
1560 rc = qedr_copy_srq_uresp(dev, srq, udata);
1561 if (rc)
1562 goto err2;
1563 }
1564
9fd15987 1565 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1566 if (rc)
1567 goto err2;
1568
1569 DP_DEBUG(dev, QEDR_MSG_SRQ,
1570 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
68e326de 1571 return 0;
3491c9e7
YB		 1572
1573err2:
1574 destroy_in_params.srq_id = srq->srq_id;
1575
1576 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1577err1:
40b173dd
YB		 1578 if (udata)
1579 qedr_free_srq_user_params(srq);
1580 else
1581 qedr_free_srq_kernel_params(srq);
3491c9e7 1582err0:
68e326de 1583 return -EFAULT;
3491c9e7
YB		 1584}
1585
68e326de 1586void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1587{
1588 struct qed_rdma_destroy_srq_in_params in_params = {};
1589 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1590 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1591
9fd15987 1592 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7
YB		 1593 in_params.srq_id = srq->srq_id;
1594 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1595
e00b64f7 1596 if (ibsrq->uobject)
40b173dd
YB		 1597 qedr_free_srq_user_params(srq);
1598 else
1599 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1600
1601 DP_DEBUG(dev, QEDR_MSG_SRQ,
1602 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1603 srq->srq_id);
3491c9e7
YB		 1604}
1605
1606int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1607 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1608{
1609 struct qed_rdma_modify_srq_in_params in_params = {};
1610 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1611 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1612 int rc;
1613
1614 if (attr_mask & IB_SRQ_MAX_WR) {
1615 DP_ERR(dev,
1616 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1617 attr_mask, srq);
1618 return -EINVAL;
1619 }
1620
1621 if (attr_mask & IB_SRQ_LIMIT) {
1622 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1623 DP_ERR(dev,
1624 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1625 attr->srq_limit, srq->hw_srq.max_wr);
1626 return -EINVAL;
1627 }
1628
1629 in_params.srq_id = srq->srq_id;
1630 in_params.wqe_limit = attr->srq_limit;
1631 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1632 if (rc)
1633 return rc;
1634 }
1635
1636 srq->srq_limit = attr->srq_limit;
1637
1638 DP_DEBUG(dev, QEDR_MSG_SRQ,
1639 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1640
1641 return 0;
cecbcddf
RA		 1642}
1643
1644static inline void
df158561
AR		 1645qedr_init_common_qp_in_params(struct qedr_dev *dev,
1646 struct qedr_pd *pd,
1647 struct qedr_qp *qp,
1648 struct ib_qp_init_attr *attrs,
1649 bool fmr_and_reserved_lkey,
1650 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1651{
df158561
AR		 1652 /* QP handle to be written in an async event */
1653 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1654 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1655
1656 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1657 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1658 params->pd = pd->pd_id;
1659 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1660 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1661 params->stats_queue = 0;
df158561
AR		 1662 params->srq_id = 0;
1663 params->use_srq = false;
3491c9e7
YB		 1664
1665 if (!qp->srq) {
1666 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1667
1668 } else {
1669 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1670 params->srq_id = qp->srq->srq_id;
1671 params->use_srq = true;
1672 }
cecbcddf
RA		 1673}
1674
df158561 1675static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1676{
df158561
AR		 1677 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1678 "qp=%p. "
1679 "sq_addr=0x%llx, "
1680 "sq_len=%zd, "
1681 "rq_addr=0x%llx, "
1682 "rq_len=%zd"
1683 "\n",
1684 qp,
1685 qp->usq.buf_addr,
1686 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1687}
cecbcddf 1688
69ad0e7f
KM		 1689static inline void
1690qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1691 struct qedr_qp *qp,
1692 struct qed_rdma_create_qp_out_params *out_params)
1693{
1694 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1695 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1696
1697 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1698 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1699 if (!qp->srq) {
1700 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1701 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1702 }
69ad0e7f
KM		 1703
1704 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1705 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1706}
1707
97f61250
MK		 1708static void qedr_cleanup_user(struct qedr_dev *dev,
1709 struct qedr_ucontext *ctx,
1710 struct qedr_qp *qp)
df158561 1711{
836a0fbb 1712 ib_umem_release(qp->usq.umem);
df158561 1713 qp->usq.umem = NULL;
cecbcddf 1714
836a0fbb 1715 ib_umem_release(qp->urq.umem);
df158561 1716 qp->urq.umem = NULL;
24e412c1
MK		 1717
1718 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1719 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1720 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1721 } else {
1722 kfree(qp->usq.pbl_tbl);
1723 kfree(qp->urq.pbl_tbl);
1724 }
97f61250
MK		 1725
1726 if (qp->usq.db_rec_data) {
1727 qedr_db_recovery_del(dev, qp->usq.db_addr,
1728 &qp->usq.db_rec_data->db_data);
1729 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1730 }
1731
1732 if (qp->urq.db_rec_data) {
1733 qedr_db_recovery_del(dev, qp->urq.db_addr,
1734 &qp->urq.db_rec_data->db_data);
1735 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1736 }
b4bc7660
MK		 1737
1738 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1739 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1740 &qp->urq.db_rec_db2_data);
cecbcddf
RA		 1741}
1742
df158561
AR		 1743static int qedr_create_user_qp(struct qedr_dev *dev,
1744 struct qedr_qp *qp,
1745 struct ib_pd *ibpd,
1746 struct ib_udata *udata,
1747 struct ib_qp_init_attr *attrs)
cecbcddf 1748{
df158561
AR		 1749 struct qed_rdma_create_qp_in_params in_params;
1750 struct qed_rdma_create_qp_out_params out_params;
1751 struct qedr_pd *pd = get_qedr_pd(ibpd);
97f61250
MK		 1752 struct qedr_create_qp_uresp uresp;
1753 struct qedr_ucontext *ctx = NULL;
df158561 1754 struct qedr_create_qp_ureq ureq;
69ad0e7f 1755 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
df158561 1756 int rc = -EINVAL;
cecbcddf 1757
82af6d19 1758 qp->create_type = QEDR_QP_CREATE_USER;
df158561 1759 memset(&ureq, 0, sizeof(ureq));
97f61250 1760 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
df158561
AR		 1761 if (rc) {
1762 DP_ERR(dev, "Problem copying data from user space\n");
1763 return rc;
1764 }
cecbcddf 1765
72b894b0 1766 /* SQ - read access only (0) */
b0ea0fa5 1767 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
72b894b0 1768 ureq.sq_len, true, 0, alloc_and_init);
df158561
AR		 1769 if (rc)
1770 return rc;
cecbcddf 1771
40b173dd 1772 if (!qp->srq) {
72b894b0 1773 /* RQ - read access only (0) */
b0ea0fa5 1774 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
72b894b0 1775 ureq.rq_len, true, 0, alloc_and_init);
40b173dd
YB		 1776 if (rc)
1777 return rc;
1778 }
df158561
AR		 1779
1780 memset(&in_params, 0, sizeof(in_params));
1781 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1782 in_params.qp_handle_lo = ureq.qp_handle_lo;
1783 in_params.qp_handle_hi = ureq.qp_handle_hi;
1784 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1785 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
40b173dd
YB		 1786 if (!qp->srq) {
1787 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1788 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1789 }
df158561
AR		 1790
1791 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1792 &in_params, &out_params);
1793
1794 if (!qp->qed_qp) {
1795 rc = -ENOMEM;
1796 goto err1;
1797 }
1798
69ad0e7f
KM		 1799 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1800 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1801
df158561
AR		 1802 qp->qp_id = out_params.qp_id;
1803 qp->icid = out_params.icid;
1804
97f61250
MK		 1805 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1806 if (rc)
1807 goto err;
1808
1809 /* db offset was calculated in copy_qp_uresp, now set in the user q */
1810 ctx = pd->uctx;
1811 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1812 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1813
b4bc7660
MK		 1814 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1815 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1816
1817 /* calculate the db_rec_db2 data since it is constant so no
1818 * need to reflect from user
1819 */
1820 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1821 qp->urq.db_rec_db2_data.data.value =
1822 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1823 }
1824
97f61250
MK		 1825 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1826 &qp->usq.db_rec_data->db_data,
1827 DB_REC_WIDTH_32B,
1828 DB_REC_USER);
1829 if (rc)
1830 goto err;
1831
1832 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1833 &qp->urq.db_rec_data->db_data,
1834 DB_REC_WIDTH_32B,
1835 DB_REC_USER);
df158561
AR		 1836 if (rc)
1837 goto err;
1838
b4bc7660
MK		 1839 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1840 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1841 &qp->urq.db_rec_db2_data,
1842 DB_REC_WIDTH_32B,
1843 DB_REC_USER);
1844 if (rc)
1845 goto err;
1846 }
df158561 1847 qedr_qp_user_print(dev, qp);
cecbcddf 1848
97f61250 1849 return rc;
df158561
AR		 1850err:
1851 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1852 if (rc)
1853 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1854
1855err1:
97f61250 1856 qedr_cleanup_user(dev, ctx, qp);
df158561 1857 return rc;
cecbcddf
RA		 1858}
1859
97f61250 1860static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
f5b1b177 1861{
97f61250
MK		 1862 int rc;
1863
f5b1b177
KM		 1864 qp->sq.db = dev->db_addr +
1865 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1866 qp->sq.db_data.data.icid = qp->icid;
1867
97f61250
MK		 1868 rc = qedr_db_recovery_add(dev, qp->sq.db,
1869 &qp->sq.db_data,
1870 DB_REC_WIDTH_32B,
1871 DB_REC_KERNEL);
1872 if (rc)
1873 return rc;
1874
f5b1b177
KM		 1875 qp->rq.db = dev->db_addr +
1876 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1877 qp->rq.db_data.data.icid = qp->icid;
1878 qp->rq.iwarp_db2 = dev->db_addr +
1879 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1880 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1881 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
97f61250
MK		 1882
1883 rc = qedr_db_recovery_add(dev, qp->rq.db,
1884 &qp->rq.db_data,
1885 DB_REC_WIDTH_32B,
1886 DB_REC_KERNEL);
b4bc7660
MK		 1887 if (rc)
1888 return rc;
1889
1890 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1891 &qp->rq.iwarp_db2_data,
1892 DB_REC_WIDTH_32B,
1893 DB_REC_KERNEL);
97f61250 1894 return rc;
f5b1b177
KM		 1895}
1896
df158561
AR		 1897static int
1898qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1899 struct qedr_qp *qp,
1900 struct qed_rdma_create_qp_in_params *in_params,
1901 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1902{
df158561 1903 struct qed_rdma_create_qp_out_params out_params;
cecbcddf
RA		 1904 int rc;
1905
cecbcddf
RA		 1906 rc = dev->ops->common->chain_alloc(dev->cdev,
1907 QED_CHAIN_USE_TO_PRODUCE,
1908 QED_CHAIN_MODE_PBL,
1909 QED_CHAIN_CNT_TYPE_U32,
1910 n_sq_elems,
1911 QEDR_SQE_ELEMENT_SIZE,
1a4a6975 1912 &qp->sq.pbl, NULL);
cecbcddf 1913
df158561
AR		 1914 if (rc)
1915 return rc;
cecbcddf 1916
df158561
AR		 1917 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1918 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1919
cecbcddf
RA		 1920 rc = dev->ops->common->chain_alloc(dev->cdev,
1921 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1922 QED_CHAIN_MODE_PBL,
1923 QED_CHAIN_CNT_TYPE_U32,
1924 n_rq_elems,
1925 QEDR_RQE_ELEMENT_SIZE,
1a4a6975 1926 &qp->rq.pbl, NULL);
df158561
AR		 1927 if (rc)
1928 return rc;
cecbcddf 1929
df158561
AR		 1930 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1931 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1932
df158561
AR		 1933 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1934 in_params, &out_params);
cecbcddf 1935
df158561
AR		 1936 if (!qp->qed_qp)
1937 return -EINVAL;
cecbcddf 1938
df158561
AR		 1939 qp->qp_id = out_params.qp_id;
1940 qp->icid = out_params.icid;
cecbcddf 1941
97f61250 1942 return qedr_set_roce_db_info(dev, qp);
f5b1b177 1943}
cecbcddf 1944
f5b1b177
KM		 1945static int
1946qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1947 struct qedr_qp *qp,
1948 struct qed_rdma_create_qp_in_params *in_params,
1949 u32 n_sq_elems, u32 n_rq_elems)
1950{
1951 struct qed_rdma_create_qp_out_params out_params;
1952 struct qed_chain_ext_pbl ext_pbl;
1953 int rc;
1954
1955 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1956 QEDR_SQE_ELEMENT_SIZE,
1957 QED_CHAIN_MODE_PBL);
1958 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1959 QEDR_RQE_ELEMENT_SIZE,
1960 QED_CHAIN_MODE_PBL);
1961
1962 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1963 in_params, &out_params);
1964
1965 if (!qp->qed_qp)
1966 return -EINVAL;
1967
1968 /* Now we allocate the chain */
1969 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1970 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1971
1972 rc = dev->ops->common->chain_alloc(dev->cdev,
1973 QED_CHAIN_USE_TO_PRODUCE,
1974 QED_CHAIN_MODE_PBL,
1975 QED_CHAIN_CNT_TYPE_U32,
1976 n_sq_elems,
1977 QEDR_SQE_ELEMENT_SIZE,
1978 &qp->sq.pbl, &ext_pbl);
1979
1980 if (rc)
1981 goto err;
1982
1983 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1984 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1985
1986 rc = dev->ops->common->chain_alloc(dev->cdev,
1987 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1988 QED_CHAIN_MODE_PBL,
1989 QED_CHAIN_CNT_TYPE_U32,
1990 n_rq_elems,
1991 QEDR_RQE_ELEMENT_SIZE,
1992 &qp->rq.pbl, &ext_pbl);
1993
1994 if (rc)
1995 goto err;
1996
1997 qp->qp_id = out_params.qp_id;
1998 qp->icid = out_params.icid;
1999
97f61250 2000 return qedr_set_iwarp_db_info(dev, qp);
f5b1b177
KM		 2001
2002err:
2003 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2004
2005 return rc;
cecbcddf
RA		 2006}
2007
df158561 2008static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 2009{
df158561
AR		 2010 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2011 kfree(qp->wqe_wr_id);
cecbcddf 2012
df158561
AR		 2013 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2014 kfree(qp->rqe_wr_id);
97f61250
MK		 2015
2016 /* GSI qp is not registered to db mechanism so no need to delete */
2017 if (qp->qp_type == IB_QPT_GSI)
2018 return;
2019
2020 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2021
b4bc7660 2022 if (!qp->srq) {
97f61250 2023 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
b4bc7660
MK		 2024
2025 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2026 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2027 &qp->rq.iwarp_db2_data);
2028 }
cecbcddf
RA		 2029}
2030
df158561
AR		 2031static int qedr_create_kernel_qp(struct qedr_dev *dev,
2032 struct qedr_qp *qp,
2033 struct ib_pd *ibpd,
2034 struct ib_qp_init_attr *attrs)
cecbcddf 2035{
df158561
AR		 2036 struct qed_rdma_create_qp_in_params in_params;
2037 struct qedr_pd *pd = get_qedr_pd(ibpd);
2038 int rc = -EINVAL;
2039 u32 n_rq_elems;
2040 u32 n_sq_elems;
2041 u32 n_sq_entries;
cecbcddf 2042
df158561 2043 memset(&in_params, 0, sizeof(in_params));
82af6d19 2044 qp->create_type = QEDR_QP_CREATE_KERNEL;
cecbcddf 2045
df158561
AR		 2046 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2047 * the ring. The ring should allow at least a single WR, even if the
2048 * user requested none, due to allocation issues.
2049 * We should add an extra WR since the prod and cons indices of
2050 * wqe_wr_id are managed in such a way that the WQ is considered full
2051 * when (prod+1)%max_wr==cons. We currently don't do that because we
2052 * double the number of entries due an iSER issue that pushes far more
2053 * WRs than indicated. If we decline its ib_post_send() then we get
2054 * error prints in the dmesg we'd like to avoid.
2055 */
2056 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2057 dev->attr.max_sqe);
cecbcddf 2058
6396bb22 2059 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 2060 GFP_KERNEL);
2061 if (!qp->wqe_wr_id) {
2062 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2063 return -ENOMEM;
2064 }
cecbcddf 2065
df158561
AR		 2066 /* QP handle to be written in CQE */
2067 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2068 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 2069
df158561
AR		 2070 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2071 * the ring. There ring should allow at least a single WR, even if the
2072 * user requested none, due to allocation issues.
2073 */
2074 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 2075
df158561 2076 /* Allocate driver internal RQ array */
6396bb22 2077 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 2078 GFP_KERNEL);
2079 if (!qp->rqe_wr_id) {
2080 DP_ERR(dev,
2081 "create qp: failed RQ shadow memory allocation\n");
2082 kfree(qp->wqe_wr_id);
2083 return -ENOMEM;
cecbcddf
RA		 2084 }
2085
df158561 2086 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 2087
df158561
AR		 2088 n_sq_entries = attrs->cap.max_send_wr;
2089 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2090 n_sq_entries = max_t(u32, n_sq_entries, 1);
2091 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 2092
df158561
AR		 2093 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2094
f5b1b177
KM		 2095 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2096 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2097 n_sq_elems, n_rq_elems);
2098 else
2099 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2100 n_sq_elems, n_rq_elems);
df158561
AR		 2101 if (rc)
2102 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 2103
2104 return rc;
2105}
2106
2107struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2108 struct ib_qp_init_attr *attrs,
2109 struct ib_udata *udata)
2110{
2111 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 2112 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 2113 struct qedr_qp *qp;
181d8015 2114 struct ib_qp *ibqp;
cecbcddf
RA		 2115 int rc = 0;
2116
2117 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2118 udata ? "user library" : "kernel", pd);
2119
e00b64f7 2120 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf
RA		 2121 if (rc)
2122 return ERR_PTR(rc);
2123
cecbcddf 2124 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 2125 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2126 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 2127 get_qedr_cq(attrs->send_cq),
2128 get_qedr_cq(attrs->send_cq)->icid,
2129 get_qedr_cq(attrs->recv_cq),
3491c9e7 2130 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 2131
df158561
AR		 2132 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2133 if (!qp) {
2134 DP_ERR(dev, "create qp: failed allocating memory\n");
2135 return ERR_PTR(-ENOMEM);
2136 }
2137
2138 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 2139
04886779 2140 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 2141 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2142 if (IS_ERR(ibqp))
2143 kfree(qp);
2144 return ibqp;
04886779
RA		 2145 }
2146
df158561
AR		 2147 if (udata)
2148 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2149 else
2150 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 2151
df158561
AR		 2152 if (rc)
2153 goto err;
cecbcddf 2154
cecbcddf
RA		 2155 qp->ibqp.qp_num = qp->qp_id;
2156
1212767e 2157 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 2158 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e
YB		 2159 if (rc)
2160 goto err;
2161 }
de0089e6 2162
cecbcddf
RA		 2163 return &qp->ibqp;
2164
df158561 2165err:
cecbcddf
RA		 2166 kfree(qp);
2167
2168 return ERR_PTR(-EFAULT);
2169}
2170
27a4b1a6 2171static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 2172{
2173 switch (qp_state) {
2174 case QED_ROCE_QP_STATE_RESET:
2175 return IB_QPS_RESET;
2176 case QED_ROCE_QP_STATE_INIT:
2177 return IB_QPS_INIT;
2178 case QED_ROCE_QP_STATE_RTR:
2179 return IB_QPS_RTR;
2180 case QED_ROCE_QP_STATE_RTS:
2181 return IB_QPS_RTS;
2182 case QED_ROCE_QP_STATE_SQD:
2183 return IB_QPS_SQD;
2184 case QED_ROCE_QP_STATE_ERR:
2185 return IB_QPS_ERR;
2186 case QED_ROCE_QP_STATE_SQE:
2187 return IB_QPS_SQE;
2188 }
2189 return IB_QPS_ERR;
2190}
2191
27a4b1a6
RA		 2192static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2193 enum ib_qp_state qp_state)
cecbcddf
RA		 2194{
2195 switch (qp_state) {
2196 case IB_QPS_RESET:
2197 return QED_ROCE_QP_STATE_RESET;
2198 case IB_QPS_INIT:
2199 return QED_ROCE_QP_STATE_INIT;
2200 case IB_QPS_RTR:
2201 return QED_ROCE_QP_STATE_RTR;
2202 case IB_QPS_RTS:
2203 return QED_ROCE_QP_STATE_RTS;
2204 case IB_QPS_SQD:
2205 return QED_ROCE_QP_STATE_SQD;
2206 case IB_QPS_ERR:
2207 return QED_ROCE_QP_STATE_ERR;
2208 default:
2209 return QED_ROCE_QP_STATE_ERR;
2210 }
2211}
2212
2213static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2214{
2215 qed_chain_reset(&qph->pbl);
2216 qph->prod = 0;
2217 qph->cons = 0;
2218 qph->wqe_cons = 0;
2219 qph->db_data.data.value = cpu_to_le16(0);
2220}
2221
2222static int qedr_update_qp_state(struct qedr_dev *dev,
2223 struct qedr_qp *qp,
caf61b1b 2224 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 2225 enum qed_roce_qp_state new_state)
2226{
2227 int status = 0;
2228
caf61b1b 2229 if (new_state == cur_state)
865cea40 2230 return 0;
cecbcddf 2231
caf61b1b 2232 switch (cur_state) {
cecbcddf
RA		 2233 case QED_ROCE_QP_STATE_RESET:
2234 switch (new_state) {
2235 case QED_ROCE_QP_STATE_INIT:
2236 qp->prev_wqe_size = 0;
2237 qedr_reset_qp_hwq_info(&qp->sq);
2238 qedr_reset_qp_hwq_info(&qp->rq);
2239 break;
2240 default:
2241 status = -EINVAL;
2242 break;
790b57f6 2243 }
cecbcddf
RA		 2244 break;
2245 case QED_ROCE_QP_STATE_INIT:
2246 switch (new_state) {
2247 case QED_ROCE_QP_STATE_RTR:
2248 /* Update doorbell (in case post_recv was
2249 * done before move to RTR)
2250 */
f5b1b177
KM		 2251
2252 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177 2253 writel(qp->rq.db_data.raw, qp->rq.db);
f5b1b177 2254 }
cecbcddf
RA		 2255 break;
2256 case QED_ROCE_QP_STATE_ERR:
2257 break;
2258 default:
2259 /* Invalid state change. */
2260 status = -EINVAL;
2261 break;
790b57f6 2262 }
cecbcddf
RA		 2263 break;
2264 case QED_ROCE_QP_STATE_RTR:
2265 /* RTR->XXX */
2266 switch (new_state) {
2267 case QED_ROCE_QP_STATE_RTS:
2268 break;
2269 case QED_ROCE_QP_STATE_ERR:
2270 break;
2271 default:
2272 /* Invalid state change. */
2273 status = -EINVAL;
2274 break;
790b57f6 2275 }
cecbcddf
RA		 2276 break;
2277 case QED_ROCE_QP_STATE_RTS:
2278 /* RTS->XXX */
2279 switch (new_state) {
2280 case QED_ROCE_QP_STATE_SQD:
2281 break;
2282 case QED_ROCE_QP_STATE_ERR:
2283 break;
2284 default:
2285 /* Invalid state change. */
2286 status = -EINVAL;
2287 break;
790b57f6 2288 }
cecbcddf
RA		 2289 break;
2290 case QED_ROCE_QP_STATE_SQD:
2291 /* SQD->XXX */
2292 switch (new_state) {
2293 case QED_ROCE_QP_STATE_RTS:
2294 case QED_ROCE_QP_STATE_ERR:
2295 break;
2296 default:
2297 /* Invalid state change. */
2298 status = -EINVAL;
2299 break;
790b57f6 2300 }
cecbcddf
RA		 2301 break;
2302 case QED_ROCE_QP_STATE_ERR:
2303 /* ERR->XXX */
2304 switch (new_state) {
2305 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2306 if ((qp->rq.prod != qp->rq.cons) ||
2307 (qp->sq.prod != qp->sq.cons)) {
2308 DP_NOTICE(dev,
2309 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2310 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2311 qp->sq.cons);
2312 status = -EINVAL;
2313 }
cecbcddf
RA		 2314 break;
2315 default:
2316 status = -EINVAL;
2317 break;
790b57f6 2318 }
cecbcddf
RA		 2319 break;
2320 default:
2321 status = -EINVAL;
2322 break;
790b57f6 2323 }
cecbcddf
RA		 2324
2325 return status;
2326}
2327
2328int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2329 int attr_mask, struct ib_udata *udata)
2330{
2331 struct qedr_qp *qp = get_qedr_qp(ibqp);
2332 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2333 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2334 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2335 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2336 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2337 int rc = 0;
2338
2339 DP_DEBUG(dev, QEDR_MSG_QP,
2340 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2341 attr->qp_state);
2342
2343 old_qp_state = qedr_get_ibqp_state(qp->state);
2344 if (attr_mask & IB_QP_STATE)
2345 new_qp_state = attr->qp_state;
2346 else
2347 new_qp_state = old_qp_state;
2348
f5b1b177
KM		 2349 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2350 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2351 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2352 DP_ERR(dev,
2353 "modify qp: invalid attribute mask=0x%x specified for\n"
2354 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2355 attr_mask, qp->qp_id, ibqp->qp_type,
2356 old_qp_state, new_qp_state);
2357 rc = -EINVAL;
2358 goto err;
2359 }
cecbcddf
RA		 2360 }
2361
2362 /* Translate the masks... */
2363 if (attr_mask & IB_QP_STATE) {
2364 SET_FIELD(qp_params.modify_flags,
2365 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2366 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2367 }
2368
2369 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2370 qp_params.sqd_async = true;
2371
2372 if (attr_mask & IB_QP_PKEY_INDEX) {
2373 SET_FIELD(qp_params.modify_flags,
2374 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2375 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2376 rc = -EINVAL;
2377 goto err;
2378 }
2379
2380 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2381 }
2382
2383 if (attr_mask & IB_QP_QKEY)
2384 qp->qkey = attr->qkey;
2385
2386 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2387 SET_FIELD(qp_params.modify_flags,
2388 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2389 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2390 IB_ACCESS_REMOTE_READ;
2391 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2392 IB_ACCESS_REMOTE_WRITE;
2393 qp_params.incoming_atomic_en = attr->qp_access_flags &
2394 IB_ACCESS_REMOTE_ATOMIC;
2395 }
2396
2397 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2398 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2399 return -EINVAL;
2400
cecbcddf
RA		 2401 if (attr_mask & IB_QP_PATH_MTU) {
2402 if (attr->path_mtu < IB_MTU_256 ||
2403 attr->path_mtu > IB_MTU_4096) {
2404 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2405 rc = -EINVAL;
2406 goto err;
2407 }
2408 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2409 ib_mtu_enum_to_int(iboe_get_mtu
2410 (dev->ndev->mtu)));
2411 }
2412
2413 if (!qp->mtu) {
2414 qp->mtu =
2415 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2416 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2417 }
2418
2419 SET_FIELD(qp_params.modify_flags,
2420 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2421
d8966fcd
DC		 2422 qp_params.traffic_class_tos = grh->traffic_class;
2423 qp_params.flow_label = grh->flow_label;
2424 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2425
d8966fcd 2426 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2427
2428 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2429 if (rc) {
2430 DP_ERR(dev,
2431 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2432 grh->sgid_index, rc);
cecbcddf
RA		 2433 return rc;
2434 }
2435
2436 rc = qedr_get_dmac(dev, &attr->ah_attr,
2437 qp_params.remote_mac_addr);
2438 if (rc)
2439 return rc;
2440
2441 qp_params.use_local_mac = true;
2442 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2443
2444 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2445 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2446 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2447 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2448 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2449 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2450 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2451 qp_params.remote_mac_addr);
cecbcddf
RA		 2452
2453 qp_params.mtu = qp->mtu;
2454 qp_params.lb_indication = false;
2455 }
2456
2457 if (!qp_params.mtu) {
2458 /* Stay with current MTU */
2459 if (qp->mtu)
2460 qp_params.mtu = qp->mtu;
2461 else
2462 qp_params.mtu =
2463 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2464 }
2465
2466 if (attr_mask & IB_QP_TIMEOUT) {
2467 SET_FIELD(qp_params.modify_flags,
2468 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2469
c3594f22
KM		 2470 /* The received timeout value is an exponent used like this:
2471 * "12.7.34 LOCAL ACK TIMEOUT
2472 * Value representing the transport (ACK) timeout for use by
2473 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2474 * The FW expects timeout in msec so we need to divide the usec
2475 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2476 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2477 * The value of zero means infinite so we use a 'max_t' to make
2478 * sure that sub 1 msec values will be configured as 1 msec.
2479 */
2480 if (attr->timeout)
2481 qp_params.ack_timeout =
2482 1 << max_t(int, attr->timeout - 8, 0);
2483 else
cecbcddf 2484 qp_params.ack_timeout = 0;
cecbcddf 2485 }
c3594f22 2486
cecbcddf
RA		 2487 if (attr_mask & IB_QP_RETRY_CNT) {
2488 SET_FIELD(qp_params.modify_flags,
2489 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2490 qp_params.retry_cnt = attr->retry_cnt;
2491 }
2492
2493 if (attr_mask & IB_QP_RNR_RETRY) {
2494 SET_FIELD(qp_params.modify_flags,
2495 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2496 qp_params.rnr_retry_cnt = attr->rnr_retry;
2497 }
2498
2499 if (attr_mask & IB_QP_RQ_PSN) {
2500 SET_FIELD(qp_params.modify_flags,
2501 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2502 qp_params.rq_psn = attr->rq_psn;
2503 qp->rq_psn = attr->rq_psn;
2504 }
2505
2506 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2507 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2508 rc = -EINVAL;
2509 DP_ERR(dev,
2510 "unsupported max_rd_atomic=%d, supported=%d\n",
2511 attr->max_rd_atomic,
2512 dev->attr.max_qp_req_rd_atomic_resc);
2513 goto err;
2514 }
2515
2516 SET_FIELD(qp_params.modify_flags,
2517 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2518 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2519 }
2520
2521 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2522 SET_FIELD(qp_params.modify_flags,
2523 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2524 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2525 }
2526
2527 if (attr_mask & IB_QP_SQ_PSN) {
2528 SET_FIELD(qp_params.modify_flags,
2529 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2530 qp_params.sq_psn = attr->sq_psn;
2531 qp->sq_psn = attr->sq_psn;
2532 }
2533
2534 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2535 if (attr->max_dest_rd_atomic >
2536 dev->attr.max_qp_resp_rd_atomic_resc) {
2537 DP_ERR(dev,
2538 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2539 attr->max_dest_rd_atomic,
2540 dev->attr.max_qp_resp_rd_atomic_resc);
2541
2542 rc = -EINVAL;
2543 goto err;
2544 }
2545
2546 SET_FIELD(qp_params.modify_flags,
2547 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2548 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2549 }
2550
2551 if (attr_mask & IB_QP_DEST_QPN) {
2552 SET_FIELD(qp_params.modify_flags,
2553 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2554
2555 qp_params.dest_qp = attr->dest_qp_num;
2556 qp->dest_qp_num = attr->dest_qp_num;
2557 }
2558
caf61b1b
KM		 2559 cur_state = qp->state;
2560
2561 /* Update the QP state before the actual ramrod to prevent a race with
2562 * fast path. Modifying the QP state to error will cause the device to
2563 * flush the CQEs and while polling the flushed CQEs will considered as
2564 * a potential issue if the QP isn't in error state.
2565 */
2566 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2567 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2568 qp->state = QED_ROCE_QP_STATE_ERR;
2569
cecbcddf
RA		 2570 if (qp->qp_type != IB_QPT_GSI)
2571 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2572 qp->qed_qp, &qp_params);
2573
2574 if (attr_mask & IB_QP_STATE) {
2575 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2576 rc = qedr_update_qp_state(dev, qp, cur_state,
2577 qp_params.new_state);
cecbcddf
RA		 2578 qp->state = qp_params.new_state;
2579 }
2580
2581err:
2582 return rc;
2583}
2584
2585static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2586{
2587 int ib_qp_acc_flags = 0;
2588
2589 if (params->incoming_rdma_write_en)
2590 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2591 if (params->incoming_rdma_read_en)
2592 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2593 if (params->incoming_atomic_en)
2594 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2595 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2596 return ib_qp_acc_flags;
2597}
2598
2599int qedr_query_qp(struct ib_qp *ibqp,
2600 struct ib_qp_attr *qp_attr,
2601 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2602{
2603 struct qed_rdma_query_qp_out_params params;
2604 struct qedr_qp *qp = get_qedr_qp(ibqp);
2605 struct qedr_dev *dev = qp->dev;
2606 int rc = 0;
2607
2608 memset(&params, 0, sizeof(params));
2609
2610 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2611 if (rc)
2612 goto err;
2613
2614 memset(qp_attr, 0, sizeof(*qp_attr));
2615 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2616
2617 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2618 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2619 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2620 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2621 qp_attr->rq_psn = params.rq_psn;
2622 qp_attr->sq_psn = params.sq_psn;
2623 qp_attr->dest_qp_num = params.dest_qp;
2624
2625 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2626
2627 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2628 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2629 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2630 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 2631 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 2632 qp_init_attr->cap = qp_attr->cap;
2633
44c58487 2634 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2635 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2636 params.flow_label, qp->sgid_idx,
2637 params.hop_limit_ttl, params.traffic_class_tos);
2638 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2639 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2640 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2641 qp_attr->timeout = params.timeout;
2642 qp_attr->rnr_retry = params.rnr_retry;
2643 qp_attr->retry_cnt = params.retry_cnt;
2644 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2645 qp_attr->pkey_index = params.pkey_index;
2646 qp_attr->port_num = 1;
d8966fcd
DC		 2647 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2648 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2649 qp_attr->alt_pkey_index = 0;
2650 qp_attr->alt_port_num = 0;
2651 qp_attr->alt_timeout = 0;
2652 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2653
2654 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2655 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2656 qp_attr->max_rd_atomic = params.max_rd_atomic;
2657 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2658
2659 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2660 qp_attr->cap.max_inline_data);
2661
2662err:
2663 return rc;
2664}
2665
bdeacabd
SR		 2666static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2667 struct ib_udata *udata)
df158561 2668{
97f61250
MK		 2669 struct qedr_ucontext *ctx =
2670 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2671 ibucontext);
2672 int rc;
df158561
AR		 2673
2674 if (qp->qp_type != IB_QPT_GSI) {
2675 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2676 if (rc)
2677 return rc;
2678 }
2679
82af6d19 2680 if (qp->create_type == QEDR_QP_CREATE_USER)
97f61250 2681 qedr_cleanup_user(dev, ctx, qp);
df158561
AR		 2682 else
2683 qedr_cleanup_kernel(dev, qp);
2684
2685 return 0;
2686}
2687
c4367a26 2688int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2689{
2690 struct qedr_qp *qp = get_qedr_qp(ibqp);
2691 struct qedr_dev *dev = qp->dev;
2692 struct ib_qp_attr attr;
2693 int attr_mask = 0;
cecbcddf
RA		 2694
2695 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2696 qp, qp->qp_type);
2697
f5b1b177
KM		 2698 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2699 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2700 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2701 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2702
f5b1b177
KM		 2703 attr.qp_state = IB_QPS_ERR;
2704 attr_mask |= IB_QP_STATE;
cecbcddf 2705
f5b1b177
KM		 2706 /* Change the QP state to ERROR */
2707 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2708 }
e411e058 2709 } else {
82af6d19
MK		 2710 /* If connection establishment started the WAIT_FOR_CONNECT
2711 * bit will be on and we need to Wait for the establishment
2712 * to complete before destroying the qp.
2713 */
2714 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2715 &qp->iwarp_cm_flags))
2716 wait_for_completion(&qp->iwarp_cm_comp);
2717
2718 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2719 * bit will be on, and we need to wait for the disconnect to
2720 * complete before continuing. We can use the same completion,
2721 * iwarp_cm_comp, since this is the only place that waits for
2722 * this completion and it is sequential. In addition,
2723 * disconnect can't occur before the connection is fully
2724 * established, therefore if WAIT_FOR_DISCONNECT is on it
2725 * means WAIT_FOR_CONNECT is also on and the completion for
2726 * CONNECT already occurred.
2727 */
2728 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2729 &qp->iwarp_cm_flags))
2730 wait_for_completion(&qp->iwarp_cm_comp);
cecbcddf
RA		 2731 }
2732
df158561 2733 if (qp->qp_type == IB_QPT_GSI)
04886779 2734 qedr_destroy_gsi_qp(dev);
cecbcddf 2735
82af6d19
MK		 2736 /* We need to remove the entry from the xarray before we release the
2737 * qp_id to avoid a race of the qp_id being reallocated and failing
2738 * on xa_insert
2739 */
2740 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2741 xa_erase(&dev->qps, qp->qp_id);
2742
bdeacabd 2743 qedr_free_qp_resources(dev, qp, udata);
cecbcddf 2744
82af6d19
MK		 2745 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2746 qedr_iw_qp_rem_ref(&qp->ibqp);
2747
cf167e5e 2748 return 0;
cecbcddf 2749}
e0290cce 2750
fa5d010c 2751int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
d3456914 2752 struct ib_udata *udata)
04886779 2753{
d3456914 2754 struct qedr_ah *ah = get_qedr_ah(ibah);
04886779 2755
fa5d010c 2756 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
04886779 2757
d3456914 2758 return 0;
04886779
RA		 2759}
2760
d3456914 2761void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
04886779
RA		 2762{
2763 struct qedr_ah *ah = get_qedr_ah(ibah);
2764
d97099fe 2765 rdma_destroy_ah_attr(&ah->attr);
04886779
RA		 2766}
2767
e0290cce
RA		 2768static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2769{
2770 struct qedr_pbl *pbl, *tmp;
2771
2772 if (info->pbl_table)
2773 list_add_tail(&info->pbl_table->list_entry,
2774 &info->free_pbl_list);
2775
2776 if (!list_empty(&info->inuse_pbl_list))
2777 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2778
2779 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2780 list_del(&pbl->list_entry);
2781 qedr_free_pbl(dev, &info->pbl_info, pbl);
2782 }
2783}
2784
2785static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2786 size_t page_list_len, bool two_layered)
2787{
2788 struct qedr_pbl *tmp;
2789 int rc;
2790
2791 INIT_LIST_HEAD(&info->free_pbl_list);
2792 INIT_LIST_HEAD(&info->inuse_pbl_list);
2793
2794 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2795 page_list_len, two_layered);
2796 if (rc)
2797 goto done;
2798
2799 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2800 if (IS_ERR(info->pbl_table)) {
2801 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2802 goto done;
2803 }
2804
2805 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2806 &info->pbl_table->pa);
2807
2808 /* in usual case we use 2 PBLs, so we add one to free
2809 * list and allocating another one
2810 */
2811 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2812 if (IS_ERR(tmp)) {
e0290cce
RA		 2813 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2814 goto done;
2815 }
2816
2817 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2818
2819 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2820
2821done:
2822 if (rc)
2823 free_mr_info(dev, info);
2824
2825 return rc;
2826}
2827
2828struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2829 u64 usr_addr, int acc, struct ib_udata *udata)
2830{
2831 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2832 struct qedr_mr *mr;
2833 struct qedr_pd *pd;
2834 int rc = -ENOMEM;
2835
2836 pd = get_qedr_pd(ibpd);
2837 DP_DEBUG(dev, QEDR_MSG_MR,
2838 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2839 pd->pd_id, start, len, usr_addr, acc);
2840
2841 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2842 return ERR_PTR(-EINVAL);
2843
2844 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2845 if (!mr)
2846 return ERR_PTR(rc);
2847
2848 mr->type = QEDR_MR_USER;
2849
c320e527 2850 mr->umem = ib_umem_get(ibpd->device, start, len, acc);
e0290cce
RA		 2851 if (IS_ERR(mr->umem)) {
2852 rc = -EFAULT;
2853 goto err0;
2854 }
2855
2856 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2857 if (rc)
2858 goto err1;
2859
2860 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2861 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2862
2863 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2864 if (rc) {
2865 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2866 goto err1;
2867 }
2868
2869 /* Index only, 18 bit long, lkey = itid << 8 | key */
2870 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2871 mr->hw_mr.key = 0;
2872 mr->hw_mr.pd = pd->pd_id;
2873 mr->hw_mr.local_read = 1;
2874 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2875 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2876 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2877 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2878 mr->hw_mr.mw_bind = false;
2879 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2880 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2881 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 2882 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 2883 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2884 mr->hw_mr.length = len;
2885 mr->hw_mr.vaddr = usr_addr;
2886 mr->hw_mr.zbva = false;
2887 mr->hw_mr.phy_mr = false;
2888 mr->hw_mr.dma_mr = false;
2889
2890 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2891 if (rc) {
2892 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2893 goto err2;
2894 }
2895
2896 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2897 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2898 mr->hw_mr.remote_atomic)
2899 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2900
2901 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2902 mr->ibmr.lkey);
2903 return &mr->ibmr;
2904
2905err2:
2906 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2907err1:
2908 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2909err0:
2910 kfree(mr);
2911 return ERR_PTR(rc);
2912}
2913
c4367a26 2914int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 2915{
2916 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2917 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2918 int rc = 0;
2919
2920 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2921 if (rc)
2922 return rc;
2923
2924 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2925
24e412c1
MK		 2926 if (mr->type != QEDR_MR_DMA)
2927 free_mr_info(dev, &mr->info);
e0290cce
RA		 2928
2929 /* it could be user registered memory. */
836a0fbb 2930 ib_umem_release(mr->umem);
e0290cce
RA		 2931
2932 kfree(mr);
2933
2934 return rc;
2935}
2936
27a4b1a6
RA		 2937static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2938 int max_page_list_len)
e0290cce
RA		 2939{
2940 struct qedr_pd *pd = get_qedr_pd(ibpd);
2941 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2942 struct qedr_mr *mr;
2943 int rc = -ENOMEM;
2944
2945 DP_DEBUG(dev, QEDR_MSG_MR,
2946 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2947 max_page_list_len);
2948
2949 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2950 if (!mr)
2951 return ERR_PTR(rc);
2952
2953 mr->dev = dev;
2954 mr->type = QEDR_MR_FRMR;
2955
2956 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2957 if (rc)
2958 goto err0;
2959
2960 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2961 if (rc) {
2962 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2963 goto err0;
2964 }
2965
2966 /* Index only, 18 bit long, lkey = itid << 8 | key */
2967 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2968 mr->hw_mr.key = 0;
2969 mr->hw_mr.pd = pd->pd_id;
2970 mr->hw_mr.local_read = 1;
2971 mr->hw_mr.local_write = 0;
2972 mr->hw_mr.remote_read = 0;
2973 mr->hw_mr.remote_write = 0;
2974 mr->hw_mr.remote_atomic = 0;
2975 mr->hw_mr.mw_bind = false;
2976 mr->hw_mr.pbl_ptr = 0;
2977 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2978 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2979 mr->hw_mr.fbo = 0;
2980 mr->hw_mr.length = 0;
2981 mr->hw_mr.vaddr = 0;
2982 mr->hw_mr.zbva = false;
2983 mr->hw_mr.phy_mr = true;
2984 mr->hw_mr.dma_mr = false;
2985
2986 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2987 if (rc) {
2988 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2989 goto err1;
2990 }
2991
2992 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2993 mr->ibmr.rkey = mr->ibmr.lkey;
2994
2995 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2996 return mr;
2997
2998err1:
2999 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3000err0:
3001 kfree(mr);
3002 return ERR_PTR(rc);
3003}
3004
c4367a26
SR		 3005struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3006 u32 max_num_sg, struct ib_udata *udata)
e0290cce 3007{
e0290cce
RA		 3008 struct qedr_mr *mr;
3009
3010 if (mr_type != IB_MR_TYPE_MEM_REG)
3011 return ERR_PTR(-EINVAL);
3012
3013 mr = __qedr_alloc_mr(ibpd, max_num_sg);
3014
3015 if (IS_ERR(mr))
3016 return ERR_PTR(-EINVAL);
3017
e0290cce
RA		 3018 return &mr->ibmr;
3019}
3020
3021static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3022{
3023 struct qedr_mr *mr = get_qedr_mr(ibmr);
3024 struct qedr_pbl *pbl_table;
3025 struct regpair *pbe;
3026 u32 pbes_in_page;
3027
3028 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 3029 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 3030 return -ENOMEM;
3031 }
3032
3033 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3034 mr->npages, addr);
3035
3036 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3037 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3038 pbe = (struct regpair *)pbl_table->va;
3039 pbe += mr->npages % pbes_in_page;
3040 pbe->lo = cpu_to_le32((u32)addr);
3041 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3042
3043 mr->npages++;
3044
3045 return 0;
3046}
3047
3048static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3049{
3050 int work = info->completed - info->completed_handled - 1;
3051
3052 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3053 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3054 struct qedr_pbl *pbl;
3055
3056 /* Free all the page list that are possible to be freed
3057 * (all the ones that were invalidated), under the assumption
3058 * that if an FMR was completed successfully that means that
3059 * if there was an invalidate operation before it also ended
3060 */
3061 pbl = list_first_entry(&info->inuse_pbl_list,
3062 struct qedr_pbl, list_entry);
aafec388 3063 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 3064 info->completed_handled++;
3065 }
3066}
3067
3068int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3069 int sg_nents, unsigned int *sg_offset)
3070{
3071 struct qedr_mr *mr = get_qedr_mr(ibmr);
3072
3073 mr->npages = 0;
3074
3075 handle_completed_mrs(mr->dev, &mr->info);
3076 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3077}
3078
3079struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3080{
3081 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3082 struct qedr_pd *pd = get_qedr_pd(ibpd);
3083 struct qedr_mr *mr;
3084 int rc;
3085
3086 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3087 if (!mr)
3088 return ERR_PTR(-ENOMEM);
3089
3090 mr->type = QEDR_MR_DMA;
3091
3092 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3093 if (rc) {
3094 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3095 goto err1;
3096 }
3097
3098 /* index only, 18 bit long, lkey = itid << 8 | key */
3099 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3100 mr->hw_mr.pd = pd->pd_id;
3101 mr->hw_mr.local_read = 1;
3102 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3103 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3104 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3105 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3106 mr->hw_mr.dma_mr = true;
3107
3108 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3109 if (rc) {
3110 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3111 goto err2;
3112 }
3113
3114 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3115 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3116 mr->hw_mr.remote_atomic)
3117 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3118
3119 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3120 return &mr->ibmr;
3121
3122err2:
3123 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3124err1:
3125 kfree(mr);
3126 return ERR_PTR(rc);
3127}
afa0e13b
RA		 3128
3129static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3130{
3131 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3132}
3133
3134static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3135{
3136 int i, len = 0;
3137
3138 for (i = 0; i < num_sge; i++)
3139 len += sg_list[i].length;
3140
3141 return len;
3142}
3143
3144static void swap_wqe_data64(u64 *p)
3145{
3146 int i;
3147
3148 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3149 *p = cpu_to_be64(cpu_to_le64(*p));
3150}
3151
3152static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3153 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 3154 const struct ib_send_wr *wr,
3155 const struct ib_send_wr **bad_wr,
3156 u8 *bits, u8 bit)
afa0e13b
RA		 3157{
3158 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3159 char *seg_prt, *wqe;
3160 int i, seg_siz;
3161
3162 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3163 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3164 *bad_wr = wr;
3165 return 0;
3166 }
3167
3168 if (!data_size)
3169 return data_size;
3170
3171 *bits |= bit;
3172
3173 seg_prt = NULL;
3174 wqe = NULL;
3175 seg_siz = 0;
3176
3177 /* Copy data inline */
3178 for (i = 0; i < wr->num_sge; i++) {
3179 u32 len = wr->sg_list[i].length;
3180 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3181
3182 while (len > 0) {
3183 u32 cur;
3184
3185 /* New segment required */
3186 if (!seg_siz) {
3187 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3188 seg_prt = wqe;
3189 seg_siz = sizeof(struct rdma_sq_common_wqe);
3190 (*wqe_size)++;
3191 }
3192
3193 /* Calculate currently allowed length */
3194 cur = min_t(u32, len, seg_siz);
3195 memcpy(seg_prt, src, cur);
3196
3197 /* Update segment variables */
3198 seg_prt += cur;
3199 seg_siz -= cur;
3200
3201 /* Update sge variables */
3202 src += cur;
3203 len -= cur;
3204
3205 /* Swap fully-completed segments */
3206 if (!seg_siz)
3207 swap_wqe_data64((u64 *)wqe);
3208 }
3209 }
3210
3211 /* swap last not completed segment */
3212 if (seg_siz)
3213 swap_wqe_data64((u64 *)wqe);
3214
3215 return data_size;
3216}
3217
3218#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3219 do { \
3220 DMA_REGPAIR_LE(sge->addr, vaddr); \
3221 (sge)->length = cpu_to_le32(vlength); \
3222 (sge)->flags = cpu_to_le32(vflags); \
3223 } while (0)
3224
3225#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3226 do { \
3227 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3228 (hdr)->num_sges = num_sge; \
3229 } while (0)
3230
3231#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3232 do { \
3233 DMA_REGPAIR_LE(sge->addr, vaddr); \
3234 (sge)->length = cpu_to_le32(vlength); \
3235 (sge)->l_key = cpu_to_le32(vlkey); \
3236 } while (0)
3237
3238static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 3239 const struct ib_send_wr *wr)
afa0e13b
RA		 3240{
3241 u32 data_size = 0;
3242 int i;
3243
3244 for (i = 0; i < wr->num_sge; i++) {
3245 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3246
3247 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3248 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3249 sge->length = cpu_to_le32(wr->sg_list[i].length);
3250 data_size += wr->sg_list[i].length;
3251 }
3252
3253 if (wqe_size)
3254 *wqe_size += wr->num_sge;
3255
3256 return data_size;
3257}
3258
3259static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3260 struct qedr_qp *qp,
3261 struct rdma_sq_rdma_wqe_1st *rwqe,
3262 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3263 const struct ib_send_wr *wr,
3264 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3265{
3266 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3267 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3268
8b0cabc6
AR		 3269 if (wr->send_flags & IB_SEND_INLINE &&
3270 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3271 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3272 u8 flags = 0;
3273
3274 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3275 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3276 bad_wr, &rwqe->flags, flags);
3277 }
3278
3279 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3280}
3281
3282static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3283 struct qedr_qp *qp,
3284 struct rdma_sq_send_wqe_1st *swqe,
3285 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3286 const struct ib_send_wr *wr,
3287 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3288{
3289 memset(swqe2, 0, sizeof(*swqe2));
3290 if (wr->send_flags & IB_SEND_INLINE) {
3291 u8 flags = 0;
3292
3293 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3294 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3295 bad_wr, &swqe->flags, flags);
3296 }
3297
3298 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3299}
3300
3301static int qedr_prepare_reg(struct qedr_qp *qp,
3302 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3303 const struct ib_reg_wr *wr)
afa0e13b
RA		 3304{
3305 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3306 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3307
3308 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3309 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3310 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3311 fwqe1->l_key = wr->key;
3312
08c4cf51
AR		 3313 fwqe2->access_ctrl = 0;
3314
afa0e13b
RA		 3315 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3316 !!(wr->access & IB_ACCESS_REMOTE_READ));
3317 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3318 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3319 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3320 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3321 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3322 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3323 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3324 fwqe2->fmr_ctrl = 0;
3325
3326 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3327 ilog2(mr->ibmr.page_size) - 12);
3328
3329 fwqe2->length_hi = 0;
3330 fwqe2->length_lo = mr->ibmr.length;
3331 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3332 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3333
3334 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3335
3336 return 0;
3337}
3338
27a4b1a6 3339static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3340{
3341 switch (opcode) {
3342 case IB_WR_RDMA_WRITE:
3343 case IB_WR_RDMA_WRITE_WITH_IMM:
3344 return IB_WC_RDMA_WRITE;
3345 case IB_WR_SEND_WITH_IMM:
3346 case IB_WR_SEND:
3347 case IB_WR_SEND_WITH_INV:
3348 return IB_WC_SEND;
3349 case IB_WR_RDMA_READ:
fb1a22be 3350 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3351 return IB_WC_RDMA_READ;
3352 case IB_WR_ATOMIC_CMP_AND_SWP:
3353 return IB_WC_COMP_SWAP;
3354 case IB_WR_ATOMIC_FETCH_AND_ADD:
3355 return IB_WC_FETCH_ADD;
3356 case IB_WR_REG_MR:
3357 return IB_WC_REG_MR;
3358 case IB_WR_LOCAL_INV:
3359 return IB_WC_LOCAL_INV;
3360 default:
3361 return IB_WC_SEND;
3362 }
3363}
3364
f696bf6d
BVA		 3365static inline bool qedr_can_post_send(struct qedr_qp *qp,
3366 const struct ib_send_wr *wr)
afa0e13b
RA		 3367{
3368 int wq_is_full, err_wr, pbl_is_full;
3369 struct qedr_dev *dev = qp->dev;
3370
3371 /* prevent SQ overflow and/or processing of a bad WR */
3372 err_wr = wr->num_sge > qp->sq.max_sges;
3373 wq_is_full = qedr_wq_is_full(&qp->sq);
3374 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3375 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3376 if (wq_is_full || err_wr || pbl_is_full) {
3377 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3378 DP_ERR(dev,
3379 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3380 qp);
3381 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3382 }
3383
3384 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3385 DP_ERR(dev,
3386 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3387 qp);
3388 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3389 }
3390
3391 if (pbl_is_full &&
3392 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3393 DP_ERR(dev,
3394 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3395 qp);
3396 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3397 }
3398 return false;
3399 }
3400 return true;
3401}
3402
d34ac5cd
BVA		 3403static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3404 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3405{
3406 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3407 struct qedr_qp *qp = get_qedr_qp(ibqp);
3408 struct rdma_sq_atomic_wqe_1st *awqe1;
3409 struct rdma_sq_atomic_wqe_2nd *awqe2;
3410 struct rdma_sq_atomic_wqe_3rd *awqe3;
3411 struct rdma_sq_send_wqe_2st *swqe2;
3412 struct rdma_sq_local_inv_wqe *iwqe;
3413 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3414 struct rdma_sq_send_wqe_1st *swqe;
3415 struct rdma_sq_rdma_wqe_1st *rwqe;
3416 struct rdma_sq_fmr_wqe_1st *fwqe1;
3417 struct rdma_sq_common_wqe *wqe;
3418 u32 length;
3419 int rc = 0;
3420 bool comp;
3421
3422 if (!qedr_can_post_send(qp, wr)) {
3423 *bad_wr = wr;
3424 return -ENOMEM;
3425 }
3426
3427 wqe = qed_chain_produce(&qp->sq.pbl);
3428 qp->wqe_wr_id[qp->sq.prod].signaled =
3429 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3430
3431 wqe->flags = 0;
3432 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3433 !!(wr->send_flags & IB_SEND_SOLICITED));
3434 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3435 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3436 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3437 !!(wr->send_flags & IB_SEND_FENCE));
3438 wqe->prev_wqe_size = qp->prev_wqe_size;
3439
3440 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3441
3442 switch (wr->opcode) {
3443 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3444 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3445 rc = -EINVAL;
3446 *bad_wr = wr;
3447 break;
3448 }
afa0e13b
RA		 3449 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3450 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3451 swqe->wqe_size = 2;
3452 swqe2 = qed_chain_produce(&qp->sq.pbl);
3453
7bed7ebc 3454 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3455 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3456 wr, bad_wr);
3457 swqe->length = cpu_to_le32(length);
3458 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3459 qp->prev_wqe_size = swqe->wqe_size;
3460 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3461 break;
3462 case IB_WR_SEND:
3463 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3464 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3465
3466 swqe->wqe_size = 2;
3467 swqe2 = qed_chain_produce(&qp->sq.pbl);
3468 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3469 wr, bad_wr);
3470 swqe->length = cpu_to_le32(length);
3471 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3472 qp->prev_wqe_size = swqe->wqe_size;
3473 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3474 break;
3475 case IB_WR_SEND_WITH_INV:
3476 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3477 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3478 swqe2 = qed_chain_produce(&qp->sq.pbl);
3479 swqe->wqe_size = 2;
3480 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3481 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3482 wr, bad_wr);
3483 swqe->length = cpu_to_le32(length);
3484 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3485 qp->prev_wqe_size = swqe->wqe_size;
3486 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3487 break;
3488
3489 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3490 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3491 rc = -EINVAL;
3492 *bad_wr = wr;
3493 break;
3494 }
afa0e13b
RA		 3495 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3496 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3497
3498 rwqe->wqe_size = 2;
3499 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3500 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3501 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3502 wr, bad_wr);
3503 rwqe->length = cpu_to_le32(length);
3504 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3505 qp->prev_wqe_size = rwqe->wqe_size;
3506 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3507 break;
3508 case IB_WR_RDMA_WRITE:
3509 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3510 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3511
3512 rwqe->wqe_size = 2;
3513 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3514 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3515 wr, bad_wr);
3516 rwqe->length = cpu_to_le32(length);
3517 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3518 qp->prev_wqe_size = rwqe->wqe_size;
3519 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3520 break;
3521 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3522 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
1b8a708b 3523 /* fallthrough -- same is identical to RDMA READ */
afa0e13b
RA		 3524
3525 case IB_WR_RDMA_READ:
3526 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3527 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3528
3529 rwqe->wqe_size = 2;
3530 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3531 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3532 wr, bad_wr);
3533 rwqe->length = cpu_to_le32(length);
3534 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3535 qp->prev_wqe_size = rwqe->wqe_size;
3536 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3537 break;
3538
3539 case IB_WR_ATOMIC_CMP_AND_SWP:
3540 case IB_WR_ATOMIC_FETCH_AND_ADD:
3541 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3542 awqe1->wqe_size = 4;
3543
3544 awqe2 = qed_chain_produce(&qp->sq.pbl);
3545 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3546 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3547
3548 awqe3 = qed_chain_produce(&qp->sq.pbl);
3549
3550 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3551 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3552 DMA_REGPAIR_LE(awqe3->swap_data,
3553 atomic_wr(wr)->compare_add);
3554 } else {
3555 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3556 DMA_REGPAIR_LE(awqe3->swap_data,
3557 atomic_wr(wr)->swap);
3558 DMA_REGPAIR_LE(awqe3->cmp_data,
3559 atomic_wr(wr)->compare_add);
3560 }
3561
3562 qedr_prepare_sq_sges(qp, NULL, wr);
3563
3564 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3565 qp->prev_wqe_size = awqe1->wqe_size;
3566 break;
3567
3568 case IB_WR_LOCAL_INV:
3569 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3570 iwqe->wqe_size = 1;
3571
3572 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3573 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3574 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3575 qp->prev_wqe_size = iwqe->wqe_size;
3576 break;
3577 case IB_WR_REG_MR:
3578 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3579 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3580 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3581 fwqe1->wqe_size = 2;
3582
3583 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3584 if (rc) {
3585 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3586 *bad_wr = wr;
3587 break;
3588 }
3589
3590 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3591 qp->prev_wqe_size = fwqe1->wqe_size;
3592 break;
3593 default:
3594 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3595 rc = -EINVAL;
3596 *bad_wr = wr;
3597 break;
3598 }
3599
3600 if (*bad_wr) {
3601 u16 value;
3602
3603 /* Restore prod to its position before
3604 * this WR was processed
3605 */
3606 value = le16_to_cpu(qp->sq.db_data.data.value);
3607 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3608
3609 /* Restore prev_wqe_size */
3610 qp->prev_wqe_size = wqe->prev_wqe_size;
3611 rc = -EINVAL;
3612 DP_ERR(dev, "POST SEND FAILED\n");
3613 }
3614
3615 return rc;
3616}
3617
d34ac5cd
BVA		 3618int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3619 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3620{
3621 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3622 struct qedr_qp *qp = get_qedr_qp(ibqp);
3623 unsigned long flags;
3624 int rc = 0;
3625
3626 *bad_wr = NULL;
3627
04886779
RA		 3628 if (qp->qp_type == IB_QPT_GSI)
3629 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3630
afa0e13b
RA		 3631 spin_lock_irqsave(&qp->q_lock, flags);
3632
f5b1b177
KM		 3633 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3634 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3635 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3636 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3637 spin_unlock_irqrestore(&qp->q_lock, flags);
3638 *bad_wr = wr;
3639 DP_DEBUG(dev, QEDR_MSG_CQ,
3640 "QP in wrong state! QP icid=0x%x state %d\n",
3641 qp->icid, qp->state);
3642 return -EINVAL;
3643 }
afa0e13b
RA		 3644 }
3645
afa0e13b
RA		 3646 while (wr) {
3647 rc = __qedr_post_send(ibqp, wr, bad_wr);
3648 if (rc)
3649 break;
3650
3651 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3652
3653 qedr_inc_sw_prod(&qp->sq);
3654
3655 qp->sq.db_data.data.value++;
3656
3657 wr = wr->next;
3658 }
3659
3660 /* Trigger doorbell
3661 * If there was a failure in the first WR then it will be triggered in
3662 * vane. However this is not harmful (as long as the producer value is
3663 * unchanged). For performance reasons we avoid checking for this
3664 * redundant doorbell.
09c4854f
KM		 3665 *
3666 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3667 * soon as we give the doorbell, we could get a completion
3668 * for this wr, therefore we need to make sure that the
3669 * memory is updated before giving the doorbell.
3670 * During qedr_poll_cq, rmb is called before accessing the
3671 * cqe. This covers for the smp_rmb as well.
afa0e13b 3672 */
09c4854f 3673 smp_wmb();
afa0e13b
RA		 3674 writel(qp->sq.db_data.raw, qp->sq.db);
3675
afa0e13b
RA		 3676 spin_unlock_irqrestore(&qp->q_lock, flags);
3677
3678 return rc;
3679}
3680
3491c9e7
YB		 3681static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3682{
3683 u32 used;
3684
3685 /* Calculate number of elements used based on producer
3686 * count and consumer count and subtract it from max
3687 * work request supported so that we get elements left.
3688 */
3689 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3690
3691 return hw_srq->max_wr - used;
3692}
3693
3694int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3695 const struct ib_recv_wr **bad_wr)
3696{
3697 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3698 struct qedr_srq_hwq_info *hw_srq;
3699 struct qedr_dev *dev = srq->dev;
3700 struct qed_chain *pbl;
3701 unsigned long flags;
3702 int status = 0;
3703 u32 num_sge;
3704 u32 offset;
3705
3706 spin_lock_irqsave(&srq->lock, flags);
3707
3708 hw_srq = &srq->hw_srq;
3709 pbl = &srq->hw_srq.pbl;
3710 while (wr) {
3711 struct rdma_srq_wqe_header *hdr;
3712 int i;
3713
3714 if (!qedr_srq_elem_left(hw_srq) ||
3715 wr->num_sge > srq->hw_srq.max_sges) {
3716 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3717 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3718 wr->num_sge, srq->hw_srq.max_sges);
3719 status = -ENOMEM;
3720 *bad_wr = wr;
3721 break;
3722 }
3723
3724 hdr = qed_chain_produce(pbl);
3725 num_sge = wr->num_sge;
3726 /* Set number of sge and work request id in header */
3727 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3728
3729 srq->hw_srq.wr_prod_cnt++;
3730 hw_srq->wqe_prod++;
3731 hw_srq->sge_prod++;
3732
3733 DP_DEBUG(dev, QEDR_MSG_SRQ,
3734 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3735 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3736
3737 for (i = 0; i < wr->num_sge; i++) {
3738 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3739
3740 /* Set SGE length, lkey and address */
3741 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3742 wr->sg_list[i].length, wr->sg_list[i].lkey);
3743
3744 DP_DEBUG(dev, QEDR_MSG_SRQ,
3745 "[%d]: len %d key %x addr %x:%x\n",
3746 i, srq_sge->length, srq_sge->l_key,
3747 srq_sge->addr.hi, srq_sge->addr.lo);
3748 hw_srq->sge_prod++;
3749 }
3750
3751 /* Flush WQE and SGE information before
3752 * updating producer.
3753 */
3754 wmb();
3755
3756 /* SRQ producer is 8 bytes. Need to update SGE producer index
3757 * in first 4 bytes and need to update WQE producer in
3758 * next 4 bytes.
3759 */
3760 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3761 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3762 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3763 hw_srq->wqe_prod;
3764
3765 /* Flush producer after updating it. */
3766 wmb();
3767 wr = wr->next;
3768 }
3769
3770 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3771 qed_chain_get_elem_left(pbl));
3772 spin_unlock_irqrestore(&srq->lock, flags);
3773
3774 return status;
3775}
3776
d34ac5cd
BVA		 3777int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3778 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3779{
3780 struct qedr_qp *qp = get_qedr_qp(ibqp);
3781 struct qedr_dev *dev = qp->dev;
3782 unsigned long flags;
3783 int status = 0;
3784
04886779
RA		 3785 if (qp->qp_type == IB_QPT_GSI)
3786 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3787
afa0e13b
RA		 3788 spin_lock_irqsave(&qp->q_lock, flags);
3789
922d9a40 3790 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3791 spin_unlock_irqrestore(&qp->q_lock, flags);
3792 *bad_wr = wr;
3793 return -EINVAL;
3794 }
3795
3796 while (wr) {
3797 int i;
3798
3799 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3800 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3801 wr->num_sge > qp->rq.max_sges) {
3802 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3803 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3804 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3805 qp->rq.max_sges);
3806 status = -ENOMEM;
3807 *bad_wr = wr;
3808 break;
3809 }
3810 for (i = 0; i < wr->num_sge; i++) {
3811 u32 flags = 0;
3812 struct rdma_rq_sge *rqe =
3813 qed_chain_produce(&qp->rq.pbl);
3814
3815 /* First one must include the number
3816 * of SGE in the list
3817 */
3818 if (!i)
3819 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3820 wr->num_sge);
3821
d52c89f1 3822 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3823 wr->sg_list[i].lkey);
3824
3825 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3826 wr->sg_list[i].length, flags);
3827 }
3828
3829 /* Special case of no sges. FW requires between 1-4 sges...
3830 * in this case we need to post 1 sge with length zero. this is
3831 * because rdma write with immediate consumes an RQ.
3832 */
3833 if (!wr->num_sge) {
3834 u32 flags = 0;
3835 struct rdma_rq_sge *rqe =
3836 qed_chain_produce(&qp->rq.pbl);
3837
3838 /* First one must include the number
3839 * of SGE in the list
3840 */
d52c89f1 3841 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3842 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3843
3844 RQ_SGE_SET(rqe, 0, 0, flags);
3845 i = 1;
3846 }
3847
3848 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3849 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3850
3851 qedr_inc_sw_prod(&qp->rq);
3852
09c4854f
KM		 3853 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3854 * soon as we give the doorbell, we could get a completion
3855 * for this wr, therefore we need to make sure that the
3856 * memory is update before giving the doorbell.
3857 * During qedr_poll_cq, rmb is called before accessing the
3858 * cqe. This covers for the smp_rmb as well.
3859 */
3860 smp_wmb();
afa0e13b
RA		 3861
3862 qp->rq.db_data.data.value++;
3863
3864 writel(qp->rq.db_data.raw, qp->rq.db);
3865
f5b1b177
KM		 3866 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3867 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
f5b1b177
KM		 3868 }
3869
afa0e13b
RA		 3870 wr = wr->next;
3871 }
3872
3873 spin_unlock_irqrestore(&qp->q_lock, flags);
3874
3875 return status;
3876}
3877
3878static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3879{
3880 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3881
3882 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3883 cq->pbl_toggle;
3884}
3885
3886static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3887{
3888 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3889 struct qedr_qp *qp;
3890
3891 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3892 resp_cqe->qp_handle.lo,
3893 u64);
3894 return qp;
3895}
3896
3897static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3898{
3899 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3900
3901 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3902}
3903
3904/* Return latest CQE (needs processing) */
3905static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3906{
3907 return cq->latest_cqe;
3908}
3909
3910/* In fmr we need to increase the number of fmr completed counter for the fmr
3911 * algorithm determining whether we can free a pbl or not.
3912 * we need to perform this whether the work request was signaled or not. for
3913 * this purpose we call this function from the condition that checks if a wr
3914 * should be skipped, to make sure we don't miss it ( possibly this fmr
3915 * operation was not signalted)
3916 */
3917static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3918{
3919 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3920 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3921}
3922
3923static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3924 struct qedr_cq *cq, int num_entries,
3925 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3926 int force)
3927{
3928 u16 cnt = 0;
3929
3930 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3931 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3932 qedr_chk_if_fmr(qp);
3933 /* skip WC */
3934 goto next_cqe;
3935 }
3936
3937 /* fill WC */
3938 wc->status = status;
27035a1b 3939 wc->vendor_err = 0;
afa0e13b
RA		 3940 wc->wc_flags = 0;
3941 wc->src_qp = qp->id;
3942 wc->qp = &qp->ibqp;
3943
3944 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3945 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3946
3947 switch (wc->opcode) {
3948 case IB_WC_RDMA_WRITE:
3949 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3950 break;
3951 case IB_WC_COMP_SWAP:
3952 case IB_WC_FETCH_ADD:
3953 wc->byte_len = 8;
3954 break;
3955 case IB_WC_REG_MR:
3956 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3957 break;
dac27386
MK		 3958 case IB_WC_RDMA_READ:
3959 case IB_WC_SEND:
3960 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3961 break;
afa0e13b
RA		 3962 default:
3963 break;
3964 }
3965
3966 num_entries--;
3967 wc++;
3968 cnt++;
3969next_cqe:
3970 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3971 qed_chain_consume(&qp->sq.pbl);
3972 qedr_inc_sw_cons(&qp->sq);
3973 }
3974
3975 return cnt;
3976}
3977
3978static int qedr_poll_cq_req(struct qedr_dev *dev,
3979 struct qedr_qp *qp, struct qedr_cq *cq,
3980 int num_entries, struct ib_wc *wc,
3981 struct rdma_cqe_requester *req)
3982{
3983 int cnt = 0;
3984
3985 switch (req->status) {
3986 case RDMA_CQE_REQ_STS_OK:
3987 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3988 IB_WC_SUCCESS, 0);
3989 break;
3990 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 3991 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 3992 DP_DEBUG(dev, QEDR_MSG_CQ,
3993 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3994 cq->icid, qp->icid);
afa0e13b 3995 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3996 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 3997 break;
3998 default:
3999 /* process all WQE before the cosumer */
4000 qp->state = QED_ROCE_QP_STATE_ERR;
4001 cnt = process_req(dev, qp, cq, num_entries, wc,
4002 req->sq_cons - 1, IB_WC_SUCCESS, 0);
4003 wc += cnt;
4004 /* if we have extra WC fill it with actual error info */
4005 if (cnt < num_entries) {
4006 enum ib_wc_status wc_status;
4007
4008 switch (req->status) {
4009 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4010 DP_ERR(dev,
4011 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4012 cq->icid, qp->icid);
4013 wc_status = IB_WC_BAD_RESP_ERR;
4014 break;
4015 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4016 DP_ERR(dev,
4017 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4018 cq->icid, qp->icid);
4019 wc_status = IB_WC_LOC_LEN_ERR;
4020 break;
4021 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4022 DP_ERR(dev,
4023 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4024 cq->icid, qp->icid);
4025 wc_status = IB_WC_LOC_QP_OP_ERR;
4026 break;
4027 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4028 DP_ERR(dev,
4029 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4030 cq->icid, qp->icid);
4031 wc_status = IB_WC_LOC_PROT_ERR;
4032 break;
4033 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4034 DP_ERR(dev,
4035 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4036 cq->icid, qp->icid);
4037 wc_status = IB_WC_MW_BIND_ERR;
4038 break;
4039 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4040 DP_ERR(dev,
4041 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4042 cq->icid, qp->icid);
4043 wc_status = IB_WC_REM_INV_REQ_ERR;
4044 break;
4045 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4046 DP_ERR(dev,
4047 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4048 cq->icid, qp->icid);
4049 wc_status = IB_WC_REM_ACCESS_ERR;
4050 break;
4051 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4052 DP_ERR(dev,
4053 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4054 cq->icid, qp->icid);
4055 wc_status = IB_WC_REM_OP_ERR;
4056 break;
4057 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4058 DP_ERR(dev,
4059 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4060 cq->icid, qp->icid);
4061 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4062 break;
4063 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4064 DP_ERR(dev,
4065 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4066 cq->icid, qp->icid);
4067 wc_status = IB_WC_RETRY_EXC_ERR;
4068 break;
4069 default:
4070 DP_ERR(dev,
4071 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4072 cq->icid, qp->icid);
4073 wc_status = IB_WC_GENERAL_ERR;
4074 }
4075 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4076 wc_status, 1);
4077 }
4078 }
4079
4080 return cnt;
4081}
4082
b6acd71f 4083static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 4084{
b6acd71f 4085 switch (status) {
afa0e13b 4086 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 4087 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 4088 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 4089 return IB_WC_LOC_LEN_ERR;
afa0e13b 4090 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 4091 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 4092 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 4093 return IB_WC_LOC_PROT_ERR;
afa0e13b 4094 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 4095 return IB_WC_MW_BIND_ERR;
afa0e13b 4096 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 4097 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 4098 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 4099 return IB_WC_SUCCESS;
4100 default:
4101 return IB_WC_GENERAL_ERR;
4102 }
4103}
afa0e13b 4104
b6acd71f
AR		 4105static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4106 struct ib_wc *wc)
4107{
4108 wc->status = IB_WC_SUCCESS;
4109 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 4110
b6acd71f 4111 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 4112 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 4113 wc->wc_flags |= IB_WC_WITH_IMM;
4114
4115 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 4116 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4117
b6acd71f
AR		 4118 if (resp->flags & QEDR_RESP_INV)
4119 return -EINVAL;
4120
4121 } else if (resp->flags & QEDR_RESP_INV) {
4122 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4123 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4124
4125 if (resp->flags & QEDR_RESP_RDMA)
4126 return -EINVAL;
4127
4128 } else if (resp->flags & QEDR_RESP_RDMA) {
4129 return -EINVAL;
4130 }
4131
4132 return 0;
4133}
4134
4135static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4136 struct qedr_cq *cq, struct ib_wc *wc,
4137 struct rdma_cqe_responder *resp, u64 wr_id)
4138{
4139 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4140 wc->opcode = IB_WC_RECV;
4141 wc->wc_flags = 0;
4142
4143 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4144 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4145 DP_ERR(dev,
4146 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4147 cq, cq->icid, resp->flags);
4148
4149 } else {
4150 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4151 if (wc->status == IB_WC_GENERAL_ERR)
4152 DP_ERR(dev,
4153 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4154 cq, cq->icid, resp->status);
afa0e13b
RA		 4155 }
4156
b6acd71f 4157 /* Fill the rest of the WC */
27035a1b 4158 wc->vendor_err = 0;
afa0e13b
RA		 4159 wc->src_qp = qp->id;
4160 wc->qp = &qp->ibqp;
4161 wc->wr_id = wr_id;
4162}
4163
3491c9e7
YB		 4164static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4165 struct qedr_cq *cq, struct ib_wc *wc,
4166 struct rdma_cqe_responder *resp)
4167{
4168 struct qedr_srq *srq = qp->srq;
4169 u64 wr_id;
4170
4171 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4172 le32_to_cpu(resp->srq_wr_id.lo), u64);
4173
4174 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4175 wc->status = IB_WC_WR_FLUSH_ERR;
4176 wc->vendor_err = 0;
4177 wc->wr_id = wr_id;
4178 wc->byte_len = 0;
4179 wc->src_qp = qp->id;
4180 wc->qp = &qp->ibqp;
4181 wc->wr_id = wr_id;
4182 } else {
4183 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4184 }
4185 srq->hw_srq.wr_cons_cnt++;
4186
4187 return 1;
4188}
afa0e13b
RA		 4189static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4190 struct qedr_cq *cq, struct ib_wc *wc,
4191 struct rdma_cqe_responder *resp)
4192{
4193 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4194
4195 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4196
4197 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4198 qed_chain_consume(&qp->rq.pbl);
4199 qedr_inc_sw_cons(&qp->rq);
4200
4201 return 1;
4202}
4203
4204static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4205 int num_entries, struct ib_wc *wc, u16 hw_cons)
4206{
4207 u16 cnt = 0;
4208
4209 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4210 /* fill WC */
4211 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 4212 wc->vendor_err = 0;
afa0e13b
RA		 4213 wc->wc_flags = 0;
4214 wc->src_qp = qp->id;
4215 wc->byte_len = 0;
4216 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4217 wc->qp = &qp->ibqp;
4218 num_entries--;
4219 wc++;
4220 cnt++;
4221 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4222 qed_chain_consume(&qp->rq.pbl);
4223 qedr_inc_sw_cons(&qp->rq);
4224 }
4225
4226 return cnt;
4227}
4228
4229static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4230 struct rdma_cqe_responder *resp, int *update)
4231{
50bc60cb 4232 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 4233 consume_cqe(cq);
4234 *update |= 1;
4235 }
4236}
4237
3491c9e7
YB		 4238static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4239 struct qedr_cq *cq, int num_entries,
4240 struct ib_wc *wc,
4241 struct rdma_cqe_responder *resp)
4242{
4243 int cnt;
4244
4245 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4246 consume_cqe(cq);
4247
4248 return cnt;
4249}
4250
afa0e13b
RA		 4251static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4252 struct qedr_cq *cq, int num_entries,
4253 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4254 int *update)
4255{
4256 int cnt;
4257
4258 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4259 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4260 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4261 try_consume_resp_cqe(cq, qp, resp, update);
4262 } else {
4263 cnt = process_resp_one(dev, qp, cq, wc, resp);
4264 consume_cqe(cq);
4265 *update |= 1;
4266 }
4267
4268 return cnt;
4269}
4270
4271static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4272 struct rdma_cqe_requester *req, int *update)
4273{
4274 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4275 consume_cqe(cq);
4276 *update |= 1;
4277 }
4278}
4279
4280int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4281{
4282 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4283 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4284 union rdma_cqe *cqe;
afa0e13b
RA		 4285 u32 old_cons, new_cons;
4286 unsigned long flags;
4287 int update = 0;
4288 int done = 0;
4289
4dd72636
AR		 4290 if (cq->destroyed) {
4291 DP_ERR(dev,
4292 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4293 cq, cq->icid);
4294 return 0;
4295 }
4296
04886779
RA		 4297 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4298 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4299
afa0e13b 4300 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4301 cqe = cq->latest_cqe;
afa0e13b
RA		 4302 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4303 while (num_entries && is_valid_cqe(cq, cqe)) {
4304 struct qedr_qp *qp;
4305 int cnt = 0;
4306
4307 /* prevent speculative reads of any field of CQE */
4308 rmb();
4309
4310 qp = cqe_get_qp(cqe);
4311 if (!qp) {
4312 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4313 break;
4314 }
4315
4316 wc->qp = &qp->ibqp;
4317
4318 switch (cqe_get_type(cqe)) {
4319 case RDMA_CQE_TYPE_REQUESTER:
4320 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4321 &cqe->req);
4322 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4323 break;
4324 case RDMA_CQE_TYPE_RESPONDER_RQ:
4325 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4326 &cqe->resp, &update);
4327 break;
3491c9e7
YB		 4328 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4329 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4330 wc, &cqe->resp);
4331 update = 1;
4332 break;
afa0e13b
RA		 4333 case RDMA_CQE_TYPE_INVALID:
4334 default:
4335 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4336 cqe_get_type(cqe));
4337 }
4338 num_entries -= cnt;
4339 wc += cnt;
4340 done += cnt;
4341
4342 cqe = get_cqe(cq);
4343 }
4344 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4345
4346 cq->cq_cons += new_cons - old_cons;
4347
4348 if (update)
4349 /* doorbell notifies abount latest VALID entry,
4350 * but chain already point to the next INVALID one
4351 */
4352 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4353
4354 spin_unlock_irqrestore(&cq->cq_lock, flags);
4355 return done;
4356}
993d1b52
RA		 4357
4358int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
e26e7b88
LR		 4359 u8 port_num, const struct ib_wc *in_wc,
4360 const struct ib_grh *in_grh, const struct ib_mad *in,
4361 struct ib_mad *out_mad, size_t *out_mad_size,
4362 u16 *out_mad_pkey_index)
993d1b52 4363{
993d1b52
RA		 4364 return IB_MAD_RESULT_SUCCESS;
4365}
Linux 6.x block layer and io_uring tree(s)