projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB: Pass uverbs_attr_bundle down ib_x destroy path
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45
be086e7c
MY		 46#include <linux/qed/common_hsi.h>
47#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 48#include <linux/qed/qed_if.h>
49#include "qedr.h"
50#include "verbs.h"
51#include <rdma/qedr-abi.h>
99d195cc 52#include "qedr_roce_cm.h"
ac1b36e5 53
3491c9e7
YB		 54#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
55#define RDMA_MAX_SGE_PER_SRQ (4)
56#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
57
a7efd777
RA		 58#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
59
c75d3ec8
AR		 60static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
61 size_t len)
62{
63 size_t min_len = min_t(size_t, len, udata->outlen);
64
65 return ib_copy_to_udata(udata, src, min_len);
66}
67
a7efd777
RA		 68int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
69{
dbe30dae 70 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 71 return -EINVAL;
72
73 *pkey = QEDR_ROCE_PKEY_DEFAULT;
74 return 0;
75}
76
e6a38c54
KM		 77int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
78 int index, union ib_gid *sgid)
79{
80 struct qedr_dev *dev = get_qedr_dev(ibdev);
81
82 memset(sgid->raw, 0, sizeof(sgid->raw));
83 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
84
85 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
86 sgid->global.interface_id, sgid->global.subnet_prefix);
87
88 return 0;
89}
90
3491c9e7
YB		 91int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
92{
93 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
94 struct qedr_device_attr *qattr = &dev->attr;
95 struct qedr_srq *srq = get_qedr_srq(ibsrq);
96
97 srq_attr->srq_limit = srq->srq_limit;
98 srq_attr->max_wr = qattr->max_srq_wr;
99 srq_attr->max_sge = qattr->max_sge;
100
101 return 0;
102}
103
ac1b36e5
RA		 104int qedr_query_device(struct ib_device *ibdev,
105 struct ib_device_attr *attr, struct ib_udata *udata)
106{
107 struct qedr_dev *dev = get_qedr_dev(ibdev);
108 struct qedr_device_attr *qattr = &dev->attr;
109
110 if (!dev->rdma_ctx) {
111 DP_ERR(dev,
112 "qedr_query_device called with invalid params rdma_ctx=%p\n",
113 dev->rdma_ctx);
114 return -EINVAL;
115 }
116
117 memset(attr, 0, sizeof(*attr));
118
119 attr->fw_ver = qattr->fw_ver;
120 attr->sys_image_guid = qattr->sys_image_guid;
121 attr->max_mr_size = qattr->max_mr_size;
122 attr->page_size_cap = qattr->page_size_caps;
123 attr->vendor_id = qattr->vendor_id;
124 attr->vendor_part_id = qattr->vendor_part_id;
125 attr->hw_ver = qattr->hw_ver;
126 attr->max_qp = qattr->max_qp;
127 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
128 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
129 IB_DEVICE_RC_RNR_NAK_GEN |
130 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
131
33023fb8
SW		 132 attr->max_send_sge = qattr->max_sge;
133 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 134 attr->max_sge_rd = qattr->max_sge;
135 attr->max_cq = qattr->max_cq;
136 attr->max_cqe = qattr->max_cqe;
137 attr->max_mr = qattr->max_mr;
138 attr->max_mw = qattr->max_mw;
139 attr->max_pd = qattr->max_pd;
140 attr->atomic_cap = dev->atomic_cap;
141 attr->max_fmr = qattr->max_fmr;
142 attr->max_map_per_fmr = 16;
143 attr->max_qp_init_rd_atom =
144 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
145 attr->max_qp_rd_atom =
146 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
147 attr->max_qp_init_rd_atom);
148
149 attr->max_srq = qattr->max_srq;
150 attr->max_srq_sge = qattr->max_srq_sge;
151 attr->max_srq_wr = qattr->max_srq_wr;
152
153 attr->local_ca_ack_delay = qattr->dev_ack_delay;
154 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
155 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
156 attr->max_ah = qattr->max_ah;
157
158 return 0;
159}
160
161#define QEDR_SPEED_SDR (1)
162#define QEDR_SPEED_DDR (2)
163#define QEDR_SPEED_QDR (4)
164#define QEDR_SPEED_FDR10 (8)
165#define QEDR_SPEED_FDR (16)
166#define QEDR_SPEED_EDR (32)
167
168static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
169 u8 *ib_width)
170{
171 switch (speed) {
172 case 1000:
173 *ib_speed = QEDR_SPEED_SDR;
174 *ib_width = IB_WIDTH_1X;
175 break;
176 case 10000:
177 *ib_speed = QEDR_SPEED_QDR;
178 *ib_width = IB_WIDTH_1X;
179 break;
180
181 case 20000:
182 *ib_speed = QEDR_SPEED_DDR;
183 *ib_width = IB_WIDTH_4X;
184 break;
185
186 case 25000:
187 *ib_speed = QEDR_SPEED_EDR;
188 *ib_width = IB_WIDTH_1X;
189 break;
190
191 case 40000:
192 *ib_speed = QEDR_SPEED_QDR;
193 *ib_width = IB_WIDTH_4X;
194 break;
195
196 case 50000:
197 *ib_speed = QEDR_SPEED_QDR;
198 *ib_width = IB_WIDTH_4X;
199 break;
200
201 case 100000:
202 *ib_speed = QEDR_SPEED_EDR;
203 *ib_width = IB_WIDTH_4X;
204 break;
205
206 default:
207 /* Unsupported */
208 *ib_speed = QEDR_SPEED_SDR;
209 *ib_width = IB_WIDTH_1X;
210 }
211}
212
213int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
214{
215 struct qedr_dev *dev;
216 struct qed_rdma_port *rdma_port;
217
218 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 219
220 if (!dev->rdma_ctx) {
221 DP_ERR(dev, "rdma_ctx is NULL\n");
222 return -EINVAL;
223 }
224
225 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 226
c4550c63 227 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 228 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
229 attr->state = IB_PORT_ACTIVE;
230 attr->phys_state = 5;
231 } else {
232 attr->state = IB_PORT_DOWN;
233 attr->phys_state = 3;
234 }
235 attr->max_mtu = IB_MTU_4096;
236 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
237 attr->lid = 0;
238 attr->lmc = 0;
239 attr->sm_lid = 0;
240 attr->sm_sl = 0;
2f944c0f 241 attr->ip_gids = true;
f5b1b177
KM		 242 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
243 attr->gid_tbl_len = 1;
244 attr->pkey_tbl_len = 1;
245 } else {
246 attr->gid_tbl_len = QEDR_MAX_SGID;
247 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
248 }
ac1b36e5
RA		 249 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
250 attr->qkey_viol_cntr = 0;
251 get_link_speed_and_width(rdma_port->link_speed,
252 &attr->active_speed, &attr->active_width);
253 attr->max_msg_sz = rdma_port->max_msg_size;
254 attr->max_vl_num = 4;
255
256 return 0;
257}
258
259int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
260 struct ib_port_modify *props)
261{
ac1b36e5
RA		 262 return 0;
263}
264
265static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
266 unsigned long len)
267{
268 struct qedr_mm *mm;
269
270 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
271 if (!mm)
272 return -ENOMEM;
273
274 mm->key.phy_addr = phy_addr;
275 /* This function might be called with a length which is not a multiple
276 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
277 * forces this granularity by increasing the requested size if needed.
278 * When qedr_mmap is called, it will search the list with the updated
279 * length as a key. To prevent search failures, the length is rounded up
280 * in advance to PAGE_SIZE.
281 */
282 mm->key.len = roundup(len, PAGE_SIZE);
283 INIT_LIST_HEAD(&mm->entry);
284
285 mutex_lock(&uctx->mm_list_lock);
286 list_add(&mm->entry, &uctx->mm_head);
287 mutex_unlock(&uctx->mm_list_lock);
288
289 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
290 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
291 (unsigned long long)mm->key.phy_addr,
292 (unsigned long)mm->key.len, uctx);
293
294 return 0;
295}
296
297static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
298 unsigned long len)
299{
300 bool found = false;
301 struct qedr_mm *mm;
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_for_each_entry(mm, &uctx->mm_head, entry) {
305 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
306 continue;
307
308 found = true;
309 break;
310 }
311 mutex_unlock(&uctx->mm_list_lock);
312 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
313 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
314 mm->key.phy_addr, mm->key.len, uctx, found);
315
316 return found;
317}
318
a2a074ef 319int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 320{
a2a074ef 321 struct ib_device *ibdev = uctx->device;
ac1b36e5 322 int rc;
a2a074ef
LR		 323 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
324 struct qedr_alloc_ucontext_resp uresp = {};
ac1b36e5
RA		 325 struct qedr_dev *dev = get_qedr_dev(ibdev);
326 struct qed_rdma_add_user_out_params oparams;
327
328 if (!udata)
a2a074ef 329 return -EFAULT;
ac1b36e5
RA		 330
331 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
332 if (rc) {
333 DP_ERR(dev,
334 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
335 rc);
a2a074ef 336 return rc;
ac1b36e5
RA		 337 }
338
339 ctx->dpi = oparams.dpi;
340 ctx->dpi_addr = oparams.dpi_addr;
341 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
342 ctx->dpi_size = oparams.dpi_size;
343 INIT_LIST_HEAD(&ctx->mm_head);
344 mutex_init(&ctx->mm_list_lock);
345
ad84dad2 346 uresp.dpm_enabled = dev->user_dpm_enabled;
67cbe353
AR		 347 uresp.wids_enabled = 1;
348 uresp.wid_count = oparams.wid_count;
ac1b36e5
RA		 349 uresp.db_pa = ctx->dpi_phys_addr;
350 uresp.db_size = ctx->dpi_size;
351 uresp.max_send_wr = dev->attr.max_sqe;
352 uresp.max_recv_wr = dev->attr.max_rqe;
353 uresp.max_srq_wr = dev->attr.max_srq_wr;
354 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
355 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
356 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
357 uresp.max_cqes = QEDR_MAX_CQES;
358
c75d3ec8 359 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 360 if (rc)
a2a074ef 361 return rc;
ac1b36e5
RA		 362
363 ctx->dev = dev;
364
365 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
366 if (rc)
a2a074ef 367 return rc;
ac1b36e5
RA		 368
369 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
370 &ctx->ibucontext);
a2a074ef 371 return 0;
ac1b36e5
RA		 372}
373
a2a074ef 374void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 375{
376 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
377 struct qedr_mm *mm, *tmp;
ac1b36e5
RA		 378
379 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
380 uctx);
381 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
382
383 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
384 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
385 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
386 mm->key.phy_addr, mm->key.len, uctx);
387 list_del(&mm->entry);
388 kfree(mm);
389 }
ac1b36e5
RA		 390}
391
392int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
393{
394 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
395 struct qedr_dev *dev = get_qedr_dev(context->device);
30bf066c 396 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
ac1b36e5 397 unsigned long len = (vma->vm_end - vma->vm_start);
30bf066c
KM		 398 unsigned long dpi_start;
399
400 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
ac1b36e5
RA		 401
402 DP_DEBUG(dev, QEDR_MSG_INIT,
30bf066c
KM		 403 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
404 (void *)vma->vm_start, (void *)vma->vm_end,
405 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
406
407 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
408 DP_ERR(dev,
367d2f07 409 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
30bf066c 410 (void *)vma->vm_start, (void *)vma->vm_end);
ac1b36e5
RA		 411 return -EINVAL;
412 }
413
30bf066c
KM		 414 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
415 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
ac1b36e5
RA		 416 vma->vm_pgoff);
417 return -EINVAL;
418 }
419
30bf066c
KM		 420 if (phys_addr < dpi_start ||
421 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
422 DP_ERR(dev,
423 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
424 (void *)phys_addr, (void *)dpi_start,
425 ucontext->dpi_size);
426 return -EINVAL;
427 }
ac1b36e5 428
30bf066c
KM		 429 if (vma->vm_flags & VM_READ) {
430 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
431 return -EINVAL;
ac1b36e5 432 }
30bf066c
KM		 433
434 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
435 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
436 vma->vm_page_prot);
ac1b36e5 437}
a7efd777 438
21a428a0
LR		 439int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
440 struct ib_udata *udata)
a7efd777 441{
21a428a0 442 struct ib_device *ibdev = ibpd->device;
a7efd777 443 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 444 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 445 u16 pd_id;
446 int rc;
447
448 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
449 (udata && context) ? "User Lib" : "Kernel");
450
451 if (!dev->rdma_ctx) {
847cb1a3 452 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 453 return -EINVAL;
a7efd777
RA		 454 }
455
9c1e0228
RA		 456 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
457 if (rc)
21a428a0 458 return rc;
a7efd777 459
a7efd777
RA		 460 pd->pd_id = pd_id;
461
462 if (udata && context) {
57939021
JG		 463 struct qedr_alloc_pd_uresp uresp = {
464 .pd_id = pd_id,
465 };
9c1e0228 466
c75d3ec8 467 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 468 if (rc) {
a7efd777 469 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 470 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 471 return rc;
9c1e0228
RA		 472 }
473
474 pd->uctx = get_qedr_ucontext(context);
475 pd->uctx->pd = pd;
a7efd777
RA		 476 }
477
21a428a0 478 return 0;
a7efd777
RA		 479}
480
c4367a26 481void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 482{
483 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
484 struct qedr_pd *pd = get_qedr_pd(ibpd);
485
a7efd777
RA		 486 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
487 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
a7efd777
RA		 488}
489
490static void qedr_free_pbl(struct qedr_dev *dev,
491 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
492{
493 struct pci_dev *pdev = dev->pdev;
494 int i;
495
496 for (i = 0; i < pbl_info->num_pbls; i++) {
497 if (!pbl[i].va)
498 continue;
499 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
500 pbl[i].va, pbl[i].pa);
501 }
502
503 kfree(pbl);
504}
505
506#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
507#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
508
509#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
510#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
511#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
512
513static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
514 struct qedr_pbl_info *pbl_info,
515 gfp_t flags)
516{
517 struct pci_dev *pdev = dev->pdev;
518 struct qedr_pbl *pbl_table;
519 dma_addr_t *pbl_main_tbl;
520 dma_addr_t pa;
521 void *va;
522 int i;
523
524 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
525 if (!pbl_table)
526 return ERR_PTR(-ENOMEM);
527
528 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 529 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
530 flags);
a7efd777
RA		 531 if (!va)
532 goto err;
533
a7efd777
RA		 534 pbl_table[i].va = va;
535 pbl_table[i].pa = pa;
536 }
537
538 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
539 * the first one with physical pointers to all of the rest
540 */
541 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
542 for (i = 0; i < pbl_info->num_pbls - 1; i++)
543 pbl_main_tbl[i] = pbl_table[i + 1].pa;
544
545 return pbl_table;
546
547err:
548 for (i--; i >= 0; i--)
549 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
550 pbl_table[i].va, pbl_table[i].pa);
551
552 qedr_free_pbl(dev, pbl_info, pbl_table);
553
554 return ERR_PTR(-ENOMEM);
555}
556
557static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
558 struct qedr_pbl_info *pbl_info,
559 u32 num_pbes, int two_layer_capable)
560{
561 u32 pbl_capacity;
562 u32 pbl_size;
563 u32 num_pbls;
564
565 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
566 if (num_pbes > MAX_PBES_TWO_LAYER) {
567 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
568 num_pbes);
569 return -EINVAL;
570 }
571
572 /* calculate required pbl page size */
573 pbl_size = MIN_FW_PBL_PAGE_SIZE;
574 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
575 NUM_PBES_ON_PAGE(pbl_size);
576
577 while (pbl_capacity < num_pbes) {
578 pbl_size *= 2;
579 pbl_capacity = pbl_size / sizeof(u64);
580 pbl_capacity = pbl_capacity * pbl_capacity;
581 }
582
583 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
584 num_pbls++; /* One for the layer0 ( points to the pbls) */
585 pbl_info->two_layered = true;
586 } else {
587 /* One layered PBL */
588 num_pbls = 1;
589 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
590 roundup_pow_of_two((num_pbes * sizeof(u64))));
591 pbl_info->two_layered = false;
592 }
593
594 pbl_info->num_pbls = num_pbls;
595 pbl_info->pbl_size = pbl_size;
596 pbl_info->num_pbes = num_pbes;
597
598 DP_DEBUG(dev, QEDR_MSG_MR,
599 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
600 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
601
602 return 0;
603}
604
605static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
606 struct qedr_pbl *pbl,
e57bb6be 607 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 608{
95ad233f 609 int pbe_cnt, total_num_pbes = 0;
e57bb6be 610 u32 fw_pg_cnt, fw_pg_per_umem_pg;
a7efd777 611 struct qedr_pbl *pbl_tbl;
95ad233f 612 struct sg_dma_page_iter sg_iter;
a7efd777 613 struct regpair *pbe;
e57bb6be 614 u64 pg_addr;
a7efd777
RA		 615
616 if (!pbl_info->num_pbes)
617 return;
618
619 /* If we have a two layered pbl, the first pbl points to the rest
620 * of the pbls and the first entry lays on the second pbl in the table
621 */
622 if (pbl_info->two_layered)
623 pbl_tbl = &pbl[1];
624 else
625 pbl_tbl = pbl;
626
627 pbe = (struct regpair *)pbl_tbl->va;
628 if (!pbe) {
629 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
630 return;
631 }
632
633 pbe_cnt = 0;
634
95ad233f
SS		 635 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
636
637 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
638 pg_addr = sg_page_iter_dma_address(&sg_iter);
639 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
640 pbe->lo = cpu_to_le32(pg_addr);
641 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
642
643 pg_addr += BIT(pg_shift);
644 pbe_cnt++;
645 total_num_pbes++;
646 pbe++;
e57bb6be 647
95ad233f
SS		 648 if (total_num_pbes == pbl_info->num_pbes)
649 return;
650
651 /* If the given pbl is full storing the pbes,
652 * move to next pbl.
653 */
654 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
655 pbl_tbl++;
656 pbe = (struct regpair *)pbl_tbl->va;
657 pbe_cnt = 0;
a7efd777 658 }
95ad233f
SS		 659
660 fw_pg_cnt++;
a7efd777
RA		 661 }
662 }
663}
664
665static int qedr_copy_cq_uresp(struct qedr_dev *dev,
666 struct qedr_cq *cq, struct ib_udata *udata)
667{
668 struct qedr_create_cq_uresp uresp;
669 int rc;
670
671 memset(&uresp, 0, sizeof(uresp));
672
673 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
674 uresp.icid = cq->icid;
675
c75d3ec8 676 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 677 if (rc)
678 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
679
680 return rc;
681}
682
683static void consume_cqe(struct qedr_cq *cq)
684{
685 if (cq->latest_cqe == cq->toggle_cqe)
686 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
687
688 cq->latest_cqe = qed_chain_consume(&cq->pbl);
689}
690
691static inline int qedr_align_cq_entries(int entries)
692{
693 u64 size, aligned_size;
694
695 /* We allocate an extra entry that we don't report to the FW. */
696 size = (entries + 1) * QEDR_CQE_SIZE;
697 aligned_size = ALIGN(size, PAGE_SIZE);
698
699 return aligned_size / QEDR_CQE_SIZE;
700}
701
b0ea0fa5 702static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 703 struct qedr_dev *dev,
b0ea0fa5
JG		 704 struct qedr_userq *q, u64 buf_addr,
705 size_t buf_len, int access, int dmasync,
69ad0e7f 706 int alloc_and_init)
a7efd777 707{
e57bb6be 708 u32 fw_pages;
a7efd777
RA		 709 int rc;
710
711 q->buf_addr = buf_addr;
712 q->buf_len = buf_len;
b0ea0fa5 713 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
a7efd777
RA		 714 if (IS_ERR(q->umem)) {
715 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
716 PTR_ERR(q->umem));
717 return PTR_ERR(q->umem);
718 }
719
e57bb6be 720 fw_pages = ib_umem_page_count(q->umem) <<
95ad233f 721 (PAGE_SHIFT - FW_PAGE_SHIFT);
e57bb6be
RA		 722
723 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 724 if (rc)
725 goto err0;
726
69ad0e7f
KM		 727 if (alloc_and_init) {
728 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
729 if (IS_ERR(q->pbl_tbl)) {
730 rc = PTR_ERR(q->pbl_tbl);
731 goto err0;
732 }
e57bb6be
RA		 733 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
734 FW_PAGE_SHIFT);
69ad0e7f
KM		 735 } else {
736 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 737 if (!q->pbl_tbl) {
738 rc = -ENOMEM;
69ad0e7f 739 goto err0;
89fd2576 740 }
69ad0e7f 741 }
a7efd777
RA		 742
743 return 0;
744
745err0:
746 ib_umem_release(q->umem);
69ad0e7f 747 q->umem = NULL;
a7efd777
RA		 748
749 return rc;
750}
751
752static inline void qedr_init_cq_params(struct qedr_cq *cq,
753 struct qedr_ucontext *ctx,
754 struct qedr_dev *dev, int vector,
755 int chain_entries, int page_cnt,
756 u64 pbl_ptr,
757 struct qed_rdma_create_cq_in_params
758 *params)
759{
760 memset(params, 0, sizeof(*params));
761 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
762 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
763 params->cnq_id = vector;
764 params->cq_size = chain_entries - 1;
765 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
766 params->pbl_num_pages = page_cnt;
767 params->pbl_ptr = pbl_ptr;
768 params->pbl_two_level = 0;
769}
770
771static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
772{
a7efd777
RA		 773 cq->db.data.agg_flags = flags;
774 cq->db.data.value = cpu_to_le32(cons);
775 writeq(cq->db.raw, cq->db_addr);
776
777 /* Make sure write would stick */
778 mmiowb();
779}
780
781int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
782{
783 struct qedr_cq *cq = get_qedr_cq(ibcq);
784 unsigned long sflags;
4dd72636
AR		 785 struct qedr_dev *dev;
786
787 dev = get_qedr_dev(ibcq->device);
788
789 if (cq->destroyed) {
790 DP_ERR(dev,
791 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
792 cq, cq->icid);
793 return -EINVAL;
794 }
795
a7efd777
RA		 796
797 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
798 return 0;
799
800 spin_lock_irqsave(&cq->cq_lock, sflags);
801
802 cq->arm_flags = 0;
803
804 if (flags & IB_CQ_SOLICITED)
805 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
806
807 if (flags & IB_CQ_NEXT_COMP)
808 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
809
810 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
811
812 spin_unlock_irqrestore(&cq->cq_lock, sflags);
813
814 return 0;
815}
816
817struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
818 const struct ib_cq_init_attr *attr,
819 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
820{
821 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
822 struct qed_rdma_destroy_cq_out_params destroy_oparams;
823 struct qed_rdma_destroy_cq_in_params destroy_iparams;
824 struct qedr_dev *dev = get_qedr_dev(ibdev);
825 struct qed_rdma_create_cq_in_params params;
826 struct qedr_create_cq_ureq ureq;
827 int vector = attr->comp_vector;
828 int entries = attr->cqe;
829 struct qedr_cq *cq;
830 int chain_entries;
831 int page_cnt;
832 u64 pbl_ptr;
833 u16 icid;
834 int rc;
835
836 DP_DEBUG(dev, QEDR_MSG_INIT,
837 "create_cq: called from %s. entries=%d, vector=%d\n",
838 udata ? "User Lib" : "Kernel", entries, vector);
839
840 if (entries > QEDR_MAX_CQES) {
841 DP_ERR(dev,
842 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
843 entries, QEDR_MAX_CQES);
844 return ERR_PTR(-EINVAL);
845 }
846
847 chain_entries = qedr_align_cq_entries(entries);
848 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
849
850 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
851 if (!cq)
852 return ERR_PTR(-ENOMEM);
853
854 if (udata) {
855 memset(&ureq, 0, sizeof(ureq));
856 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
857 DP_ERR(dev,
858 "create cq: problem copying data from user space\n");
859 goto err0;
860 }
861
862 if (!ureq.len) {
863 DP_ERR(dev,
864 "create cq: cannot create a cq with 0 entries\n");
865 goto err0;
866 }
867
868 cq->cq_type = QEDR_CQ_TYPE_USER;
869
b0ea0fa5
JG		 870 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
871 ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
872 1);
a7efd777
RA		 873 if (rc)
874 goto err0;
875
876 pbl_ptr = cq->q.pbl_tbl->pa;
877 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 878
879 cq->ibcq.cqe = chain_entries;
a7efd777
RA		 880 } else {
881 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
882
883 rc = dev->ops->common->chain_alloc(dev->cdev,
884 QED_CHAIN_USE_TO_CONSUME,
885 QED_CHAIN_MODE_PBL,
886 QED_CHAIN_CNT_TYPE_U32,
887 chain_entries,
888 sizeof(union rdma_cqe),
1a4a6975 889 &cq->pbl, NULL);
a7efd777
RA		 890 if (rc)
891 goto err1;
892
893 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
894 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 895 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 896 }
897
898 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
899 pbl_ptr, &params);
900
901 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
902 if (rc)
903 goto err2;
904
905 cq->icid = icid;
906 cq->sig = QEDR_CQ_MAGIC_NUMBER;
907 spin_lock_init(&cq->cq_lock);
908
909 if (ib_ctx) {
910 rc = qedr_copy_cq_uresp(dev, cq, udata);
911 if (rc)
912 goto err3;
913 } else {
914 /* Generate doorbell address. */
915 cq->db_addr = dev->db_addr +
916 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
917 cq->db.data.icid = cq->icid;
918 cq->db.data.params = DB_AGG_CMD_SET <<
919 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
920
921 /* point to the very last element, passing it we will toggle */
922 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
923 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
924 cq->latest_cqe = NULL;
925 consume_cqe(cq);
926 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
927 }
928
929 DP_DEBUG(dev, QEDR_MSG_CQ,
930 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
931 cq->icid, cq, params.cq_size);
932
933 return &cq->ibcq;
934
935err3:
936 destroy_iparams.icid = cq->icid;
937 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
938 &destroy_oparams);
939err2:
940 if (udata)
941 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
942 else
943 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
944err1:
945 if (udata)
946 ib_umem_release(cq->q.umem);
947err0:
948 kfree(cq);
949 return ERR_PTR(-EINVAL);
950}
951
952int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
953{
954 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
955 struct qedr_cq *cq = get_qedr_cq(ibcq);
956
957 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
958
959 return 0;
960}
961
4dd72636
AR		 962#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
963#define QEDR_DESTROY_CQ_ITER_DURATION (10)
964
c4367a26 965int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 966{
967 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
968 struct qed_rdma_destroy_cq_out_params oparams;
969 struct qed_rdma_destroy_cq_in_params iparams;
970 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 971 int iter;
942b3b2c 972 int rc;
a7efd777 973
942b3b2c 974 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 975
4dd72636
AR		 976 cq->destroyed = 1;
977
a7efd777 978 /* GSIs CQs are handled by driver, so they don't exist in the FW */
942b3b2c
AR		 979 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
980 goto done;
a1211359 981
942b3b2c
AR		 982 iparams.icid = cq->icid;
983 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
984 if (rc)
985 return rc;
986
987 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777
RA		 988
989 if (ibcq->uobject && ibcq->uobject->context) {
990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
991 ib_umem_release(cq->q.umem);
992 }
993
4dd72636
AR		 994 /* We don't want the IRQ handler to handle a non-existing CQ so we
995 * wait until all CNQ interrupts, if any, are received. This will always
996 * happen and will always happen very fast. If not, then a serious error
997 * has occured. That is why we can use a long delay.
998 * We spin for a short time so we don’t lose time on context switching
999 * in case all the completions are handled in that span. Otherwise
1000 * we sleep for a while and check again. Since the CNQ may be
1001 * associated with (only) the current CPU we use msleep to allow the
1002 * current CPU to be freed.
1003 * The CNQ notification is increased in qedr_irq_handler().
1004 */
1005 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1006 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1007 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1008 iter--;
1009 }
1010
1011 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1012 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1013 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1014 iter--;
1015 }
1016
1017 if (oparams.num_cq_notif != cq->cnq_notif)
1018 goto err;
1019
1020 /* Note that we don't need to have explicit code to wait for the
1021 * completion of the event handler because it is invoked from the EQ.
1022 * Since the destroy CQ ramrod has also been received on the EQ we can
1023 * be certain that there's no event handler in process.
1024 */
942b3b2c 1025done:
4dd72636
AR		 1026 cq->sig = ~cq->sig;
1027
a7efd777
RA		 1028 kfree(cq);
1029
1030 return 0;
4dd72636
AR		 1031
1032err:
1033 DP_ERR(dev,
1034 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1035 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1036
1037 return -EINVAL;
a7efd777 1038}
cecbcddf
RA		 1039
1040static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1041 struct ib_qp_attr *attr,
1042 int attr_mask,
1043 struct qed_rdma_modify_qp_in_params
1044 *qp_params)
1045{
47ec3866 1046 const struct ib_gid_attr *gid_attr;
cecbcddf 1047 enum rdma_network_type nw_type;
d8966fcd 1048 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1049 u32 ipv4_addr;
cecbcddf
RA		 1050 int i;
1051
47ec3866
PP		 1052 gid_attr = grh->sgid_attr;
1053 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev);
3e44e0ee 1054
47ec3866 1055 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1056 switch (nw_type) {
1057 case RDMA_NETWORK_IPV6:
47ec3866 1058 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1059 sizeof(qp_params->sgid));
1060 memcpy(&qp_params->dgid.bytes[0],
1061 &grh->dgid,
1062 sizeof(qp_params->dgid));
1063 qp_params->roce_mode = ROCE_V2_IPV6;
1064 SET_FIELD(qp_params->modify_flags,
1065 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1066 break;
1067 case RDMA_NETWORK_IB:
47ec3866 1068 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1069 sizeof(qp_params->sgid));
1070 memcpy(&qp_params->dgid.bytes[0],
1071 &grh->dgid,
1072 sizeof(qp_params->dgid));
1073 qp_params->roce_mode = ROCE_V1;
1074 break;
1075 case RDMA_NETWORK_IPV4:
1076 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1077 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1078 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1079 qp_params->sgid.ipv4_addr = ipv4_addr;
1080 ipv4_addr =
1081 qedr_get_ipv4_from_gid(grh->dgid.raw);
1082 qp_params->dgid.ipv4_addr = ipv4_addr;
1083 SET_FIELD(qp_params->modify_flags,
1084 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1085 qp_params->roce_mode = ROCE_V2_IPV4;
1086 break;
cecbcddf
RA		 1087 }
1088
1089 for (i = 0; i < 4; i++) {
1090 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1091 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1092 }
1093
1094 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1095 qp_params->vlan_id = 0;
1096
1097 return 0;
1098}
1099
cecbcddf 1100static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1101 struct ib_qp_init_attr *attrs,
1102 struct ib_udata *udata)
cecbcddf
RA		 1103{
1104 struct qedr_device_attr *qattr = &dev->attr;
1105
1106 /* QP0... attrs->qp_type == IB_QPT_GSI */
1107 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1108 DP_DEBUG(dev, QEDR_MSG_QP,
1109 "create qp: unsupported qp type=0x%x requested\n",
1110 attrs->qp_type);
1111 return -EINVAL;
1112 }
1113
1114 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1115 DP_ERR(dev,
1116 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1117 attrs->cap.max_send_wr, qattr->max_sqe);
1118 return -EINVAL;
1119 }
1120
1121 if (attrs->cap.max_inline_data > qattr->max_inline) {
1122 DP_ERR(dev,
1123 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1124 attrs->cap.max_inline_data, qattr->max_inline);
1125 return -EINVAL;
1126 }
1127
1128 if (attrs->cap.max_send_sge > qattr->max_sge) {
1129 DP_ERR(dev,
1130 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1131 attrs->cap.max_send_sge, qattr->max_sge);
1132 return -EINVAL;
1133 }
1134
1135 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1136 DP_ERR(dev,
1137 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1138 attrs->cap.max_recv_sge, qattr->max_sge);
1139 return -EINVAL;
1140 }
1141
1142 /* Unprivileged user space cannot create special QP */
e00b64f7 1143 if (udata && attrs->qp_type == IB_QPT_GSI) {
cecbcddf
RA		 1144 DP_ERR(dev,
1145 "create qp: userspace can't create special QPs of type=0x%x\n",
1146 attrs->qp_type);
1147 return -EINVAL;
1148 }
1149
1150 return 0;
1151}
1152
40b173dd
YB		 1153static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1154 struct qedr_srq *srq, struct ib_udata *udata)
1155{
1156 struct qedr_create_srq_uresp uresp = {};
1157 int rc;
1158
1159 uresp.srq_id = srq->srq_id;
1160
1161 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1162 if (rc)
1163 DP_ERR(dev, "create srq: problem copying data to user space\n");
1164
1165 return rc;
1166}
1167
69ad0e7f
KM		 1168static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1169 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1170 struct qedr_qp *qp)
1171{
69ad0e7f
KM		 1172 /* iWARP requires two doorbells per RQ. */
1173 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1174 uresp->rq_db_offset =
1175 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1176 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1177 } else {
1178 uresp->rq_db_offset =
1179 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1180 }
1181
cecbcddf
RA		 1182 uresp->rq_icid = qp->icid;
1183}
1184
69ad0e7f
KM		 1185static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1186 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1187 struct qedr_qp *qp)
1188{
1189 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1190
1191 /* iWARP uses the same cid for rq and sq */
1192 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1193 uresp->sq_icid = qp->icid;
1194 else
1195 uresp->sq_icid = qp->icid + 1;
cecbcddf
RA		 1196}
1197
1198static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1199 struct qedr_qp *qp, struct ib_udata *udata)
1200{
1201 struct qedr_create_qp_uresp uresp;
1202 int rc;
1203
1204 memset(&uresp, 0, sizeof(uresp));
69ad0e7f
KM		 1205 qedr_copy_sq_uresp(dev, &uresp, qp);
1206 qedr_copy_rq_uresp(dev, &uresp, qp);
cecbcddf
RA		 1207
1208 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1209 uresp.qp_id = qp->qp_id;
1210
c75d3ec8 1211 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
cecbcddf
RA		 1212 if (rc)
1213 DP_ERR(dev,
1214 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1215 qp->icid);
1216
1217 return rc;
1218}
1219
df158561
AR		 1220static void qedr_set_common_qp_params(struct qedr_dev *dev,
1221 struct qedr_qp *qp,
1222 struct qedr_pd *pd,
1223 struct ib_qp_init_attr *attrs)
cecbcddf 1224{
cecbcddf 1225 spin_lock_init(&qp->q_lock);
de0089e6 1226 atomic_set(&qp->refcnt, 1);
df158561 1227 qp->pd = pd;
cecbcddf
RA		 1228 qp->qp_type = attrs->qp_type;
1229 qp->max_inline_data = attrs->cap.max_inline_data;
1230 qp->sq.max_sges = attrs->cap.max_send_sge;
1231 qp->state = QED_ROCE_QP_STATE_RESET;
1232 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1233 qp->sq_cq = get_qedr_cq(attrs->send_cq);
cecbcddf
RA		 1234 qp->dev = dev;
1235
3491c9e7
YB		 1236 if (attrs->srq) {
1237 qp->srq = get_qedr_srq(attrs->srq);
1238 } else {
1239 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1240 qp->rq.max_sges = attrs->cap.max_recv_sge;
1241 DP_DEBUG(dev, QEDR_MSG_QP,
1242 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1243 qp->rq.max_sges, qp->rq_cq->icid);
1244 }
1245
cecbcddf
RA		 1246 DP_DEBUG(dev, QEDR_MSG_QP,
1247 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1248 pd->pd_id, qp->qp_type, qp->max_inline_data,
1249 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1250 DP_DEBUG(dev, QEDR_MSG_QP,
1251 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1252 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1253}
1254
df158561 1255static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf
RA		 1256{
1257 qp->sq.db = dev->db_addr +
1258 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1259 qp->sq.db_data.data.icid = qp->icid + 1;
3491c9e7
YB		 1260 if (!qp->srq) {
1261 qp->rq.db = dev->db_addr +
1262 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1263 qp->rq.db_data.data.icid = qp->icid;
1264 }
1265}
1266
1267static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev,
1268 struct ib_srq_init_attr *attrs,
1269 struct ib_udata *udata)
1270{
1271 struct qedr_device_attr *qattr = &dev->attr;
1272
1273 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1274 DP_ERR(dev,
1275 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1276 attrs->attr.max_wr, qattr->max_srq_wr);
1277 return -EINVAL;
1278 }
1279
1280 if (attrs->attr.max_sge > qattr->max_sge) {
1281 DP_ERR(dev,
1282 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1283 attrs->attr.max_sge, qattr->max_sge);
1284 return -EINVAL;
1285 }
1286
1287 return 0;
1288}
1289
40b173dd
YB		 1290static void qedr_free_srq_user_params(struct qedr_srq *srq)
1291{
1292 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1293 ib_umem_release(srq->usrq.umem);
1294 ib_umem_release(srq->prod_umem);
1295}
1296
3491c9e7
YB		 1297static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1298{
1299 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1300 struct qedr_dev *dev = srq->dev;
1301
1302 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1303
1304 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1305 hw_srq->virt_prod_pair_addr,
1306 hw_srq->phy_prod_pair_addr);
1307}
1308
b0ea0fa5 1309static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1310 struct qedr_srq *srq,
1311 struct qedr_create_srq_ureq *ureq,
1312 int access, int dmasync)
1313{
1314 struct scatterlist *sg;
1315 int rc;
1316
b0ea0fa5 1317 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
40b173dd
YB		 1318 ureq->srq_len, access, dmasync, 1);
1319 if (rc)
1320 return rc;
1321
b0ea0fa5
JG		 1322 srq->prod_umem =
1323 ib_umem_get(udata, ureq->prod_pair_addr,
1324 sizeof(struct rdma_srq_producers), access, dmasync);
40b173dd
YB		 1325 if (IS_ERR(srq->prod_umem)) {
1326 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1327 ib_umem_release(srq->usrq.umem);
1328 DP_ERR(srq->dev,
1329 "create srq: failed ib_umem_get for producer, got %ld\n",
1330 PTR_ERR(srq->prod_umem));
1331 return PTR_ERR(srq->prod_umem);
1332 }
1333
1334 sg = srq->prod_umem->sg_head.sgl;
1335 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1336
1337 return 0;
1338}
1339
3491c9e7
YB		 1340static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1341 struct qedr_dev *dev,
1342 struct ib_srq_init_attr *init_attr)
1343{
1344 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1345 dma_addr_t phy_prod_pair_addr;
1346 u32 num_elems;
1347 void *va;
1348 int rc;
1349
1350 va = dma_alloc_coherent(&dev->pdev->dev,
1351 sizeof(struct rdma_srq_producers),
1352 &phy_prod_pair_addr, GFP_KERNEL);
1353 if (!va) {
1354 DP_ERR(dev,
1355 "create srq: failed to allocate dma memory for producer\n");
1356 return -ENOMEM;
1357 }
1358
1359 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1360 hw_srq->virt_prod_pair_addr = va;
1361
1362 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1363 rc = dev->ops->common->chain_alloc(dev->cdev,
1364 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1365 QED_CHAIN_MODE_PBL,
1366 QED_CHAIN_CNT_TYPE_U32,
1367 num_elems,
1368 QEDR_SRQ_WQE_ELEM_SIZE,
1369 &hw_srq->pbl, NULL);
1370 if (rc)
1371 goto err0;
1372
1373 hw_srq->num_elems = num_elems;
1374
1375 return 0;
1376
1377err0:
1378 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1379 va, phy_prod_pair_addr);
1380 return rc;
1381}
1382
3491c9e7
YB		 1383struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
1384 struct ib_srq_init_attr *init_attr,
1385 struct ib_udata *udata)
1386{
1387 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1388 struct qed_rdma_create_srq_in_params in_params = {};
1389 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1390 struct qed_rdma_create_srq_out_params out_params;
1391 struct qedr_pd *pd = get_qedr_pd(ibpd);
40b173dd 1392 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1393 u64 pbl_base_addr, phy_prod_pair_addr;
1394 struct qedr_srq_hwq_info *hw_srq;
1395 u32 page_cnt, page_size;
3491c9e7
YB		 1396 struct qedr_srq *srq;
1397 int rc = 0;
1398
1399 DP_DEBUG(dev, QEDR_MSG_QP,
1400 "create SRQ called from %s (pd %p)\n",
1401 (udata) ? "User lib" : "kernel", pd);
1402
1403 rc = qedr_check_srq_params(ibpd, dev, init_attr, udata);
1404 if (rc)
1405 return ERR_PTR(-EINVAL);
1406
1407 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1408 if (!srq)
1409 return ERR_PTR(-ENOMEM);
1410
1411 srq->dev = dev;
1412 hw_srq = &srq->hw_srq;
1413 spin_lock_init(&srq->lock);
1414
1415 hw_srq->max_wr = init_attr->attr.max_wr;
1416 hw_srq->max_sges = init_attr->attr.max_sge;
1417
89944450 1418 if (udata) {
40b173dd
YB		 1419 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1420 DP_ERR(dev,
1421 "create srq: problem copying data from user space\n");
1422 goto err0;
1423 }
1424
b0ea0fa5 1425 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
40b173dd
YB		 1426 if (rc)
1427 goto err0;
1428
1429 page_cnt = srq->usrq.pbl_info.num_pbes;
1430 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1431 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1432 page_size = PAGE_SIZE;
40b173dd
YB		 1433 } else {
1434 struct qed_chain *pbl;
1435
1436 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1437 if (rc)
1438 goto err0;
1439
1440 pbl = &hw_srq->pbl;
1441 page_cnt = qed_chain_get_page_cnt(pbl);
1442 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1443 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1444 page_size = QED_CHAIN_PAGE_SIZE;
1445 }
3491c9e7 1446
3491c9e7
YB		 1447 in_params.pd_id = pd->pd_id;
1448 in_params.pbl_base_addr = pbl_base_addr;
1449 in_params.prod_pair_addr = phy_prod_pair_addr;
1450 in_params.num_pages = page_cnt;
1451 in_params.page_size = page_size;
1452
1453 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1454 if (rc)
1455 goto err1;
1456
1457 srq->srq_id = out_params.srq_id;
1458
40b173dd
YB		 1459 if (udata) {
1460 rc = qedr_copy_srq_uresp(dev, srq, udata);
1461 if (rc)
1462 goto err2;
1463 }
1464
9fd15987 1465 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1466 if (rc)
1467 goto err2;
1468
1469 DP_DEBUG(dev, QEDR_MSG_SRQ,
1470 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1471 return &srq->ibsrq;
1472
1473err2:
1474 destroy_in_params.srq_id = srq->srq_id;
1475
1476 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1477err1:
40b173dd
YB		 1478 if (udata)
1479 qedr_free_srq_user_params(srq);
1480 else
1481 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1482err0:
1483 kfree(srq);
1484
1485 return ERR_PTR(-EFAULT);
1486}
1487
c4367a26 1488int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1489{
1490 struct qed_rdma_destroy_srq_in_params in_params = {};
1491 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1492 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1493
9fd15987 1494 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7
YB		 1495 in_params.srq_id = srq->srq_id;
1496 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1497
e00b64f7 1498 if (ibsrq->uobject)
40b173dd
YB		 1499 qedr_free_srq_user_params(srq);
1500 else
1501 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1502
1503 DP_DEBUG(dev, QEDR_MSG_SRQ,
1504 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1505 srq->srq_id);
1506 kfree(srq);
1507
1508 return 0;
1509}
1510
1511int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1512 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1513{
1514 struct qed_rdma_modify_srq_in_params in_params = {};
1515 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1516 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1517 int rc;
1518
1519 if (attr_mask & IB_SRQ_MAX_WR) {
1520 DP_ERR(dev,
1521 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1522 attr_mask, srq);
1523 return -EINVAL;
1524 }
1525
1526 if (attr_mask & IB_SRQ_LIMIT) {
1527 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1528 DP_ERR(dev,
1529 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1530 attr->srq_limit, srq->hw_srq.max_wr);
1531 return -EINVAL;
1532 }
1533
1534 in_params.srq_id = srq->srq_id;
1535 in_params.wqe_limit = attr->srq_limit;
1536 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1537 if (rc)
1538 return rc;
1539 }
1540
1541 srq->srq_limit = attr->srq_limit;
1542
1543 DP_DEBUG(dev, QEDR_MSG_SRQ,
1544 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1545
1546 return 0;
cecbcddf
RA		 1547}
1548
1549static inline void
df158561
AR		 1550qedr_init_common_qp_in_params(struct qedr_dev *dev,
1551 struct qedr_pd *pd,
1552 struct qedr_qp *qp,
1553 struct ib_qp_init_attr *attrs,
1554 bool fmr_and_reserved_lkey,
1555 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1556{
df158561
AR		 1557 /* QP handle to be written in an async event */
1558 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1559 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1560
1561 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1562 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1563 params->pd = pd->pd_id;
1564 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1565 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1566 params->stats_queue = 0;
df158561
AR		 1567 params->srq_id = 0;
1568 params->use_srq = false;
3491c9e7
YB		 1569
1570 if (!qp->srq) {
1571 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1572
1573 } else {
1574 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1575 params->srq_id = qp->srq->srq_id;
1576 params->use_srq = true;
1577 }
cecbcddf
RA		 1578}
1579
df158561 1580static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1581{
df158561
AR		 1582 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1583 "qp=%p. "
1584 "sq_addr=0x%llx, "
1585 "sq_len=%zd, "
1586 "rq_addr=0x%llx, "
1587 "rq_len=%zd"
1588 "\n",
1589 qp,
1590 qp->usq.buf_addr,
1591 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1592}
cecbcddf 1593
69ad0e7f
KM		 1594static inline void
1595qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1596 struct qedr_qp *qp,
1597 struct qed_rdma_create_qp_out_params *out_params)
1598{
1599 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1600 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1601
1602 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1603 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1604 if (!qp->srq) {
1605 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1606 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1607 }
69ad0e7f
KM		 1608
1609 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1610 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1611}
1612
df158561
AR		 1613static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1614{
1615 if (qp->usq.umem)
1616 ib_umem_release(qp->usq.umem);
1617 qp->usq.umem = NULL;
cecbcddf 1618
df158561
AR		 1619 if (qp->urq.umem)
1620 ib_umem_release(qp->urq.umem);
1621 qp->urq.umem = NULL;
cecbcddf
RA		 1622}
1623
df158561
AR		 1624static int qedr_create_user_qp(struct qedr_dev *dev,
1625 struct qedr_qp *qp,
1626 struct ib_pd *ibpd,
1627 struct ib_udata *udata,
1628 struct ib_qp_init_attr *attrs)
cecbcddf 1629{
df158561
AR		 1630 struct qed_rdma_create_qp_in_params in_params;
1631 struct qed_rdma_create_qp_out_params out_params;
1632 struct qedr_pd *pd = get_qedr_pd(ibpd);
df158561 1633 struct qedr_create_qp_ureq ureq;
69ad0e7f 1634 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
df158561 1635 int rc = -EINVAL;
cecbcddf 1636
df158561
AR		 1637 memset(&ureq, 0, sizeof(ureq));
1638 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1639 if (rc) {
1640 DP_ERR(dev, "Problem copying data from user space\n");
1641 return rc;
1642 }
cecbcddf 1643
df158561 1644 /* SQ - read access only (0), dma sync not required (0) */
b0ea0fa5 1645 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
69ad0e7f 1646 ureq.sq_len, 0, 0, alloc_and_init);
df158561
AR		 1647 if (rc)
1648 return rc;
cecbcddf 1649
40b173dd
YB		 1650 if (!qp->srq) {
1651 /* RQ - read access only (0), dma sync not required (0) */
b0ea0fa5 1652 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
40b173dd
YB		 1653 ureq.rq_len, 0, 0, alloc_and_init);
1654 if (rc)
1655 return rc;
1656 }
df158561
AR		 1657
1658 memset(&in_params, 0, sizeof(in_params));
1659 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1660 in_params.qp_handle_lo = ureq.qp_handle_lo;
1661 in_params.qp_handle_hi = ureq.qp_handle_hi;
1662 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1663 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
40b173dd
YB		 1664 if (!qp->srq) {
1665 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1666 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1667 }
df158561
AR		 1668
1669 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1670 &in_params, &out_params);
1671
1672 if (!qp->qed_qp) {
1673 rc = -ENOMEM;
1674 goto err1;
1675 }
1676
69ad0e7f
KM		 1677 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1678 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1679
df158561
AR		 1680 qp->qp_id = out_params.qp_id;
1681 qp->icid = out_params.icid;
1682
1683 rc = qedr_copy_qp_uresp(dev, qp, udata);
1684 if (rc)
1685 goto err;
1686
1687 qedr_qp_user_print(dev, qp);
cecbcddf
RA		 1688
1689 return 0;
df158561
AR		 1690err:
1691 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1692 if (rc)
1693 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1694
1695err1:
1696 qedr_cleanup_user(dev, qp);
1697 return rc;
cecbcddf
RA		 1698}
1699
f5b1b177
KM		 1700static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1701{
1702 qp->sq.db = dev->db_addr +
1703 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1704 qp->sq.db_data.data.icid = qp->icid;
1705
1706 qp->rq.db = dev->db_addr +
1707 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1708 qp->rq.db_data.data.icid = qp->icid;
1709 qp->rq.iwarp_db2 = dev->db_addr +
1710 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1711 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1712 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1713}
1714
df158561
AR		 1715static int
1716qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1717 struct qedr_qp *qp,
1718 struct qed_rdma_create_qp_in_params *in_params,
1719 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1720{
df158561 1721 struct qed_rdma_create_qp_out_params out_params;
cecbcddf
RA		 1722 int rc;
1723
cecbcddf
RA		 1724 rc = dev->ops->common->chain_alloc(dev->cdev,
1725 QED_CHAIN_USE_TO_PRODUCE,
1726 QED_CHAIN_MODE_PBL,
1727 QED_CHAIN_CNT_TYPE_U32,
1728 n_sq_elems,
1729 QEDR_SQE_ELEMENT_SIZE,
1a4a6975 1730 &qp->sq.pbl, NULL);
cecbcddf 1731
df158561
AR		 1732 if (rc)
1733 return rc;
cecbcddf 1734
df158561
AR		 1735 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1736 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1737
cecbcddf
RA		 1738 rc = dev->ops->common->chain_alloc(dev->cdev,
1739 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1740 QED_CHAIN_MODE_PBL,
1741 QED_CHAIN_CNT_TYPE_U32,
1742 n_rq_elems,
1743 QEDR_RQE_ELEMENT_SIZE,
1a4a6975 1744 &qp->rq.pbl, NULL);
df158561
AR		 1745 if (rc)
1746 return rc;
cecbcddf 1747
df158561
AR		 1748 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1749 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1750
df158561
AR		 1751 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1752 in_params, &out_params);
cecbcddf 1753
df158561
AR		 1754 if (!qp->qed_qp)
1755 return -EINVAL;
cecbcddf 1756
df158561
AR		 1757 qp->qp_id = out_params.qp_id;
1758 qp->icid = out_params.icid;
cecbcddf 1759
df158561 1760 qedr_set_roce_db_info(dev, qp);
f5b1b177
KM		 1761 return rc;
1762}
cecbcddf 1763
f5b1b177
KM		 1764static int
1765qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1766 struct qedr_qp *qp,
1767 struct qed_rdma_create_qp_in_params *in_params,
1768 u32 n_sq_elems, u32 n_rq_elems)
1769{
1770 struct qed_rdma_create_qp_out_params out_params;
1771 struct qed_chain_ext_pbl ext_pbl;
1772 int rc;
1773
1774 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1775 QEDR_SQE_ELEMENT_SIZE,
1776 QED_CHAIN_MODE_PBL);
1777 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1778 QEDR_RQE_ELEMENT_SIZE,
1779 QED_CHAIN_MODE_PBL);
1780
1781 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1782 in_params, &out_params);
1783
1784 if (!qp->qed_qp)
1785 return -EINVAL;
1786
1787 /* Now we allocate the chain */
1788 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1789 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1790
1791 rc = dev->ops->common->chain_alloc(dev->cdev,
1792 QED_CHAIN_USE_TO_PRODUCE,
1793 QED_CHAIN_MODE_PBL,
1794 QED_CHAIN_CNT_TYPE_U32,
1795 n_sq_elems,
1796 QEDR_SQE_ELEMENT_SIZE,
1797 &qp->sq.pbl, &ext_pbl);
1798
1799 if (rc)
1800 goto err;
1801
1802 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1803 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1804
1805 rc = dev->ops->common->chain_alloc(dev->cdev,
1806 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1807 QED_CHAIN_MODE_PBL,
1808 QED_CHAIN_CNT_TYPE_U32,
1809 n_rq_elems,
1810 QEDR_RQE_ELEMENT_SIZE,
1811 &qp->rq.pbl, &ext_pbl);
1812
1813 if (rc)
1814 goto err;
1815
1816 qp->qp_id = out_params.qp_id;
1817 qp->icid = out_params.icid;
1818
1819 qedr_set_iwarp_db_info(dev, qp);
1820 return rc;
1821
1822err:
1823 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1824
1825 return rc;
cecbcddf
RA		 1826}
1827
df158561 1828static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1829{
df158561
AR		 1830 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1831 kfree(qp->wqe_wr_id);
cecbcddf 1832
df158561
AR		 1833 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1834 kfree(qp->rqe_wr_id);
cecbcddf
RA		 1835}
1836
df158561
AR		 1837static int qedr_create_kernel_qp(struct qedr_dev *dev,
1838 struct qedr_qp *qp,
1839 struct ib_pd *ibpd,
1840 struct ib_qp_init_attr *attrs)
cecbcddf 1841{
df158561
AR		 1842 struct qed_rdma_create_qp_in_params in_params;
1843 struct qedr_pd *pd = get_qedr_pd(ibpd);
1844 int rc = -EINVAL;
1845 u32 n_rq_elems;
1846 u32 n_sq_elems;
1847 u32 n_sq_entries;
cecbcddf 1848
df158561 1849 memset(&in_params, 0, sizeof(in_params));
cecbcddf 1850
df158561
AR		 1851 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1852 * the ring. The ring should allow at least a single WR, even if the
1853 * user requested none, due to allocation issues.
1854 * We should add an extra WR since the prod and cons indices of
1855 * wqe_wr_id are managed in such a way that the WQ is considered full
1856 * when (prod+1)%max_wr==cons. We currently don't do that because we
1857 * double the number of entries due an iSER issue that pushes far more
1858 * WRs than indicated. If we decline its ib_post_send() then we get
1859 * error prints in the dmesg we'd like to avoid.
1860 */
1861 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1862 dev->attr.max_sqe);
cecbcddf 1863
6396bb22 1864 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 1865 GFP_KERNEL);
1866 if (!qp->wqe_wr_id) {
1867 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1868 return -ENOMEM;
1869 }
cecbcddf 1870
df158561
AR		 1871 /* QP handle to be written in CQE */
1872 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1873 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 1874
df158561
AR		 1875 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1876 * the ring. There ring should allow at least a single WR, even if the
1877 * user requested none, due to allocation issues.
1878 */
1879 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 1880
df158561 1881 /* Allocate driver internal RQ array */
6396bb22 1882 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 1883 GFP_KERNEL);
1884 if (!qp->rqe_wr_id) {
1885 DP_ERR(dev,
1886 "create qp: failed RQ shadow memory allocation\n");
1887 kfree(qp->wqe_wr_id);
1888 return -ENOMEM;
cecbcddf
RA		 1889 }
1890
df158561 1891 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 1892
df158561
AR		 1893 n_sq_entries = attrs->cap.max_send_wr;
1894 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1895 n_sq_entries = max_t(u32, n_sq_entries, 1);
1896 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 1897
df158561
AR		 1898 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1899
f5b1b177
KM		 1900 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1901 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1902 n_sq_elems, n_rq_elems);
1903 else
1904 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1905 n_sq_elems, n_rq_elems);
df158561
AR		 1906 if (rc)
1907 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 1908
1909 return rc;
1910}
1911
1912struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1913 struct ib_qp_init_attr *attrs,
1914 struct ib_udata *udata)
1915{
1916 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 1917 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 1918 struct qedr_qp *qp;
181d8015 1919 struct ib_qp *ibqp;
cecbcddf
RA		 1920 int rc = 0;
1921
1922 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1923 udata ? "user library" : "kernel", pd);
1924
e00b64f7 1925 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf
RA		 1926 if (rc)
1927 return ERR_PTR(rc);
1928
cecbcddf 1929 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 1930 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1931 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 1932 get_qedr_cq(attrs->send_cq),
1933 get_qedr_cq(attrs->send_cq)->icid,
1934 get_qedr_cq(attrs->recv_cq),
3491c9e7 1935 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 1936
df158561
AR		 1937 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1938 if (!qp) {
1939 DP_ERR(dev, "create qp: failed allocating memory\n");
1940 return ERR_PTR(-ENOMEM);
1941 }
1942
1943 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 1944
04886779 1945 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 1946 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1947 if (IS_ERR(ibqp))
1948 kfree(qp);
1949 return ibqp;
04886779
RA		 1950 }
1951
df158561
AR		 1952 if (udata)
1953 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1954 else
1955 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 1956
df158561
AR		 1957 if (rc)
1958 goto err;
cecbcddf 1959
cecbcddf
RA		 1960 qp->ibqp.qp_num = qp->qp_id;
1961
1212767e 1962 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
b6014f9e 1963 rc = xa_insert_irq(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e
YB		 1964 if (rc)
1965 goto err;
1966 }
de0089e6 1967
cecbcddf
RA		 1968 return &qp->ibqp;
1969
df158561 1970err:
cecbcddf
RA		 1971 kfree(qp);
1972
1973 return ERR_PTR(-EFAULT);
1974}
1975
27a4b1a6 1976static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 1977{
1978 switch (qp_state) {
1979 case QED_ROCE_QP_STATE_RESET:
1980 return IB_QPS_RESET;
1981 case QED_ROCE_QP_STATE_INIT:
1982 return IB_QPS_INIT;
1983 case QED_ROCE_QP_STATE_RTR:
1984 return IB_QPS_RTR;
1985 case QED_ROCE_QP_STATE_RTS:
1986 return IB_QPS_RTS;
1987 case QED_ROCE_QP_STATE_SQD:
1988 return IB_QPS_SQD;
1989 case QED_ROCE_QP_STATE_ERR:
1990 return IB_QPS_ERR;
1991 case QED_ROCE_QP_STATE_SQE:
1992 return IB_QPS_SQE;
1993 }
1994 return IB_QPS_ERR;
1995}
1996
27a4b1a6
RA		 1997static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1998 enum ib_qp_state qp_state)
cecbcddf
RA		 1999{
2000 switch (qp_state) {
2001 case IB_QPS_RESET:
2002 return QED_ROCE_QP_STATE_RESET;
2003 case IB_QPS_INIT:
2004 return QED_ROCE_QP_STATE_INIT;
2005 case IB_QPS_RTR:
2006 return QED_ROCE_QP_STATE_RTR;
2007 case IB_QPS_RTS:
2008 return QED_ROCE_QP_STATE_RTS;
2009 case IB_QPS_SQD:
2010 return QED_ROCE_QP_STATE_SQD;
2011 case IB_QPS_ERR:
2012 return QED_ROCE_QP_STATE_ERR;
2013 default:
2014 return QED_ROCE_QP_STATE_ERR;
2015 }
2016}
2017
2018static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2019{
2020 qed_chain_reset(&qph->pbl);
2021 qph->prod = 0;
2022 qph->cons = 0;
2023 qph->wqe_cons = 0;
2024 qph->db_data.data.value = cpu_to_le16(0);
2025}
2026
2027static int qedr_update_qp_state(struct qedr_dev *dev,
2028 struct qedr_qp *qp,
caf61b1b 2029 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 2030 enum qed_roce_qp_state new_state)
2031{
2032 int status = 0;
2033
caf61b1b 2034 if (new_state == cur_state)
865cea40 2035 return 0;
cecbcddf 2036
caf61b1b 2037 switch (cur_state) {
cecbcddf
RA		 2038 case QED_ROCE_QP_STATE_RESET:
2039 switch (new_state) {
2040 case QED_ROCE_QP_STATE_INIT:
2041 qp->prev_wqe_size = 0;
2042 qedr_reset_qp_hwq_info(&qp->sq);
2043 qedr_reset_qp_hwq_info(&qp->rq);
2044 break;
2045 default:
2046 status = -EINVAL;
2047 break;
790b57f6 2048 }
cecbcddf
RA		 2049 break;
2050 case QED_ROCE_QP_STATE_INIT:
2051 switch (new_state) {
2052 case QED_ROCE_QP_STATE_RTR:
2053 /* Update doorbell (in case post_recv was
2054 * done before move to RTR)
2055 */
f5b1b177
KM		 2056
2057 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177
KM		 2058 writel(qp->rq.db_data.raw, qp->rq.db);
2059 /* Make sure write takes effect */
2060 mmiowb();
2061 }
cecbcddf
RA		 2062 break;
2063 case QED_ROCE_QP_STATE_ERR:
2064 break;
2065 default:
2066 /* Invalid state change. */
2067 status = -EINVAL;
2068 break;
790b57f6 2069 }
cecbcddf
RA		 2070 break;
2071 case QED_ROCE_QP_STATE_RTR:
2072 /* RTR->XXX */
2073 switch (new_state) {
2074 case QED_ROCE_QP_STATE_RTS:
2075 break;
2076 case QED_ROCE_QP_STATE_ERR:
2077 break;
2078 default:
2079 /* Invalid state change. */
2080 status = -EINVAL;
2081 break;
790b57f6 2082 }
cecbcddf
RA		 2083 break;
2084 case QED_ROCE_QP_STATE_RTS:
2085 /* RTS->XXX */
2086 switch (new_state) {
2087 case QED_ROCE_QP_STATE_SQD:
2088 break;
2089 case QED_ROCE_QP_STATE_ERR:
2090 break;
2091 default:
2092 /* Invalid state change. */
2093 status = -EINVAL;
2094 break;
790b57f6 2095 }
cecbcddf
RA		 2096 break;
2097 case QED_ROCE_QP_STATE_SQD:
2098 /* SQD->XXX */
2099 switch (new_state) {
2100 case QED_ROCE_QP_STATE_RTS:
2101 case QED_ROCE_QP_STATE_ERR:
2102 break;
2103 default:
2104 /* Invalid state change. */
2105 status = -EINVAL;
2106 break;
790b57f6 2107 }
cecbcddf
RA		 2108 break;
2109 case QED_ROCE_QP_STATE_ERR:
2110 /* ERR->XXX */
2111 switch (new_state) {
2112 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2113 if ((qp->rq.prod != qp->rq.cons) ||
2114 (qp->sq.prod != qp->sq.cons)) {
2115 DP_NOTICE(dev,
2116 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2117 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2118 qp->sq.cons);
2119 status = -EINVAL;
2120 }
cecbcddf
RA		 2121 break;
2122 default:
2123 status = -EINVAL;
2124 break;
790b57f6 2125 }
cecbcddf
RA		 2126 break;
2127 default:
2128 status = -EINVAL;
2129 break;
790b57f6 2130 }
cecbcddf
RA		 2131
2132 return status;
2133}
2134
2135int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2136 int attr_mask, struct ib_udata *udata)
2137{
2138 struct qedr_qp *qp = get_qedr_qp(ibqp);
2139 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2140 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2141 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2142 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2143 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2144 int rc = 0;
2145
2146 DP_DEBUG(dev, QEDR_MSG_QP,
2147 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2148 attr->qp_state);
2149
2150 old_qp_state = qedr_get_ibqp_state(qp->state);
2151 if (attr_mask & IB_QP_STATE)
2152 new_qp_state = attr->qp_state;
2153 else
2154 new_qp_state = old_qp_state;
2155
f5b1b177
KM		 2156 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2157 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2158 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2159 DP_ERR(dev,
2160 "modify qp: invalid attribute mask=0x%x specified for\n"
2161 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2162 attr_mask, qp->qp_id, ibqp->qp_type,
2163 old_qp_state, new_qp_state);
2164 rc = -EINVAL;
2165 goto err;
2166 }
cecbcddf
RA		 2167 }
2168
2169 /* Translate the masks... */
2170 if (attr_mask & IB_QP_STATE) {
2171 SET_FIELD(qp_params.modify_flags,
2172 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2173 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2174 }
2175
2176 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2177 qp_params.sqd_async = true;
2178
2179 if (attr_mask & IB_QP_PKEY_INDEX) {
2180 SET_FIELD(qp_params.modify_flags,
2181 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2182 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2183 rc = -EINVAL;
2184 goto err;
2185 }
2186
2187 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2188 }
2189
2190 if (attr_mask & IB_QP_QKEY)
2191 qp->qkey = attr->qkey;
2192
2193 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2194 SET_FIELD(qp_params.modify_flags,
2195 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2196 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2197 IB_ACCESS_REMOTE_READ;
2198 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2199 IB_ACCESS_REMOTE_WRITE;
2200 qp_params.incoming_atomic_en = attr->qp_access_flags &
2201 IB_ACCESS_REMOTE_ATOMIC;
2202 }
2203
2204 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2205 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2206 return -EINVAL;
2207
cecbcddf
RA		 2208 if (attr_mask & IB_QP_PATH_MTU) {
2209 if (attr->path_mtu < IB_MTU_256 ||
2210 attr->path_mtu > IB_MTU_4096) {
2211 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2212 rc = -EINVAL;
2213 goto err;
2214 }
2215 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2216 ib_mtu_enum_to_int(iboe_get_mtu
2217 (dev->ndev->mtu)));
2218 }
2219
2220 if (!qp->mtu) {
2221 qp->mtu =
2222 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2223 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2224 }
2225
2226 SET_FIELD(qp_params.modify_flags,
2227 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2228
d8966fcd
DC		 2229 qp_params.traffic_class_tos = grh->traffic_class;
2230 qp_params.flow_label = grh->flow_label;
2231 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2232
d8966fcd 2233 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2234
2235 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2236 if (rc) {
2237 DP_ERR(dev,
2238 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2239 grh->sgid_index, rc);
cecbcddf
RA		 2240 return rc;
2241 }
2242
2243 rc = qedr_get_dmac(dev, &attr->ah_attr,
2244 qp_params.remote_mac_addr);
2245 if (rc)
2246 return rc;
2247
2248 qp_params.use_local_mac = true;
2249 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2250
2251 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2252 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2253 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2254 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2255 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2256 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2257 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2258 qp_params.remote_mac_addr);
cecbcddf
RA		 2259
2260 qp_params.mtu = qp->mtu;
2261 qp_params.lb_indication = false;
2262 }
2263
2264 if (!qp_params.mtu) {
2265 /* Stay with current MTU */
2266 if (qp->mtu)
2267 qp_params.mtu = qp->mtu;
2268 else
2269 qp_params.mtu =
2270 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2271 }
2272
2273 if (attr_mask & IB_QP_TIMEOUT) {
2274 SET_FIELD(qp_params.modify_flags,
2275 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2276
c3594f22
KM		 2277 /* The received timeout value is an exponent used like this:
2278 * "12.7.34 LOCAL ACK TIMEOUT
2279 * Value representing the transport (ACK) timeout for use by
2280 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2281 * The FW expects timeout in msec so we need to divide the usec
2282 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2283 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2284 * The value of zero means infinite so we use a 'max_t' to make
2285 * sure that sub 1 msec values will be configured as 1 msec.
2286 */
2287 if (attr->timeout)
2288 qp_params.ack_timeout =
2289 1 << max_t(int, attr->timeout - 8, 0);
2290 else
cecbcddf 2291 qp_params.ack_timeout = 0;
cecbcddf 2292 }
c3594f22 2293
cecbcddf
RA		 2294 if (attr_mask & IB_QP_RETRY_CNT) {
2295 SET_FIELD(qp_params.modify_flags,
2296 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2297 qp_params.retry_cnt = attr->retry_cnt;
2298 }
2299
2300 if (attr_mask & IB_QP_RNR_RETRY) {
2301 SET_FIELD(qp_params.modify_flags,
2302 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2303 qp_params.rnr_retry_cnt = attr->rnr_retry;
2304 }
2305
2306 if (attr_mask & IB_QP_RQ_PSN) {
2307 SET_FIELD(qp_params.modify_flags,
2308 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2309 qp_params.rq_psn = attr->rq_psn;
2310 qp->rq_psn = attr->rq_psn;
2311 }
2312
2313 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2314 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2315 rc = -EINVAL;
2316 DP_ERR(dev,
2317 "unsupported max_rd_atomic=%d, supported=%d\n",
2318 attr->max_rd_atomic,
2319 dev->attr.max_qp_req_rd_atomic_resc);
2320 goto err;
2321 }
2322
2323 SET_FIELD(qp_params.modify_flags,
2324 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2325 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2326 }
2327
2328 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2329 SET_FIELD(qp_params.modify_flags,
2330 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2331 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2332 }
2333
2334 if (attr_mask & IB_QP_SQ_PSN) {
2335 SET_FIELD(qp_params.modify_flags,
2336 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2337 qp_params.sq_psn = attr->sq_psn;
2338 qp->sq_psn = attr->sq_psn;
2339 }
2340
2341 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2342 if (attr->max_dest_rd_atomic >
2343 dev->attr.max_qp_resp_rd_atomic_resc) {
2344 DP_ERR(dev,
2345 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2346 attr->max_dest_rd_atomic,
2347 dev->attr.max_qp_resp_rd_atomic_resc);
2348
2349 rc = -EINVAL;
2350 goto err;
2351 }
2352
2353 SET_FIELD(qp_params.modify_flags,
2354 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2355 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2356 }
2357
2358 if (attr_mask & IB_QP_DEST_QPN) {
2359 SET_FIELD(qp_params.modify_flags,
2360 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2361
2362 qp_params.dest_qp = attr->dest_qp_num;
2363 qp->dest_qp_num = attr->dest_qp_num;
2364 }
2365
caf61b1b
KM		 2366 cur_state = qp->state;
2367
2368 /* Update the QP state before the actual ramrod to prevent a race with
2369 * fast path. Modifying the QP state to error will cause the device to
2370 * flush the CQEs and while polling the flushed CQEs will considered as
2371 * a potential issue if the QP isn't in error state.
2372 */
2373 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2374 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2375 qp->state = QED_ROCE_QP_STATE_ERR;
2376
cecbcddf
RA		 2377 if (qp->qp_type != IB_QPT_GSI)
2378 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2379 qp->qed_qp, &qp_params);
2380
2381 if (attr_mask & IB_QP_STATE) {
2382 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2383 rc = qedr_update_qp_state(dev, qp, cur_state,
2384 qp_params.new_state);
cecbcddf
RA		 2385 qp->state = qp_params.new_state;
2386 }
2387
2388err:
2389 return rc;
2390}
2391
2392static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2393{
2394 int ib_qp_acc_flags = 0;
2395
2396 if (params->incoming_rdma_write_en)
2397 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2398 if (params->incoming_rdma_read_en)
2399 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2400 if (params->incoming_atomic_en)
2401 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2402 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2403 return ib_qp_acc_flags;
2404}
2405
2406int qedr_query_qp(struct ib_qp *ibqp,
2407 struct ib_qp_attr *qp_attr,
2408 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2409{
2410 struct qed_rdma_query_qp_out_params params;
2411 struct qedr_qp *qp = get_qedr_qp(ibqp);
2412 struct qedr_dev *dev = qp->dev;
2413 int rc = 0;
2414
2415 memset(&params, 0, sizeof(params));
2416
2417 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2418 if (rc)
2419 goto err;
2420
2421 memset(qp_attr, 0, sizeof(*qp_attr));
2422 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2423
2424 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2425 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2426 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2427 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2428 qp_attr->rq_psn = params.rq_psn;
2429 qp_attr->sq_psn = params.sq_psn;
2430 qp_attr->dest_qp_num = params.dest_qp;
2431
2432 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2433
2434 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2435 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2436 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2437 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 2438 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 2439 qp_init_attr->cap = qp_attr->cap;
2440
44c58487 2441 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2442 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2443 params.flow_label, qp->sgid_idx,
2444 params.hop_limit_ttl, params.traffic_class_tos);
2445 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2446 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2447 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2448 qp_attr->timeout = params.timeout;
2449 qp_attr->rnr_retry = params.rnr_retry;
2450 qp_attr->retry_cnt = params.retry_cnt;
2451 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2452 qp_attr->pkey_index = params.pkey_index;
2453 qp_attr->port_num = 1;
d8966fcd
DC		 2454 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2455 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2456 qp_attr->alt_pkey_index = 0;
2457 qp_attr->alt_port_num = 0;
2458 qp_attr->alt_timeout = 0;
2459 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2460
2461 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2462 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2463 qp_attr->max_rd_atomic = params.max_rd_atomic;
2464 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2465
2466 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2467 qp_attr->cap.max_inline_data);
2468
2469err:
2470 return rc;
2471}
2472
0089985e 2473static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
df158561
AR		 2474{
2475 int rc = 0;
2476
2477 if (qp->qp_type != IB_QPT_GSI) {
2478 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2479 if (rc)
2480 return rc;
2481 }
2482
2483 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2484 qedr_cleanup_user(dev, qp);
2485 else
2486 qedr_cleanup_kernel(dev, qp);
2487
2488 return 0;
2489}
2490
c4367a26 2491int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2492{
2493 struct qedr_qp *qp = get_qedr_qp(ibqp);
2494 struct qedr_dev *dev = qp->dev;
2495 struct ib_qp_attr attr;
2496 int attr_mask = 0;
2497 int rc = 0;
2498
2499 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2500 qp, qp->qp_type);
2501
f5b1b177
KM		 2502 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2503 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2504 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2505 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2506
f5b1b177
KM		 2507 attr.qp_state = IB_QPS_ERR;
2508 attr_mask |= IB_QP_STATE;
cecbcddf 2509
f5b1b177
KM		 2510 /* Change the QP state to ERROR */
2511 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2512 }
e411e058
KM		 2513 } else {
2514 /* Wait for the connect/accept to complete */
2515 if (qp->ep) {
2516 int wait_count = 1;
2517
2518 while (qp->ep->during_connect) {
2519 DP_DEBUG(dev, QEDR_MSG_QP,
2520 "Still in during connect/accept\n");
2521
2522 msleep(100);
2523 if (wait_count++ > 200) {
2524 DP_NOTICE(dev,
2525 "during connect timeout\n");
2526 break;
2527 }
2528 }
2529 }
cecbcddf
RA		 2530 }
2531
df158561 2532 if (qp->qp_type == IB_QPT_GSI)
04886779 2533 qedr_destroy_gsi_qp(dev);
cecbcddf 2534
df158561 2535 qedr_free_qp_resources(dev, qp);
cecbcddf 2536
1212767e
YB		 2537 if (atomic_dec_and_test(&qp->refcnt) &&
2538 rdma_protocol_iwarp(&dev->ibdev, 1)) {
b6014f9e 2539 xa_erase_irq(&dev->qps, qp->qp_id);
de0089e6
KM		 2540 kfree(qp);
2541 }
cecbcddf
RA		 2542 return rc;
2543}
e0290cce 2544
90898850 2545struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
b090c4e3 2546 u32 flags, struct ib_udata *udata)
04886779
RA		 2547{
2548 struct qedr_ah *ah;
2549
2550 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2551 if (!ah)
2552 return ERR_PTR(-ENOMEM);
2553
d97099fe 2554 rdma_copy_ah_attr(&ah->attr, attr);
04886779
RA		 2555
2556 return &ah->ibah;
2557}
2558
c4367a26 2559int qedr_destroy_ah(struct ib_ah *ibah, u32 flags, struct ib_udata *udata)
04886779
RA		 2560{
2561 struct qedr_ah *ah = get_qedr_ah(ibah);
2562
d97099fe 2563 rdma_destroy_ah_attr(&ah->attr);
04886779
RA		 2564 kfree(ah);
2565 return 0;
2566}
2567
e0290cce
RA		 2568static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2569{
2570 struct qedr_pbl *pbl, *tmp;
2571
2572 if (info->pbl_table)
2573 list_add_tail(&info->pbl_table->list_entry,
2574 &info->free_pbl_list);
2575
2576 if (!list_empty(&info->inuse_pbl_list))
2577 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2578
2579 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2580 list_del(&pbl->list_entry);
2581 qedr_free_pbl(dev, &info->pbl_info, pbl);
2582 }
2583}
2584
2585static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2586 size_t page_list_len, bool two_layered)
2587{
2588 struct qedr_pbl *tmp;
2589 int rc;
2590
2591 INIT_LIST_HEAD(&info->free_pbl_list);
2592 INIT_LIST_HEAD(&info->inuse_pbl_list);
2593
2594 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2595 page_list_len, two_layered);
2596 if (rc)
2597 goto done;
2598
2599 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2600 if (IS_ERR(info->pbl_table)) {
2601 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2602 goto done;
2603 }
2604
2605 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2606 &info->pbl_table->pa);
2607
2608 /* in usual case we use 2 PBLs, so we add one to free
2609 * list and allocating another one
2610 */
2611 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2612 if (IS_ERR(tmp)) {
e0290cce
RA		 2613 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2614 goto done;
2615 }
2616
2617 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2618
2619 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2620
2621done:
2622 if (rc)
2623 free_mr_info(dev, info);
2624
2625 return rc;
2626}
2627
2628struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2629 u64 usr_addr, int acc, struct ib_udata *udata)
2630{
2631 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2632 struct qedr_mr *mr;
2633 struct qedr_pd *pd;
2634 int rc = -ENOMEM;
2635
2636 pd = get_qedr_pd(ibpd);
2637 DP_DEBUG(dev, QEDR_MSG_MR,
2638 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2639 pd->pd_id, start, len, usr_addr, acc);
2640
2641 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2642 return ERR_PTR(-EINVAL);
2643
2644 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2645 if (!mr)
2646 return ERR_PTR(rc);
2647
2648 mr->type = QEDR_MR_USER;
2649
b0ea0fa5 2650 mr->umem = ib_umem_get(udata, start, len, acc, 0);
e0290cce
RA		 2651 if (IS_ERR(mr->umem)) {
2652 rc = -EFAULT;
2653 goto err0;
2654 }
2655
2656 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2657 if (rc)
2658 goto err1;
2659
2660 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2661 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2662
2663 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2664 if (rc) {
2665 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2666 goto err1;
2667 }
2668
2669 /* Index only, 18 bit long, lkey = itid << 8 | key */
2670 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2671 mr->hw_mr.key = 0;
2672 mr->hw_mr.pd = pd->pd_id;
2673 mr->hw_mr.local_read = 1;
2674 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2675 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2676 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2677 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2678 mr->hw_mr.mw_bind = false;
2679 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2680 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2681 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 2682 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 2683 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2684 mr->hw_mr.length = len;
2685 mr->hw_mr.vaddr = usr_addr;
2686 mr->hw_mr.zbva = false;
2687 mr->hw_mr.phy_mr = false;
2688 mr->hw_mr.dma_mr = false;
2689
2690 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2691 if (rc) {
2692 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2693 goto err2;
2694 }
2695
2696 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2697 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2698 mr->hw_mr.remote_atomic)
2699 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2700
2701 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2702 mr->ibmr.lkey);
2703 return &mr->ibmr;
2704
2705err2:
2706 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2707err1:
2708 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2709err0:
2710 kfree(mr);
2711 return ERR_PTR(rc);
2712}
2713
c4367a26 2714int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 2715{
2716 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2717 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2718 int rc = 0;
2719
2720 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2721 if (rc)
2722 return rc;
2723
2724 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2725
2726 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2727 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2728
2729 /* it could be user registered memory. */
2730 if (mr->umem)
2731 ib_umem_release(mr->umem);
2732
2733 kfree(mr);
2734
2735 return rc;
2736}
2737
27a4b1a6
RA		 2738static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2739 int max_page_list_len)
e0290cce
RA		 2740{
2741 struct qedr_pd *pd = get_qedr_pd(ibpd);
2742 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2743 struct qedr_mr *mr;
2744 int rc = -ENOMEM;
2745
2746 DP_DEBUG(dev, QEDR_MSG_MR,
2747 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2748 max_page_list_len);
2749
2750 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2751 if (!mr)
2752 return ERR_PTR(rc);
2753
2754 mr->dev = dev;
2755 mr->type = QEDR_MR_FRMR;
2756
2757 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2758 if (rc)
2759 goto err0;
2760
2761 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2762 if (rc) {
2763 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2764 goto err0;
2765 }
2766
2767 /* Index only, 18 bit long, lkey = itid << 8 | key */
2768 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2769 mr->hw_mr.key = 0;
2770 mr->hw_mr.pd = pd->pd_id;
2771 mr->hw_mr.local_read = 1;
2772 mr->hw_mr.local_write = 0;
2773 mr->hw_mr.remote_read = 0;
2774 mr->hw_mr.remote_write = 0;
2775 mr->hw_mr.remote_atomic = 0;
2776 mr->hw_mr.mw_bind = false;
2777 mr->hw_mr.pbl_ptr = 0;
2778 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2779 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2780 mr->hw_mr.fbo = 0;
2781 mr->hw_mr.length = 0;
2782 mr->hw_mr.vaddr = 0;
2783 mr->hw_mr.zbva = false;
2784 mr->hw_mr.phy_mr = true;
2785 mr->hw_mr.dma_mr = false;
2786
2787 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2788 if (rc) {
2789 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2790 goto err1;
2791 }
2792
2793 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2794 mr->ibmr.rkey = mr->ibmr.lkey;
2795
2796 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2797 return mr;
2798
2799err1:
2800 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2801err0:
2802 kfree(mr);
2803 return ERR_PTR(rc);
2804}
2805
c4367a26
SR		 2806struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2807 u32 max_num_sg, struct ib_udata *udata)
e0290cce 2808{
e0290cce
RA		 2809 struct qedr_mr *mr;
2810
2811 if (mr_type != IB_MR_TYPE_MEM_REG)
2812 return ERR_PTR(-EINVAL);
2813
2814 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2815
2816 if (IS_ERR(mr))
2817 return ERR_PTR(-EINVAL);
2818
e0290cce
RA		 2819 return &mr->ibmr;
2820}
2821
2822static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2823{
2824 struct qedr_mr *mr = get_qedr_mr(ibmr);
2825 struct qedr_pbl *pbl_table;
2826 struct regpair *pbe;
2827 u32 pbes_in_page;
2828
2829 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 2830 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 2831 return -ENOMEM;
2832 }
2833
2834 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2835 mr->npages, addr);
2836
2837 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2838 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2839 pbe = (struct regpair *)pbl_table->va;
2840 pbe += mr->npages % pbes_in_page;
2841 pbe->lo = cpu_to_le32((u32)addr);
2842 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2843
2844 mr->npages++;
2845
2846 return 0;
2847}
2848
2849static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2850{
2851 int work = info->completed - info->completed_handled - 1;
2852
2853 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2854 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2855 struct qedr_pbl *pbl;
2856
2857 /* Free all the page list that are possible to be freed
2858 * (all the ones that were invalidated), under the assumption
2859 * that if an FMR was completed successfully that means that
2860 * if there was an invalidate operation before it also ended
2861 */
2862 pbl = list_first_entry(&info->inuse_pbl_list,
2863 struct qedr_pbl, list_entry);
aafec388 2864 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 2865 info->completed_handled++;
2866 }
2867}
2868
2869int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2870 int sg_nents, unsigned int *sg_offset)
2871{
2872 struct qedr_mr *mr = get_qedr_mr(ibmr);
2873
2874 mr->npages = 0;
2875
2876 handle_completed_mrs(mr->dev, &mr->info);
2877 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2878}
2879
2880struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2881{
2882 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2883 struct qedr_pd *pd = get_qedr_pd(ibpd);
2884 struct qedr_mr *mr;
2885 int rc;
2886
2887 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2888 if (!mr)
2889 return ERR_PTR(-ENOMEM);
2890
2891 mr->type = QEDR_MR_DMA;
2892
2893 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2894 if (rc) {
2895 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2896 goto err1;
2897 }
2898
2899 /* index only, 18 bit long, lkey = itid << 8 | key */
2900 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2901 mr->hw_mr.pd = pd->pd_id;
2902 mr->hw_mr.local_read = 1;
2903 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2904 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2905 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2906 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2907 mr->hw_mr.dma_mr = true;
2908
2909 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2910 if (rc) {
2911 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2912 goto err2;
2913 }
2914
2915 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2916 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2917 mr->hw_mr.remote_atomic)
2918 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2919
2920 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2921 return &mr->ibmr;
2922
2923err2:
2924 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2925err1:
2926 kfree(mr);
2927 return ERR_PTR(rc);
2928}
afa0e13b
RA		 2929
2930static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2931{
2932 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2933}
2934
2935static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2936{
2937 int i, len = 0;
2938
2939 for (i = 0; i < num_sge; i++)
2940 len += sg_list[i].length;
2941
2942 return len;
2943}
2944
2945static void swap_wqe_data64(u64 *p)
2946{
2947 int i;
2948
2949 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2950 *p = cpu_to_be64(cpu_to_le64(*p));
2951}
2952
2953static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2954 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 2955 const struct ib_send_wr *wr,
2956 const struct ib_send_wr **bad_wr,
2957 u8 *bits, u8 bit)
afa0e13b
RA		 2958{
2959 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2960 char *seg_prt, *wqe;
2961 int i, seg_siz;
2962
2963 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2964 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2965 *bad_wr = wr;
2966 return 0;
2967 }
2968
2969 if (!data_size)
2970 return data_size;
2971
2972 *bits |= bit;
2973
2974 seg_prt = NULL;
2975 wqe = NULL;
2976 seg_siz = 0;
2977
2978 /* Copy data inline */
2979 for (i = 0; i < wr->num_sge; i++) {
2980 u32 len = wr->sg_list[i].length;
2981 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2982
2983 while (len > 0) {
2984 u32 cur;
2985
2986 /* New segment required */
2987 if (!seg_siz) {
2988 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2989 seg_prt = wqe;
2990 seg_siz = sizeof(struct rdma_sq_common_wqe);
2991 (*wqe_size)++;
2992 }
2993
2994 /* Calculate currently allowed length */
2995 cur = min_t(u32, len, seg_siz);
2996 memcpy(seg_prt, src, cur);
2997
2998 /* Update segment variables */
2999 seg_prt += cur;
3000 seg_siz -= cur;
3001
3002 /* Update sge variables */
3003 src += cur;
3004 len -= cur;
3005
3006 /* Swap fully-completed segments */
3007 if (!seg_siz)
3008 swap_wqe_data64((u64 *)wqe);
3009 }
3010 }
3011
3012 /* swap last not completed segment */
3013 if (seg_siz)
3014 swap_wqe_data64((u64 *)wqe);
3015
3016 return data_size;
3017}
3018
3019#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3020 do { \
3021 DMA_REGPAIR_LE(sge->addr, vaddr); \
3022 (sge)->length = cpu_to_le32(vlength); \
3023 (sge)->flags = cpu_to_le32(vflags); \
3024 } while (0)
3025
3026#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3027 do { \
3028 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3029 (hdr)->num_sges = num_sge; \
3030 } while (0)
3031
3032#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3033 do { \
3034 DMA_REGPAIR_LE(sge->addr, vaddr); \
3035 (sge)->length = cpu_to_le32(vlength); \
3036 (sge)->l_key = cpu_to_le32(vlkey); \
3037 } while (0)
3038
3039static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 3040 const struct ib_send_wr *wr)
afa0e13b
RA		 3041{
3042 u32 data_size = 0;
3043 int i;
3044
3045 for (i = 0; i < wr->num_sge; i++) {
3046 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3047
3048 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3049 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3050 sge->length = cpu_to_le32(wr->sg_list[i].length);
3051 data_size += wr->sg_list[i].length;
3052 }
3053
3054 if (wqe_size)
3055 *wqe_size += wr->num_sge;
3056
3057 return data_size;
3058}
3059
3060static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3061 struct qedr_qp *qp,
3062 struct rdma_sq_rdma_wqe_1st *rwqe,
3063 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3064 const struct ib_send_wr *wr,
3065 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3066{
3067 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3068 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3069
8b0cabc6
AR		 3070 if (wr->send_flags & IB_SEND_INLINE &&
3071 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3072 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3073 u8 flags = 0;
3074
3075 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3076 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3077 bad_wr, &rwqe->flags, flags);
3078 }
3079
3080 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3081}
3082
3083static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3084 struct qedr_qp *qp,
3085 struct rdma_sq_send_wqe_1st *swqe,
3086 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3087 const struct ib_send_wr *wr,
3088 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3089{
3090 memset(swqe2, 0, sizeof(*swqe2));
3091 if (wr->send_flags & IB_SEND_INLINE) {
3092 u8 flags = 0;
3093
3094 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3095 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3096 bad_wr, &swqe->flags, flags);
3097 }
3098
3099 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3100}
3101
3102static int qedr_prepare_reg(struct qedr_qp *qp,
3103 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3104 const struct ib_reg_wr *wr)
afa0e13b
RA		 3105{
3106 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3107 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3108
3109 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3110 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3111 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3112 fwqe1->l_key = wr->key;
3113
08c4cf51
AR		 3114 fwqe2->access_ctrl = 0;
3115
afa0e13b
RA		 3116 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3117 !!(wr->access & IB_ACCESS_REMOTE_READ));
3118 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3119 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3120 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3121 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3122 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3123 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3124 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3125 fwqe2->fmr_ctrl = 0;
3126
3127 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3128 ilog2(mr->ibmr.page_size) - 12);
3129
3130 fwqe2->length_hi = 0;
3131 fwqe2->length_lo = mr->ibmr.length;
3132 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3133 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3134
3135 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3136
3137 return 0;
3138}
3139
27a4b1a6 3140static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3141{
3142 switch (opcode) {
3143 case IB_WR_RDMA_WRITE:
3144 case IB_WR_RDMA_WRITE_WITH_IMM:
3145 return IB_WC_RDMA_WRITE;
3146 case IB_WR_SEND_WITH_IMM:
3147 case IB_WR_SEND:
3148 case IB_WR_SEND_WITH_INV:
3149 return IB_WC_SEND;
3150 case IB_WR_RDMA_READ:
fb1a22be 3151 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3152 return IB_WC_RDMA_READ;
3153 case IB_WR_ATOMIC_CMP_AND_SWP:
3154 return IB_WC_COMP_SWAP;
3155 case IB_WR_ATOMIC_FETCH_AND_ADD:
3156 return IB_WC_FETCH_ADD;
3157 case IB_WR_REG_MR:
3158 return IB_WC_REG_MR;
3159 case IB_WR_LOCAL_INV:
3160 return IB_WC_LOCAL_INV;
3161 default:
3162 return IB_WC_SEND;
3163 }
3164}
3165
f696bf6d
BVA		 3166static inline bool qedr_can_post_send(struct qedr_qp *qp,
3167 const struct ib_send_wr *wr)
afa0e13b
RA		 3168{
3169 int wq_is_full, err_wr, pbl_is_full;
3170 struct qedr_dev *dev = qp->dev;
3171
3172 /* prevent SQ overflow and/or processing of a bad WR */
3173 err_wr = wr->num_sge > qp->sq.max_sges;
3174 wq_is_full = qedr_wq_is_full(&qp->sq);
3175 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3176 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3177 if (wq_is_full || err_wr || pbl_is_full) {
3178 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3179 DP_ERR(dev,
3180 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3181 qp);
3182 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3183 }
3184
3185 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3186 DP_ERR(dev,
3187 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3188 qp);
3189 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3190 }
3191
3192 if (pbl_is_full &&
3193 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3194 DP_ERR(dev,
3195 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3196 qp);
3197 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3198 }
3199 return false;
3200 }
3201 return true;
3202}
3203
d34ac5cd
BVA		 3204static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3205 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3206{
3207 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3208 struct qedr_qp *qp = get_qedr_qp(ibqp);
3209 struct rdma_sq_atomic_wqe_1st *awqe1;
3210 struct rdma_sq_atomic_wqe_2nd *awqe2;
3211 struct rdma_sq_atomic_wqe_3rd *awqe3;
3212 struct rdma_sq_send_wqe_2st *swqe2;
3213 struct rdma_sq_local_inv_wqe *iwqe;
3214 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3215 struct rdma_sq_send_wqe_1st *swqe;
3216 struct rdma_sq_rdma_wqe_1st *rwqe;
3217 struct rdma_sq_fmr_wqe_1st *fwqe1;
3218 struct rdma_sq_common_wqe *wqe;
3219 u32 length;
3220 int rc = 0;
3221 bool comp;
3222
3223 if (!qedr_can_post_send(qp, wr)) {
3224 *bad_wr = wr;
3225 return -ENOMEM;
3226 }
3227
3228 wqe = qed_chain_produce(&qp->sq.pbl);
3229 qp->wqe_wr_id[qp->sq.prod].signaled =
3230 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3231
3232 wqe->flags = 0;
3233 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3234 !!(wr->send_flags & IB_SEND_SOLICITED));
3235 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3236 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3237 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3238 !!(wr->send_flags & IB_SEND_FENCE));
3239 wqe->prev_wqe_size = qp->prev_wqe_size;
3240
3241 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3242
3243 switch (wr->opcode) {
3244 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3245 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3246 rc = -EINVAL;
3247 *bad_wr = wr;
3248 break;
3249 }
afa0e13b
RA		 3250 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3251 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3252 swqe->wqe_size = 2;
3253 swqe2 = qed_chain_produce(&qp->sq.pbl);
3254
7bed7ebc 3255 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3256 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3257 wr, bad_wr);
3258 swqe->length = cpu_to_le32(length);
3259 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3260 qp->prev_wqe_size = swqe->wqe_size;
3261 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3262 break;
3263 case IB_WR_SEND:
3264 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3265 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3266
3267 swqe->wqe_size = 2;
3268 swqe2 = qed_chain_produce(&qp->sq.pbl);
3269 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3270 wr, bad_wr);
3271 swqe->length = cpu_to_le32(length);
3272 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3273 qp->prev_wqe_size = swqe->wqe_size;
3274 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3275 break;
3276 case IB_WR_SEND_WITH_INV:
3277 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3278 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3279 swqe2 = qed_chain_produce(&qp->sq.pbl);
3280 swqe->wqe_size = 2;
3281 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3282 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3283 wr, bad_wr);
3284 swqe->length = cpu_to_le32(length);
3285 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3286 qp->prev_wqe_size = swqe->wqe_size;
3287 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3288 break;
3289
3290 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3291 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3292 rc = -EINVAL;
3293 *bad_wr = wr;
3294 break;
3295 }
afa0e13b
RA		 3296 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3297 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3298
3299 rwqe->wqe_size = 2;
3300 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3301 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3302 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3303 wr, bad_wr);
3304 rwqe->length = cpu_to_le32(length);
3305 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3306 qp->prev_wqe_size = rwqe->wqe_size;
3307 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3308 break;
3309 case IB_WR_RDMA_WRITE:
3310 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3311 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3312
3313 rwqe->wqe_size = 2;
3314 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3315 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3316 wr, bad_wr);
3317 rwqe->length = cpu_to_le32(length);
3318 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3319 qp->prev_wqe_size = rwqe->wqe_size;
3320 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3321 break;
3322 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3323 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
1b8a708b 3324 /* fallthrough -- same is identical to RDMA READ */
afa0e13b
RA		 3325
3326 case IB_WR_RDMA_READ:
3327 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3328 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3329
3330 rwqe->wqe_size = 2;
3331 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3332 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3333 wr, bad_wr);
3334 rwqe->length = cpu_to_le32(length);
3335 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3336 qp->prev_wqe_size = rwqe->wqe_size;
3337 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3338 break;
3339
3340 case IB_WR_ATOMIC_CMP_AND_SWP:
3341 case IB_WR_ATOMIC_FETCH_AND_ADD:
3342 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3343 awqe1->wqe_size = 4;
3344
3345 awqe2 = qed_chain_produce(&qp->sq.pbl);
3346 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3347 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3348
3349 awqe3 = qed_chain_produce(&qp->sq.pbl);
3350
3351 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3352 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3353 DMA_REGPAIR_LE(awqe3->swap_data,
3354 atomic_wr(wr)->compare_add);
3355 } else {
3356 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3357 DMA_REGPAIR_LE(awqe3->swap_data,
3358 atomic_wr(wr)->swap);
3359 DMA_REGPAIR_LE(awqe3->cmp_data,
3360 atomic_wr(wr)->compare_add);
3361 }
3362
3363 qedr_prepare_sq_sges(qp, NULL, wr);
3364
3365 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3366 qp->prev_wqe_size = awqe1->wqe_size;
3367 break;
3368
3369 case IB_WR_LOCAL_INV:
3370 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3371 iwqe->wqe_size = 1;
3372
3373 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3374 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3375 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3376 qp->prev_wqe_size = iwqe->wqe_size;
3377 break;
3378 case IB_WR_REG_MR:
3379 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3380 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3381 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3382 fwqe1->wqe_size = 2;
3383
3384 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3385 if (rc) {
3386 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3387 *bad_wr = wr;
3388 break;
3389 }
3390
3391 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3392 qp->prev_wqe_size = fwqe1->wqe_size;
3393 break;
3394 default:
3395 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3396 rc = -EINVAL;
3397 *bad_wr = wr;
3398 break;
3399 }
3400
3401 if (*bad_wr) {
3402 u16 value;
3403
3404 /* Restore prod to its position before
3405 * this WR was processed
3406 */
3407 value = le16_to_cpu(qp->sq.db_data.data.value);
3408 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3409
3410 /* Restore prev_wqe_size */
3411 qp->prev_wqe_size = wqe->prev_wqe_size;
3412 rc = -EINVAL;
3413 DP_ERR(dev, "POST SEND FAILED\n");
3414 }
3415
3416 return rc;
3417}
3418
d34ac5cd
BVA		 3419int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3420 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3421{
3422 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3423 struct qedr_qp *qp = get_qedr_qp(ibqp);
3424 unsigned long flags;
3425 int rc = 0;
3426
3427 *bad_wr = NULL;
3428
04886779
RA		 3429 if (qp->qp_type == IB_QPT_GSI)
3430 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3431
afa0e13b
RA		 3432 spin_lock_irqsave(&qp->q_lock, flags);
3433
f5b1b177
KM		 3434 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3435 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3436 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3437 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3438 spin_unlock_irqrestore(&qp->q_lock, flags);
3439 *bad_wr = wr;
3440 DP_DEBUG(dev, QEDR_MSG_CQ,
3441 "QP in wrong state! QP icid=0x%x state %d\n",
3442 qp->icid, qp->state);
3443 return -EINVAL;
3444 }
afa0e13b
RA		 3445 }
3446
afa0e13b
RA		 3447 while (wr) {
3448 rc = __qedr_post_send(ibqp, wr, bad_wr);
3449 if (rc)
3450 break;
3451
3452 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3453
3454 qedr_inc_sw_prod(&qp->sq);
3455
3456 qp->sq.db_data.data.value++;
3457
3458 wr = wr->next;
3459 }
3460
3461 /* Trigger doorbell
3462 * If there was a failure in the first WR then it will be triggered in
3463 * vane. However this is not harmful (as long as the producer value is
3464 * unchanged). For performance reasons we avoid checking for this
3465 * redundant doorbell.
09c4854f
KM		 3466 *
3467 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3468 * soon as we give the doorbell, we could get a completion
3469 * for this wr, therefore we need to make sure that the
3470 * memory is updated before giving the doorbell.
3471 * During qedr_poll_cq, rmb is called before accessing the
3472 * cqe. This covers for the smp_rmb as well.
afa0e13b 3473 */
09c4854f 3474 smp_wmb();
afa0e13b
RA		 3475 writel(qp->sq.db_data.raw, qp->sq.db);
3476
3477 /* Make sure write sticks */
3478 mmiowb();
3479
3480 spin_unlock_irqrestore(&qp->q_lock, flags);
3481
3482 return rc;
3483}
3484
3491c9e7
YB		 3485static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3486{
3487 u32 used;
3488
3489 /* Calculate number of elements used based on producer
3490 * count and consumer count and subtract it from max
3491 * work request supported so that we get elements left.
3492 */
3493 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3494
3495 return hw_srq->max_wr - used;
3496}
3497
3498int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3499 const struct ib_recv_wr **bad_wr)
3500{
3501 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3502 struct qedr_srq_hwq_info *hw_srq;
3503 struct qedr_dev *dev = srq->dev;
3504 struct qed_chain *pbl;
3505 unsigned long flags;
3506 int status = 0;
3507 u32 num_sge;
3508 u32 offset;
3509
3510 spin_lock_irqsave(&srq->lock, flags);
3511
3512 hw_srq = &srq->hw_srq;
3513 pbl = &srq->hw_srq.pbl;
3514 while (wr) {
3515 struct rdma_srq_wqe_header *hdr;
3516 int i;
3517
3518 if (!qedr_srq_elem_left(hw_srq) ||
3519 wr->num_sge > srq->hw_srq.max_sges) {
3520 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3521 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3522 wr->num_sge, srq->hw_srq.max_sges);
3523 status = -ENOMEM;
3524 *bad_wr = wr;
3525 break;
3526 }
3527
3528 hdr = qed_chain_produce(pbl);
3529 num_sge = wr->num_sge;
3530 /* Set number of sge and work request id in header */
3531 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3532
3533 srq->hw_srq.wr_prod_cnt++;
3534 hw_srq->wqe_prod++;
3535 hw_srq->sge_prod++;
3536
3537 DP_DEBUG(dev, QEDR_MSG_SRQ,
3538 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3539 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3540
3541 for (i = 0; i < wr->num_sge; i++) {
3542 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3543
3544 /* Set SGE length, lkey and address */
3545 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3546 wr->sg_list[i].length, wr->sg_list[i].lkey);
3547
3548 DP_DEBUG(dev, QEDR_MSG_SRQ,
3549 "[%d]: len %d key %x addr %x:%x\n",
3550 i, srq_sge->length, srq_sge->l_key,
3551 srq_sge->addr.hi, srq_sge->addr.lo);
3552 hw_srq->sge_prod++;
3553 }
3554
3555 /* Flush WQE and SGE information before
3556 * updating producer.
3557 */
3558 wmb();
3559
3560 /* SRQ producer is 8 bytes. Need to update SGE producer index
3561 * in first 4 bytes and need to update WQE producer in
3562 * next 4 bytes.
3563 */
3564 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3565 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3566 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3567 hw_srq->wqe_prod;
3568
3569 /* Flush producer after updating it. */
3570 wmb();
3571 wr = wr->next;
3572 }
3573
3574 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3575 qed_chain_get_elem_left(pbl));
3576 spin_unlock_irqrestore(&srq->lock, flags);
3577
3578 return status;
3579}
3580
d34ac5cd
BVA		 3581int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3582 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3583{
3584 struct qedr_qp *qp = get_qedr_qp(ibqp);
3585 struct qedr_dev *dev = qp->dev;
3586 unsigned long flags;
3587 int status = 0;
3588
04886779
RA		 3589 if (qp->qp_type == IB_QPT_GSI)
3590 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3591
afa0e13b
RA		 3592 spin_lock_irqsave(&qp->q_lock, flags);
3593
922d9a40 3594 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3595 spin_unlock_irqrestore(&qp->q_lock, flags);
3596 *bad_wr = wr;
3597 return -EINVAL;
3598 }
3599
3600 while (wr) {
3601 int i;
3602
3603 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3604 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3605 wr->num_sge > qp->rq.max_sges) {
3606 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3607 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3608 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3609 qp->rq.max_sges);
3610 status = -ENOMEM;
3611 *bad_wr = wr;
3612 break;
3613 }
3614 for (i = 0; i < wr->num_sge; i++) {
3615 u32 flags = 0;
3616 struct rdma_rq_sge *rqe =
3617 qed_chain_produce(&qp->rq.pbl);
3618
3619 /* First one must include the number
3620 * of SGE in the list
3621 */
3622 if (!i)
3623 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3624 wr->num_sge);
3625
d52c89f1 3626 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3627 wr->sg_list[i].lkey);
3628
3629 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3630 wr->sg_list[i].length, flags);
3631 }
3632
3633 /* Special case of no sges. FW requires between 1-4 sges...
3634 * in this case we need to post 1 sge with length zero. this is
3635 * because rdma write with immediate consumes an RQ.
3636 */
3637 if (!wr->num_sge) {
3638 u32 flags = 0;
3639 struct rdma_rq_sge *rqe =
3640 qed_chain_produce(&qp->rq.pbl);
3641
3642 /* First one must include the number
3643 * of SGE in the list
3644 */
d52c89f1 3645 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3646 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3647
3648 RQ_SGE_SET(rqe, 0, 0, flags);
3649 i = 1;
3650 }
3651
3652 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3653 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3654
3655 qedr_inc_sw_prod(&qp->rq);
3656
09c4854f
KM		 3657 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3658 * soon as we give the doorbell, we could get a completion
3659 * for this wr, therefore we need to make sure that the
3660 * memory is update before giving the doorbell.
3661 * During qedr_poll_cq, rmb is called before accessing the
3662 * cqe. This covers for the smp_rmb as well.
3663 */
3664 smp_wmb();
afa0e13b
RA		 3665
3666 qp->rq.db_data.data.value++;
3667
3668 writel(qp->rq.db_data.raw, qp->rq.db);
3669
3670 /* Make sure write sticks */
3671 mmiowb();
3672
f5b1b177
KM		 3673 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3674 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3675 mmiowb(); /* for second doorbell */
3676 }
3677
afa0e13b
RA		 3678 wr = wr->next;
3679 }
3680
3681 spin_unlock_irqrestore(&qp->q_lock, flags);
3682
3683 return status;
3684}
3685
3686static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3687{
3688 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3689
3690 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3691 cq->pbl_toggle;
3692}
3693
3694static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3695{
3696 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3697 struct qedr_qp *qp;
3698
3699 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3700 resp_cqe->qp_handle.lo,
3701 u64);
3702 return qp;
3703}
3704
3705static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3706{
3707 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3708
3709 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3710}
3711
3712/* Return latest CQE (needs processing) */
3713static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3714{
3715 return cq->latest_cqe;
3716}
3717
3718/* In fmr we need to increase the number of fmr completed counter for the fmr
3719 * algorithm determining whether we can free a pbl or not.
3720 * we need to perform this whether the work request was signaled or not. for
3721 * this purpose we call this function from the condition that checks if a wr
3722 * should be skipped, to make sure we don't miss it ( possibly this fmr
3723 * operation was not signalted)
3724 */
3725static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3726{
3727 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3728 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3729}
3730
3731static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3732 struct qedr_cq *cq, int num_entries,
3733 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3734 int force)
3735{
3736 u16 cnt = 0;
3737
3738 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3739 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3740 qedr_chk_if_fmr(qp);
3741 /* skip WC */
3742 goto next_cqe;
3743 }
3744
3745 /* fill WC */
3746 wc->status = status;
27035a1b 3747 wc->vendor_err = 0;
afa0e13b
RA		 3748 wc->wc_flags = 0;
3749 wc->src_qp = qp->id;
3750 wc->qp = &qp->ibqp;
3751
3752 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3753 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3754
3755 switch (wc->opcode) {
3756 case IB_WC_RDMA_WRITE:
3757 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3758 break;
3759 case IB_WC_COMP_SWAP:
3760 case IB_WC_FETCH_ADD:
3761 wc->byte_len = 8;
3762 break;
3763 case IB_WC_REG_MR:
3764 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3765 break;
dac27386
MK		 3766 case IB_WC_RDMA_READ:
3767 case IB_WC_SEND:
3768 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3769 break;
afa0e13b
RA		 3770 default:
3771 break;
3772 }
3773
3774 num_entries--;
3775 wc++;
3776 cnt++;
3777next_cqe:
3778 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3779 qed_chain_consume(&qp->sq.pbl);
3780 qedr_inc_sw_cons(&qp->sq);
3781 }
3782
3783 return cnt;
3784}
3785
3786static int qedr_poll_cq_req(struct qedr_dev *dev,
3787 struct qedr_qp *qp, struct qedr_cq *cq,
3788 int num_entries, struct ib_wc *wc,
3789 struct rdma_cqe_requester *req)
3790{
3791 int cnt = 0;
3792
3793 switch (req->status) {
3794 case RDMA_CQE_REQ_STS_OK:
3795 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3796 IB_WC_SUCCESS, 0);
3797 break;
3798 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 3799 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 3800 DP_DEBUG(dev, QEDR_MSG_CQ,
3801 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3802 cq->icid, qp->icid);
afa0e13b 3803 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3804 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 3805 break;
3806 default:
3807 /* process all WQE before the cosumer */
3808 qp->state = QED_ROCE_QP_STATE_ERR;
3809 cnt = process_req(dev, qp, cq, num_entries, wc,
3810 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3811 wc += cnt;
3812 /* if we have extra WC fill it with actual error info */
3813 if (cnt < num_entries) {
3814 enum ib_wc_status wc_status;
3815
3816 switch (req->status) {
3817 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3818 DP_ERR(dev,
3819 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3820 cq->icid, qp->icid);
3821 wc_status = IB_WC_BAD_RESP_ERR;
3822 break;
3823 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3824 DP_ERR(dev,
3825 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3826 cq->icid, qp->icid);
3827 wc_status = IB_WC_LOC_LEN_ERR;
3828 break;
3829 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3830 DP_ERR(dev,
3831 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3832 cq->icid, qp->icid);
3833 wc_status = IB_WC_LOC_QP_OP_ERR;
3834 break;
3835 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3836 DP_ERR(dev,
3837 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3838 cq->icid, qp->icid);
3839 wc_status = IB_WC_LOC_PROT_ERR;
3840 break;
3841 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3842 DP_ERR(dev,
3843 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3844 cq->icid, qp->icid);
3845 wc_status = IB_WC_MW_BIND_ERR;
3846 break;
3847 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3848 DP_ERR(dev,
3849 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3850 cq->icid, qp->icid);
3851 wc_status = IB_WC_REM_INV_REQ_ERR;
3852 break;
3853 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3854 DP_ERR(dev,
3855 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3856 cq->icid, qp->icid);
3857 wc_status = IB_WC_REM_ACCESS_ERR;
3858 break;
3859 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3860 DP_ERR(dev,
3861 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3862 cq->icid, qp->icid);
3863 wc_status = IB_WC_REM_OP_ERR;
3864 break;
3865 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3866 DP_ERR(dev,
3867 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3868 cq->icid, qp->icid);
3869 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3870 break;
3871 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3872 DP_ERR(dev,
3873 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3874 cq->icid, qp->icid);
3875 wc_status = IB_WC_RETRY_EXC_ERR;
3876 break;
3877 default:
3878 DP_ERR(dev,
3879 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3880 cq->icid, qp->icid);
3881 wc_status = IB_WC_GENERAL_ERR;
3882 }
3883 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3884 wc_status, 1);
3885 }
3886 }
3887
3888 return cnt;
3889}
3890
b6acd71f 3891static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 3892{
b6acd71f 3893 switch (status) {
afa0e13b 3894 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 3895 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 3896 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 3897 return IB_WC_LOC_LEN_ERR;
afa0e13b 3898 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 3899 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 3900 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 3901 return IB_WC_LOC_PROT_ERR;
afa0e13b 3902 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 3903 return IB_WC_MW_BIND_ERR;
afa0e13b 3904 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 3905 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 3906 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 3907 return IB_WC_SUCCESS;
3908 default:
3909 return IB_WC_GENERAL_ERR;
3910 }
3911}
afa0e13b 3912
b6acd71f
AR		 3913static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3914 struct ib_wc *wc)
3915{
3916 wc->status = IB_WC_SUCCESS;
3917 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 3918
b6acd71f 3919 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 3920 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 3921 wc->wc_flags |= IB_WC_WITH_IMM;
3922
3923 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 3924 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3925
b6acd71f
AR		 3926 if (resp->flags & QEDR_RESP_INV)
3927 return -EINVAL;
3928
3929 } else if (resp->flags & QEDR_RESP_INV) {
3930 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3931 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3932
3933 if (resp->flags & QEDR_RESP_RDMA)
3934 return -EINVAL;
3935
3936 } else if (resp->flags & QEDR_RESP_RDMA) {
3937 return -EINVAL;
3938 }
3939
3940 return 0;
3941}
3942
3943static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3944 struct qedr_cq *cq, struct ib_wc *wc,
3945 struct rdma_cqe_responder *resp, u64 wr_id)
3946{
3947 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3948 wc->opcode = IB_WC_RECV;
3949 wc->wc_flags = 0;
3950
3951 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3952 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3953 DP_ERR(dev,
3954 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3955 cq, cq->icid, resp->flags);
3956
3957 } else {
3958 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3959 if (wc->status == IB_WC_GENERAL_ERR)
3960 DP_ERR(dev,
3961 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3962 cq, cq->icid, resp->status);
afa0e13b
RA		 3963 }
3964
b6acd71f 3965 /* Fill the rest of the WC */
27035a1b 3966 wc->vendor_err = 0;
afa0e13b
RA		 3967 wc->src_qp = qp->id;
3968 wc->qp = &qp->ibqp;
3969 wc->wr_id = wr_id;
3970}
3971
3491c9e7
YB		 3972static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3973 struct qedr_cq *cq, struct ib_wc *wc,
3974 struct rdma_cqe_responder *resp)
3975{
3976 struct qedr_srq *srq = qp->srq;
3977 u64 wr_id;
3978
3979 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
3980 le32_to_cpu(resp->srq_wr_id.lo), u64);
3981
3982 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3983 wc->status = IB_WC_WR_FLUSH_ERR;
3984 wc->vendor_err = 0;
3985 wc->wr_id = wr_id;
3986 wc->byte_len = 0;
3987 wc->src_qp = qp->id;
3988 wc->qp = &qp->ibqp;
3989 wc->wr_id = wr_id;
3990 } else {
3991 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3992 }
3993 srq->hw_srq.wr_cons_cnt++;
3994
3995 return 1;
3996}
afa0e13b
RA		 3997static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3998 struct qedr_cq *cq, struct ib_wc *wc,
3999 struct rdma_cqe_responder *resp)
4000{
4001 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4002
4003 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4004
4005 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4006 qed_chain_consume(&qp->rq.pbl);
4007 qedr_inc_sw_cons(&qp->rq);
4008
4009 return 1;
4010}
4011
4012static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4013 int num_entries, struct ib_wc *wc, u16 hw_cons)
4014{
4015 u16 cnt = 0;
4016
4017 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4018 /* fill WC */
4019 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 4020 wc->vendor_err = 0;
afa0e13b
RA		 4021 wc->wc_flags = 0;
4022 wc->src_qp = qp->id;
4023 wc->byte_len = 0;
4024 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4025 wc->qp = &qp->ibqp;
4026 num_entries--;
4027 wc++;
4028 cnt++;
4029 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4030 qed_chain_consume(&qp->rq.pbl);
4031 qedr_inc_sw_cons(&qp->rq);
4032 }
4033
4034 return cnt;
4035}
4036
4037static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4038 struct rdma_cqe_responder *resp, int *update)
4039{
50bc60cb 4040 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 4041 consume_cqe(cq);
4042 *update |= 1;
4043 }
4044}
4045
3491c9e7
YB		 4046static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4047 struct qedr_cq *cq, int num_entries,
4048 struct ib_wc *wc,
4049 struct rdma_cqe_responder *resp)
4050{
4051 int cnt;
4052
4053 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4054 consume_cqe(cq);
4055
4056 return cnt;
4057}
4058
afa0e13b
RA		 4059static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4060 struct qedr_cq *cq, int num_entries,
4061 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4062 int *update)
4063{
4064 int cnt;
4065
4066 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4067 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4068 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4069 try_consume_resp_cqe(cq, qp, resp, update);
4070 } else {
4071 cnt = process_resp_one(dev, qp, cq, wc, resp);
4072 consume_cqe(cq);
4073 *update |= 1;
4074 }
4075
4076 return cnt;
4077}
4078
4079static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4080 struct rdma_cqe_requester *req, int *update)
4081{
4082 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4083 consume_cqe(cq);
4084 *update |= 1;
4085 }
4086}
4087
4088int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4089{
4090 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4091 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4092 union rdma_cqe *cqe;
afa0e13b
RA		 4093 u32 old_cons, new_cons;
4094 unsigned long flags;
4095 int update = 0;
4096 int done = 0;
4097
4dd72636
AR		 4098 if (cq->destroyed) {
4099 DP_ERR(dev,
4100 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4101 cq, cq->icid);
4102 return 0;
4103 }
4104
04886779
RA		 4105 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4106 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4107
afa0e13b 4108 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4109 cqe = cq->latest_cqe;
afa0e13b
RA		 4110 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4111 while (num_entries && is_valid_cqe(cq, cqe)) {
4112 struct qedr_qp *qp;
4113 int cnt = 0;
4114
4115 /* prevent speculative reads of any field of CQE */
4116 rmb();
4117
4118 qp = cqe_get_qp(cqe);
4119 if (!qp) {
4120 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4121 break;
4122 }
4123
4124 wc->qp = &qp->ibqp;
4125
4126 switch (cqe_get_type(cqe)) {
4127 case RDMA_CQE_TYPE_REQUESTER:
4128 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4129 &cqe->req);
4130 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4131 break;
4132 case RDMA_CQE_TYPE_RESPONDER_RQ:
4133 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4134 &cqe->resp, &update);
4135 break;
3491c9e7
YB		 4136 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4137 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4138 wc, &cqe->resp);
4139 update = 1;
4140 break;
afa0e13b
RA		 4141 case RDMA_CQE_TYPE_INVALID:
4142 default:
4143 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4144 cqe_get_type(cqe));
4145 }
4146 num_entries -= cnt;
4147 wc += cnt;
4148 done += cnt;
4149
4150 cqe = get_cqe(cq);
4151 }
4152 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4153
4154 cq->cq_cons += new_cons - old_cons;
4155
4156 if (update)
4157 /* doorbell notifies abount latest VALID entry,
4158 * but chain already point to the next INVALID one
4159 */
4160 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4161
4162 spin_unlock_irqrestore(&cq->cq_lock, flags);
4163 return done;
4164}
993d1b52
RA		 4165
4166int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4167 u8 port_num,
4168 const struct ib_wc *in_wc,
4169 const struct ib_grh *in_grh,
4170 const struct ib_mad_hdr *mad_hdr,
4171 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4172 size_t *out_mad_size, u16 *out_mad_pkey_index)
4173{
4174 struct qedr_dev *dev = get_qedr_dev(ibdev);
4175
4176 DP_DEBUG(dev, QEDR_MSG_GSI,
4177 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4178 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4179 mad_hdr->class_specific, mad_hdr->class_version,
4180 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4181 return IB_MAD_RESULT_SUCCESS;
4182}
Linux 6.x block layer and io_uring tree(s)