projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/qedr: Fix qpids xarray api used
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
ff23dfa1 45#include <rdma/uverbs_ioctl.h>
ac1b36e5 46
be086e7c
MY		 47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
99d195cc 53#include "qedr_roce_cm.h"
ac1b36e5 54
3491c9e7
YB		 55#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
56#define RDMA_MAX_SGE_PER_SRQ (4)
57#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
58
a7efd777
RA		 59#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
60
c75d3ec8
AR		 61static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
62 size_t len)
63{
64 size_t min_len = min_t(size_t, len, udata->outlen);
65
66 return ib_copy_to_udata(udata, src, min_len);
67}
68
a7efd777
RA		 69int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
70{
dbe30dae 71 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 72 return -EINVAL;
73
74 *pkey = QEDR_ROCE_PKEY_DEFAULT;
75 return 0;
76}
77
e6a38c54
KM		 78int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
79 int index, union ib_gid *sgid)
80{
81 struct qedr_dev *dev = get_qedr_dev(ibdev);
82
83 memset(sgid->raw, 0, sizeof(sgid->raw));
84 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
85
86 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
87 sgid->global.interface_id, sgid->global.subnet_prefix);
88
89 return 0;
90}
91
3491c9e7
YB		 92int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
93{
94 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
95 struct qedr_device_attr *qattr = &dev->attr;
96 struct qedr_srq *srq = get_qedr_srq(ibsrq);
97
98 srq_attr->srq_limit = srq->srq_limit;
99 srq_attr->max_wr = qattr->max_srq_wr;
100 srq_attr->max_sge = qattr->max_sge;
101
102 return 0;
103}
104
ac1b36e5
RA		 105int qedr_query_device(struct ib_device *ibdev,
106 struct ib_device_attr *attr, struct ib_udata *udata)
107{
108 struct qedr_dev *dev = get_qedr_dev(ibdev);
109 struct qedr_device_attr *qattr = &dev->attr;
110
111 if (!dev->rdma_ctx) {
112 DP_ERR(dev,
113 "qedr_query_device called with invalid params rdma_ctx=%p\n",
114 dev->rdma_ctx);
115 return -EINVAL;
116 }
117
118 memset(attr, 0, sizeof(*attr));
119
120 attr->fw_ver = qattr->fw_ver;
121 attr->sys_image_guid = qattr->sys_image_guid;
122 attr->max_mr_size = qattr->max_mr_size;
123 attr->page_size_cap = qattr->page_size_caps;
124 attr->vendor_id = qattr->vendor_id;
125 attr->vendor_part_id = qattr->vendor_part_id;
126 attr->hw_ver = qattr->hw_ver;
127 attr->max_qp = qattr->max_qp;
128 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
129 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
130 IB_DEVICE_RC_RNR_NAK_GEN |
131 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
132
33023fb8
SW		 133 attr->max_send_sge = qattr->max_sge;
134 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 135 attr->max_sge_rd = qattr->max_sge;
136 attr->max_cq = qattr->max_cq;
137 attr->max_cqe = qattr->max_cqe;
138 attr->max_mr = qattr->max_mr;
139 attr->max_mw = qattr->max_mw;
140 attr->max_pd = qattr->max_pd;
141 attr->atomic_cap = dev->atomic_cap;
142 attr->max_fmr = qattr->max_fmr;
143 attr->max_map_per_fmr = 16;
144 attr->max_qp_init_rd_atom =
145 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
146 attr->max_qp_rd_atom =
147 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
148 attr->max_qp_init_rd_atom);
149
150 attr->max_srq = qattr->max_srq;
151 attr->max_srq_sge = qattr->max_srq_sge;
152 attr->max_srq_wr = qattr->max_srq_wr;
153
154 attr->local_ca_ack_delay = qattr->dev_ack_delay;
155 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
156 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
157 attr->max_ah = qattr->max_ah;
158
159 return 0;
160}
161
ac1b36e5
RA		 162static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
163 u8 *ib_width)
164{
165 switch (speed) {
166 case 1000:
69054666 167 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 168 *ib_width = IB_WIDTH_1X;
169 break;
170 case 10000:
69054666 171 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 172 *ib_width = IB_WIDTH_1X;
173 break;
174
175 case 20000:
69054666 176 *ib_speed = IB_SPEED_DDR;
ac1b36e5
RA		 177 *ib_width = IB_WIDTH_4X;
178 break;
179
180 case 25000:
69054666 181 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 182 *ib_width = IB_WIDTH_1X;
183 break;
184
185 case 40000:
69054666 186 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 187 *ib_width = IB_WIDTH_4X;
188 break;
189
190 case 50000:
69054666
SO		 191 *ib_speed = IB_SPEED_HDR;
192 *ib_width = IB_WIDTH_1X;
ac1b36e5
RA		 193 break;
194
195 case 100000:
69054666 196 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 197 *ib_width = IB_WIDTH_4X;
198 break;
199
200 default:
201 /* Unsupported */
69054666 202 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 203 *ib_width = IB_WIDTH_1X;
204 }
205}
206
207int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
208{
209 struct qedr_dev *dev;
210 struct qed_rdma_port *rdma_port;
211
212 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 213
214 if (!dev->rdma_ctx) {
215 DP_ERR(dev, "rdma_ctx is NULL\n");
216 return -EINVAL;
217 }
218
219 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 220
c4550c63 221 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 222 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
223 attr->state = IB_PORT_ACTIVE;
72a7720f 224 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
ac1b36e5
RA		 225 } else {
226 attr->state = IB_PORT_DOWN;
72a7720f 227 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
ac1b36e5
RA		 228 }
229 attr->max_mtu = IB_MTU_4096;
230 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
231 attr->lid = 0;
232 attr->lmc = 0;
233 attr->sm_lid = 0;
234 attr->sm_sl = 0;
2f944c0f 235 attr->ip_gids = true;
f5b1b177
KM		 236 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
237 attr->gid_tbl_len = 1;
238 attr->pkey_tbl_len = 1;
239 } else {
240 attr->gid_tbl_len = QEDR_MAX_SGID;
241 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
242 }
ac1b36e5
RA		 243 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
244 attr->qkey_viol_cntr = 0;
245 get_link_speed_and_width(rdma_port->link_speed,
246 &attr->active_speed, &attr->active_width);
247 attr->max_msg_sz = rdma_port->max_msg_size;
248 attr->max_vl_num = 4;
249
250 return 0;
251}
252
253int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
254 struct ib_port_modify *props)
255{
ac1b36e5
RA		 256 return 0;
257}
258
259static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
260 unsigned long len)
261{
262 struct qedr_mm *mm;
263
264 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
265 if (!mm)
266 return -ENOMEM;
267
268 mm->key.phy_addr = phy_addr;
269 /* This function might be called with a length which is not a multiple
270 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
271 * forces this granularity by increasing the requested size if needed.
272 * When qedr_mmap is called, it will search the list with the updated
273 * length as a key. To prevent search failures, the length is rounded up
274 * in advance to PAGE_SIZE.
275 */
276 mm->key.len = roundup(len, PAGE_SIZE);
277 INIT_LIST_HEAD(&mm->entry);
278
279 mutex_lock(&uctx->mm_list_lock);
280 list_add(&mm->entry, &uctx->mm_head);
281 mutex_unlock(&uctx->mm_list_lock);
282
283 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
284 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
285 (unsigned long long)mm->key.phy_addr,
286 (unsigned long)mm->key.len, uctx);
287
288 return 0;
289}
290
291static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
292 unsigned long len)
293{
294 bool found = false;
295 struct qedr_mm *mm;
296
297 mutex_lock(&uctx->mm_list_lock);
298 list_for_each_entry(mm, &uctx->mm_head, entry) {
299 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
300 continue;
301
302 found = true;
303 break;
304 }
305 mutex_unlock(&uctx->mm_list_lock);
306 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
307 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
308 mm->key.phy_addr, mm->key.len, uctx, found);
309
310 return found;
311}
312
a2a074ef 313int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 314{
a2a074ef 315 struct ib_device *ibdev = uctx->device;
ac1b36e5 316 int rc;
a2a074ef
LR		 317 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
318 struct qedr_alloc_ucontext_resp uresp = {};
ac1b36e5
RA		 319 struct qedr_dev *dev = get_qedr_dev(ibdev);
320 struct qed_rdma_add_user_out_params oparams;
321
322 if (!udata)
a2a074ef 323 return -EFAULT;
ac1b36e5
RA		 324
325 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
326 if (rc) {
327 DP_ERR(dev,
328 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
329 rc);
a2a074ef 330 return rc;
ac1b36e5
RA		 331 }
332
333 ctx->dpi = oparams.dpi;
334 ctx->dpi_addr = oparams.dpi_addr;
335 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
336 ctx->dpi_size = oparams.dpi_size;
337 INIT_LIST_HEAD(&ctx->mm_head);
338 mutex_init(&ctx->mm_list_lock);
339
ad84dad2 340 uresp.dpm_enabled = dev->user_dpm_enabled;
67cbe353
AR		 341 uresp.wids_enabled = 1;
342 uresp.wid_count = oparams.wid_count;
ac1b36e5
RA		 343 uresp.db_pa = ctx->dpi_phys_addr;
344 uresp.db_size = ctx->dpi_size;
345 uresp.max_send_wr = dev->attr.max_sqe;
346 uresp.max_recv_wr = dev->attr.max_rqe;
347 uresp.max_srq_wr = dev->attr.max_srq_wr;
348 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
349 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
350 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
351 uresp.max_cqes = QEDR_MAX_CQES;
352
c75d3ec8 353 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 354 if (rc)
a2a074ef 355 return rc;
ac1b36e5
RA		 356
357 ctx->dev = dev;
358
359 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
360 if (rc)
a2a074ef 361 return rc;
ac1b36e5
RA		 362
363 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
364 &ctx->ibucontext);
a2a074ef 365 return 0;
ac1b36e5
RA		 366}
367
a2a074ef 368void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 369{
370 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
371 struct qedr_mm *mm, *tmp;
ac1b36e5
RA		 372
373 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
374 uctx);
375 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
376
377 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
378 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
379 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
380 mm->key.phy_addr, mm->key.len, uctx);
381 list_del(&mm->entry);
382 kfree(mm);
383 }
ac1b36e5
RA		 384}
385
386int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
387{
388 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
389 struct qedr_dev *dev = get_qedr_dev(context->device);
30bf066c 390 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
ac1b36e5 391 unsigned long len = (vma->vm_end - vma->vm_start);
30bf066c
KM		 392 unsigned long dpi_start;
393
394 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
ac1b36e5
RA		 395
396 DP_DEBUG(dev, QEDR_MSG_INIT,
30bf066c
KM		 397 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
398 (void *)vma->vm_start, (void *)vma->vm_end,
399 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
400
401 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
402 DP_ERR(dev,
367d2f07 403 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
30bf066c 404 (void *)vma->vm_start, (void *)vma->vm_end);
ac1b36e5
RA		 405 return -EINVAL;
406 }
407
30bf066c
KM		 408 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
409 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
ac1b36e5
RA		 410 vma->vm_pgoff);
411 return -EINVAL;
412 }
413
30bf066c
KM		 414 if (phys_addr < dpi_start ||
415 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
416 DP_ERR(dev,
417 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
418 (void *)phys_addr, (void *)dpi_start,
419 ucontext->dpi_size);
420 return -EINVAL;
421 }
ac1b36e5 422
30bf066c
KM		 423 if (vma->vm_flags & VM_READ) {
424 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
425 return -EINVAL;
ac1b36e5 426 }
30bf066c
KM		 427
428 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
429 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
430 vma->vm_page_prot);
ac1b36e5 431}
a7efd777 432
ff23dfa1 433int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777 434{
21a428a0 435 struct ib_device *ibdev = ibpd->device;
a7efd777 436 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 437 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 438 u16 pd_id;
439 int rc;
440
441 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
ff23dfa1 442 udata ? "User Lib" : "Kernel");
a7efd777
RA		 443
444 if (!dev->rdma_ctx) {
847cb1a3 445 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 446 return -EINVAL;
a7efd777
RA		 447 }
448
9c1e0228
RA		 449 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
450 if (rc)
21a428a0 451 return rc;
a7efd777 452
a7efd777
RA		 453 pd->pd_id = pd_id;
454
ff23dfa1 455 if (udata) {
57939021
JG		 456 struct qedr_alloc_pd_uresp uresp = {
457 .pd_id = pd_id,
458 };
ff23dfa1
SR		 459 struct qedr_ucontext *context = rdma_udata_to_drv_context(
460 udata, struct qedr_ucontext, ibucontext);
9c1e0228 461
c75d3ec8 462 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 463 if (rc) {
a7efd777 464 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 465 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 466 return rc;
9c1e0228
RA		 467 }
468
ff23dfa1 469 pd->uctx = context;
9c1e0228 470 pd->uctx->pd = pd;
a7efd777
RA		 471 }
472
21a428a0 473 return 0;
a7efd777
RA		 474}
475
c4367a26 476void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 477{
478 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
479 struct qedr_pd *pd = get_qedr_pd(ibpd);
480
a7efd777
RA		 481 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
482 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
a7efd777
RA		 483}
484
485static void qedr_free_pbl(struct qedr_dev *dev,
486 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
487{
488 struct pci_dev *pdev = dev->pdev;
489 int i;
490
491 for (i = 0; i < pbl_info->num_pbls; i++) {
492 if (!pbl[i].va)
493 continue;
494 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
495 pbl[i].va, pbl[i].pa);
496 }
497
498 kfree(pbl);
499}
500
501#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
502#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
503
504#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
505#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
506#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
507
508static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
509 struct qedr_pbl_info *pbl_info,
510 gfp_t flags)
511{
512 struct pci_dev *pdev = dev->pdev;
513 struct qedr_pbl *pbl_table;
514 dma_addr_t *pbl_main_tbl;
515 dma_addr_t pa;
516 void *va;
517 int i;
518
519 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
520 if (!pbl_table)
521 return ERR_PTR(-ENOMEM);
522
523 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 524 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
525 flags);
a7efd777
RA		 526 if (!va)
527 goto err;
528
a7efd777
RA		 529 pbl_table[i].va = va;
530 pbl_table[i].pa = pa;
531 }
532
533 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
534 * the first one with physical pointers to all of the rest
535 */
536 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
537 for (i = 0; i < pbl_info->num_pbls - 1; i++)
538 pbl_main_tbl[i] = pbl_table[i + 1].pa;
539
540 return pbl_table;
541
542err:
543 for (i--; i >= 0; i--)
544 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
545 pbl_table[i].va, pbl_table[i].pa);
546
547 qedr_free_pbl(dev, pbl_info, pbl_table);
548
549 return ERR_PTR(-ENOMEM);
550}
551
552static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
553 struct qedr_pbl_info *pbl_info,
554 u32 num_pbes, int two_layer_capable)
555{
556 u32 pbl_capacity;
557 u32 pbl_size;
558 u32 num_pbls;
559
560 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
561 if (num_pbes > MAX_PBES_TWO_LAYER) {
562 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
563 num_pbes);
564 return -EINVAL;
565 }
566
567 /* calculate required pbl page size */
568 pbl_size = MIN_FW_PBL_PAGE_SIZE;
569 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
570 NUM_PBES_ON_PAGE(pbl_size);
571
572 while (pbl_capacity < num_pbes) {
573 pbl_size *= 2;
574 pbl_capacity = pbl_size / sizeof(u64);
575 pbl_capacity = pbl_capacity * pbl_capacity;
576 }
577
578 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
579 num_pbls++; /* One for the layer0 ( points to the pbls) */
580 pbl_info->two_layered = true;
581 } else {
582 /* One layered PBL */
583 num_pbls = 1;
584 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
585 roundup_pow_of_two((num_pbes * sizeof(u64))));
586 pbl_info->two_layered = false;
587 }
588
589 pbl_info->num_pbls = num_pbls;
590 pbl_info->pbl_size = pbl_size;
591 pbl_info->num_pbes = num_pbes;
592
593 DP_DEBUG(dev, QEDR_MSG_MR,
594 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
595 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
596
597 return 0;
598}
599
600static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
601 struct qedr_pbl *pbl,
e57bb6be 602 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 603{
95ad233f 604 int pbe_cnt, total_num_pbes = 0;
e57bb6be 605 u32 fw_pg_cnt, fw_pg_per_umem_pg;
a7efd777 606 struct qedr_pbl *pbl_tbl;
95ad233f 607 struct sg_dma_page_iter sg_iter;
a7efd777 608 struct regpair *pbe;
e57bb6be 609 u64 pg_addr;
a7efd777
RA		 610
611 if (!pbl_info->num_pbes)
612 return;
613
614 /* If we have a two layered pbl, the first pbl points to the rest
615 * of the pbls and the first entry lays on the second pbl in the table
616 */
617 if (pbl_info->two_layered)
618 pbl_tbl = &pbl[1];
619 else
620 pbl_tbl = pbl;
621
622 pbe = (struct regpair *)pbl_tbl->va;
623 if (!pbe) {
624 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
625 return;
626 }
627
628 pbe_cnt = 0;
629
95ad233f
SS		 630 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
631
632 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
633 pg_addr = sg_page_iter_dma_address(&sg_iter);
634 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
635 pbe->lo = cpu_to_le32(pg_addr);
636 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
637
638 pg_addr += BIT(pg_shift);
639 pbe_cnt++;
640 total_num_pbes++;
641 pbe++;
e57bb6be 642
95ad233f
SS		 643 if (total_num_pbes == pbl_info->num_pbes)
644 return;
645
646 /* If the given pbl is full storing the pbes,
647 * move to next pbl.
648 */
649 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
650 pbl_tbl++;
651 pbe = (struct regpair *)pbl_tbl->va;
652 pbe_cnt = 0;
a7efd777 653 }
95ad233f
SS		 654
655 fw_pg_cnt++;
a7efd777
RA		 656 }
657 }
658}
659
660static int qedr_copy_cq_uresp(struct qedr_dev *dev,
661 struct qedr_cq *cq, struct ib_udata *udata)
662{
663 struct qedr_create_cq_uresp uresp;
664 int rc;
665
666 memset(&uresp, 0, sizeof(uresp));
667
668 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
669 uresp.icid = cq->icid;
670
c75d3ec8 671 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 672 if (rc)
673 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
674
675 return rc;
676}
677
678static void consume_cqe(struct qedr_cq *cq)
679{
680 if (cq->latest_cqe == cq->toggle_cqe)
681 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
682
683 cq->latest_cqe = qed_chain_consume(&cq->pbl);
684}
685
686static inline int qedr_align_cq_entries(int entries)
687{
688 u64 size, aligned_size;
689
690 /* We allocate an extra entry that we don't report to the FW. */
691 size = (entries + 1) * QEDR_CQE_SIZE;
692 aligned_size = ALIGN(size, PAGE_SIZE);
693
694 return aligned_size / QEDR_CQE_SIZE;
695}
696
b0ea0fa5 697static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 698 struct qedr_dev *dev,
b0ea0fa5
JG		 699 struct qedr_userq *q, u64 buf_addr,
700 size_t buf_len, int access, int dmasync,
69ad0e7f 701 int alloc_and_init)
a7efd777 702{
e57bb6be 703 u32 fw_pages;
a7efd777
RA		 704 int rc;
705
706 q->buf_addr = buf_addr;
707 q->buf_len = buf_len;
b0ea0fa5 708 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
a7efd777
RA		 709 if (IS_ERR(q->umem)) {
710 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
711 PTR_ERR(q->umem));
712 return PTR_ERR(q->umem);
713 }
714
e57bb6be 715 fw_pages = ib_umem_page_count(q->umem) <<
95ad233f 716 (PAGE_SHIFT - FW_PAGE_SHIFT);
e57bb6be
RA		 717
718 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 719 if (rc)
720 goto err0;
721
69ad0e7f
KM		 722 if (alloc_and_init) {
723 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
724 if (IS_ERR(q->pbl_tbl)) {
725 rc = PTR_ERR(q->pbl_tbl);
726 goto err0;
727 }
e57bb6be
RA		 728 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
729 FW_PAGE_SHIFT);
69ad0e7f
KM		 730 } else {
731 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 732 if (!q->pbl_tbl) {
733 rc = -ENOMEM;
69ad0e7f 734 goto err0;
89fd2576 735 }
69ad0e7f 736 }
a7efd777
RA		 737
738 return 0;
739
740err0:
741 ib_umem_release(q->umem);
69ad0e7f 742 q->umem = NULL;
a7efd777
RA		 743
744 return rc;
745}
746
747static inline void qedr_init_cq_params(struct qedr_cq *cq,
748 struct qedr_ucontext *ctx,
749 struct qedr_dev *dev, int vector,
750 int chain_entries, int page_cnt,
751 u64 pbl_ptr,
752 struct qed_rdma_create_cq_in_params
753 *params)
754{
755 memset(params, 0, sizeof(*params));
756 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
757 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
758 params->cnq_id = vector;
759 params->cq_size = chain_entries - 1;
760 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
761 params->pbl_num_pages = page_cnt;
762 params->pbl_ptr = pbl_ptr;
763 params->pbl_two_level = 0;
764}
765
766static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
767{
a7efd777
RA		 768 cq->db.data.agg_flags = flags;
769 cq->db.data.value = cpu_to_le32(cons);
770 writeq(cq->db.raw, cq->db_addr);
a7efd777
RA		 771}
772
773int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
774{
775 struct qedr_cq *cq = get_qedr_cq(ibcq);
776 unsigned long sflags;
4dd72636
AR		 777 struct qedr_dev *dev;
778
779 dev = get_qedr_dev(ibcq->device);
780
781 if (cq->destroyed) {
782 DP_ERR(dev,
783 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
784 cq, cq->icid);
785 return -EINVAL;
786 }
787
a7efd777
RA		 788
789 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
790 return 0;
791
792 spin_lock_irqsave(&cq->cq_lock, sflags);
793
794 cq->arm_flags = 0;
795
796 if (flags & IB_CQ_SOLICITED)
797 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
798
799 if (flags & IB_CQ_NEXT_COMP)
800 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
801
802 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
803
804 spin_unlock_irqrestore(&cq->cq_lock, sflags);
805
806 return 0;
807}
808
e39afe3d
LR		 809int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
810 struct ib_udata *udata)
a7efd777 811{
e39afe3d 812 struct ib_device *ibdev = ibcq->device;
ff23dfa1
SR		 813 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
814 udata, struct qedr_ucontext, ibucontext);
a7efd777
RA		 815 struct qed_rdma_destroy_cq_out_params destroy_oparams;
816 struct qed_rdma_destroy_cq_in_params destroy_iparams;
817 struct qedr_dev *dev = get_qedr_dev(ibdev);
818 struct qed_rdma_create_cq_in_params params;
e39afe3d 819 struct qedr_create_cq_ureq ureq = {};
a7efd777
RA		 820 int vector = attr->comp_vector;
821 int entries = attr->cqe;
e39afe3d 822 struct qedr_cq *cq = get_qedr_cq(ibcq);
a7efd777
RA		 823 int chain_entries;
824 int page_cnt;
825 u64 pbl_ptr;
826 u16 icid;
827 int rc;
828
829 DP_DEBUG(dev, QEDR_MSG_INIT,
830 "create_cq: called from %s. entries=%d, vector=%d\n",
831 udata ? "User Lib" : "Kernel", entries, vector);
832
833 if (entries > QEDR_MAX_CQES) {
834 DP_ERR(dev,
835 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
836 entries, QEDR_MAX_CQES);
e39afe3d 837 return -EINVAL;
a7efd777
RA		 838 }
839
840 chain_entries = qedr_align_cq_entries(entries);
841 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
842
a7efd777 843 if (udata) {
a7efd777
RA		 844 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
845 DP_ERR(dev,
846 "create cq: problem copying data from user space\n");
847 goto err0;
848 }
849
850 if (!ureq.len) {
851 DP_ERR(dev,
852 "create cq: cannot create a cq with 0 entries\n");
853 goto err0;
854 }
855
856 cq->cq_type = QEDR_CQ_TYPE_USER;
857
b0ea0fa5
JG		 858 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
859 ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
860 1);
a7efd777
RA		 861 if (rc)
862 goto err0;
863
864 pbl_ptr = cq->q.pbl_tbl->pa;
865 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 866
867 cq->ibcq.cqe = chain_entries;
a7efd777
RA		 868 } else {
869 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
870
871 rc = dev->ops->common->chain_alloc(dev->cdev,
872 QED_CHAIN_USE_TO_CONSUME,
873 QED_CHAIN_MODE_PBL,
874 QED_CHAIN_CNT_TYPE_U32,
875 chain_entries,
876 sizeof(union rdma_cqe),
1a4a6975 877 &cq->pbl, NULL);
a7efd777
RA		 878 if (rc)
879 goto err1;
880
881 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
882 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 883 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 884 }
885
886 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
887 pbl_ptr, &params);
888
889 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
890 if (rc)
891 goto err2;
892
893 cq->icid = icid;
894 cq->sig = QEDR_CQ_MAGIC_NUMBER;
895 spin_lock_init(&cq->cq_lock);
896
ff23dfa1 897 if (udata) {
a7efd777
RA		 898 rc = qedr_copy_cq_uresp(dev, cq, udata);
899 if (rc)
900 goto err3;
901 } else {
902 /* Generate doorbell address. */
903 cq->db_addr = dev->db_addr +
904 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
905 cq->db.data.icid = cq->icid;
906 cq->db.data.params = DB_AGG_CMD_SET <<
907 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
908
909 /* point to the very last element, passing it we will toggle */
910 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
911 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
912 cq->latest_cqe = NULL;
913 consume_cqe(cq);
914 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
915 }
916
917 DP_DEBUG(dev, QEDR_MSG_CQ,
918 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
919 cq->icid, cq, params.cq_size);
920
e39afe3d 921 return 0;
a7efd777
RA		 922
923err3:
924 destroy_iparams.icid = cq->icid;
925 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
926 &destroy_oparams);
927err2:
928 if (udata)
929 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
930 else
931 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
932err1:
933 if (udata)
934 ib_umem_release(cq->q.umem);
935err0:
e39afe3d 936 return -EINVAL;
a7efd777
RA		 937}
938
939int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
940{
941 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
942 struct qedr_cq *cq = get_qedr_cq(ibcq);
943
944 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
945
946 return 0;
947}
948
4dd72636
AR		 949#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
950#define QEDR_DESTROY_CQ_ITER_DURATION (10)
951
a52c8e24 952void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 953{
954 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
955 struct qed_rdma_destroy_cq_out_params oparams;
956 struct qed_rdma_destroy_cq_in_params iparams;
957 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 958 int iter;
a7efd777 959
942b3b2c 960 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 961
4dd72636
AR		 962 cq->destroyed = 1;
963
a7efd777 964 /* GSIs CQs are handled by driver, so they don't exist in the FW */
942b3b2c 965 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
e39afe3d 966 return;
a1211359 967
942b3b2c 968 iparams.icid = cq->icid;
a52c8e24 969 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
942b3b2c 970 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777 971
bdeacabd 972 if (udata) {
a7efd777
RA		 973 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
974 ib_umem_release(cq->q.umem);
975 }
976
4dd72636
AR		 977 /* We don't want the IRQ handler to handle a non-existing CQ so we
978 * wait until all CNQ interrupts, if any, are received. This will always
979 * happen and will always happen very fast. If not, then a serious error
980 * has occured. That is why we can use a long delay.
981 * We spin for a short time so we don’t lose time on context switching
982 * in case all the completions are handled in that span. Otherwise
983 * we sleep for a while and check again. Since the CNQ may be
984 * associated with (only) the current CPU we use msleep to allow the
985 * current CPU to be freed.
986 * The CNQ notification is increased in qedr_irq_handler().
987 */
988 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
989 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
990 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
991 iter--;
992 }
993
994 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
995 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
996 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
997 iter--;
998 }
999
4dd72636
AR		 1000 /* Note that we don't need to have explicit code to wait for the
1001 * completion of the event handler because it is invoked from the EQ.
1002 * Since the destroy CQ ramrod has also been received on the EQ we can
1003 * be certain that there's no event handler in process.
1004 */
a7efd777 1005}
cecbcddf
RA		 1006
1007static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1008 struct ib_qp_attr *attr,
1009 int attr_mask,
1010 struct qed_rdma_modify_qp_in_params
1011 *qp_params)
1012{
47ec3866 1013 const struct ib_gid_attr *gid_attr;
cecbcddf 1014 enum rdma_network_type nw_type;
d8966fcd 1015 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1016 u32 ipv4_addr;
a70c0739 1017 int ret;
cecbcddf
RA		 1018 int i;
1019
47ec3866 1020 gid_attr = grh->sgid_attr;
a70c0739
PP		 1021 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1022 if (ret)
1023 return ret;
3e44e0ee 1024
47ec3866 1025 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1026 switch (nw_type) {
1027 case RDMA_NETWORK_IPV6:
47ec3866 1028 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1029 sizeof(qp_params->sgid));
1030 memcpy(&qp_params->dgid.bytes[0],
1031 &grh->dgid,
1032 sizeof(qp_params->dgid));
1033 qp_params->roce_mode = ROCE_V2_IPV6;
1034 SET_FIELD(qp_params->modify_flags,
1035 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1036 break;
1037 case RDMA_NETWORK_IB:
47ec3866 1038 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1039 sizeof(qp_params->sgid));
1040 memcpy(&qp_params->dgid.bytes[0],
1041 &grh->dgid,
1042 sizeof(qp_params->dgid));
1043 qp_params->roce_mode = ROCE_V1;
1044 break;
1045 case RDMA_NETWORK_IPV4:
1046 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1047 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1048 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1049 qp_params->sgid.ipv4_addr = ipv4_addr;
1050 ipv4_addr =
1051 qedr_get_ipv4_from_gid(grh->dgid.raw);
1052 qp_params->dgid.ipv4_addr = ipv4_addr;
1053 SET_FIELD(qp_params->modify_flags,
1054 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1055 qp_params->roce_mode = ROCE_V2_IPV4;
1056 break;
cecbcddf
RA		 1057 }
1058
1059 for (i = 0; i < 4; i++) {
1060 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1061 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1062 }
1063
1064 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1065 qp_params->vlan_id = 0;
1066
1067 return 0;
1068}
1069
cecbcddf 1070static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1071 struct ib_qp_init_attr *attrs,
1072 struct ib_udata *udata)
cecbcddf
RA		 1073{
1074 struct qedr_device_attr *qattr = &dev->attr;
1075
1076 /* QP0... attrs->qp_type == IB_QPT_GSI */
1077 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1078 DP_DEBUG(dev, QEDR_MSG_QP,
1079 "create qp: unsupported qp type=0x%x requested\n",
1080 attrs->qp_type);
1081 return -EINVAL;
1082 }
1083
1084 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1085 DP_ERR(dev,
1086 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1087 attrs->cap.max_send_wr, qattr->max_sqe);
1088 return -EINVAL;
1089 }
1090
1091 if (attrs->cap.max_inline_data > qattr->max_inline) {
1092 DP_ERR(dev,
1093 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1094 attrs->cap.max_inline_data, qattr->max_inline);
1095 return -EINVAL;
1096 }
1097
1098 if (attrs->cap.max_send_sge > qattr->max_sge) {
1099 DP_ERR(dev,
1100 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1101 attrs->cap.max_send_sge, qattr->max_sge);
1102 return -EINVAL;
1103 }
1104
1105 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1106 DP_ERR(dev,
1107 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1108 attrs->cap.max_recv_sge, qattr->max_sge);
1109 return -EINVAL;
1110 }
1111
1112 /* Unprivileged user space cannot create special QP */
e00b64f7 1113 if (udata && attrs->qp_type == IB_QPT_GSI) {
cecbcddf
RA		 1114 DP_ERR(dev,
1115 "create qp: userspace can't create special QPs of type=0x%x\n",
1116 attrs->qp_type);
1117 return -EINVAL;
1118 }
1119
1120 return 0;
1121}
1122
40b173dd
YB		 1123static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1124 struct qedr_srq *srq, struct ib_udata *udata)
1125{
1126 struct qedr_create_srq_uresp uresp = {};
1127 int rc;
1128
1129 uresp.srq_id = srq->srq_id;
1130
1131 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1132 if (rc)
1133 DP_ERR(dev, "create srq: problem copying data to user space\n");
1134
1135 return rc;
1136}
1137
69ad0e7f
KM		 1138static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1139 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1140 struct qedr_qp *qp)
1141{
69ad0e7f
KM		 1142 /* iWARP requires two doorbells per RQ. */
1143 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1144 uresp->rq_db_offset =
1145 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1146 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1147 } else {
1148 uresp->rq_db_offset =
1149 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1150 }
1151
cecbcddf
RA		 1152 uresp->rq_icid = qp->icid;
1153}
1154
69ad0e7f
KM		 1155static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1156 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1157 struct qedr_qp *qp)
1158{
1159 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1160
1161 /* iWARP uses the same cid for rq and sq */
1162 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1163 uresp->sq_icid = qp->icid;
1164 else
1165 uresp->sq_icid = qp->icid + 1;
cecbcddf
RA		 1166}
1167
1168static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1169 struct qedr_qp *qp, struct ib_udata *udata)
1170{
1171 struct qedr_create_qp_uresp uresp;
1172 int rc;
1173
1174 memset(&uresp, 0, sizeof(uresp));
69ad0e7f
KM		 1175 qedr_copy_sq_uresp(dev, &uresp, qp);
1176 qedr_copy_rq_uresp(dev, &uresp, qp);
cecbcddf
RA		 1177
1178 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1179 uresp.qp_id = qp->qp_id;
1180
c75d3ec8 1181 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
cecbcddf
RA		 1182 if (rc)
1183 DP_ERR(dev,
1184 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1185 qp->icid);
1186
1187 return rc;
1188}
1189
df158561
AR		 1190static void qedr_set_common_qp_params(struct qedr_dev *dev,
1191 struct qedr_qp *qp,
1192 struct qedr_pd *pd,
1193 struct ib_qp_init_attr *attrs)
cecbcddf 1194{
cecbcddf 1195 spin_lock_init(&qp->q_lock);
de0089e6 1196 atomic_set(&qp->refcnt, 1);
df158561 1197 qp->pd = pd;
cecbcddf
RA		 1198 qp->qp_type = attrs->qp_type;
1199 qp->max_inline_data = attrs->cap.max_inline_data;
1200 qp->sq.max_sges = attrs->cap.max_send_sge;
1201 qp->state = QED_ROCE_QP_STATE_RESET;
1202 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1203 qp->sq_cq = get_qedr_cq(attrs->send_cq);
cecbcddf
RA		 1204 qp->dev = dev;
1205
3491c9e7
YB		 1206 if (attrs->srq) {
1207 qp->srq = get_qedr_srq(attrs->srq);
1208 } else {
1209 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1210 qp->rq.max_sges = attrs->cap.max_recv_sge;
1211 DP_DEBUG(dev, QEDR_MSG_QP,
1212 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1213 qp->rq.max_sges, qp->rq_cq->icid);
1214 }
1215
cecbcddf
RA		 1216 DP_DEBUG(dev, QEDR_MSG_QP,
1217 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1218 pd->pd_id, qp->qp_type, qp->max_inline_data,
1219 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1220 DP_DEBUG(dev, QEDR_MSG_QP,
1221 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1222 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1223}
1224
df158561 1225static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf
RA		 1226{
1227 qp->sq.db = dev->db_addr +
1228 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1229 qp->sq.db_data.data.icid = qp->icid + 1;
3491c9e7
YB		 1230 if (!qp->srq) {
1231 qp->rq.db = dev->db_addr +
1232 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1233 qp->rq.db_data.data.icid = qp->icid;
1234 }
1235}
1236
68e326de 1237static int qedr_check_srq_params(struct qedr_dev *dev,
3491c9e7
YB		 1238 struct ib_srq_init_attr *attrs,
1239 struct ib_udata *udata)
1240{
1241 struct qedr_device_attr *qattr = &dev->attr;
1242
1243 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1244 DP_ERR(dev,
1245 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1246 attrs->attr.max_wr, qattr->max_srq_wr);
1247 return -EINVAL;
1248 }
1249
1250 if (attrs->attr.max_sge > qattr->max_sge) {
1251 DP_ERR(dev,
1252 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1253 attrs->attr.max_sge, qattr->max_sge);
1254 return -EINVAL;
1255 }
1256
1257 return 0;
1258}
1259
40b173dd
YB		 1260static void qedr_free_srq_user_params(struct qedr_srq *srq)
1261{
1262 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1263 ib_umem_release(srq->usrq.umem);
1264 ib_umem_release(srq->prod_umem);
1265}
1266
3491c9e7
YB		 1267static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1268{
1269 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1270 struct qedr_dev *dev = srq->dev;
1271
1272 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1273
1274 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1275 hw_srq->virt_prod_pair_addr,
1276 hw_srq->phy_prod_pair_addr);
1277}
1278
b0ea0fa5 1279static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1280 struct qedr_srq *srq,
1281 struct qedr_create_srq_ureq *ureq,
1282 int access, int dmasync)
1283{
1284 struct scatterlist *sg;
1285 int rc;
1286
b0ea0fa5 1287 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
40b173dd
YB		 1288 ureq->srq_len, access, dmasync, 1);
1289 if (rc)
1290 return rc;
1291
b0ea0fa5
JG		 1292 srq->prod_umem =
1293 ib_umem_get(udata, ureq->prod_pair_addr,
1294 sizeof(struct rdma_srq_producers), access, dmasync);
40b173dd
YB		 1295 if (IS_ERR(srq->prod_umem)) {
1296 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1297 ib_umem_release(srq->usrq.umem);
1298 DP_ERR(srq->dev,
1299 "create srq: failed ib_umem_get for producer, got %ld\n",
1300 PTR_ERR(srq->prod_umem));
1301 return PTR_ERR(srq->prod_umem);
1302 }
1303
1304 sg = srq->prod_umem->sg_head.sgl;
1305 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1306
1307 return 0;
1308}
1309
3491c9e7
YB		 1310static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1311 struct qedr_dev *dev,
1312 struct ib_srq_init_attr *init_attr)
1313{
1314 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1315 dma_addr_t phy_prod_pair_addr;
1316 u32 num_elems;
1317 void *va;
1318 int rc;
1319
1320 va = dma_alloc_coherent(&dev->pdev->dev,
1321 sizeof(struct rdma_srq_producers),
1322 &phy_prod_pair_addr, GFP_KERNEL);
1323 if (!va) {
1324 DP_ERR(dev,
1325 "create srq: failed to allocate dma memory for producer\n");
1326 return -ENOMEM;
1327 }
1328
1329 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1330 hw_srq->virt_prod_pair_addr = va;
1331
1332 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1333 rc = dev->ops->common->chain_alloc(dev->cdev,
1334 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1335 QED_CHAIN_MODE_PBL,
1336 QED_CHAIN_CNT_TYPE_U32,
1337 num_elems,
1338 QEDR_SRQ_WQE_ELEM_SIZE,
1339 &hw_srq->pbl, NULL);
1340 if (rc)
1341 goto err0;
1342
1343 hw_srq->num_elems = num_elems;
1344
1345 return 0;
1346
1347err0:
1348 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1349 va, phy_prod_pair_addr);
1350 return rc;
1351}
1352
68e326de
LR		 1353int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1354 struct ib_udata *udata)
3491c9e7
YB		 1355{
1356 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1357 struct qed_rdma_create_srq_in_params in_params = {};
68e326de 1358 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
3491c9e7 1359 struct qed_rdma_create_srq_out_params out_params;
68e326de 1360 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
40b173dd 1361 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1362 u64 pbl_base_addr, phy_prod_pair_addr;
1363 struct qedr_srq_hwq_info *hw_srq;
1364 u32 page_cnt, page_size;
68e326de 1365 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3491c9e7
YB		 1366 int rc = 0;
1367
1368 DP_DEBUG(dev, QEDR_MSG_QP,
1369 "create SRQ called from %s (pd %p)\n",
1370 (udata) ? "User lib" : "kernel", pd);
1371
68e326de 1372 rc = qedr_check_srq_params(dev, init_attr, udata);
3491c9e7 1373 if (rc)
68e326de 1374 return -EINVAL;
3491c9e7
YB		 1375
1376 srq->dev = dev;
1377 hw_srq = &srq->hw_srq;
1378 spin_lock_init(&srq->lock);
1379
1380 hw_srq->max_wr = init_attr->attr.max_wr;
1381 hw_srq->max_sges = init_attr->attr.max_sge;
1382
89944450 1383 if (udata) {
40b173dd
YB		 1384 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1385 DP_ERR(dev,
1386 "create srq: problem copying data from user space\n");
1387 goto err0;
1388 }
1389
b0ea0fa5 1390 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
40b173dd
YB		 1391 if (rc)
1392 goto err0;
1393
1394 page_cnt = srq->usrq.pbl_info.num_pbes;
1395 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1396 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1397 page_size = PAGE_SIZE;
40b173dd
YB		 1398 } else {
1399 struct qed_chain *pbl;
1400
1401 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1402 if (rc)
1403 goto err0;
1404
1405 pbl = &hw_srq->pbl;
1406 page_cnt = qed_chain_get_page_cnt(pbl);
1407 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1408 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1409 page_size = QED_CHAIN_PAGE_SIZE;
1410 }
3491c9e7 1411
3491c9e7
YB		 1412 in_params.pd_id = pd->pd_id;
1413 in_params.pbl_base_addr = pbl_base_addr;
1414 in_params.prod_pair_addr = phy_prod_pair_addr;
1415 in_params.num_pages = page_cnt;
1416 in_params.page_size = page_size;
1417
1418 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1419 if (rc)
1420 goto err1;
1421
1422 srq->srq_id = out_params.srq_id;
1423
40b173dd
YB		 1424 if (udata) {
1425 rc = qedr_copy_srq_uresp(dev, srq, udata);
1426 if (rc)
1427 goto err2;
1428 }
1429
9fd15987 1430 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1431 if (rc)
1432 goto err2;
1433
1434 DP_DEBUG(dev, QEDR_MSG_SRQ,
1435 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
68e326de 1436 return 0;
3491c9e7
YB		 1437
1438err2:
1439 destroy_in_params.srq_id = srq->srq_id;
1440
1441 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1442err1:
40b173dd
YB		 1443 if (udata)
1444 qedr_free_srq_user_params(srq);
1445 else
1446 qedr_free_srq_kernel_params(srq);
3491c9e7 1447err0:
68e326de 1448 return -EFAULT;
3491c9e7
YB		 1449}
1450
68e326de 1451void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1452{
1453 struct qed_rdma_destroy_srq_in_params in_params = {};
1454 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1455 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1456
9fd15987 1457 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7
YB		 1458 in_params.srq_id = srq->srq_id;
1459 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1460
e00b64f7 1461 if (ibsrq->uobject)
40b173dd
YB		 1462 qedr_free_srq_user_params(srq);
1463 else
1464 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1465
1466 DP_DEBUG(dev, QEDR_MSG_SRQ,
1467 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1468 srq->srq_id);
3491c9e7
YB		 1469}
1470
1471int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1472 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1473{
1474 struct qed_rdma_modify_srq_in_params in_params = {};
1475 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1476 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1477 int rc;
1478
1479 if (attr_mask & IB_SRQ_MAX_WR) {
1480 DP_ERR(dev,
1481 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1482 attr_mask, srq);
1483 return -EINVAL;
1484 }
1485
1486 if (attr_mask & IB_SRQ_LIMIT) {
1487 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1488 DP_ERR(dev,
1489 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1490 attr->srq_limit, srq->hw_srq.max_wr);
1491 return -EINVAL;
1492 }
1493
1494 in_params.srq_id = srq->srq_id;
1495 in_params.wqe_limit = attr->srq_limit;
1496 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1497 if (rc)
1498 return rc;
1499 }
1500
1501 srq->srq_limit = attr->srq_limit;
1502
1503 DP_DEBUG(dev, QEDR_MSG_SRQ,
1504 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1505
1506 return 0;
cecbcddf
RA		 1507}
1508
1509static inline void
df158561
AR		 1510qedr_init_common_qp_in_params(struct qedr_dev *dev,
1511 struct qedr_pd *pd,
1512 struct qedr_qp *qp,
1513 struct ib_qp_init_attr *attrs,
1514 bool fmr_and_reserved_lkey,
1515 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1516{
df158561
AR		 1517 /* QP handle to be written in an async event */
1518 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1519 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1520
1521 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1522 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1523 params->pd = pd->pd_id;
1524 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1525 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1526 params->stats_queue = 0;
df158561
AR		 1527 params->srq_id = 0;
1528 params->use_srq = false;
3491c9e7
YB		 1529
1530 if (!qp->srq) {
1531 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1532
1533 } else {
1534 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1535 params->srq_id = qp->srq->srq_id;
1536 params->use_srq = true;
1537 }
cecbcddf
RA		 1538}
1539
df158561 1540static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1541{
df158561
AR		 1542 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1543 "qp=%p. "
1544 "sq_addr=0x%llx, "
1545 "sq_len=%zd, "
1546 "rq_addr=0x%llx, "
1547 "rq_len=%zd"
1548 "\n",
1549 qp,
1550 qp->usq.buf_addr,
1551 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1552}
cecbcddf 1553
69ad0e7f
KM		 1554static inline void
1555qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1556 struct qedr_qp *qp,
1557 struct qed_rdma_create_qp_out_params *out_params)
1558{
1559 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1560 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1561
1562 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1563 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1564 if (!qp->srq) {
1565 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1566 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1567 }
69ad0e7f
KM		 1568
1569 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1570 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1571}
1572
df158561
AR		 1573static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1574{
836a0fbb 1575 ib_umem_release(qp->usq.umem);
df158561 1576 qp->usq.umem = NULL;
cecbcddf 1577
836a0fbb 1578 ib_umem_release(qp->urq.umem);
df158561 1579 qp->urq.umem = NULL;
cecbcddf
RA		 1580}
1581
df158561
AR		 1582static int qedr_create_user_qp(struct qedr_dev *dev,
1583 struct qedr_qp *qp,
1584 struct ib_pd *ibpd,
1585 struct ib_udata *udata,
1586 struct ib_qp_init_attr *attrs)
cecbcddf 1587{
df158561
AR		 1588 struct qed_rdma_create_qp_in_params in_params;
1589 struct qed_rdma_create_qp_out_params out_params;
1590 struct qedr_pd *pd = get_qedr_pd(ibpd);
df158561 1591 struct qedr_create_qp_ureq ureq;
69ad0e7f 1592 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
df158561 1593 int rc = -EINVAL;
cecbcddf 1594
df158561
AR		 1595 memset(&ureq, 0, sizeof(ureq));
1596 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1597 if (rc) {
1598 DP_ERR(dev, "Problem copying data from user space\n");
1599 return rc;
1600 }
cecbcddf 1601
df158561 1602 /* SQ - read access only (0), dma sync not required (0) */
b0ea0fa5 1603 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
69ad0e7f 1604 ureq.sq_len, 0, 0, alloc_and_init);
df158561
AR		 1605 if (rc)
1606 return rc;
cecbcddf 1607
40b173dd
YB		 1608 if (!qp->srq) {
1609 /* RQ - read access only (0), dma sync not required (0) */
b0ea0fa5 1610 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
40b173dd
YB		 1611 ureq.rq_len, 0, 0, alloc_and_init);
1612 if (rc)
1613 return rc;
1614 }
df158561
AR		 1615
1616 memset(&in_params, 0, sizeof(in_params));
1617 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1618 in_params.qp_handle_lo = ureq.qp_handle_lo;
1619 in_params.qp_handle_hi = ureq.qp_handle_hi;
1620 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1621 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
40b173dd
YB		 1622 if (!qp->srq) {
1623 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1624 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1625 }
df158561
AR		 1626
1627 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1628 &in_params, &out_params);
1629
1630 if (!qp->qed_qp) {
1631 rc = -ENOMEM;
1632 goto err1;
1633 }
1634
69ad0e7f
KM		 1635 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1636 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1637
df158561
AR		 1638 qp->qp_id = out_params.qp_id;
1639 qp->icid = out_params.icid;
1640
1641 rc = qedr_copy_qp_uresp(dev, qp, udata);
1642 if (rc)
1643 goto err;
1644
1645 qedr_qp_user_print(dev, qp);
cecbcddf
RA		 1646
1647 return 0;
df158561
AR		 1648err:
1649 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1650 if (rc)
1651 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1652
1653err1:
1654 qedr_cleanup_user(dev, qp);
1655 return rc;
cecbcddf
RA		 1656}
1657
f5b1b177
KM		 1658static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1659{
1660 qp->sq.db = dev->db_addr +
1661 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1662 qp->sq.db_data.data.icid = qp->icid;
1663
1664 qp->rq.db = dev->db_addr +
1665 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1666 qp->rq.db_data.data.icid = qp->icid;
1667 qp->rq.iwarp_db2 = dev->db_addr +
1668 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1669 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1670 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1671}
1672
df158561
AR		 1673static int
1674qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1675 struct qedr_qp *qp,
1676 struct qed_rdma_create_qp_in_params *in_params,
1677 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1678{
df158561 1679 struct qed_rdma_create_qp_out_params out_params;
cecbcddf
RA		 1680 int rc;
1681
cecbcddf
RA		 1682 rc = dev->ops->common->chain_alloc(dev->cdev,
1683 QED_CHAIN_USE_TO_PRODUCE,
1684 QED_CHAIN_MODE_PBL,
1685 QED_CHAIN_CNT_TYPE_U32,
1686 n_sq_elems,
1687 QEDR_SQE_ELEMENT_SIZE,
1a4a6975 1688 &qp->sq.pbl, NULL);
cecbcddf 1689
df158561
AR		 1690 if (rc)
1691 return rc;
cecbcddf 1692
df158561
AR		 1693 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1694 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1695
cecbcddf
RA		 1696 rc = dev->ops->common->chain_alloc(dev->cdev,
1697 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1698 QED_CHAIN_MODE_PBL,
1699 QED_CHAIN_CNT_TYPE_U32,
1700 n_rq_elems,
1701 QEDR_RQE_ELEMENT_SIZE,
1a4a6975 1702 &qp->rq.pbl, NULL);
df158561
AR		 1703 if (rc)
1704 return rc;
cecbcddf 1705
df158561
AR		 1706 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1707 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1708
df158561
AR		 1709 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1710 in_params, &out_params);
cecbcddf 1711
df158561
AR		 1712 if (!qp->qed_qp)
1713 return -EINVAL;
cecbcddf 1714
df158561
AR		 1715 qp->qp_id = out_params.qp_id;
1716 qp->icid = out_params.icid;
cecbcddf 1717
df158561 1718 qedr_set_roce_db_info(dev, qp);
f5b1b177
KM		 1719 return rc;
1720}
cecbcddf 1721
f5b1b177
KM		 1722static int
1723qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1724 struct qedr_qp *qp,
1725 struct qed_rdma_create_qp_in_params *in_params,
1726 u32 n_sq_elems, u32 n_rq_elems)
1727{
1728 struct qed_rdma_create_qp_out_params out_params;
1729 struct qed_chain_ext_pbl ext_pbl;
1730 int rc;
1731
1732 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1733 QEDR_SQE_ELEMENT_SIZE,
1734 QED_CHAIN_MODE_PBL);
1735 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1736 QEDR_RQE_ELEMENT_SIZE,
1737 QED_CHAIN_MODE_PBL);
1738
1739 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1740 in_params, &out_params);
1741
1742 if (!qp->qed_qp)
1743 return -EINVAL;
1744
1745 /* Now we allocate the chain */
1746 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1747 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1748
1749 rc = dev->ops->common->chain_alloc(dev->cdev,
1750 QED_CHAIN_USE_TO_PRODUCE,
1751 QED_CHAIN_MODE_PBL,
1752 QED_CHAIN_CNT_TYPE_U32,
1753 n_sq_elems,
1754 QEDR_SQE_ELEMENT_SIZE,
1755 &qp->sq.pbl, &ext_pbl);
1756
1757 if (rc)
1758 goto err;
1759
1760 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1761 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1762
1763 rc = dev->ops->common->chain_alloc(dev->cdev,
1764 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1765 QED_CHAIN_MODE_PBL,
1766 QED_CHAIN_CNT_TYPE_U32,
1767 n_rq_elems,
1768 QEDR_RQE_ELEMENT_SIZE,
1769 &qp->rq.pbl, &ext_pbl);
1770
1771 if (rc)
1772 goto err;
1773
1774 qp->qp_id = out_params.qp_id;
1775 qp->icid = out_params.icid;
1776
1777 qedr_set_iwarp_db_info(dev, qp);
1778 return rc;
1779
1780err:
1781 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1782
1783 return rc;
cecbcddf
RA		 1784}
1785
df158561 1786static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1787{
df158561
AR		 1788 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1789 kfree(qp->wqe_wr_id);
cecbcddf 1790
df158561
AR		 1791 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1792 kfree(qp->rqe_wr_id);
cecbcddf
RA		 1793}
1794
df158561
AR		 1795static int qedr_create_kernel_qp(struct qedr_dev *dev,
1796 struct qedr_qp *qp,
1797 struct ib_pd *ibpd,
1798 struct ib_qp_init_attr *attrs)
cecbcddf 1799{
df158561
AR		 1800 struct qed_rdma_create_qp_in_params in_params;
1801 struct qedr_pd *pd = get_qedr_pd(ibpd);
1802 int rc = -EINVAL;
1803 u32 n_rq_elems;
1804 u32 n_sq_elems;
1805 u32 n_sq_entries;
cecbcddf 1806
df158561 1807 memset(&in_params, 0, sizeof(in_params));
cecbcddf 1808
df158561
AR		 1809 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1810 * the ring. The ring should allow at least a single WR, even if the
1811 * user requested none, due to allocation issues.
1812 * We should add an extra WR since the prod and cons indices of
1813 * wqe_wr_id are managed in such a way that the WQ is considered full
1814 * when (prod+1)%max_wr==cons. We currently don't do that because we
1815 * double the number of entries due an iSER issue that pushes far more
1816 * WRs than indicated. If we decline its ib_post_send() then we get
1817 * error prints in the dmesg we'd like to avoid.
1818 */
1819 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1820 dev->attr.max_sqe);
cecbcddf 1821
6396bb22 1822 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 1823 GFP_KERNEL);
1824 if (!qp->wqe_wr_id) {
1825 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1826 return -ENOMEM;
1827 }
cecbcddf 1828
df158561
AR		 1829 /* QP handle to be written in CQE */
1830 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1831 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 1832
df158561
AR		 1833 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1834 * the ring. There ring should allow at least a single WR, even if the
1835 * user requested none, due to allocation issues.
1836 */
1837 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 1838
df158561 1839 /* Allocate driver internal RQ array */
6396bb22 1840 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 1841 GFP_KERNEL);
1842 if (!qp->rqe_wr_id) {
1843 DP_ERR(dev,
1844 "create qp: failed RQ shadow memory allocation\n");
1845 kfree(qp->wqe_wr_id);
1846 return -ENOMEM;
cecbcddf
RA		 1847 }
1848
df158561 1849 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 1850
df158561
AR		 1851 n_sq_entries = attrs->cap.max_send_wr;
1852 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1853 n_sq_entries = max_t(u32, n_sq_entries, 1);
1854 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 1855
df158561
AR		 1856 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1857
f5b1b177
KM		 1858 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1859 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1860 n_sq_elems, n_rq_elems);
1861 else
1862 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1863 n_sq_elems, n_rq_elems);
df158561
AR		 1864 if (rc)
1865 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 1866
1867 return rc;
1868}
1869
1870struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1871 struct ib_qp_init_attr *attrs,
1872 struct ib_udata *udata)
1873{
1874 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 1875 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 1876 struct qedr_qp *qp;
181d8015 1877 struct ib_qp *ibqp;
cecbcddf
RA		 1878 int rc = 0;
1879
1880 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1881 udata ? "user library" : "kernel", pd);
1882
e00b64f7 1883 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf
RA		 1884 if (rc)
1885 return ERR_PTR(rc);
1886
cecbcddf 1887 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 1888 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1889 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 1890 get_qedr_cq(attrs->send_cq),
1891 get_qedr_cq(attrs->send_cq)->icid,
1892 get_qedr_cq(attrs->recv_cq),
3491c9e7 1893 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 1894
df158561
AR		 1895 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1896 if (!qp) {
1897 DP_ERR(dev, "create qp: failed allocating memory\n");
1898 return ERR_PTR(-ENOMEM);
1899 }
1900
1901 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 1902
04886779 1903 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 1904 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1905 if (IS_ERR(ibqp))
1906 kfree(qp);
1907 return ibqp;
04886779
RA		 1908 }
1909
df158561
AR		 1910 if (udata)
1911 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1912 else
1913 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 1914
df158561
AR		 1915 if (rc)
1916 goto err;
cecbcddf 1917
cecbcddf
RA		 1918 qp->ibqp.qp_num = qp->qp_id;
1919
1212767e 1920 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 1921 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e
YB		 1922 if (rc)
1923 goto err;
1924 }
de0089e6 1925
cecbcddf
RA		 1926 return &qp->ibqp;
1927
df158561 1928err:
cecbcddf
RA		 1929 kfree(qp);
1930
1931 return ERR_PTR(-EFAULT);
1932}
1933
27a4b1a6 1934static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 1935{
1936 switch (qp_state) {
1937 case QED_ROCE_QP_STATE_RESET:
1938 return IB_QPS_RESET;
1939 case QED_ROCE_QP_STATE_INIT:
1940 return IB_QPS_INIT;
1941 case QED_ROCE_QP_STATE_RTR:
1942 return IB_QPS_RTR;
1943 case QED_ROCE_QP_STATE_RTS:
1944 return IB_QPS_RTS;
1945 case QED_ROCE_QP_STATE_SQD:
1946 return IB_QPS_SQD;
1947 case QED_ROCE_QP_STATE_ERR:
1948 return IB_QPS_ERR;
1949 case QED_ROCE_QP_STATE_SQE:
1950 return IB_QPS_SQE;
1951 }
1952 return IB_QPS_ERR;
1953}
1954
27a4b1a6
RA		 1955static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1956 enum ib_qp_state qp_state)
cecbcddf
RA		 1957{
1958 switch (qp_state) {
1959 case IB_QPS_RESET:
1960 return QED_ROCE_QP_STATE_RESET;
1961 case IB_QPS_INIT:
1962 return QED_ROCE_QP_STATE_INIT;
1963 case IB_QPS_RTR:
1964 return QED_ROCE_QP_STATE_RTR;
1965 case IB_QPS_RTS:
1966 return QED_ROCE_QP_STATE_RTS;
1967 case IB_QPS_SQD:
1968 return QED_ROCE_QP_STATE_SQD;
1969 case IB_QPS_ERR:
1970 return QED_ROCE_QP_STATE_ERR;
1971 default:
1972 return QED_ROCE_QP_STATE_ERR;
1973 }
1974}
1975
1976static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1977{
1978 qed_chain_reset(&qph->pbl);
1979 qph->prod = 0;
1980 qph->cons = 0;
1981 qph->wqe_cons = 0;
1982 qph->db_data.data.value = cpu_to_le16(0);
1983}
1984
1985static int qedr_update_qp_state(struct qedr_dev *dev,
1986 struct qedr_qp *qp,
caf61b1b 1987 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 1988 enum qed_roce_qp_state new_state)
1989{
1990 int status = 0;
1991
caf61b1b 1992 if (new_state == cur_state)
865cea40 1993 return 0;
cecbcddf 1994
caf61b1b 1995 switch (cur_state) {
cecbcddf
RA		 1996 case QED_ROCE_QP_STATE_RESET:
1997 switch (new_state) {
1998 case QED_ROCE_QP_STATE_INIT:
1999 qp->prev_wqe_size = 0;
2000 qedr_reset_qp_hwq_info(&qp->sq);
2001 qedr_reset_qp_hwq_info(&qp->rq);
2002 break;
2003 default:
2004 status = -EINVAL;
2005 break;
790b57f6 2006 }
cecbcddf
RA		 2007 break;
2008 case QED_ROCE_QP_STATE_INIT:
2009 switch (new_state) {
2010 case QED_ROCE_QP_STATE_RTR:
2011 /* Update doorbell (in case post_recv was
2012 * done before move to RTR)
2013 */
f5b1b177
KM		 2014
2015 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177 2016 writel(qp->rq.db_data.raw, qp->rq.db);
f5b1b177 2017 }
cecbcddf
RA		 2018 break;
2019 case QED_ROCE_QP_STATE_ERR:
2020 break;
2021 default:
2022 /* Invalid state change. */
2023 status = -EINVAL;
2024 break;
790b57f6 2025 }
cecbcddf
RA		 2026 break;
2027 case QED_ROCE_QP_STATE_RTR:
2028 /* RTR->XXX */
2029 switch (new_state) {
2030 case QED_ROCE_QP_STATE_RTS:
2031 break;
2032 case QED_ROCE_QP_STATE_ERR:
2033 break;
2034 default:
2035 /* Invalid state change. */
2036 status = -EINVAL;
2037 break;
790b57f6 2038 }
cecbcddf
RA		 2039 break;
2040 case QED_ROCE_QP_STATE_RTS:
2041 /* RTS->XXX */
2042 switch (new_state) {
2043 case QED_ROCE_QP_STATE_SQD:
2044 break;
2045 case QED_ROCE_QP_STATE_ERR:
2046 break;
2047 default:
2048 /* Invalid state change. */
2049 status = -EINVAL;
2050 break;
790b57f6 2051 }
cecbcddf
RA		 2052 break;
2053 case QED_ROCE_QP_STATE_SQD:
2054 /* SQD->XXX */
2055 switch (new_state) {
2056 case QED_ROCE_QP_STATE_RTS:
2057 case QED_ROCE_QP_STATE_ERR:
2058 break;
2059 default:
2060 /* Invalid state change. */
2061 status = -EINVAL;
2062 break;
790b57f6 2063 }
cecbcddf
RA		 2064 break;
2065 case QED_ROCE_QP_STATE_ERR:
2066 /* ERR->XXX */
2067 switch (new_state) {
2068 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2069 if ((qp->rq.prod != qp->rq.cons) ||
2070 (qp->sq.prod != qp->sq.cons)) {
2071 DP_NOTICE(dev,
2072 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2073 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2074 qp->sq.cons);
2075 status = -EINVAL;
2076 }
cecbcddf
RA		 2077 break;
2078 default:
2079 status = -EINVAL;
2080 break;
790b57f6 2081 }
cecbcddf
RA		 2082 break;
2083 default:
2084 status = -EINVAL;
2085 break;
790b57f6 2086 }
cecbcddf
RA		 2087
2088 return status;
2089}
2090
2091int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2092 int attr_mask, struct ib_udata *udata)
2093{
2094 struct qedr_qp *qp = get_qedr_qp(ibqp);
2095 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2096 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2097 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2098 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2099 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2100 int rc = 0;
2101
2102 DP_DEBUG(dev, QEDR_MSG_QP,
2103 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2104 attr->qp_state);
2105
2106 old_qp_state = qedr_get_ibqp_state(qp->state);
2107 if (attr_mask & IB_QP_STATE)
2108 new_qp_state = attr->qp_state;
2109 else
2110 new_qp_state = old_qp_state;
2111
f5b1b177
KM		 2112 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2113 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2114 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2115 DP_ERR(dev,
2116 "modify qp: invalid attribute mask=0x%x specified for\n"
2117 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2118 attr_mask, qp->qp_id, ibqp->qp_type,
2119 old_qp_state, new_qp_state);
2120 rc = -EINVAL;
2121 goto err;
2122 }
cecbcddf
RA		 2123 }
2124
2125 /* Translate the masks... */
2126 if (attr_mask & IB_QP_STATE) {
2127 SET_FIELD(qp_params.modify_flags,
2128 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2129 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2130 }
2131
2132 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2133 qp_params.sqd_async = true;
2134
2135 if (attr_mask & IB_QP_PKEY_INDEX) {
2136 SET_FIELD(qp_params.modify_flags,
2137 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2138 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2139 rc = -EINVAL;
2140 goto err;
2141 }
2142
2143 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2144 }
2145
2146 if (attr_mask & IB_QP_QKEY)
2147 qp->qkey = attr->qkey;
2148
2149 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2150 SET_FIELD(qp_params.modify_flags,
2151 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2152 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2153 IB_ACCESS_REMOTE_READ;
2154 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2155 IB_ACCESS_REMOTE_WRITE;
2156 qp_params.incoming_atomic_en = attr->qp_access_flags &
2157 IB_ACCESS_REMOTE_ATOMIC;
2158 }
2159
2160 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2161 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2162 return -EINVAL;
2163
cecbcddf
RA		 2164 if (attr_mask & IB_QP_PATH_MTU) {
2165 if (attr->path_mtu < IB_MTU_256 ||
2166 attr->path_mtu > IB_MTU_4096) {
2167 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2168 rc = -EINVAL;
2169 goto err;
2170 }
2171 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2172 ib_mtu_enum_to_int(iboe_get_mtu
2173 (dev->ndev->mtu)));
2174 }
2175
2176 if (!qp->mtu) {
2177 qp->mtu =
2178 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2179 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2180 }
2181
2182 SET_FIELD(qp_params.modify_flags,
2183 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2184
d8966fcd
DC		 2185 qp_params.traffic_class_tos = grh->traffic_class;
2186 qp_params.flow_label = grh->flow_label;
2187 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2188
d8966fcd 2189 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2190
2191 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2192 if (rc) {
2193 DP_ERR(dev,
2194 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2195 grh->sgid_index, rc);
cecbcddf
RA		 2196 return rc;
2197 }
2198
2199 rc = qedr_get_dmac(dev, &attr->ah_attr,
2200 qp_params.remote_mac_addr);
2201 if (rc)
2202 return rc;
2203
2204 qp_params.use_local_mac = true;
2205 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2206
2207 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2208 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2209 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2210 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2211 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2212 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2213 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2214 qp_params.remote_mac_addr);
cecbcddf
RA		 2215
2216 qp_params.mtu = qp->mtu;
2217 qp_params.lb_indication = false;
2218 }
2219
2220 if (!qp_params.mtu) {
2221 /* Stay with current MTU */
2222 if (qp->mtu)
2223 qp_params.mtu = qp->mtu;
2224 else
2225 qp_params.mtu =
2226 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2227 }
2228
2229 if (attr_mask & IB_QP_TIMEOUT) {
2230 SET_FIELD(qp_params.modify_flags,
2231 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2232
c3594f22
KM		 2233 /* The received timeout value is an exponent used like this:
2234 * "12.7.34 LOCAL ACK TIMEOUT
2235 * Value representing the transport (ACK) timeout for use by
2236 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2237 * The FW expects timeout in msec so we need to divide the usec
2238 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2239 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2240 * The value of zero means infinite so we use a 'max_t' to make
2241 * sure that sub 1 msec values will be configured as 1 msec.
2242 */
2243 if (attr->timeout)
2244 qp_params.ack_timeout =
2245 1 << max_t(int, attr->timeout - 8, 0);
2246 else
cecbcddf 2247 qp_params.ack_timeout = 0;
cecbcddf 2248 }
c3594f22 2249
cecbcddf
RA		 2250 if (attr_mask & IB_QP_RETRY_CNT) {
2251 SET_FIELD(qp_params.modify_flags,
2252 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2253 qp_params.retry_cnt = attr->retry_cnt;
2254 }
2255
2256 if (attr_mask & IB_QP_RNR_RETRY) {
2257 SET_FIELD(qp_params.modify_flags,
2258 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2259 qp_params.rnr_retry_cnt = attr->rnr_retry;
2260 }
2261
2262 if (attr_mask & IB_QP_RQ_PSN) {
2263 SET_FIELD(qp_params.modify_flags,
2264 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2265 qp_params.rq_psn = attr->rq_psn;
2266 qp->rq_psn = attr->rq_psn;
2267 }
2268
2269 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2270 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2271 rc = -EINVAL;
2272 DP_ERR(dev,
2273 "unsupported max_rd_atomic=%d, supported=%d\n",
2274 attr->max_rd_atomic,
2275 dev->attr.max_qp_req_rd_atomic_resc);
2276 goto err;
2277 }
2278
2279 SET_FIELD(qp_params.modify_flags,
2280 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2281 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2282 }
2283
2284 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2285 SET_FIELD(qp_params.modify_flags,
2286 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2287 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2288 }
2289
2290 if (attr_mask & IB_QP_SQ_PSN) {
2291 SET_FIELD(qp_params.modify_flags,
2292 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2293 qp_params.sq_psn = attr->sq_psn;
2294 qp->sq_psn = attr->sq_psn;
2295 }
2296
2297 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2298 if (attr->max_dest_rd_atomic >
2299 dev->attr.max_qp_resp_rd_atomic_resc) {
2300 DP_ERR(dev,
2301 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2302 attr->max_dest_rd_atomic,
2303 dev->attr.max_qp_resp_rd_atomic_resc);
2304
2305 rc = -EINVAL;
2306 goto err;
2307 }
2308
2309 SET_FIELD(qp_params.modify_flags,
2310 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2311 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2312 }
2313
2314 if (attr_mask & IB_QP_DEST_QPN) {
2315 SET_FIELD(qp_params.modify_flags,
2316 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2317
2318 qp_params.dest_qp = attr->dest_qp_num;
2319 qp->dest_qp_num = attr->dest_qp_num;
2320 }
2321
caf61b1b
KM		 2322 cur_state = qp->state;
2323
2324 /* Update the QP state before the actual ramrod to prevent a race with
2325 * fast path. Modifying the QP state to error will cause the device to
2326 * flush the CQEs and while polling the flushed CQEs will considered as
2327 * a potential issue if the QP isn't in error state.
2328 */
2329 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2330 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2331 qp->state = QED_ROCE_QP_STATE_ERR;
2332
cecbcddf
RA		 2333 if (qp->qp_type != IB_QPT_GSI)
2334 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2335 qp->qed_qp, &qp_params);
2336
2337 if (attr_mask & IB_QP_STATE) {
2338 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2339 rc = qedr_update_qp_state(dev, qp, cur_state,
2340 qp_params.new_state);
cecbcddf
RA		 2341 qp->state = qp_params.new_state;
2342 }
2343
2344err:
2345 return rc;
2346}
2347
2348static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2349{
2350 int ib_qp_acc_flags = 0;
2351
2352 if (params->incoming_rdma_write_en)
2353 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2354 if (params->incoming_rdma_read_en)
2355 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2356 if (params->incoming_atomic_en)
2357 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2358 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2359 return ib_qp_acc_flags;
2360}
2361
2362int qedr_query_qp(struct ib_qp *ibqp,
2363 struct ib_qp_attr *qp_attr,
2364 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2365{
2366 struct qed_rdma_query_qp_out_params params;
2367 struct qedr_qp *qp = get_qedr_qp(ibqp);
2368 struct qedr_dev *dev = qp->dev;
2369 int rc = 0;
2370
2371 memset(&params, 0, sizeof(params));
2372
2373 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2374 if (rc)
2375 goto err;
2376
2377 memset(qp_attr, 0, sizeof(*qp_attr));
2378 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2379
2380 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2381 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2382 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2383 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2384 qp_attr->rq_psn = params.rq_psn;
2385 qp_attr->sq_psn = params.sq_psn;
2386 qp_attr->dest_qp_num = params.dest_qp;
2387
2388 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2389
2390 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2391 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2392 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2393 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 2394 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 2395 qp_init_attr->cap = qp_attr->cap;
2396
44c58487 2397 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2398 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2399 params.flow_label, qp->sgid_idx,
2400 params.hop_limit_ttl, params.traffic_class_tos);
2401 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2402 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2403 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2404 qp_attr->timeout = params.timeout;
2405 qp_attr->rnr_retry = params.rnr_retry;
2406 qp_attr->retry_cnt = params.retry_cnt;
2407 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2408 qp_attr->pkey_index = params.pkey_index;
2409 qp_attr->port_num = 1;
d8966fcd
DC		 2410 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2411 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2412 qp_attr->alt_pkey_index = 0;
2413 qp_attr->alt_port_num = 0;
2414 qp_attr->alt_timeout = 0;
2415 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2416
2417 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2418 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2419 qp_attr->max_rd_atomic = params.max_rd_atomic;
2420 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2421
2422 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2423 qp_attr->cap.max_inline_data);
2424
2425err:
2426 return rc;
2427}
2428
bdeacabd
SR		 2429static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2430 struct ib_udata *udata)
df158561
AR		 2431{
2432 int rc = 0;
2433
2434 if (qp->qp_type != IB_QPT_GSI) {
2435 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2436 if (rc)
2437 return rc;
2438 }
2439
bdeacabd 2440 if (udata)
df158561
AR		 2441 qedr_cleanup_user(dev, qp);
2442 else
2443 qedr_cleanup_kernel(dev, qp);
2444
2445 return 0;
2446}
2447
c4367a26 2448int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2449{
2450 struct qedr_qp *qp = get_qedr_qp(ibqp);
2451 struct qedr_dev *dev = qp->dev;
2452 struct ib_qp_attr attr;
2453 int attr_mask = 0;
cecbcddf
RA		 2454
2455 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2456 qp, qp->qp_type);
2457
f5b1b177
KM		 2458 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2459 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2460 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2461 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2462
f5b1b177
KM		 2463 attr.qp_state = IB_QPS_ERR;
2464 attr_mask |= IB_QP_STATE;
cecbcddf 2465
f5b1b177
KM		 2466 /* Change the QP state to ERROR */
2467 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2468 }
e411e058
KM		 2469 } else {
2470 /* Wait for the connect/accept to complete */
2471 if (qp->ep) {
2472 int wait_count = 1;
2473
2474 while (qp->ep->during_connect) {
2475 DP_DEBUG(dev, QEDR_MSG_QP,
2476 "Still in during connect/accept\n");
2477
2478 msleep(100);
2479 if (wait_count++ > 200) {
2480 DP_NOTICE(dev,
2481 "during connect timeout\n");
2482 break;
2483 }
2484 }
2485 }
cecbcddf
RA		 2486 }
2487
df158561 2488 if (qp->qp_type == IB_QPT_GSI)
04886779 2489 qedr_destroy_gsi_qp(dev);
cecbcddf 2490
bdeacabd 2491 qedr_free_qp_resources(dev, qp, udata);
cecbcddf 2492
1212767e
YB		 2493 if (atomic_dec_and_test(&qp->refcnt) &&
2494 rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 2495 xa_erase(&dev->qps, qp->qp_id);
de0089e6
KM		 2496 kfree(qp);
2497 }
cf167e5e 2498 return 0;
cecbcddf 2499}
e0290cce 2500
d3456914
LR		 2501int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
2502 struct ib_udata *udata)
04886779 2503{
d3456914 2504 struct qedr_ah *ah = get_qedr_ah(ibah);
04886779 2505
d97099fe 2506 rdma_copy_ah_attr(&ah->attr, attr);
04886779 2507
d3456914 2508 return 0;
04886779
RA		 2509}
2510
d3456914 2511void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
04886779
RA		 2512{
2513 struct qedr_ah *ah = get_qedr_ah(ibah);
2514
d97099fe 2515 rdma_destroy_ah_attr(&ah->attr);
04886779
RA		 2516}
2517
e0290cce
RA		 2518static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2519{
2520 struct qedr_pbl *pbl, *tmp;
2521
2522 if (info->pbl_table)
2523 list_add_tail(&info->pbl_table->list_entry,
2524 &info->free_pbl_list);
2525
2526 if (!list_empty(&info->inuse_pbl_list))
2527 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2528
2529 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2530 list_del(&pbl->list_entry);
2531 qedr_free_pbl(dev, &info->pbl_info, pbl);
2532 }
2533}
2534
2535static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2536 size_t page_list_len, bool two_layered)
2537{
2538 struct qedr_pbl *tmp;
2539 int rc;
2540
2541 INIT_LIST_HEAD(&info->free_pbl_list);
2542 INIT_LIST_HEAD(&info->inuse_pbl_list);
2543
2544 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2545 page_list_len, two_layered);
2546 if (rc)
2547 goto done;
2548
2549 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2550 if (IS_ERR(info->pbl_table)) {
2551 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2552 goto done;
2553 }
2554
2555 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2556 &info->pbl_table->pa);
2557
2558 /* in usual case we use 2 PBLs, so we add one to free
2559 * list and allocating another one
2560 */
2561 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2562 if (IS_ERR(tmp)) {
e0290cce
RA		 2563 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2564 goto done;
2565 }
2566
2567 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2568
2569 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2570
2571done:
2572 if (rc)
2573 free_mr_info(dev, info);
2574
2575 return rc;
2576}
2577
2578struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2579 u64 usr_addr, int acc, struct ib_udata *udata)
2580{
2581 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2582 struct qedr_mr *mr;
2583 struct qedr_pd *pd;
2584 int rc = -ENOMEM;
2585
2586 pd = get_qedr_pd(ibpd);
2587 DP_DEBUG(dev, QEDR_MSG_MR,
2588 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2589 pd->pd_id, start, len, usr_addr, acc);
2590
2591 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2592 return ERR_PTR(-EINVAL);
2593
2594 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2595 if (!mr)
2596 return ERR_PTR(rc);
2597
2598 mr->type = QEDR_MR_USER;
2599
b0ea0fa5 2600 mr->umem = ib_umem_get(udata, start, len, acc, 0);
e0290cce
RA		 2601 if (IS_ERR(mr->umem)) {
2602 rc = -EFAULT;
2603 goto err0;
2604 }
2605
2606 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2607 if (rc)
2608 goto err1;
2609
2610 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2611 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2612
2613 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2614 if (rc) {
2615 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2616 goto err1;
2617 }
2618
2619 /* Index only, 18 bit long, lkey = itid << 8 | key */
2620 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2621 mr->hw_mr.key = 0;
2622 mr->hw_mr.pd = pd->pd_id;
2623 mr->hw_mr.local_read = 1;
2624 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2625 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2626 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2627 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2628 mr->hw_mr.mw_bind = false;
2629 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2630 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2631 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 2632 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 2633 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2634 mr->hw_mr.length = len;
2635 mr->hw_mr.vaddr = usr_addr;
2636 mr->hw_mr.zbva = false;
2637 mr->hw_mr.phy_mr = false;
2638 mr->hw_mr.dma_mr = false;
2639
2640 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2641 if (rc) {
2642 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2643 goto err2;
2644 }
2645
2646 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2647 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2648 mr->hw_mr.remote_atomic)
2649 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2650
2651 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2652 mr->ibmr.lkey);
2653 return &mr->ibmr;
2654
2655err2:
2656 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2657err1:
2658 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2659err0:
2660 kfree(mr);
2661 return ERR_PTR(rc);
2662}
2663
c4367a26 2664int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 2665{
2666 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2667 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2668 int rc = 0;
2669
2670 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2671 if (rc)
2672 return rc;
2673
2674 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2675
2676 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2677 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2678
2679 /* it could be user registered memory. */
836a0fbb 2680 ib_umem_release(mr->umem);
e0290cce
RA		 2681
2682 kfree(mr);
2683
2684 return rc;
2685}
2686
27a4b1a6
RA		 2687static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2688 int max_page_list_len)
e0290cce
RA		 2689{
2690 struct qedr_pd *pd = get_qedr_pd(ibpd);
2691 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2692 struct qedr_mr *mr;
2693 int rc = -ENOMEM;
2694
2695 DP_DEBUG(dev, QEDR_MSG_MR,
2696 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2697 max_page_list_len);
2698
2699 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2700 if (!mr)
2701 return ERR_PTR(rc);
2702
2703 mr->dev = dev;
2704 mr->type = QEDR_MR_FRMR;
2705
2706 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2707 if (rc)
2708 goto err0;
2709
2710 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2711 if (rc) {
2712 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2713 goto err0;
2714 }
2715
2716 /* Index only, 18 bit long, lkey = itid << 8 | key */
2717 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2718 mr->hw_mr.key = 0;
2719 mr->hw_mr.pd = pd->pd_id;
2720 mr->hw_mr.local_read = 1;
2721 mr->hw_mr.local_write = 0;
2722 mr->hw_mr.remote_read = 0;
2723 mr->hw_mr.remote_write = 0;
2724 mr->hw_mr.remote_atomic = 0;
2725 mr->hw_mr.mw_bind = false;
2726 mr->hw_mr.pbl_ptr = 0;
2727 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2728 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2729 mr->hw_mr.fbo = 0;
2730 mr->hw_mr.length = 0;
2731 mr->hw_mr.vaddr = 0;
2732 mr->hw_mr.zbva = false;
2733 mr->hw_mr.phy_mr = true;
2734 mr->hw_mr.dma_mr = false;
2735
2736 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2737 if (rc) {
2738 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2739 goto err1;
2740 }
2741
2742 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2743 mr->ibmr.rkey = mr->ibmr.lkey;
2744
2745 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2746 return mr;
2747
2748err1:
2749 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2750err0:
2751 kfree(mr);
2752 return ERR_PTR(rc);
2753}
2754
c4367a26
SR		 2755struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2756 u32 max_num_sg, struct ib_udata *udata)
e0290cce 2757{
e0290cce
RA		 2758 struct qedr_mr *mr;
2759
2760 if (mr_type != IB_MR_TYPE_MEM_REG)
2761 return ERR_PTR(-EINVAL);
2762
2763 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2764
2765 if (IS_ERR(mr))
2766 return ERR_PTR(-EINVAL);
2767
e0290cce
RA		 2768 return &mr->ibmr;
2769}
2770
2771static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2772{
2773 struct qedr_mr *mr = get_qedr_mr(ibmr);
2774 struct qedr_pbl *pbl_table;
2775 struct regpair *pbe;
2776 u32 pbes_in_page;
2777
2778 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 2779 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 2780 return -ENOMEM;
2781 }
2782
2783 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2784 mr->npages, addr);
2785
2786 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2787 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2788 pbe = (struct regpair *)pbl_table->va;
2789 pbe += mr->npages % pbes_in_page;
2790 pbe->lo = cpu_to_le32((u32)addr);
2791 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2792
2793 mr->npages++;
2794
2795 return 0;
2796}
2797
2798static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2799{
2800 int work = info->completed - info->completed_handled - 1;
2801
2802 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2803 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2804 struct qedr_pbl *pbl;
2805
2806 /* Free all the page list that are possible to be freed
2807 * (all the ones that were invalidated), under the assumption
2808 * that if an FMR was completed successfully that means that
2809 * if there was an invalidate operation before it also ended
2810 */
2811 pbl = list_first_entry(&info->inuse_pbl_list,
2812 struct qedr_pbl, list_entry);
aafec388 2813 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 2814 info->completed_handled++;
2815 }
2816}
2817
2818int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2819 int sg_nents, unsigned int *sg_offset)
2820{
2821 struct qedr_mr *mr = get_qedr_mr(ibmr);
2822
2823 mr->npages = 0;
2824
2825 handle_completed_mrs(mr->dev, &mr->info);
2826 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2827}
2828
2829struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2830{
2831 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2832 struct qedr_pd *pd = get_qedr_pd(ibpd);
2833 struct qedr_mr *mr;
2834 int rc;
2835
2836 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2837 if (!mr)
2838 return ERR_PTR(-ENOMEM);
2839
2840 mr->type = QEDR_MR_DMA;
2841
2842 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2843 if (rc) {
2844 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2845 goto err1;
2846 }
2847
2848 /* index only, 18 bit long, lkey = itid << 8 | key */
2849 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2850 mr->hw_mr.pd = pd->pd_id;
2851 mr->hw_mr.local_read = 1;
2852 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2853 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2854 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2855 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2856 mr->hw_mr.dma_mr = true;
2857
2858 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2859 if (rc) {
2860 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2861 goto err2;
2862 }
2863
2864 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2865 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2866 mr->hw_mr.remote_atomic)
2867 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2868
2869 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2870 return &mr->ibmr;
2871
2872err2:
2873 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2874err1:
2875 kfree(mr);
2876 return ERR_PTR(rc);
2877}
afa0e13b
RA		 2878
2879static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2880{
2881 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2882}
2883
2884static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2885{
2886 int i, len = 0;
2887
2888 for (i = 0; i < num_sge; i++)
2889 len += sg_list[i].length;
2890
2891 return len;
2892}
2893
2894static void swap_wqe_data64(u64 *p)
2895{
2896 int i;
2897
2898 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2899 *p = cpu_to_be64(cpu_to_le64(*p));
2900}
2901
2902static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2903 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 2904 const struct ib_send_wr *wr,
2905 const struct ib_send_wr **bad_wr,
2906 u8 *bits, u8 bit)
afa0e13b
RA		 2907{
2908 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2909 char *seg_prt, *wqe;
2910 int i, seg_siz;
2911
2912 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2913 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2914 *bad_wr = wr;
2915 return 0;
2916 }
2917
2918 if (!data_size)
2919 return data_size;
2920
2921 *bits |= bit;
2922
2923 seg_prt = NULL;
2924 wqe = NULL;
2925 seg_siz = 0;
2926
2927 /* Copy data inline */
2928 for (i = 0; i < wr->num_sge; i++) {
2929 u32 len = wr->sg_list[i].length;
2930 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2931
2932 while (len > 0) {
2933 u32 cur;
2934
2935 /* New segment required */
2936 if (!seg_siz) {
2937 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2938 seg_prt = wqe;
2939 seg_siz = sizeof(struct rdma_sq_common_wqe);
2940 (*wqe_size)++;
2941 }
2942
2943 /* Calculate currently allowed length */
2944 cur = min_t(u32, len, seg_siz);
2945 memcpy(seg_prt, src, cur);
2946
2947 /* Update segment variables */
2948 seg_prt += cur;
2949 seg_siz -= cur;
2950
2951 /* Update sge variables */
2952 src += cur;
2953 len -= cur;
2954
2955 /* Swap fully-completed segments */
2956 if (!seg_siz)
2957 swap_wqe_data64((u64 *)wqe);
2958 }
2959 }
2960
2961 /* swap last not completed segment */
2962 if (seg_siz)
2963 swap_wqe_data64((u64 *)wqe);
2964
2965 return data_size;
2966}
2967
2968#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2969 do { \
2970 DMA_REGPAIR_LE(sge->addr, vaddr); \
2971 (sge)->length = cpu_to_le32(vlength); \
2972 (sge)->flags = cpu_to_le32(vflags); \
2973 } while (0)
2974
2975#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2976 do { \
2977 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2978 (hdr)->num_sges = num_sge; \
2979 } while (0)
2980
2981#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2982 do { \
2983 DMA_REGPAIR_LE(sge->addr, vaddr); \
2984 (sge)->length = cpu_to_le32(vlength); \
2985 (sge)->l_key = cpu_to_le32(vlkey); \
2986 } while (0)
2987
2988static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 2989 const struct ib_send_wr *wr)
afa0e13b
RA		 2990{
2991 u32 data_size = 0;
2992 int i;
2993
2994 for (i = 0; i < wr->num_sge; i++) {
2995 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2996
2997 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2998 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2999 sge->length = cpu_to_le32(wr->sg_list[i].length);
3000 data_size += wr->sg_list[i].length;
3001 }
3002
3003 if (wqe_size)
3004 *wqe_size += wr->num_sge;
3005
3006 return data_size;
3007}
3008
3009static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3010 struct qedr_qp *qp,
3011 struct rdma_sq_rdma_wqe_1st *rwqe,
3012 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3013 const struct ib_send_wr *wr,
3014 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3015{
3016 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3017 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3018
8b0cabc6
AR		 3019 if (wr->send_flags & IB_SEND_INLINE &&
3020 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3021 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3022 u8 flags = 0;
3023
3024 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3025 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3026 bad_wr, &rwqe->flags, flags);
3027 }
3028
3029 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3030}
3031
3032static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3033 struct qedr_qp *qp,
3034 struct rdma_sq_send_wqe_1st *swqe,
3035 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3036 const struct ib_send_wr *wr,
3037 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3038{
3039 memset(swqe2, 0, sizeof(*swqe2));
3040 if (wr->send_flags & IB_SEND_INLINE) {
3041 u8 flags = 0;
3042
3043 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3044 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3045 bad_wr, &swqe->flags, flags);
3046 }
3047
3048 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3049}
3050
3051static int qedr_prepare_reg(struct qedr_qp *qp,
3052 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3053 const struct ib_reg_wr *wr)
afa0e13b
RA		 3054{
3055 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3056 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3057
3058 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3059 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3060 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3061 fwqe1->l_key = wr->key;
3062
08c4cf51
AR		 3063 fwqe2->access_ctrl = 0;
3064
afa0e13b
RA		 3065 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3066 !!(wr->access & IB_ACCESS_REMOTE_READ));
3067 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3068 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3069 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3070 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3071 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3072 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3073 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3074 fwqe2->fmr_ctrl = 0;
3075
3076 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3077 ilog2(mr->ibmr.page_size) - 12);
3078
3079 fwqe2->length_hi = 0;
3080 fwqe2->length_lo = mr->ibmr.length;
3081 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3082 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3083
3084 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3085
3086 return 0;
3087}
3088
27a4b1a6 3089static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3090{
3091 switch (opcode) {
3092 case IB_WR_RDMA_WRITE:
3093 case IB_WR_RDMA_WRITE_WITH_IMM:
3094 return IB_WC_RDMA_WRITE;
3095 case IB_WR_SEND_WITH_IMM:
3096 case IB_WR_SEND:
3097 case IB_WR_SEND_WITH_INV:
3098 return IB_WC_SEND;
3099 case IB_WR_RDMA_READ:
fb1a22be 3100 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3101 return IB_WC_RDMA_READ;
3102 case IB_WR_ATOMIC_CMP_AND_SWP:
3103 return IB_WC_COMP_SWAP;
3104 case IB_WR_ATOMIC_FETCH_AND_ADD:
3105 return IB_WC_FETCH_ADD;
3106 case IB_WR_REG_MR:
3107 return IB_WC_REG_MR;
3108 case IB_WR_LOCAL_INV:
3109 return IB_WC_LOCAL_INV;
3110 default:
3111 return IB_WC_SEND;
3112 }
3113}
3114
f696bf6d
BVA		 3115static inline bool qedr_can_post_send(struct qedr_qp *qp,
3116 const struct ib_send_wr *wr)
afa0e13b
RA		 3117{
3118 int wq_is_full, err_wr, pbl_is_full;
3119 struct qedr_dev *dev = qp->dev;
3120
3121 /* prevent SQ overflow and/or processing of a bad WR */
3122 err_wr = wr->num_sge > qp->sq.max_sges;
3123 wq_is_full = qedr_wq_is_full(&qp->sq);
3124 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3125 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3126 if (wq_is_full || err_wr || pbl_is_full) {
3127 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3128 DP_ERR(dev,
3129 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3130 qp);
3131 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3132 }
3133
3134 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3135 DP_ERR(dev,
3136 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3137 qp);
3138 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3139 }
3140
3141 if (pbl_is_full &&
3142 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3143 DP_ERR(dev,
3144 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3145 qp);
3146 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3147 }
3148 return false;
3149 }
3150 return true;
3151}
3152
d34ac5cd
BVA		 3153static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3154 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3155{
3156 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3157 struct qedr_qp *qp = get_qedr_qp(ibqp);
3158 struct rdma_sq_atomic_wqe_1st *awqe1;
3159 struct rdma_sq_atomic_wqe_2nd *awqe2;
3160 struct rdma_sq_atomic_wqe_3rd *awqe3;
3161 struct rdma_sq_send_wqe_2st *swqe2;
3162 struct rdma_sq_local_inv_wqe *iwqe;
3163 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3164 struct rdma_sq_send_wqe_1st *swqe;
3165 struct rdma_sq_rdma_wqe_1st *rwqe;
3166 struct rdma_sq_fmr_wqe_1st *fwqe1;
3167 struct rdma_sq_common_wqe *wqe;
3168 u32 length;
3169 int rc = 0;
3170 bool comp;
3171
3172 if (!qedr_can_post_send(qp, wr)) {
3173 *bad_wr = wr;
3174 return -ENOMEM;
3175 }
3176
3177 wqe = qed_chain_produce(&qp->sq.pbl);
3178 qp->wqe_wr_id[qp->sq.prod].signaled =
3179 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3180
3181 wqe->flags = 0;
3182 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3183 !!(wr->send_flags & IB_SEND_SOLICITED));
3184 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3185 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3186 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3187 !!(wr->send_flags & IB_SEND_FENCE));
3188 wqe->prev_wqe_size = qp->prev_wqe_size;
3189
3190 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3191
3192 switch (wr->opcode) {
3193 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3194 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3195 rc = -EINVAL;
3196 *bad_wr = wr;
3197 break;
3198 }
afa0e13b
RA		 3199 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3200 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3201 swqe->wqe_size = 2;
3202 swqe2 = qed_chain_produce(&qp->sq.pbl);
3203
7bed7ebc 3204 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3205 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3206 wr, bad_wr);
3207 swqe->length = cpu_to_le32(length);
3208 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3209 qp->prev_wqe_size = swqe->wqe_size;
3210 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3211 break;
3212 case IB_WR_SEND:
3213 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3214 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3215
3216 swqe->wqe_size = 2;
3217 swqe2 = qed_chain_produce(&qp->sq.pbl);
3218 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3219 wr, bad_wr);
3220 swqe->length = cpu_to_le32(length);
3221 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3222 qp->prev_wqe_size = swqe->wqe_size;
3223 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3224 break;
3225 case IB_WR_SEND_WITH_INV:
3226 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3227 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3228 swqe2 = qed_chain_produce(&qp->sq.pbl);
3229 swqe->wqe_size = 2;
3230 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3231 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3232 wr, bad_wr);
3233 swqe->length = cpu_to_le32(length);
3234 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3235 qp->prev_wqe_size = swqe->wqe_size;
3236 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3237 break;
3238
3239 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3240 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3241 rc = -EINVAL;
3242 *bad_wr = wr;
3243 break;
3244 }
afa0e13b
RA		 3245 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3246 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3247
3248 rwqe->wqe_size = 2;
3249 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3250 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3251 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3252 wr, bad_wr);
3253 rwqe->length = cpu_to_le32(length);
3254 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3255 qp->prev_wqe_size = rwqe->wqe_size;
3256 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3257 break;
3258 case IB_WR_RDMA_WRITE:
3259 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3260 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3261
3262 rwqe->wqe_size = 2;
3263 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3264 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3265 wr, bad_wr);
3266 rwqe->length = cpu_to_le32(length);
3267 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3268 qp->prev_wqe_size = rwqe->wqe_size;
3269 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3270 break;
3271 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3272 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
1b8a708b 3273 /* fallthrough -- same is identical to RDMA READ */
afa0e13b
RA		 3274
3275 case IB_WR_RDMA_READ:
3276 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3277 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3278
3279 rwqe->wqe_size = 2;
3280 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3281 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3282 wr, bad_wr);
3283 rwqe->length = cpu_to_le32(length);
3284 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3285 qp->prev_wqe_size = rwqe->wqe_size;
3286 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3287 break;
3288
3289 case IB_WR_ATOMIC_CMP_AND_SWP:
3290 case IB_WR_ATOMIC_FETCH_AND_ADD:
3291 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3292 awqe1->wqe_size = 4;
3293
3294 awqe2 = qed_chain_produce(&qp->sq.pbl);
3295 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3296 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3297
3298 awqe3 = qed_chain_produce(&qp->sq.pbl);
3299
3300 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3301 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3302 DMA_REGPAIR_LE(awqe3->swap_data,
3303 atomic_wr(wr)->compare_add);
3304 } else {
3305 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3306 DMA_REGPAIR_LE(awqe3->swap_data,
3307 atomic_wr(wr)->swap);
3308 DMA_REGPAIR_LE(awqe3->cmp_data,
3309 atomic_wr(wr)->compare_add);
3310 }
3311
3312 qedr_prepare_sq_sges(qp, NULL, wr);
3313
3314 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3315 qp->prev_wqe_size = awqe1->wqe_size;
3316 break;
3317
3318 case IB_WR_LOCAL_INV:
3319 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3320 iwqe->wqe_size = 1;
3321
3322 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3323 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3324 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3325 qp->prev_wqe_size = iwqe->wqe_size;
3326 break;
3327 case IB_WR_REG_MR:
3328 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3329 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3330 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3331 fwqe1->wqe_size = 2;
3332
3333 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3334 if (rc) {
3335 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3336 *bad_wr = wr;
3337 break;
3338 }
3339
3340 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3341 qp->prev_wqe_size = fwqe1->wqe_size;
3342 break;
3343 default:
3344 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3345 rc = -EINVAL;
3346 *bad_wr = wr;
3347 break;
3348 }
3349
3350 if (*bad_wr) {
3351 u16 value;
3352
3353 /* Restore prod to its position before
3354 * this WR was processed
3355 */
3356 value = le16_to_cpu(qp->sq.db_data.data.value);
3357 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3358
3359 /* Restore prev_wqe_size */
3360 qp->prev_wqe_size = wqe->prev_wqe_size;
3361 rc = -EINVAL;
3362 DP_ERR(dev, "POST SEND FAILED\n");
3363 }
3364
3365 return rc;
3366}
3367
d34ac5cd
BVA		 3368int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3369 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3370{
3371 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3372 struct qedr_qp *qp = get_qedr_qp(ibqp);
3373 unsigned long flags;
3374 int rc = 0;
3375
3376 *bad_wr = NULL;
3377
04886779
RA		 3378 if (qp->qp_type == IB_QPT_GSI)
3379 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3380
afa0e13b
RA		 3381 spin_lock_irqsave(&qp->q_lock, flags);
3382
f5b1b177
KM		 3383 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3384 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3385 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3386 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3387 spin_unlock_irqrestore(&qp->q_lock, flags);
3388 *bad_wr = wr;
3389 DP_DEBUG(dev, QEDR_MSG_CQ,
3390 "QP in wrong state! QP icid=0x%x state %d\n",
3391 qp->icid, qp->state);
3392 return -EINVAL;
3393 }
afa0e13b
RA		 3394 }
3395
afa0e13b
RA		 3396 while (wr) {
3397 rc = __qedr_post_send(ibqp, wr, bad_wr);
3398 if (rc)
3399 break;
3400
3401 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3402
3403 qedr_inc_sw_prod(&qp->sq);
3404
3405 qp->sq.db_data.data.value++;
3406
3407 wr = wr->next;
3408 }
3409
3410 /* Trigger doorbell
3411 * If there was a failure in the first WR then it will be triggered in
3412 * vane. However this is not harmful (as long as the producer value is
3413 * unchanged). For performance reasons we avoid checking for this
3414 * redundant doorbell.
09c4854f
KM		 3415 *
3416 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3417 * soon as we give the doorbell, we could get a completion
3418 * for this wr, therefore we need to make sure that the
3419 * memory is updated before giving the doorbell.
3420 * During qedr_poll_cq, rmb is called before accessing the
3421 * cqe. This covers for the smp_rmb as well.
afa0e13b 3422 */
09c4854f 3423 smp_wmb();
afa0e13b
RA		 3424 writel(qp->sq.db_data.raw, qp->sq.db);
3425
afa0e13b
RA		 3426 spin_unlock_irqrestore(&qp->q_lock, flags);
3427
3428 return rc;
3429}
3430
3491c9e7
YB		 3431static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3432{
3433 u32 used;
3434
3435 /* Calculate number of elements used based on producer
3436 * count and consumer count and subtract it from max
3437 * work request supported so that we get elements left.
3438 */
3439 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3440
3441 return hw_srq->max_wr - used;
3442}
3443
3444int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3445 const struct ib_recv_wr **bad_wr)
3446{
3447 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3448 struct qedr_srq_hwq_info *hw_srq;
3449 struct qedr_dev *dev = srq->dev;
3450 struct qed_chain *pbl;
3451 unsigned long flags;
3452 int status = 0;
3453 u32 num_sge;
3454 u32 offset;
3455
3456 spin_lock_irqsave(&srq->lock, flags);
3457
3458 hw_srq = &srq->hw_srq;
3459 pbl = &srq->hw_srq.pbl;
3460 while (wr) {
3461 struct rdma_srq_wqe_header *hdr;
3462 int i;
3463
3464 if (!qedr_srq_elem_left(hw_srq) ||
3465 wr->num_sge > srq->hw_srq.max_sges) {
3466 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3467 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3468 wr->num_sge, srq->hw_srq.max_sges);
3469 status = -ENOMEM;
3470 *bad_wr = wr;
3471 break;
3472 }
3473
3474 hdr = qed_chain_produce(pbl);
3475 num_sge = wr->num_sge;
3476 /* Set number of sge and work request id in header */
3477 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3478
3479 srq->hw_srq.wr_prod_cnt++;
3480 hw_srq->wqe_prod++;
3481 hw_srq->sge_prod++;
3482
3483 DP_DEBUG(dev, QEDR_MSG_SRQ,
3484 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3485 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3486
3487 for (i = 0; i < wr->num_sge; i++) {
3488 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3489
3490 /* Set SGE length, lkey and address */
3491 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3492 wr->sg_list[i].length, wr->sg_list[i].lkey);
3493
3494 DP_DEBUG(dev, QEDR_MSG_SRQ,
3495 "[%d]: len %d key %x addr %x:%x\n",
3496 i, srq_sge->length, srq_sge->l_key,
3497 srq_sge->addr.hi, srq_sge->addr.lo);
3498 hw_srq->sge_prod++;
3499 }
3500
3501 /* Flush WQE and SGE information before
3502 * updating producer.
3503 */
3504 wmb();
3505
3506 /* SRQ producer is 8 bytes. Need to update SGE producer index
3507 * in first 4 bytes and need to update WQE producer in
3508 * next 4 bytes.
3509 */
3510 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3511 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3512 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3513 hw_srq->wqe_prod;
3514
3515 /* Flush producer after updating it. */
3516 wmb();
3517 wr = wr->next;
3518 }
3519
3520 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3521 qed_chain_get_elem_left(pbl));
3522 spin_unlock_irqrestore(&srq->lock, flags);
3523
3524 return status;
3525}
3526
d34ac5cd
BVA		 3527int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3528 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3529{
3530 struct qedr_qp *qp = get_qedr_qp(ibqp);
3531 struct qedr_dev *dev = qp->dev;
3532 unsigned long flags;
3533 int status = 0;
3534
04886779
RA		 3535 if (qp->qp_type == IB_QPT_GSI)
3536 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3537
afa0e13b
RA		 3538 spin_lock_irqsave(&qp->q_lock, flags);
3539
922d9a40 3540 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3541 spin_unlock_irqrestore(&qp->q_lock, flags);
3542 *bad_wr = wr;
3543 return -EINVAL;
3544 }
3545
3546 while (wr) {
3547 int i;
3548
3549 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3550 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3551 wr->num_sge > qp->rq.max_sges) {
3552 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3553 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3554 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3555 qp->rq.max_sges);
3556 status = -ENOMEM;
3557 *bad_wr = wr;
3558 break;
3559 }
3560 for (i = 0; i < wr->num_sge; i++) {
3561 u32 flags = 0;
3562 struct rdma_rq_sge *rqe =
3563 qed_chain_produce(&qp->rq.pbl);
3564
3565 /* First one must include the number
3566 * of SGE in the list
3567 */
3568 if (!i)
3569 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3570 wr->num_sge);
3571
d52c89f1 3572 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3573 wr->sg_list[i].lkey);
3574
3575 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3576 wr->sg_list[i].length, flags);
3577 }
3578
3579 /* Special case of no sges. FW requires between 1-4 sges...
3580 * in this case we need to post 1 sge with length zero. this is
3581 * because rdma write with immediate consumes an RQ.
3582 */
3583 if (!wr->num_sge) {
3584 u32 flags = 0;
3585 struct rdma_rq_sge *rqe =
3586 qed_chain_produce(&qp->rq.pbl);
3587
3588 /* First one must include the number
3589 * of SGE in the list
3590 */
d52c89f1 3591 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3592 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3593
3594 RQ_SGE_SET(rqe, 0, 0, flags);
3595 i = 1;
3596 }
3597
3598 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3599 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3600
3601 qedr_inc_sw_prod(&qp->rq);
3602
09c4854f
KM		 3603 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3604 * soon as we give the doorbell, we could get a completion
3605 * for this wr, therefore we need to make sure that the
3606 * memory is update before giving the doorbell.
3607 * During qedr_poll_cq, rmb is called before accessing the
3608 * cqe. This covers for the smp_rmb as well.
3609 */
3610 smp_wmb();
afa0e13b
RA		 3611
3612 qp->rq.db_data.data.value++;
3613
3614 writel(qp->rq.db_data.raw, qp->rq.db);
3615
f5b1b177
KM		 3616 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3617 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
f5b1b177
KM		 3618 }
3619
afa0e13b
RA		 3620 wr = wr->next;
3621 }
3622
3623 spin_unlock_irqrestore(&qp->q_lock, flags);
3624
3625 return status;
3626}
3627
3628static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3629{
3630 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3631
3632 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3633 cq->pbl_toggle;
3634}
3635
3636static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3637{
3638 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3639 struct qedr_qp *qp;
3640
3641 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3642 resp_cqe->qp_handle.lo,
3643 u64);
3644 return qp;
3645}
3646
3647static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3648{
3649 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3650
3651 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3652}
3653
3654/* Return latest CQE (needs processing) */
3655static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3656{
3657 return cq->latest_cqe;
3658}
3659
3660/* In fmr we need to increase the number of fmr completed counter for the fmr
3661 * algorithm determining whether we can free a pbl or not.
3662 * we need to perform this whether the work request was signaled or not. for
3663 * this purpose we call this function from the condition that checks if a wr
3664 * should be skipped, to make sure we don't miss it ( possibly this fmr
3665 * operation was not signalted)
3666 */
3667static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3668{
3669 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3670 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3671}
3672
3673static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3674 struct qedr_cq *cq, int num_entries,
3675 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3676 int force)
3677{
3678 u16 cnt = 0;
3679
3680 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3681 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3682 qedr_chk_if_fmr(qp);
3683 /* skip WC */
3684 goto next_cqe;
3685 }
3686
3687 /* fill WC */
3688 wc->status = status;
27035a1b 3689 wc->vendor_err = 0;
afa0e13b
RA		 3690 wc->wc_flags = 0;
3691 wc->src_qp = qp->id;
3692 wc->qp = &qp->ibqp;
3693
3694 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3695 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3696
3697 switch (wc->opcode) {
3698 case IB_WC_RDMA_WRITE:
3699 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3700 break;
3701 case IB_WC_COMP_SWAP:
3702 case IB_WC_FETCH_ADD:
3703 wc->byte_len = 8;
3704 break;
3705 case IB_WC_REG_MR:
3706 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3707 break;
dac27386
MK		 3708 case IB_WC_RDMA_READ:
3709 case IB_WC_SEND:
3710 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3711 break;
afa0e13b
RA		 3712 default:
3713 break;
3714 }
3715
3716 num_entries--;
3717 wc++;
3718 cnt++;
3719next_cqe:
3720 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3721 qed_chain_consume(&qp->sq.pbl);
3722 qedr_inc_sw_cons(&qp->sq);
3723 }
3724
3725 return cnt;
3726}
3727
3728static int qedr_poll_cq_req(struct qedr_dev *dev,
3729 struct qedr_qp *qp, struct qedr_cq *cq,
3730 int num_entries, struct ib_wc *wc,
3731 struct rdma_cqe_requester *req)
3732{
3733 int cnt = 0;
3734
3735 switch (req->status) {
3736 case RDMA_CQE_REQ_STS_OK:
3737 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3738 IB_WC_SUCCESS, 0);
3739 break;
3740 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 3741 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 3742 DP_DEBUG(dev, QEDR_MSG_CQ,
3743 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3744 cq->icid, qp->icid);
afa0e13b 3745 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3746 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 3747 break;
3748 default:
3749 /* process all WQE before the cosumer */
3750 qp->state = QED_ROCE_QP_STATE_ERR;
3751 cnt = process_req(dev, qp, cq, num_entries, wc,
3752 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3753 wc += cnt;
3754 /* if we have extra WC fill it with actual error info */
3755 if (cnt < num_entries) {
3756 enum ib_wc_status wc_status;
3757
3758 switch (req->status) {
3759 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3760 DP_ERR(dev,
3761 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3762 cq->icid, qp->icid);
3763 wc_status = IB_WC_BAD_RESP_ERR;
3764 break;
3765 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3766 DP_ERR(dev,
3767 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3768 cq->icid, qp->icid);
3769 wc_status = IB_WC_LOC_LEN_ERR;
3770 break;
3771 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3772 DP_ERR(dev,
3773 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3774 cq->icid, qp->icid);
3775 wc_status = IB_WC_LOC_QP_OP_ERR;
3776 break;
3777 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3778 DP_ERR(dev,
3779 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3780 cq->icid, qp->icid);
3781 wc_status = IB_WC_LOC_PROT_ERR;
3782 break;
3783 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3784 DP_ERR(dev,
3785 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3786 cq->icid, qp->icid);
3787 wc_status = IB_WC_MW_BIND_ERR;
3788 break;
3789 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3790 DP_ERR(dev,
3791 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3792 cq->icid, qp->icid);
3793 wc_status = IB_WC_REM_INV_REQ_ERR;
3794 break;
3795 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3796 DP_ERR(dev,
3797 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3798 cq->icid, qp->icid);
3799 wc_status = IB_WC_REM_ACCESS_ERR;
3800 break;
3801 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3802 DP_ERR(dev,
3803 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3804 cq->icid, qp->icid);
3805 wc_status = IB_WC_REM_OP_ERR;
3806 break;
3807 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3808 DP_ERR(dev,
3809 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3810 cq->icid, qp->icid);
3811 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3812 break;
3813 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3814 DP_ERR(dev,
3815 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3816 cq->icid, qp->icid);
3817 wc_status = IB_WC_RETRY_EXC_ERR;
3818 break;
3819 default:
3820 DP_ERR(dev,
3821 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3822 cq->icid, qp->icid);
3823 wc_status = IB_WC_GENERAL_ERR;
3824 }
3825 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3826 wc_status, 1);
3827 }
3828 }
3829
3830 return cnt;
3831}
3832
b6acd71f 3833static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 3834{
b6acd71f 3835 switch (status) {
afa0e13b 3836 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 3837 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 3838 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 3839 return IB_WC_LOC_LEN_ERR;
afa0e13b 3840 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 3841 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 3842 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 3843 return IB_WC_LOC_PROT_ERR;
afa0e13b 3844 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 3845 return IB_WC_MW_BIND_ERR;
afa0e13b 3846 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 3847 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 3848 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 3849 return IB_WC_SUCCESS;
3850 default:
3851 return IB_WC_GENERAL_ERR;
3852 }
3853}
afa0e13b 3854
b6acd71f
AR		 3855static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3856 struct ib_wc *wc)
3857{
3858 wc->status = IB_WC_SUCCESS;
3859 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 3860
b6acd71f 3861 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 3862 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 3863 wc->wc_flags |= IB_WC_WITH_IMM;
3864
3865 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 3866 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3867
b6acd71f
AR		 3868 if (resp->flags & QEDR_RESP_INV)
3869 return -EINVAL;
3870
3871 } else if (resp->flags & QEDR_RESP_INV) {
3872 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3873 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3874
3875 if (resp->flags & QEDR_RESP_RDMA)
3876 return -EINVAL;
3877
3878 } else if (resp->flags & QEDR_RESP_RDMA) {
3879 return -EINVAL;
3880 }
3881
3882 return 0;
3883}
3884
3885static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3886 struct qedr_cq *cq, struct ib_wc *wc,
3887 struct rdma_cqe_responder *resp, u64 wr_id)
3888{
3889 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3890 wc->opcode = IB_WC_RECV;
3891 wc->wc_flags = 0;
3892
3893 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3894 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3895 DP_ERR(dev,
3896 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3897 cq, cq->icid, resp->flags);
3898
3899 } else {
3900 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3901 if (wc->status == IB_WC_GENERAL_ERR)
3902 DP_ERR(dev,
3903 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3904 cq, cq->icid, resp->status);
afa0e13b
RA		 3905 }
3906
b6acd71f 3907 /* Fill the rest of the WC */
27035a1b 3908 wc->vendor_err = 0;
afa0e13b
RA		 3909 wc->src_qp = qp->id;
3910 wc->qp = &qp->ibqp;
3911 wc->wr_id = wr_id;
3912}
3913
3491c9e7
YB		 3914static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3915 struct qedr_cq *cq, struct ib_wc *wc,
3916 struct rdma_cqe_responder *resp)
3917{
3918 struct qedr_srq *srq = qp->srq;
3919 u64 wr_id;
3920
3921 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
3922 le32_to_cpu(resp->srq_wr_id.lo), u64);
3923
3924 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3925 wc->status = IB_WC_WR_FLUSH_ERR;
3926 wc->vendor_err = 0;
3927 wc->wr_id = wr_id;
3928 wc->byte_len = 0;
3929 wc->src_qp = qp->id;
3930 wc->qp = &qp->ibqp;
3931 wc->wr_id = wr_id;
3932 } else {
3933 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3934 }
3935 srq->hw_srq.wr_cons_cnt++;
3936
3937 return 1;
3938}
afa0e13b
RA		 3939static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3940 struct qedr_cq *cq, struct ib_wc *wc,
3941 struct rdma_cqe_responder *resp)
3942{
3943 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3944
3945 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3946
3947 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3948 qed_chain_consume(&qp->rq.pbl);
3949 qedr_inc_sw_cons(&qp->rq);
3950
3951 return 1;
3952}
3953
3954static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3955 int num_entries, struct ib_wc *wc, u16 hw_cons)
3956{
3957 u16 cnt = 0;
3958
3959 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3960 /* fill WC */
3961 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 3962 wc->vendor_err = 0;
afa0e13b
RA		 3963 wc->wc_flags = 0;
3964 wc->src_qp = qp->id;
3965 wc->byte_len = 0;
3966 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3967 wc->qp = &qp->ibqp;
3968 num_entries--;
3969 wc++;
3970 cnt++;
3971 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3972 qed_chain_consume(&qp->rq.pbl);
3973 qedr_inc_sw_cons(&qp->rq);
3974 }
3975
3976 return cnt;
3977}
3978
3979static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3980 struct rdma_cqe_responder *resp, int *update)
3981{
50bc60cb 3982 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 3983 consume_cqe(cq);
3984 *update |= 1;
3985 }
3986}
3987
3491c9e7
YB		 3988static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3989 struct qedr_cq *cq, int num_entries,
3990 struct ib_wc *wc,
3991 struct rdma_cqe_responder *resp)
3992{
3993 int cnt;
3994
3995 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
3996 consume_cqe(cq);
3997
3998 return cnt;
3999}
4000
afa0e13b
RA		 4001static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4002 struct qedr_cq *cq, int num_entries,
4003 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4004 int *update)
4005{
4006 int cnt;
4007
4008 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4009 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4010 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4011 try_consume_resp_cqe(cq, qp, resp, update);
4012 } else {
4013 cnt = process_resp_one(dev, qp, cq, wc, resp);
4014 consume_cqe(cq);
4015 *update |= 1;
4016 }
4017
4018 return cnt;
4019}
4020
4021static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4022 struct rdma_cqe_requester *req, int *update)
4023{
4024 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4025 consume_cqe(cq);
4026 *update |= 1;
4027 }
4028}
4029
4030int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4031{
4032 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4033 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4034 union rdma_cqe *cqe;
afa0e13b
RA		 4035 u32 old_cons, new_cons;
4036 unsigned long flags;
4037 int update = 0;
4038 int done = 0;
4039
4dd72636
AR		 4040 if (cq->destroyed) {
4041 DP_ERR(dev,
4042 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4043 cq, cq->icid);
4044 return 0;
4045 }
4046
04886779
RA		 4047 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4048 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4049
afa0e13b 4050 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4051 cqe = cq->latest_cqe;
afa0e13b
RA		 4052 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4053 while (num_entries && is_valid_cqe(cq, cqe)) {
4054 struct qedr_qp *qp;
4055 int cnt = 0;
4056
4057 /* prevent speculative reads of any field of CQE */
4058 rmb();
4059
4060 qp = cqe_get_qp(cqe);
4061 if (!qp) {
4062 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4063 break;
4064 }
4065
4066 wc->qp = &qp->ibqp;
4067
4068 switch (cqe_get_type(cqe)) {
4069 case RDMA_CQE_TYPE_REQUESTER:
4070 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4071 &cqe->req);
4072 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4073 break;
4074 case RDMA_CQE_TYPE_RESPONDER_RQ:
4075 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4076 &cqe->resp, &update);
4077 break;
3491c9e7
YB		 4078 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4079 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4080 wc, &cqe->resp);
4081 update = 1;
4082 break;
afa0e13b
RA		 4083 case RDMA_CQE_TYPE_INVALID:
4084 default:
4085 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4086 cqe_get_type(cqe));
4087 }
4088 num_entries -= cnt;
4089 wc += cnt;
4090 done += cnt;
4091
4092 cqe = get_cqe(cq);
4093 }
4094 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4095
4096 cq->cq_cons += new_cons - old_cons;
4097
4098 if (update)
4099 /* doorbell notifies abount latest VALID entry,
4100 * but chain already point to the next INVALID one
4101 */
4102 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4103
4104 spin_unlock_irqrestore(&cq->cq_lock, flags);
4105 return done;
4106}
993d1b52
RA		 4107
4108int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4109 u8 port_num,
4110 const struct ib_wc *in_wc,
4111 const struct ib_grh *in_grh,
4112 const struct ib_mad_hdr *mad_hdr,
4113 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4114 size_t *out_mad_size, u16 *out_mad_pkey_index)
4115{
4116 struct qedr_dev *dev = get_qedr_dev(ibdev);
4117
4118 DP_DEBUG(dev, QEDR_MSG_GSI,
4119 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4120 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4121 mad_hdr->class_specific, mad_hdr->class_version,
4122 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4123 return IB_MAD_RESULT_SUCCESS;
4124}
Linux 6.x block layer and io_uring tree(s)