projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/bnxt_re: add DCB dependency
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45
46#include "qedr_hsi.h"
47#include <linux/qed/qed_if.h>
48#include "qedr.h"
49#include "verbs.h"
50#include <rdma/qedr-abi.h>
cecbcddf 51#include "qedr_cm.h"
ac1b36e5 52
a7efd777
RA		 53#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56{
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62}
63
ac1b36e5
RA		 64int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66{
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83}
84
85int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88{
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99}
100
101int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103{
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114}
115
116int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118{
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170}
171
172#define QEDR_SPEED_SDR (1)
173#define QEDR_SPEED_DDR (2)
174#define QEDR_SPEED_QDR (4)
175#define QEDR_SPEED_FDR10 (8)
176#define QEDR_SPEED_FDR (16)
177#define QEDR_SPEED_EDR (32)
178
179static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181{
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222}
223
224int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225{
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241 memset(attr, 0, sizeof(*attr));
242
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267}
268
269int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271{
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281}
282
283static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285{
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313}
314
315static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317{
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335}
336
337struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339{
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397}
398
399int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400{
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419}
420
421int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422{
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468}
a7efd777
RA		 469
470struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472{
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
a7efd777
RA		 474 struct qedr_pd *pd;
475 u16 pd_id;
476 int rc;
477
478 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
479 (udata && context) ? "User Lib" : "Kernel");
480
481 if (!dev->rdma_ctx) {
482 DP_ERR(dev, "invlaid RDMA context\n");
483 return ERR_PTR(-EINVAL);
484 }
485
486 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
487 if (!pd)
488 return ERR_PTR(-ENOMEM);
489
9c1e0228
RA		 490 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
491 if (rc)
492 goto err;
a7efd777 493
a7efd777
RA		 494 pd->pd_id = pd_id;
495
496 if (udata && context) {
9c1e0228
RA		 497 struct qedr_alloc_pd_uresp uresp;
498
499 uresp.pd_id = pd_id;
500
a7efd777 501 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 502 if (rc) {
a7efd777 503 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228
RA		 504 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
505 goto err;
506 }
507
508 pd->uctx = get_qedr_ucontext(context);
509 pd->uctx->pd = pd;
a7efd777
RA		 510 }
511
512 return &pd->ibpd;
9c1e0228
RA		 513
514err:
515 kfree(pd);
516 return ERR_PTR(rc);
a7efd777
RA		 517}
518
519int qedr_dealloc_pd(struct ib_pd *ibpd)
520{
521 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
522 struct qedr_pd *pd = get_qedr_pd(ibpd);
523
ea7ef2ac 524 if (!pd) {
a7efd777 525 pr_err("Invalid PD received in dealloc_pd\n");
ea7ef2ac
CIK		 526 return -EINVAL;
527 }
a7efd777
RA		 528
529 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
530 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
531
532 kfree(pd);
533
534 return 0;
535}
536
537static void qedr_free_pbl(struct qedr_dev *dev,
538 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
539{
540 struct pci_dev *pdev = dev->pdev;
541 int i;
542
543 for (i = 0; i < pbl_info->num_pbls; i++) {
544 if (!pbl[i].va)
545 continue;
546 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
547 pbl[i].va, pbl[i].pa);
548 }
549
550 kfree(pbl);
551}
552
553#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
554#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
555
556#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
557#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
558#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
559
560static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
561 struct qedr_pbl_info *pbl_info,
562 gfp_t flags)
563{
564 struct pci_dev *pdev = dev->pdev;
565 struct qedr_pbl *pbl_table;
566 dma_addr_t *pbl_main_tbl;
567 dma_addr_t pa;
568 void *va;
569 int i;
570
571 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
572 if (!pbl_table)
573 return ERR_PTR(-ENOMEM);
574
575 for (i = 0; i < pbl_info->num_pbls; i++) {
576 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
577 &pa, flags);
578 if (!va)
579 goto err;
580
581 memset(va, 0, pbl_info->pbl_size);
582 pbl_table[i].va = va;
583 pbl_table[i].pa = pa;
584 }
585
586 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
587 * the first one with physical pointers to all of the rest
588 */
589 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
590 for (i = 0; i < pbl_info->num_pbls - 1; i++)
591 pbl_main_tbl[i] = pbl_table[i + 1].pa;
592
593 return pbl_table;
594
595err:
596 for (i--; i >= 0; i--)
597 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
598 pbl_table[i].va, pbl_table[i].pa);
599
600 qedr_free_pbl(dev, pbl_info, pbl_table);
601
602 return ERR_PTR(-ENOMEM);
603}
604
605static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
606 struct qedr_pbl_info *pbl_info,
607 u32 num_pbes, int two_layer_capable)
608{
609 u32 pbl_capacity;
610 u32 pbl_size;
611 u32 num_pbls;
612
613 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
614 if (num_pbes > MAX_PBES_TWO_LAYER) {
615 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
616 num_pbes);
617 return -EINVAL;
618 }
619
620 /* calculate required pbl page size */
621 pbl_size = MIN_FW_PBL_PAGE_SIZE;
622 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
623 NUM_PBES_ON_PAGE(pbl_size);
624
625 while (pbl_capacity < num_pbes) {
626 pbl_size *= 2;
627 pbl_capacity = pbl_size / sizeof(u64);
628 pbl_capacity = pbl_capacity * pbl_capacity;
629 }
630
631 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
632 num_pbls++; /* One for the layer0 ( points to the pbls) */
633 pbl_info->two_layered = true;
634 } else {
635 /* One layered PBL */
636 num_pbls = 1;
637 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
638 roundup_pow_of_two((num_pbes * sizeof(u64))));
639 pbl_info->two_layered = false;
640 }
641
642 pbl_info->num_pbls = num_pbls;
643 pbl_info->pbl_size = pbl_size;
644 pbl_info->num_pbes = num_pbes;
645
646 DP_DEBUG(dev, QEDR_MSG_MR,
647 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
648 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
649
650 return 0;
651}
652
653static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
654 struct qedr_pbl *pbl,
655 struct qedr_pbl_info *pbl_info)
656{
657 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
658 struct qedr_pbl *pbl_tbl;
659 struct scatterlist *sg;
660 struct regpair *pbe;
661 int entry;
662 u32 addr;
663
664 if (!pbl_info->num_pbes)
665 return;
666
667 /* If we have a two layered pbl, the first pbl points to the rest
668 * of the pbls and the first entry lays on the second pbl in the table
669 */
670 if (pbl_info->two_layered)
671 pbl_tbl = &pbl[1];
672 else
673 pbl_tbl = pbl;
674
675 pbe = (struct regpair *)pbl_tbl->va;
676 if (!pbe) {
677 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
678 return;
679 }
680
681 pbe_cnt = 0;
682
683 shift = ilog2(umem->page_size);
684
685 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
686 pages = sg_dma_len(sg) >> shift;
687 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
688 /* store the page address in pbe */
689 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
690 umem->page_size * pg_cnt);
691 addr = upper_32_bits(sg_dma_address(sg) +
692 umem->page_size * pg_cnt);
693 pbe->hi = cpu_to_le32(addr);
694 pbe_cnt++;
695 total_num_pbes++;
696 pbe++;
697
698 if (total_num_pbes == pbl_info->num_pbes)
699 return;
700
701 /* If the given pbl is full storing the pbes,
702 * move to next pbl.
703 */
704 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
705 pbl_tbl++;
706 pbe = (struct regpair *)pbl_tbl->va;
707 pbe_cnt = 0;
708 }
709 }
710 }
711}
712
713static int qedr_copy_cq_uresp(struct qedr_dev *dev,
714 struct qedr_cq *cq, struct ib_udata *udata)
715{
716 struct qedr_create_cq_uresp uresp;
717 int rc;
718
719 memset(&uresp, 0, sizeof(uresp));
720
721 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
722 uresp.icid = cq->icid;
723
724 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
725 if (rc)
726 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
727
728 return rc;
729}
730
731static void consume_cqe(struct qedr_cq *cq)
732{
733 if (cq->latest_cqe == cq->toggle_cqe)
734 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
735
736 cq->latest_cqe = qed_chain_consume(&cq->pbl);
737}
738
739static inline int qedr_align_cq_entries(int entries)
740{
741 u64 size, aligned_size;
742
743 /* We allocate an extra entry that we don't report to the FW. */
744 size = (entries + 1) * QEDR_CQE_SIZE;
745 aligned_size = ALIGN(size, PAGE_SIZE);
746
747 return aligned_size / QEDR_CQE_SIZE;
748}
749
750static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
751 struct qedr_dev *dev,
752 struct qedr_userq *q,
753 u64 buf_addr, size_t buf_len,
754 int access, int dmasync)
755{
756 int page_cnt;
757 int rc;
758
759 q->buf_addr = buf_addr;
760 q->buf_len = buf_len;
761 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
762 if (IS_ERR(q->umem)) {
763 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
764 PTR_ERR(q->umem));
765 return PTR_ERR(q->umem);
766 }
767
768 page_cnt = ib_umem_page_count(q->umem);
769 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
770 if (rc)
771 goto err0;
772
773 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
774 if (IS_ERR_OR_NULL(q->pbl_tbl))
775 goto err0;
776
777 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
778
779 return 0;
780
781err0:
782 ib_umem_release(q->umem);
783
784 return rc;
785}
786
787static inline void qedr_init_cq_params(struct qedr_cq *cq,
788 struct qedr_ucontext *ctx,
789 struct qedr_dev *dev, int vector,
790 int chain_entries, int page_cnt,
791 u64 pbl_ptr,
792 struct qed_rdma_create_cq_in_params
793 *params)
794{
795 memset(params, 0, sizeof(*params));
796 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
797 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
798 params->cnq_id = vector;
799 params->cq_size = chain_entries - 1;
800 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
801 params->pbl_num_pages = page_cnt;
802 params->pbl_ptr = pbl_ptr;
803 params->pbl_two_level = 0;
804}
805
806static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
807{
808 /* Flush data before signalling doorbell */
809 wmb();
810 cq->db.data.agg_flags = flags;
811 cq->db.data.value = cpu_to_le32(cons);
812 writeq(cq->db.raw, cq->db_addr);
813
814 /* Make sure write would stick */
815 mmiowb();
816}
817
818int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
819{
820 struct qedr_cq *cq = get_qedr_cq(ibcq);
821 unsigned long sflags;
822
823 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
824 return 0;
825
826 spin_lock_irqsave(&cq->cq_lock, sflags);
827
828 cq->arm_flags = 0;
829
830 if (flags & IB_CQ_SOLICITED)
831 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
832
833 if (flags & IB_CQ_NEXT_COMP)
834 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
835
836 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
837
838 spin_unlock_irqrestore(&cq->cq_lock, sflags);
839
840 return 0;
841}
842
843struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
844 const struct ib_cq_init_attr *attr,
845 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
846{
847 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
848 struct qed_rdma_destroy_cq_out_params destroy_oparams;
849 struct qed_rdma_destroy_cq_in_params destroy_iparams;
850 struct qedr_dev *dev = get_qedr_dev(ibdev);
851 struct qed_rdma_create_cq_in_params params;
852 struct qedr_create_cq_ureq ureq;
853 int vector = attr->comp_vector;
854 int entries = attr->cqe;
855 struct qedr_cq *cq;
856 int chain_entries;
857 int page_cnt;
858 u64 pbl_ptr;
859 u16 icid;
860 int rc;
861
862 DP_DEBUG(dev, QEDR_MSG_INIT,
863 "create_cq: called from %s. entries=%d, vector=%d\n",
864 udata ? "User Lib" : "Kernel", entries, vector);
865
866 if (entries > QEDR_MAX_CQES) {
867 DP_ERR(dev,
868 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
869 entries, QEDR_MAX_CQES);
870 return ERR_PTR(-EINVAL);
871 }
872
873 chain_entries = qedr_align_cq_entries(entries);
874 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
875
876 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
877 if (!cq)
878 return ERR_PTR(-ENOMEM);
879
880 if (udata) {
881 memset(&ureq, 0, sizeof(ureq));
882 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
883 DP_ERR(dev,
884 "create cq: problem copying data from user space\n");
885 goto err0;
886 }
887
888 if (!ureq.len) {
889 DP_ERR(dev,
890 "create cq: cannot create a cq with 0 entries\n");
891 goto err0;
892 }
893
894 cq->cq_type = QEDR_CQ_TYPE_USER;
895
896 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
897 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
898 if (rc)
899 goto err0;
900
901 pbl_ptr = cq->q.pbl_tbl->pa;
902 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 903
904 cq->ibcq.cqe = chain_entries;
a7efd777
RA		 905 } else {
906 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
907
908 rc = dev->ops->common->chain_alloc(dev->cdev,
909 QED_CHAIN_USE_TO_CONSUME,
910 QED_CHAIN_MODE_PBL,
911 QED_CHAIN_CNT_TYPE_U32,
912 chain_entries,
913 sizeof(union rdma_cqe),
914 &cq->pbl);
915 if (rc)
916 goto err1;
917
918 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
919 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 920 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 921 }
922
923 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
924 pbl_ptr, &params);
925
926 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
927 if (rc)
928 goto err2;
929
930 cq->icid = icid;
931 cq->sig = QEDR_CQ_MAGIC_NUMBER;
932 spin_lock_init(&cq->cq_lock);
933
934 if (ib_ctx) {
935 rc = qedr_copy_cq_uresp(dev, cq, udata);
936 if (rc)
937 goto err3;
938 } else {
939 /* Generate doorbell address. */
940 cq->db_addr = dev->db_addr +
941 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
942 cq->db.data.icid = cq->icid;
943 cq->db.data.params = DB_AGG_CMD_SET <<
944 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
945
946 /* point to the very last element, passing it we will toggle */
947 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
948 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
949 cq->latest_cqe = NULL;
950 consume_cqe(cq);
951 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
952 }
953
954 DP_DEBUG(dev, QEDR_MSG_CQ,
955 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
956 cq->icid, cq, params.cq_size);
957
958 return &cq->ibcq;
959
960err3:
961 destroy_iparams.icid = cq->icid;
962 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
963 &destroy_oparams);
964err2:
965 if (udata)
966 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
967 else
968 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
969err1:
970 if (udata)
971 ib_umem_release(cq->q.umem);
972err0:
973 kfree(cq);
974 return ERR_PTR(-EINVAL);
975}
976
977int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
978{
979 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
980 struct qedr_cq *cq = get_qedr_cq(ibcq);
981
982 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
983
984 return 0;
985}
986
987int qedr_destroy_cq(struct ib_cq *ibcq)
988{
989 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
990 struct qed_rdma_destroy_cq_out_params oparams;
991 struct qed_rdma_destroy_cq_in_params iparams;
992 struct qedr_cq *cq = get_qedr_cq(ibcq);
993
994 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
995
996 /* GSIs CQs are handled by driver, so they don't exist in the FW */
997 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
a1211359
AR		 998 int rc;
999
a7efd777 1000 iparams.icid = cq->icid;
a1211359
AR		 1001 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams,
1002 &oparams);
1003 if (rc)
1004 return rc;
a7efd777
RA		 1005 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1006 }
1007
1008 if (ibcq->uobject && ibcq->uobject->context) {
1009 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1010 ib_umem_release(cq->q.umem);
1011 }
1012
1013 kfree(cq);
1014
1015 return 0;
1016}
cecbcddf
RA		 1017
1018static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1019 struct ib_qp_attr *attr,
1020 int attr_mask,
1021 struct qed_rdma_modify_qp_in_params
1022 *qp_params)
1023{
1024 enum rdma_network_type nw_type;
1025 struct ib_gid_attr gid_attr;
1026 union ib_gid gid;
1027 u32 ipv4_addr;
1028 int rc = 0;
1029 int i;
1030
1031 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1032 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1033 if (rc)
1034 return rc;
1035
1036 if (!memcmp(&gid, &zgid, sizeof(gid)))
1037 return -ENOENT;
1038
1039 if (gid_attr.ndev) {
1040 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1041
1042 dev_put(gid_attr.ndev);
1043 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1044 switch (nw_type) {
1045 case RDMA_NETWORK_IPV6:
1046 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1047 sizeof(qp_params->sgid));
1048 memcpy(&qp_params->dgid.bytes[0],
1049 &attr->ah_attr.grh.dgid,
1050 sizeof(qp_params->dgid));
1051 qp_params->roce_mode = ROCE_V2_IPV6;
1052 SET_FIELD(qp_params->modify_flags,
1053 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1054 break;
1055 case RDMA_NETWORK_IB:
1056 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1057 sizeof(qp_params->sgid));
1058 memcpy(&qp_params->dgid.bytes[0],
1059 &attr->ah_attr.grh.dgid,
1060 sizeof(qp_params->dgid));
1061 qp_params->roce_mode = ROCE_V1;
1062 break;
1063 case RDMA_NETWORK_IPV4:
1064 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1065 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1066 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1067 qp_params->sgid.ipv4_addr = ipv4_addr;
1068 ipv4_addr =
1069 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1070 qp_params->dgid.ipv4_addr = ipv4_addr;
1071 SET_FIELD(qp_params->modify_flags,
1072 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1073 qp_params->roce_mode = ROCE_V2_IPV4;
1074 break;
1075 }
1076 }
1077
1078 for (i = 0; i < 4; i++) {
1079 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1080 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1081 }
1082
1083 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1084 qp_params->vlan_id = 0;
1085
1086 return 0;
1087}
1088
cecbcddf
RA		 1089static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1090 struct ib_qp_init_attr *attrs)
1091{
1092 struct qedr_device_attr *qattr = &dev->attr;
1093
1094 /* QP0... attrs->qp_type == IB_QPT_GSI */
1095 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1096 DP_DEBUG(dev, QEDR_MSG_QP,
1097 "create qp: unsupported qp type=0x%x requested\n",
1098 attrs->qp_type);
1099 return -EINVAL;
1100 }
1101
1102 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1103 DP_ERR(dev,
1104 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1105 attrs->cap.max_send_wr, qattr->max_sqe);
1106 return -EINVAL;
1107 }
1108
1109 if (attrs->cap.max_inline_data > qattr->max_inline) {
1110 DP_ERR(dev,
1111 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1112 attrs->cap.max_inline_data, qattr->max_inline);
1113 return -EINVAL;
1114 }
1115
1116 if (attrs->cap.max_send_sge > qattr->max_sge) {
1117 DP_ERR(dev,
1118 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1119 attrs->cap.max_send_sge, qattr->max_sge);
1120 return -EINVAL;
1121 }
1122
1123 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1124 DP_ERR(dev,
1125 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1126 attrs->cap.max_recv_sge, qattr->max_sge);
1127 return -EINVAL;
1128 }
1129
1130 /* Unprivileged user space cannot create special QP */
1131 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1132 DP_ERR(dev,
1133 "create qp: userspace can't create special QPs of type=0x%x\n",
1134 attrs->qp_type);
1135 return -EINVAL;
1136 }
1137
1138 return 0;
1139}
1140
1141static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1142 struct qedr_qp *qp)
1143{
1144 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1145 uresp->rq_icid = qp->icid;
1146}
1147
1148static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1149 struct qedr_qp *qp)
1150{
1151 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1152 uresp->sq_icid = qp->icid + 1;
1153}
1154
1155static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1156 struct qedr_qp *qp, struct ib_udata *udata)
1157{
1158 struct qedr_create_qp_uresp uresp;
1159 int rc;
1160
1161 memset(&uresp, 0, sizeof(uresp));
1162 qedr_copy_sq_uresp(&uresp, qp);
1163 qedr_copy_rq_uresp(&uresp, qp);
1164
1165 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1166 uresp.qp_id = qp->qp_id;
1167
1168 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1169 if (rc)
1170 DP_ERR(dev,
1171 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1172 qp->icid);
1173
1174 return rc;
1175}
1176
df158561
AR		 1177static void qedr_set_common_qp_params(struct qedr_dev *dev,
1178 struct qedr_qp *qp,
1179 struct qedr_pd *pd,
1180 struct ib_qp_init_attr *attrs)
cecbcddf 1181{
cecbcddf 1182 spin_lock_init(&qp->q_lock);
df158561 1183 qp->pd = pd;
cecbcddf
RA		 1184 qp->qp_type = attrs->qp_type;
1185 qp->max_inline_data = attrs->cap.max_inline_data;
1186 qp->sq.max_sges = attrs->cap.max_send_sge;
1187 qp->state = QED_ROCE_QP_STATE_RESET;
1188 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1189 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1190 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1191 qp->dev = dev;
df158561 1192 qp->rq.max_sges = attrs->cap.max_recv_sge;
cecbcddf 1193
df158561
AR		 1194 DP_DEBUG(dev, QEDR_MSG_QP,
1195 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1196 qp->rq.max_sges, qp->rq_cq->icid);
cecbcddf
RA		 1197 DP_DEBUG(dev, QEDR_MSG_QP,
1198 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1199 pd->pd_id, qp->qp_type, qp->max_inline_data,
1200 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1201 DP_DEBUG(dev, QEDR_MSG_QP,
1202 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1203 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1204}
1205
df158561 1206static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf
RA		 1207{
1208 qp->sq.db = dev->db_addr +
1209 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1210 qp->sq.db_data.data.icid = qp->icid + 1;
df158561
AR		 1211 qp->rq.db = dev->db_addr +
1212 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1213 qp->rq.db_data.data.icid = qp->icid;
cecbcddf
RA		 1214}
1215
1216static inline void
df158561
AR		 1217qedr_init_common_qp_in_params(struct qedr_dev *dev,
1218 struct qedr_pd *pd,
1219 struct qedr_qp *qp,
1220 struct ib_qp_init_attr *attrs,
1221 bool fmr_and_reserved_lkey,
1222 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1223{
df158561
AR		 1224 /* QP handle to be written in an async event */
1225 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1226 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1227
1228 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1229 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1230 params->pd = pd->pd_id;
1231 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1232 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1233 params->stats_queue = 0;
1234 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1235 params->srq_id = 0;
1236 params->use_srq = false;
cecbcddf
RA		 1237}
1238
df158561 1239static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1240{
df158561
AR		 1241 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1242 "qp=%p. "
1243 "sq_addr=0x%llx, "
1244 "sq_len=%zd, "
1245 "rq_addr=0x%llx, "
1246 "rq_len=%zd"
1247 "\n",
1248 qp,
1249 qp->usq.buf_addr,
1250 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1251}
cecbcddf 1252
df158561
AR		 1253static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1254{
1255 if (qp->usq.umem)
1256 ib_umem_release(qp->usq.umem);
1257 qp->usq.umem = NULL;
cecbcddf 1258
df158561
AR		 1259 if (qp->urq.umem)
1260 ib_umem_release(qp->urq.umem);
1261 qp->urq.umem = NULL;
cecbcddf
RA		 1262}
1263
df158561
AR		 1264static int qedr_create_user_qp(struct qedr_dev *dev,
1265 struct qedr_qp *qp,
1266 struct ib_pd *ibpd,
1267 struct ib_udata *udata,
1268 struct ib_qp_init_attr *attrs)
cecbcddf 1269{
df158561
AR		 1270 struct qed_rdma_create_qp_in_params in_params;
1271 struct qed_rdma_create_qp_out_params out_params;
1272 struct qedr_pd *pd = get_qedr_pd(ibpd);
1273 struct ib_ucontext *ib_ctx = NULL;
1274 struct qedr_ucontext *ctx = NULL;
1275 struct qedr_create_qp_ureq ureq;
1276 int rc = -EINVAL;
cecbcddf 1277
df158561
AR		 1278 ib_ctx = ibpd->uobject->context;
1279 ctx = get_qedr_ucontext(ib_ctx);
cecbcddf 1280
df158561
AR		 1281 memset(&ureq, 0, sizeof(ureq));
1282 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1283 if (rc) {
1284 DP_ERR(dev, "Problem copying data from user space\n");
1285 return rc;
1286 }
cecbcddf 1287
df158561
AR		 1288 /* SQ - read access only (0), dma sync not required (0) */
1289 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1290 ureq.sq_len, 0, 0);
1291 if (rc)
1292 return rc;
cecbcddf 1293
df158561
AR		 1294 /* RQ - read access only (0), dma sync not required (0) */
1295 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1296 ureq.rq_len, 0, 0);
1297
1298 if (rc)
1299 return rc;
1300
1301 memset(&in_params, 0, sizeof(in_params));
1302 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1303 in_params.qp_handle_lo = ureq.qp_handle_lo;
1304 in_params.qp_handle_hi = ureq.qp_handle_hi;
1305 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1306 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1307 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1308 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1309
1310 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1311 &in_params, &out_params);
1312
1313 if (!qp->qed_qp) {
1314 rc = -ENOMEM;
1315 goto err1;
1316 }
1317
1318 qp->qp_id = out_params.qp_id;
1319 qp->icid = out_params.icid;
1320
1321 rc = qedr_copy_qp_uresp(dev, qp, udata);
1322 if (rc)
1323 goto err;
1324
1325 qedr_qp_user_print(dev, qp);
cecbcddf
RA		 1326
1327 return 0;
df158561
AR		 1328err:
1329 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1330 if (rc)
1331 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1332
1333err1:
1334 qedr_cleanup_user(dev, qp);
1335 return rc;
cecbcddf
RA		 1336}
1337
df158561
AR		 1338static int
1339qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1340 struct qedr_qp *qp,
1341 struct qed_rdma_create_qp_in_params *in_params,
1342 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1343{
df158561 1344 struct qed_rdma_create_qp_out_params out_params;
cecbcddf
RA		 1345 int rc;
1346
cecbcddf
RA		 1347 rc = dev->ops->common->chain_alloc(dev->cdev,
1348 QED_CHAIN_USE_TO_PRODUCE,
1349 QED_CHAIN_MODE_PBL,
1350 QED_CHAIN_CNT_TYPE_U32,
1351 n_sq_elems,
1352 QEDR_SQE_ELEMENT_SIZE,
1353 &qp->sq.pbl);
cecbcddf 1354
df158561
AR		 1355 if (rc)
1356 return rc;
cecbcddf 1357
df158561
AR		 1358 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1359 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1360
cecbcddf
RA		 1361 rc = dev->ops->common->chain_alloc(dev->cdev,
1362 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1363 QED_CHAIN_MODE_PBL,
1364 QED_CHAIN_CNT_TYPE_U32,
1365 n_rq_elems,
1366 QEDR_RQE_ELEMENT_SIZE,
1367 &qp->rq.pbl);
df158561
AR		 1368 if (rc)
1369 return rc;
cecbcddf 1370
df158561
AR		 1371 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1372 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1373
df158561
AR		 1374 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1375 in_params, &out_params);
cecbcddf 1376
df158561
AR		 1377 if (!qp->qed_qp)
1378 return -EINVAL;
cecbcddf 1379
df158561
AR		 1380 qp->qp_id = out_params.qp_id;
1381 qp->icid = out_params.icid;
cecbcddf 1382
df158561 1383 qedr_set_roce_db_info(dev, qp);
cecbcddf 1384
df158561 1385 return 0;
cecbcddf
RA		 1386}
1387
df158561 1388static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1389{
df158561
AR		 1390 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1391 kfree(qp->wqe_wr_id);
cecbcddf 1392
df158561
AR		 1393 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1394 kfree(qp->rqe_wr_id);
cecbcddf
RA		 1395}
1396
df158561
AR		 1397static int qedr_create_kernel_qp(struct qedr_dev *dev,
1398 struct qedr_qp *qp,
1399 struct ib_pd *ibpd,
1400 struct ib_qp_init_attr *attrs)
cecbcddf 1401{
df158561
AR		 1402 struct qed_rdma_create_qp_in_params in_params;
1403 struct qedr_pd *pd = get_qedr_pd(ibpd);
1404 int rc = -EINVAL;
1405 u32 n_rq_elems;
1406 u32 n_sq_elems;
1407 u32 n_sq_entries;
cecbcddf 1408
df158561 1409 memset(&in_params, 0, sizeof(in_params));
cecbcddf 1410
df158561
AR		 1411 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1412 * the ring. The ring should allow at least a single WR, even if the
1413 * user requested none, due to allocation issues.
1414 * We should add an extra WR since the prod and cons indices of
1415 * wqe_wr_id are managed in such a way that the WQ is considered full
1416 * when (prod+1)%max_wr==cons. We currently don't do that because we
1417 * double the number of entries due an iSER issue that pushes far more
1418 * WRs than indicated. If we decline its ib_post_send() then we get
1419 * error prints in the dmesg we'd like to avoid.
1420 */
1421 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1422 dev->attr.max_sqe);
cecbcddf 1423
df158561
AR		 1424 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1425 GFP_KERNEL);
1426 if (!qp->wqe_wr_id) {
1427 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1428 return -ENOMEM;
1429 }
cecbcddf 1430
df158561
AR		 1431 /* QP handle to be written in CQE */
1432 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1433 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 1434
df158561
AR		 1435 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1436 * the ring. There ring should allow at least a single WR, even if the
1437 * user requested none, due to allocation issues.
1438 */
1439 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 1440
df158561
AR		 1441 /* Allocate driver internal RQ array */
1442 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1443 GFP_KERNEL);
1444 if (!qp->rqe_wr_id) {
1445 DP_ERR(dev,
1446 "create qp: failed RQ shadow memory allocation\n");
1447 kfree(qp->wqe_wr_id);
1448 return -ENOMEM;
cecbcddf
RA		 1449 }
1450
df158561 1451 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 1452
df158561
AR		 1453 n_sq_entries = attrs->cap.max_send_wr;
1454 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1455 n_sq_entries = max_t(u32, n_sq_entries, 1);
1456 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 1457
df158561
AR		 1458 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1459
1460 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1461 n_sq_elems, n_rq_elems);
1462 if (rc)
1463 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 1464
1465 return rc;
1466}
1467
1468struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1469 struct ib_qp_init_attr *attrs,
1470 struct ib_udata *udata)
1471{
1472 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 1473 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 1474 struct qedr_qp *qp;
181d8015 1475 struct ib_qp *ibqp;
cecbcddf
RA		 1476 int rc = 0;
1477
1478 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1479 udata ? "user library" : "kernel", pd);
1480
1481 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1482 if (rc)
1483 return ERR_PTR(rc);
1484
181d8015
WY		 1485 if (attrs->srq)
1486 return ERR_PTR(-EINVAL);
1487
cecbcddf 1488 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 1489 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1490 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 1491 get_qedr_cq(attrs->send_cq),
1492 get_qedr_cq(attrs->send_cq)->icid,
1493 get_qedr_cq(attrs->recv_cq),
1494 get_qedr_cq(attrs->recv_cq)->icid);
1495
df158561
AR		 1496 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1497 if (!qp) {
1498 DP_ERR(dev, "create qp: failed allocating memory\n");
1499 return ERR_PTR(-ENOMEM);
1500 }
1501
1502 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 1503
04886779 1504 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 1505 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1506 if (IS_ERR(ibqp))
1507 kfree(qp);
1508 return ibqp;
04886779
RA		 1509 }
1510
df158561
AR		 1511 if (udata)
1512 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1513 else
1514 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 1515
df158561
AR		 1516 if (rc)
1517 goto err;
cecbcddf 1518
cecbcddf
RA		 1519 qp->ibqp.qp_num = qp->qp_id;
1520
cecbcddf
RA		 1521 return &qp->ibqp;
1522
df158561 1523err:
cecbcddf
RA		 1524 kfree(qp);
1525
1526 return ERR_PTR(-EFAULT);
1527}
1528
27a4b1a6 1529static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 1530{
1531 switch (qp_state) {
1532 case QED_ROCE_QP_STATE_RESET:
1533 return IB_QPS_RESET;
1534 case QED_ROCE_QP_STATE_INIT:
1535 return IB_QPS_INIT;
1536 case QED_ROCE_QP_STATE_RTR:
1537 return IB_QPS_RTR;
1538 case QED_ROCE_QP_STATE_RTS:
1539 return IB_QPS_RTS;
1540 case QED_ROCE_QP_STATE_SQD:
1541 return IB_QPS_SQD;
1542 case QED_ROCE_QP_STATE_ERR:
1543 return IB_QPS_ERR;
1544 case QED_ROCE_QP_STATE_SQE:
1545 return IB_QPS_SQE;
1546 }
1547 return IB_QPS_ERR;
1548}
1549
27a4b1a6
RA		 1550static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1551 enum ib_qp_state qp_state)
cecbcddf
RA		 1552{
1553 switch (qp_state) {
1554 case IB_QPS_RESET:
1555 return QED_ROCE_QP_STATE_RESET;
1556 case IB_QPS_INIT:
1557 return QED_ROCE_QP_STATE_INIT;
1558 case IB_QPS_RTR:
1559 return QED_ROCE_QP_STATE_RTR;
1560 case IB_QPS_RTS:
1561 return QED_ROCE_QP_STATE_RTS;
1562 case IB_QPS_SQD:
1563 return QED_ROCE_QP_STATE_SQD;
1564 case IB_QPS_ERR:
1565 return QED_ROCE_QP_STATE_ERR;
1566 default:
1567 return QED_ROCE_QP_STATE_ERR;
1568 }
1569}
1570
1571static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1572{
1573 qed_chain_reset(&qph->pbl);
1574 qph->prod = 0;
1575 qph->cons = 0;
1576 qph->wqe_cons = 0;
1577 qph->db_data.data.value = cpu_to_le16(0);
1578}
1579
1580static int qedr_update_qp_state(struct qedr_dev *dev,
1581 struct qedr_qp *qp,
1582 enum qed_roce_qp_state new_state)
1583{
1584 int status = 0;
1585
1586 if (new_state == qp->state)
865cea40 1587 return 0;
cecbcddf
RA		 1588
1589 switch (qp->state) {
1590 case QED_ROCE_QP_STATE_RESET:
1591 switch (new_state) {
1592 case QED_ROCE_QP_STATE_INIT:
1593 qp->prev_wqe_size = 0;
1594 qedr_reset_qp_hwq_info(&qp->sq);
1595 qedr_reset_qp_hwq_info(&qp->rq);
1596 break;
1597 default:
1598 status = -EINVAL;
1599 break;
1600 };
1601 break;
1602 case QED_ROCE_QP_STATE_INIT:
1603 switch (new_state) {
1604 case QED_ROCE_QP_STATE_RTR:
1605 /* Update doorbell (in case post_recv was
1606 * done before move to RTR)
1607 */
1608 wmb();
1609 writel(qp->rq.db_data.raw, qp->rq.db);
1610 /* Make sure write takes effect */
1611 mmiowb();
1612 break;
1613 case QED_ROCE_QP_STATE_ERR:
1614 break;
1615 default:
1616 /* Invalid state change. */
1617 status = -EINVAL;
1618 break;
1619 };
1620 break;
1621 case QED_ROCE_QP_STATE_RTR:
1622 /* RTR->XXX */
1623 switch (new_state) {
1624 case QED_ROCE_QP_STATE_RTS:
1625 break;
1626 case QED_ROCE_QP_STATE_ERR:
1627 break;
1628 default:
1629 /* Invalid state change. */
1630 status = -EINVAL;
1631 break;
1632 };
1633 break;
1634 case QED_ROCE_QP_STATE_RTS:
1635 /* RTS->XXX */
1636 switch (new_state) {
1637 case QED_ROCE_QP_STATE_SQD:
1638 break;
1639 case QED_ROCE_QP_STATE_ERR:
1640 break;
1641 default:
1642 /* Invalid state change. */
1643 status = -EINVAL;
1644 break;
1645 };
1646 break;
1647 case QED_ROCE_QP_STATE_SQD:
1648 /* SQD->XXX */
1649 switch (new_state) {
1650 case QED_ROCE_QP_STATE_RTS:
1651 case QED_ROCE_QP_STATE_ERR:
1652 break;
1653 default:
1654 /* Invalid state change. */
1655 status = -EINVAL;
1656 break;
1657 };
1658 break;
1659 case QED_ROCE_QP_STATE_ERR:
1660 /* ERR->XXX */
1661 switch (new_state) {
1662 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 1663 if ((qp->rq.prod != qp->rq.cons) ||
1664 (qp->sq.prod != qp->sq.cons)) {
1665 DP_NOTICE(dev,
1666 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1667 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1668 qp->sq.cons);
1669 status = -EINVAL;
1670 }
cecbcddf
RA		 1671 break;
1672 default:
1673 status = -EINVAL;
1674 break;
1675 };
1676 break;
1677 default:
1678 status = -EINVAL;
1679 break;
1680 };
1681
1682 return status;
1683}
1684
1685int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1686 int attr_mask, struct ib_udata *udata)
1687{
1688 struct qedr_qp *qp = get_qedr_qp(ibqp);
1689 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1690 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1691 enum ib_qp_state old_qp_state, new_qp_state;
1692 int rc = 0;
1693
1694 DP_DEBUG(dev, QEDR_MSG_QP,
1695 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1696 attr->qp_state);
1697
1698 old_qp_state = qedr_get_ibqp_state(qp->state);
1699 if (attr_mask & IB_QP_STATE)
1700 new_qp_state = attr->qp_state;
1701 else
1702 new_qp_state = old_qp_state;
1703
1704 if (!ib_modify_qp_is_ok
1705 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1706 IB_LINK_LAYER_ETHERNET)) {
1707 DP_ERR(dev,
1708 "modify qp: invalid attribute mask=0x%x specified for\n"
1709 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1710 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1711 new_qp_state);
1712 rc = -EINVAL;
1713 goto err;
1714 }
1715
1716 /* Translate the masks... */
1717 if (attr_mask & IB_QP_STATE) {
1718 SET_FIELD(qp_params.modify_flags,
1719 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1720 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1721 }
1722
1723 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1724 qp_params.sqd_async = true;
1725
1726 if (attr_mask & IB_QP_PKEY_INDEX) {
1727 SET_FIELD(qp_params.modify_flags,
1728 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1729 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1730 rc = -EINVAL;
1731 goto err;
1732 }
1733
1734 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1735 }
1736
1737 if (attr_mask & IB_QP_QKEY)
1738 qp->qkey = attr->qkey;
1739
1740 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1741 SET_FIELD(qp_params.modify_flags,
1742 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1743 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1744 IB_ACCESS_REMOTE_READ;
1745 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1746 IB_ACCESS_REMOTE_WRITE;
1747 qp_params.incoming_atomic_en = attr->qp_access_flags &
1748 IB_ACCESS_REMOTE_ATOMIC;
1749 }
1750
1751 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1752 if (attr_mask & IB_QP_PATH_MTU) {
1753 if (attr->path_mtu < IB_MTU_256 ||
1754 attr->path_mtu > IB_MTU_4096) {
1755 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1756 rc = -EINVAL;
1757 goto err;
1758 }
1759 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1760 ib_mtu_enum_to_int(iboe_get_mtu
1761 (dev->ndev->mtu)));
1762 }
1763
1764 if (!qp->mtu) {
1765 qp->mtu =
1766 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1767 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1768 }
1769
1770 SET_FIELD(qp_params.modify_flags,
1771 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1772
1773 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1774 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1775 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1776
1777 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1778
1779 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1780 if (rc) {
1781 DP_ERR(dev,
1782 "modify qp: problems with GID index %d (rc=%d)\n",
1783 attr->ah_attr.grh.sgid_index, rc);
1784 return rc;
1785 }
1786
1787 rc = qedr_get_dmac(dev, &attr->ah_attr,
1788 qp_params.remote_mac_addr);
1789 if (rc)
1790 return rc;
1791
1792 qp_params.use_local_mac = true;
1793 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1794
1795 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1796 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1797 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1798 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1799 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1800 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1801 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1802 qp_params.remote_mac_addr);
cecbcddf
RA		 1803
1804 qp_params.mtu = qp->mtu;
1805 qp_params.lb_indication = false;
1806 }
1807
1808 if (!qp_params.mtu) {
1809 /* Stay with current MTU */
1810 if (qp->mtu)
1811 qp_params.mtu = qp->mtu;
1812 else
1813 qp_params.mtu =
1814 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1815 }
1816
1817 if (attr_mask & IB_QP_TIMEOUT) {
1818 SET_FIELD(qp_params.modify_flags,
1819 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1820
1821 qp_params.ack_timeout = attr->timeout;
1822 if (attr->timeout) {
1823 u32 temp;
1824
1825 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1826 /* FW requires [msec] */
1827 qp_params.ack_timeout = temp;
1828 } else {
1829 /* Infinite */
1830 qp_params.ack_timeout = 0;
1831 }
1832 }
1833 if (attr_mask & IB_QP_RETRY_CNT) {
1834 SET_FIELD(qp_params.modify_flags,
1835 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1836 qp_params.retry_cnt = attr->retry_cnt;
1837 }
1838
1839 if (attr_mask & IB_QP_RNR_RETRY) {
1840 SET_FIELD(qp_params.modify_flags,
1841 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1842 qp_params.rnr_retry_cnt = attr->rnr_retry;
1843 }
1844
1845 if (attr_mask & IB_QP_RQ_PSN) {
1846 SET_FIELD(qp_params.modify_flags,
1847 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1848 qp_params.rq_psn = attr->rq_psn;
1849 qp->rq_psn = attr->rq_psn;
1850 }
1851
1852 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1853 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1854 rc = -EINVAL;
1855 DP_ERR(dev,
1856 "unsupported max_rd_atomic=%d, supported=%d\n",
1857 attr->max_rd_atomic,
1858 dev->attr.max_qp_req_rd_atomic_resc);
1859 goto err;
1860 }
1861
1862 SET_FIELD(qp_params.modify_flags,
1863 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1864 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1865 }
1866
1867 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1868 SET_FIELD(qp_params.modify_flags,
1869 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1870 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1871 }
1872
1873 if (attr_mask & IB_QP_SQ_PSN) {
1874 SET_FIELD(qp_params.modify_flags,
1875 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1876 qp_params.sq_psn = attr->sq_psn;
1877 qp->sq_psn = attr->sq_psn;
1878 }
1879
1880 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1881 if (attr->max_dest_rd_atomic >
1882 dev->attr.max_qp_resp_rd_atomic_resc) {
1883 DP_ERR(dev,
1884 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1885 attr->max_dest_rd_atomic,
1886 dev->attr.max_qp_resp_rd_atomic_resc);
1887
1888 rc = -EINVAL;
1889 goto err;
1890 }
1891
1892 SET_FIELD(qp_params.modify_flags,
1893 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1894 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1895 }
1896
1897 if (attr_mask & IB_QP_DEST_QPN) {
1898 SET_FIELD(qp_params.modify_flags,
1899 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1900
1901 qp_params.dest_qp = attr->dest_qp_num;
1902 qp->dest_qp_num = attr->dest_qp_num;
1903 }
1904
1905 if (qp->qp_type != IB_QPT_GSI)
1906 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1907 qp->qed_qp, &qp_params);
1908
1909 if (attr_mask & IB_QP_STATE) {
1910 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
d6ebbf29 1911 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
cecbcddf
RA		 1912 qp->state = qp_params.new_state;
1913 }
1914
1915err:
1916 return rc;
1917}
1918
1919static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1920{
1921 int ib_qp_acc_flags = 0;
1922
1923 if (params->incoming_rdma_write_en)
1924 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1925 if (params->incoming_rdma_read_en)
1926 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
1927 if (params->incoming_atomic_en)
1928 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
1929 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1930 return ib_qp_acc_flags;
1931}
1932
1933int qedr_query_qp(struct ib_qp *ibqp,
1934 struct ib_qp_attr *qp_attr,
1935 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1936{
1937 struct qed_rdma_query_qp_out_params params;
1938 struct qedr_qp *qp = get_qedr_qp(ibqp);
1939 struct qedr_dev *dev = qp->dev;
1940 int rc = 0;
1941
1942 memset(&params, 0, sizeof(params));
1943
1944 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
1945 if (rc)
1946 goto err;
1947
1948 memset(qp_attr, 0, sizeof(*qp_attr));
1949 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
1950
1951 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
1952 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 1953 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 1954 qp_attr->path_mig_state = IB_MIG_MIGRATED;
1955 qp_attr->rq_psn = params.rq_psn;
1956 qp_attr->sq_psn = params.sq_psn;
1957 qp_attr->dest_qp_num = params.dest_qp;
1958
1959 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
1960
1961 qp_attr->cap.max_send_wr = qp->sq.max_wr;
1962 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
1963 qp_attr->cap.max_send_sge = qp->sq.max_sges;
1964 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 1965 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 1966 qp_init_attr->cap = qp_attr->cap;
1967
1968 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
1969 sizeof(qp_attr->ah_attr.grh.dgid.raw));
1970
1971 qp_attr->ah_attr.grh.flow_label = params.flow_label;
1972 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
1973 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
1974 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
1975
1976 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
1977 qp_attr->ah_attr.port_num = 1;
1978 qp_attr->ah_attr.sl = 0;
1979 qp_attr->timeout = params.timeout;
1980 qp_attr->rnr_retry = params.rnr_retry;
1981 qp_attr->retry_cnt = params.retry_cnt;
1982 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
1983 qp_attr->pkey_index = params.pkey_index;
1984 qp_attr->port_num = 1;
1985 qp_attr->ah_attr.src_path_bits = 0;
1986 qp_attr->ah_attr.static_rate = 0;
1987 qp_attr->alt_pkey_index = 0;
1988 qp_attr->alt_port_num = 0;
1989 qp_attr->alt_timeout = 0;
1990 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1991
1992 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
1993 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
1994 qp_attr->max_rd_atomic = params.max_rd_atomic;
1995 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
1996
1997 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
1998 qp_attr->cap.max_inline_data);
1999
2000err:
2001 return rc;
2002}
2003
df158561
AR		 2004int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2005{
2006 int rc = 0;
2007
2008 if (qp->qp_type != IB_QPT_GSI) {
2009 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2010 if (rc)
2011 return rc;
2012 }
2013
2014 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2015 qedr_cleanup_user(dev, qp);
2016 else
2017 qedr_cleanup_kernel(dev, qp);
2018
2019 return 0;
2020}
2021
cecbcddf
RA		 2022int qedr_destroy_qp(struct ib_qp *ibqp)
2023{
2024 struct qedr_qp *qp = get_qedr_qp(ibqp);
2025 struct qedr_dev *dev = qp->dev;
2026 struct ib_qp_attr attr;
2027 int attr_mask = 0;
2028 int rc = 0;
2029
2030 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2031 qp, qp->qp_type);
2032
b4c2cc48
AR		 2033 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2034 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2035 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2036
cecbcddf
RA		 2037 attr.qp_state = IB_QPS_ERR;
2038 attr_mask |= IB_QP_STATE;
2039
2040 /* Change the QP state to ERROR */
2041 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2042 }
2043
df158561 2044 if (qp->qp_type == IB_QPT_GSI)
04886779 2045 qedr_destroy_gsi_qp(dev);
cecbcddf 2046
df158561 2047 qedr_free_qp_resources(dev, qp);
cecbcddf
RA		 2048
2049 kfree(qp);
2050
2051 return rc;
2052}
e0290cce 2053
477864c8
MS		 2054struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
2055 struct ib_udata *udata)
04886779
RA		 2056{
2057 struct qedr_ah *ah;
2058
2059 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2060 if (!ah)
2061 return ERR_PTR(-ENOMEM);
2062
2063 ah->attr = *attr;
2064
2065 return &ah->ibah;
2066}
2067
2068int qedr_destroy_ah(struct ib_ah *ibah)
2069{
2070 struct qedr_ah *ah = get_qedr_ah(ibah);
2071
2072 kfree(ah);
2073 return 0;
2074}
2075
e0290cce
RA		 2076static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2077{
2078 struct qedr_pbl *pbl, *tmp;
2079
2080 if (info->pbl_table)
2081 list_add_tail(&info->pbl_table->list_entry,
2082 &info->free_pbl_list);
2083
2084 if (!list_empty(&info->inuse_pbl_list))
2085 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2086
2087 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2088 list_del(&pbl->list_entry);
2089 qedr_free_pbl(dev, &info->pbl_info, pbl);
2090 }
2091}
2092
2093static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2094 size_t page_list_len, bool two_layered)
2095{
2096 struct qedr_pbl *tmp;
2097 int rc;
2098
2099 INIT_LIST_HEAD(&info->free_pbl_list);
2100 INIT_LIST_HEAD(&info->inuse_pbl_list);
2101
2102 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2103 page_list_len, two_layered);
2104 if (rc)
2105 goto done;
2106
2107 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2108 if (!info->pbl_table) {
2109 rc = -ENOMEM;
2110 goto done;
2111 }
2112
2113 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2114 &info->pbl_table->pa);
2115
2116 /* in usual case we use 2 PBLs, so we add one to free
2117 * list and allocating another one
2118 */
2119 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2120 if (!tmp) {
2121 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2122 goto done;
2123 }
2124
2125 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2126
2127 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2128
2129done:
2130 if (rc)
2131 free_mr_info(dev, info);
2132
2133 return rc;
2134}
2135
2136struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2137 u64 usr_addr, int acc, struct ib_udata *udata)
2138{
2139 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2140 struct qedr_mr *mr;
2141 struct qedr_pd *pd;
2142 int rc = -ENOMEM;
2143
2144 pd = get_qedr_pd(ibpd);
2145 DP_DEBUG(dev, QEDR_MSG_MR,
2146 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2147 pd->pd_id, start, len, usr_addr, acc);
2148
2149 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2150 return ERR_PTR(-EINVAL);
2151
2152 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2153 if (!mr)
2154 return ERR_PTR(rc);
2155
2156 mr->type = QEDR_MR_USER;
2157
2158 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2159 if (IS_ERR(mr->umem)) {
2160 rc = -EFAULT;
2161 goto err0;
2162 }
2163
2164 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2165 if (rc)
2166 goto err1;
2167
2168 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2169 &mr->info.pbl_info);
2170
2171 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2172 if (rc) {
2173 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2174 goto err1;
2175 }
2176
2177 /* Index only, 18 bit long, lkey = itid << 8 | key */
2178 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2179 mr->hw_mr.key = 0;
2180 mr->hw_mr.pd = pd->pd_id;
2181 mr->hw_mr.local_read = 1;
2182 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2183 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2184 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2185 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2186 mr->hw_mr.mw_bind = false;
2187 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2188 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2189 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2190 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size);
2191 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2192 mr->hw_mr.length = len;
2193 mr->hw_mr.vaddr = usr_addr;
2194 mr->hw_mr.zbva = false;
2195 mr->hw_mr.phy_mr = false;
2196 mr->hw_mr.dma_mr = false;
2197
2198 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2199 if (rc) {
2200 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2201 goto err2;
2202 }
2203
2204 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2205 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2206 mr->hw_mr.remote_atomic)
2207 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2208
2209 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2210 mr->ibmr.lkey);
2211 return &mr->ibmr;
2212
2213err2:
2214 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2215err1:
2216 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2217err0:
2218 kfree(mr);
2219 return ERR_PTR(rc);
2220}
2221
2222int qedr_dereg_mr(struct ib_mr *ib_mr)
2223{
2224 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2225 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2226 int rc = 0;
2227
2228 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2229 if (rc)
2230 return rc;
2231
2232 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2233
2234 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2235 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2236
2237 /* it could be user registered memory. */
2238 if (mr->umem)
2239 ib_umem_release(mr->umem);
2240
2241 kfree(mr);
2242
2243 return rc;
2244}
2245
27a4b1a6
RA		 2246static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2247 int max_page_list_len)
e0290cce
RA		 2248{
2249 struct qedr_pd *pd = get_qedr_pd(ibpd);
2250 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2251 struct qedr_mr *mr;
2252 int rc = -ENOMEM;
2253
2254 DP_DEBUG(dev, QEDR_MSG_MR,
2255 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2256 max_page_list_len);
2257
2258 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2259 if (!mr)
2260 return ERR_PTR(rc);
2261
2262 mr->dev = dev;
2263 mr->type = QEDR_MR_FRMR;
2264
2265 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2266 if (rc)
2267 goto err0;
2268
2269 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2270 if (rc) {
2271 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2272 goto err0;
2273 }
2274
2275 /* Index only, 18 bit long, lkey = itid << 8 | key */
2276 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2277 mr->hw_mr.key = 0;
2278 mr->hw_mr.pd = pd->pd_id;
2279 mr->hw_mr.local_read = 1;
2280 mr->hw_mr.local_write = 0;
2281 mr->hw_mr.remote_read = 0;
2282 mr->hw_mr.remote_write = 0;
2283 mr->hw_mr.remote_atomic = 0;
2284 mr->hw_mr.mw_bind = false;
2285 mr->hw_mr.pbl_ptr = 0;
2286 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2287 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2288 mr->hw_mr.fbo = 0;
2289 mr->hw_mr.length = 0;
2290 mr->hw_mr.vaddr = 0;
2291 mr->hw_mr.zbva = false;
2292 mr->hw_mr.phy_mr = true;
2293 mr->hw_mr.dma_mr = false;
2294
2295 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2296 if (rc) {
2297 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2298 goto err1;
2299 }
2300
2301 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2302 mr->ibmr.rkey = mr->ibmr.lkey;
2303
2304 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2305 return mr;
2306
2307err1:
2308 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2309err0:
2310 kfree(mr);
2311 return ERR_PTR(rc);
2312}
2313
2314struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2315 enum ib_mr_type mr_type, u32 max_num_sg)
2316{
2317 struct qedr_dev *dev;
2318 struct qedr_mr *mr;
2319
2320 if (mr_type != IB_MR_TYPE_MEM_REG)
2321 return ERR_PTR(-EINVAL);
2322
2323 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2324
2325 if (IS_ERR(mr))
2326 return ERR_PTR(-EINVAL);
2327
2328 dev = mr->dev;
2329
2330 return &mr->ibmr;
2331}
2332
2333static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2334{
2335 struct qedr_mr *mr = get_qedr_mr(ibmr);
2336 struct qedr_pbl *pbl_table;
2337 struct regpair *pbe;
2338 u32 pbes_in_page;
2339
2340 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2341 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2342 return -ENOMEM;
2343 }
2344
2345 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2346 mr->npages, addr);
2347
2348 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2349 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2350 pbe = (struct regpair *)pbl_table->va;
2351 pbe += mr->npages % pbes_in_page;
2352 pbe->lo = cpu_to_le32((u32)addr);
2353 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2354
2355 mr->npages++;
2356
2357 return 0;
2358}
2359
2360static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2361{
2362 int work = info->completed - info->completed_handled - 1;
2363
2364 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2365 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2366 struct qedr_pbl *pbl;
2367
2368 /* Free all the page list that are possible to be freed
2369 * (all the ones that were invalidated), under the assumption
2370 * that if an FMR was completed successfully that means that
2371 * if there was an invalidate operation before it also ended
2372 */
2373 pbl = list_first_entry(&info->inuse_pbl_list,
2374 struct qedr_pbl, list_entry);
aafec388 2375 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 2376 info->completed_handled++;
2377 }
2378}
2379
2380int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2381 int sg_nents, unsigned int *sg_offset)
2382{
2383 struct qedr_mr *mr = get_qedr_mr(ibmr);
2384
2385 mr->npages = 0;
2386
2387 handle_completed_mrs(mr->dev, &mr->info);
2388 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2389}
2390
2391struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2392{
2393 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2394 struct qedr_pd *pd = get_qedr_pd(ibpd);
2395 struct qedr_mr *mr;
2396 int rc;
2397
2398 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2399 if (!mr)
2400 return ERR_PTR(-ENOMEM);
2401
2402 mr->type = QEDR_MR_DMA;
2403
2404 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2405 if (rc) {
2406 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2407 goto err1;
2408 }
2409
2410 /* index only, 18 bit long, lkey = itid << 8 | key */
2411 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2412 mr->hw_mr.pd = pd->pd_id;
2413 mr->hw_mr.local_read = 1;
2414 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2415 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2416 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2417 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2418 mr->hw_mr.dma_mr = true;
2419
2420 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2421 if (rc) {
2422 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2423 goto err2;
2424 }
2425
2426 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2427 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2428 mr->hw_mr.remote_atomic)
2429 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2430
2431 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2432 return &mr->ibmr;
2433
2434err2:
2435 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2436err1:
2437 kfree(mr);
2438 return ERR_PTR(rc);
2439}
afa0e13b
RA		 2440
2441static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2442{
2443 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2444}
2445
2446static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2447{
2448 int i, len = 0;
2449
2450 for (i = 0; i < num_sge; i++)
2451 len += sg_list[i].length;
2452
2453 return len;
2454}
2455
2456static void swap_wqe_data64(u64 *p)
2457{
2458 int i;
2459
2460 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2461 *p = cpu_to_be64(cpu_to_le64(*p));
2462}
2463
2464static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2465 struct qedr_qp *qp, u8 *wqe_size,
2466 struct ib_send_wr *wr,
2467 struct ib_send_wr **bad_wr, u8 *bits,
2468 u8 bit)
2469{
2470 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2471 char *seg_prt, *wqe;
2472 int i, seg_siz;
2473
2474 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2475 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2476 *bad_wr = wr;
2477 return 0;
2478 }
2479
2480 if (!data_size)
2481 return data_size;
2482
2483 *bits |= bit;
2484
2485 seg_prt = NULL;
2486 wqe = NULL;
2487 seg_siz = 0;
2488
2489 /* Copy data inline */
2490 for (i = 0; i < wr->num_sge; i++) {
2491 u32 len = wr->sg_list[i].length;
2492 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2493
2494 while (len > 0) {
2495 u32 cur;
2496
2497 /* New segment required */
2498 if (!seg_siz) {
2499 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2500 seg_prt = wqe;
2501 seg_siz = sizeof(struct rdma_sq_common_wqe);
2502 (*wqe_size)++;
2503 }
2504
2505 /* Calculate currently allowed length */
2506 cur = min_t(u32, len, seg_siz);
2507 memcpy(seg_prt, src, cur);
2508
2509 /* Update segment variables */
2510 seg_prt += cur;
2511 seg_siz -= cur;
2512
2513 /* Update sge variables */
2514 src += cur;
2515 len -= cur;
2516
2517 /* Swap fully-completed segments */
2518 if (!seg_siz)
2519 swap_wqe_data64((u64 *)wqe);
2520 }
2521 }
2522
2523 /* swap last not completed segment */
2524 if (seg_siz)
2525 swap_wqe_data64((u64 *)wqe);
2526
2527 return data_size;
2528}
2529
2530#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2531 do { \
2532 DMA_REGPAIR_LE(sge->addr, vaddr); \
2533 (sge)->length = cpu_to_le32(vlength); \
2534 (sge)->flags = cpu_to_le32(vflags); \
2535 } while (0)
2536
2537#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2538 do { \
2539 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2540 (hdr)->num_sges = num_sge; \
2541 } while (0)
2542
2543#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2544 do { \
2545 DMA_REGPAIR_LE(sge->addr, vaddr); \
2546 (sge)->length = cpu_to_le32(vlength); \
2547 (sge)->l_key = cpu_to_le32(vlkey); \
2548 } while (0)
2549
2550static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2551 struct ib_send_wr *wr)
2552{
2553 u32 data_size = 0;
2554 int i;
2555
2556 for (i = 0; i < wr->num_sge; i++) {
2557 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2558
2559 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2560 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2561 sge->length = cpu_to_le32(wr->sg_list[i].length);
2562 data_size += wr->sg_list[i].length;
2563 }
2564
2565 if (wqe_size)
2566 *wqe_size += wr->num_sge;
2567
2568 return data_size;
2569}
2570
2571static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2572 struct qedr_qp *qp,
2573 struct rdma_sq_rdma_wqe_1st *rwqe,
2574 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2575 struct ib_send_wr *wr,
2576 struct ib_send_wr **bad_wr)
2577{
2578 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2579 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2580
8b0cabc6
AR		 2581 if (wr->send_flags & IB_SEND_INLINE &&
2582 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2583 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 2584 u8 flags = 0;
2585
2586 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2587 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2588 bad_wr, &rwqe->flags, flags);
2589 }
2590
2591 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2592}
2593
2594static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2595 struct qedr_qp *qp,
2596 struct rdma_sq_send_wqe_1st *swqe,
2597 struct rdma_sq_send_wqe_2st *swqe2,
2598 struct ib_send_wr *wr,
2599 struct ib_send_wr **bad_wr)
2600{
2601 memset(swqe2, 0, sizeof(*swqe2));
2602 if (wr->send_flags & IB_SEND_INLINE) {
2603 u8 flags = 0;
2604
2605 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2606 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2607 bad_wr, &swqe->flags, flags);
2608 }
2609
2610 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2611}
2612
2613static int qedr_prepare_reg(struct qedr_qp *qp,
2614 struct rdma_sq_fmr_wqe_1st *fwqe1,
2615 struct ib_reg_wr *wr)
2616{
2617 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2618 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2619
2620 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2621 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2622 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2623 fwqe1->l_key = wr->key;
2624
2625 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2626 !!(wr->access & IB_ACCESS_REMOTE_READ));
2627 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2628 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2629 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2630 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2631 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2632 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2633 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2634 fwqe2->fmr_ctrl = 0;
2635
2636 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2637 ilog2(mr->ibmr.page_size) - 12);
2638
2639 fwqe2->length_hi = 0;
2640 fwqe2->length_lo = mr->ibmr.length;
2641 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2642 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2643
2644 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2645
2646 return 0;
2647}
2648
27a4b1a6 2649static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 2650{
2651 switch (opcode) {
2652 case IB_WR_RDMA_WRITE:
2653 case IB_WR_RDMA_WRITE_WITH_IMM:
2654 return IB_WC_RDMA_WRITE;
2655 case IB_WR_SEND_WITH_IMM:
2656 case IB_WR_SEND:
2657 case IB_WR_SEND_WITH_INV:
2658 return IB_WC_SEND;
2659 case IB_WR_RDMA_READ:
2660 return IB_WC_RDMA_READ;
2661 case IB_WR_ATOMIC_CMP_AND_SWP:
2662 return IB_WC_COMP_SWAP;
2663 case IB_WR_ATOMIC_FETCH_AND_ADD:
2664 return IB_WC_FETCH_ADD;
2665 case IB_WR_REG_MR:
2666 return IB_WC_REG_MR;
2667 case IB_WR_LOCAL_INV:
2668 return IB_WC_LOCAL_INV;
2669 default:
2670 return IB_WC_SEND;
2671 }
2672}
2673
27a4b1a6 2674static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
afa0e13b
RA		 2675{
2676 int wq_is_full, err_wr, pbl_is_full;
2677 struct qedr_dev *dev = qp->dev;
2678
2679 /* prevent SQ overflow and/or processing of a bad WR */
2680 err_wr = wr->num_sge > qp->sq.max_sges;
2681 wq_is_full = qedr_wq_is_full(&qp->sq);
2682 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2683 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2684 if (wq_is_full || err_wr || pbl_is_full) {
2685 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2686 DP_ERR(dev,
2687 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2688 qp);
2689 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2690 }
2691
2692 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2693 DP_ERR(dev,
2694 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2695 qp);
2696 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2697 }
2698
2699 if (pbl_is_full &&
2700 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2701 DP_ERR(dev,
2702 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2703 qp);
2704 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2705 }
2706 return false;
2707 }
2708 return true;
2709}
2710
27a4b1a6 2711static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
afa0e13b
RA		 2712 struct ib_send_wr **bad_wr)
2713{
2714 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2715 struct qedr_qp *qp = get_qedr_qp(ibqp);
2716 struct rdma_sq_atomic_wqe_1st *awqe1;
2717 struct rdma_sq_atomic_wqe_2nd *awqe2;
2718 struct rdma_sq_atomic_wqe_3rd *awqe3;
2719 struct rdma_sq_send_wqe_2st *swqe2;
2720 struct rdma_sq_local_inv_wqe *iwqe;
2721 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2722 struct rdma_sq_send_wqe_1st *swqe;
2723 struct rdma_sq_rdma_wqe_1st *rwqe;
2724 struct rdma_sq_fmr_wqe_1st *fwqe1;
2725 struct rdma_sq_common_wqe *wqe;
2726 u32 length;
2727 int rc = 0;
2728 bool comp;
2729
2730 if (!qedr_can_post_send(qp, wr)) {
2731 *bad_wr = wr;
2732 return -ENOMEM;
2733 }
2734
2735 wqe = qed_chain_produce(&qp->sq.pbl);
2736 qp->wqe_wr_id[qp->sq.prod].signaled =
2737 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2738
2739 wqe->flags = 0;
2740 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2741 !!(wr->send_flags & IB_SEND_SOLICITED));
2742 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2743 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2744 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2745 !!(wr->send_flags & IB_SEND_FENCE));
2746 wqe->prev_wqe_size = qp->prev_wqe_size;
2747
2748 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2749
2750 switch (wr->opcode) {
2751 case IB_WR_SEND_WITH_IMM:
2752 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2753 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2754 swqe->wqe_size = 2;
2755 swqe2 = qed_chain_produce(&qp->sq.pbl);
2756
2757 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2758 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2759 wr, bad_wr);
2760 swqe->length = cpu_to_le32(length);
2761 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2762 qp->prev_wqe_size = swqe->wqe_size;
2763 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2764 break;
2765 case IB_WR_SEND:
2766 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2767 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2768
2769 swqe->wqe_size = 2;
2770 swqe2 = qed_chain_produce(&qp->sq.pbl);
2771 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2772 wr, bad_wr);
2773 swqe->length = cpu_to_le32(length);
2774 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2775 qp->prev_wqe_size = swqe->wqe_size;
2776 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2777 break;
2778 case IB_WR_SEND_WITH_INV:
2779 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2780 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2781 swqe2 = qed_chain_produce(&qp->sq.pbl);
2782 swqe->wqe_size = 2;
2783 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2784 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2785 wr, bad_wr);
2786 swqe->length = cpu_to_le32(length);
2787 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2788 qp->prev_wqe_size = swqe->wqe_size;
2789 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2790 break;
2791
2792 case IB_WR_RDMA_WRITE_WITH_IMM:
2793 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2794 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2795
2796 rwqe->wqe_size = 2;
2797 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2798 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2799 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2800 wr, bad_wr);
2801 rwqe->length = cpu_to_le32(length);
2802 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2803 qp->prev_wqe_size = rwqe->wqe_size;
2804 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2805 break;
2806 case IB_WR_RDMA_WRITE:
2807 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2808 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2809
2810 rwqe->wqe_size = 2;
2811 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2812 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2813 wr, bad_wr);
2814 rwqe->length = cpu_to_le32(length);
2815 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2816 qp->prev_wqe_size = rwqe->wqe_size;
2817 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2818 break;
2819 case IB_WR_RDMA_READ_WITH_INV:
2820 DP_ERR(dev,
2821 "RDMA READ WITH INVALIDATE not supported\n");
2822 *bad_wr = wr;
2823 rc = -EINVAL;
2824 break;
2825
2826 case IB_WR_RDMA_READ:
2827 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2828 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2829
2830 rwqe->wqe_size = 2;
2831 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2832 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2833 wr, bad_wr);
2834 rwqe->length = cpu_to_le32(length);
2835 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2836 qp->prev_wqe_size = rwqe->wqe_size;
2837 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2838 break;
2839
2840 case IB_WR_ATOMIC_CMP_AND_SWP:
2841 case IB_WR_ATOMIC_FETCH_AND_ADD:
2842 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2843 awqe1->wqe_size = 4;
2844
2845 awqe2 = qed_chain_produce(&qp->sq.pbl);
2846 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2847 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2848
2849 awqe3 = qed_chain_produce(&qp->sq.pbl);
2850
2851 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2852 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2853 DMA_REGPAIR_LE(awqe3->swap_data,
2854 atomic_wr(wr)->compare_add);
2855 } else {
2856 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2857 DMA_REGPAIR_LE(awqe3->swap_data,
2858 atomic_wr(wr)->swap);
2859 DMA_REGPAIR_LE(awqe3->cmp_data,
2860 atomic_wr(wr)->compare_add);
2861 }
2862
2863 qedr_prepare_sq_sges(qp, NULL, wr);
2864
2865 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2866 qp->prev_wqe_size = awqe1->wqe_size;
2867 break;
2868
2869 case IB_WR_LOCAL_INV:
2870 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2871 iwqe->wqe_size = 1;
2872
2873 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2874 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2875 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2876 qp->prev_wqe_size = iwqe->wqe_size;
2877 break;
2878 case IB_WR_REG_MR:
2879 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2880 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2881 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2882 fwqe1->wqe_size = 2;
2883
2884 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2885 if (rc) {
2886 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2887 *bad_wr = wr;
2888 break;
2889 }
2890
2891 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2892 qp->prev_wqe_size = fwqe1->wqe_size;
2893 break;
2894 default:
2895 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2896 rc = -EINVAL;
2897 *bad_wr = wr;
2898 break;
2899 }
2900
2901 if (*bad_wr) {
2902 u16 value;
2903
2904 /* Restore prod to its position before
2905 * this WR was processed
2906 */
2907 value = le16_to_cpu(qp->sq.db_data.data.value);
2908 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2909
2910 /* Restore prev_wqe_size */
2911 qp->prev_wqe_size = wqe->prev_wqe_size;
2912 rc = -EINVAL;
2913 DP_ERR(dev, "POST SEND FAILED\n");
2914 }
2915
2916 return rc;
2917}
2918
2919int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2920 struct ib_send_wr **bad_wr)
2921{
2922 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2923 struct qedr_qp *qp = get_qedr_qp(ibqp);
2924 unsigned long flags;
2925 int rc = 0;
2926
2927 *bad_wr = NULL;
2928
04886779
RA		 2929 if (qp->qp_type == IB_QPT_GSI)
2930 return qedr_gsi_post_send(ibqp, wr, bad_wr);
2931
afa0e13b
RA		 2932 spin_lock_irqsave(&qp->q_lock, flags);
2933
922d9a40
AR		 2934 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
2935 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2936 (qp->state != QED_ROCE_QP_STATE_SQD)) {
afa0e13b
RA		 2937 spin_unlock_irqrestore(&qp->q_lock, flags);
2938 *bad_wr = wr;
2939 DP_DEBUG(dev, QEDR_MSG_CQ,
2940 "QP in wrong state! QP icid=0x%x state %d\n",
2941 qp->icid, qp->state);
2942 return -EINVAL;
2943 }
2944
afa0e13b
RA		 2945 while (wr) {
2946 rc = __qedr_post_send(ibqp, wr, bad_wr);
2947 if (rc)
2948 break;
2949
2950 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
2951
2952 qedr_inc_sw_prod(&qp->sq);
2953
2954 qp->sq.db_data.data.value++;
2955
2956 wr = wr->next;
2957 }
2958
2959 /* Trigger doorbell
2960 * If there was a failure in the first WR then it will be triggered in
2961 * vane. However this is not harmful (as long as the producer value is
2962 * unchanged). For performance reasons we avoid checking for this
2963 * redundant doorbell.
2964 */
2965 wmb();
2966 writel(qp->sq.db_data.raw, qp->sq.db);
2967
2968 /* Make sure write sticks */
2969 mmiowb();
2970
2971 spin_unlock_irqrestore(&qp->q_lock, flags);
2972
2973 return rc;
2974}
2975
2976int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
2977 struct ib_recv_wr **bad_wr)
2978{
2979 struct qedr_qp *qp = get_qedr_qp(ibqp);
2980 struct qedr_dev *dev = qp->dev;
2981 unsigned long flags;
2982 int status = 0;
2983
04886779
RA		 2984 if (qp->qp_type == IB_QPT_GSI)
2985 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
2986
afa0e13b
RA		 2987 spin_lock_irqsave(&qp->q_lock, flags);
2988
922d9a40 2989 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 2990 spin_unlock_irqrestore(&qp->q_lock, flags);
2991 *bad_wr = wr;
2992 return -EINVAL;
2993 }
2994
2995 while (wr) {
2996 int i;
2997
2998 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
2999 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3000 wr->num_sge > qp->rq.max_sges) {
3001 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3002 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3003 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3004 qp->rq.max_sges);
3005 status = -ENOMEM;
3006 *bad_wr = wr;
3007 break;
3008 }
3009 for (i = 0; i < wr->num_sge; i++) {
3010 u32 flags = 0;
3011 struct rdma_rq_sge *rqe =
3012 qed_chain_produce(&qp->rq.pbl);
3013
3014 /* First one must include the number
3015 * of SGE in the list
3016 */
3017 if (!i)
3018 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3019 wr->num_sge);
3020
3021 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3022 wr->sg_list[i].lkey);
3023
3024 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3025 wr->sg_list[i].length, flags);
3026 }
3027
3028 /* Special case of no sges. FW requires between 1-4 sges...
3029 * in this case we need to post 1 sge with length zero. this is
3030 * because rdma write with immediate consumes an RQ.
3031 */
3032 if (!wr->num_sge) {
3033 u32 flags = 0;
3034 struct rdma_rq_sge *rqe =
3035 qed_chain_produce(&qp->rq.pbl);
3036
3037 /* First one must include the number
3038 * of SGE in the list
3039 */
3040 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3041 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3042
3043 RQ_SGE_SET(rqe, 0, 0, flags);
3044 i = 1;
3045 }
3046
3047 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3048 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3049
3050 qedr_inc_sw_prod(&qp->rq);
3051
3052 /* Flush all the writes before signalling doorbell */
3053 wmb();
3054
3055 qp->rq.db_data.data.value++;
3056
3057 writel(qp->rq.db_data.raw, qp->rq.db);
3058
3059 /* Make sure write sticks */
3060 mmiowb();
3061
3062 wr = wr->next;
3063 }
3064
3065 spin_unlock_irqrestore(&qp->q_lock, flags);
3066
3067 return status;
3068}
3069
3070static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3071{
3072 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3073
3074 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3075 cq->pbl_toggle;
3076}
3077
3078static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3079{
3080 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3081 struct qedr_qp *qp;
3082
3083 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3084 resp_cqe->qp_handle.lo,
3085 u64);
3086 return qp;
3087}
3088
3089static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3090{
3091 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3092
3093 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3094}
3095
3096/* Return latest CQE (needs processing) */
3097static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3098{
3099 return cq->latest_cqe;
3100}
3101
3102/* In fmr we need to increase the number of fmr completed counter for the fmr
3103 * algorithm determining whether we can free a pbl or not.
3104 * we need to perform this whether the work request was signaled or not. for
3105 * this purpose we call this function from the condition that checks if a wr
3106 * should be skipped, to make sure we don't miss it ( possibly this fmr
3107 * operation was not signalted)
3108 */
3109static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3110{
3111 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3112 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3113}
3114
3115static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3116 struct qedr_cq *cq, int num_entries,
3117 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3118 int force)
3119{
3120 u16 cnt = 0;
3121
3122 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3123 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3124 qedr_chk_if_fmr(qp);
3125 /* skip WC */
3126 goto next_cqe;
3127 }
3128
3129 /* fill WC */
3130 wc->status = status;
27035a1b 3131 wc->vendor_err = 0;
afa0e13b
RA		 3132 wc->wc_flags = 0;
3133 wc->src_qp = qp->id;
3134 wc->qp = &qp->ibqp;
3135
3136 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3137 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3138
3139 switch (wc->opcode) {
3140 case IB_WC_RDMA_WRITE:
3141 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3142 break;
3143 case IB_WC_COMP_SWAP:
3144 case IB_WC_FETCH_ADD:
3145 wc->byte_len = 8;
3146 break;
3147 case IB_WC_REG_MR:
3148 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3149 break;
3150 default:
3151 break;
3152 }
3153
3154 num_entries--;
3155 wc++;
3156 cnt++;
3157next_cqe:
3158 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3159 qed_chain_consume(&qp->sq.pbl);
3160 qedr_inc_sw_cons(&qp->sq);
3161 }
3162
3163 return cnt;
3164}
3165
3166static int qedr_poll_cq_req(struct qedr_dev *dev,
3167 struct qedr_qp *qp, struct qedr_cq *cq,
3168 int num_entries, struct ib_wc *wc,
3169 struct rdma_cqe_requester *req)
3170{
3171 int cnt = 0;
3172
3173 switch (req->status) {
3174 case RDMA_CQE_REQ_STS_OK:
3175 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3176 IB_WC_SUCCESS, 0);
3177 break;
3178 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149
RA		 3179 if (qp->state != QED_ROCE_QP_STATE_ERR)
3180 DP_ERR(dev,
3181 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3182 cq->icid, qp->icid);
afa0e13b 3183 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3184 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 3185 break;
3186 default:
3187 /* process all WQE before the cosumer */
3188 qp->state = QED_ROCE_QP_STATE_ERR;
3189 cnt = process_req(dev, qp, cq, num_entries, wc,
3190 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3191 wc += cnt;
3192 /* if we have extra WC fill it with actual error info */
3193 if (cnt < num_entries) {
3194 enum ib_wc_status wc_status;
3195
3196 switch (req->status) {
3197 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3198 DP_ERR(dev,
3199 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3200 cq->icid, qp->icid);
3201 wc_status = IB_WC_BAD_RESP_ERR;
3202 break;
3203 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3204 DP_ERR(dev,
3205 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3206 cq->icid, qp->icid);
3207 wc_status = IB_WC_LOC_LEN_ERR;
3208 break;
3209 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3210 DP_ERR(dev,
3211 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3212 cq->icid, qp->icid);
3213 wc_status = IB_WC_LOC_QP_OP_ERR;
3214 break;
3215 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3216 DP_ERR(dev,
3217 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3218 cq->icid, qp->icid);
3219 wc_status = IB_WC_LOC_PROT_ERR;
3220 break;
3221 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3222 DP_ERR(dev,
3223 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3224 cq->icid, qp->icid);
3225 wc_status = IB_WC_MW_BIND_ERR;
3226 break;
3227 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3228 DP_ERR(dev,
3229 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3230 cq->icid, qp->icid);
3231 wc_status = IB_WC_REM_INV_REQ_ERR;
3232 break;
3233 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3234 DP_ERR(dev,
3235 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3236 cq->icid, qp->icid);
3237 wc_status = IB_WC_REM_ACCESS_ERR;
3238 break;
3239 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3240 DP_ERR(dev,
3241 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3242 cq->icid, qp->icid);
3243 wc_status = IB_WC_REM_OP_ERR;
3244 break;
3245 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3246 DP_ERR(dev,
3247 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3248 cq->icid, qp->icid);
3249 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3250 break;
3251 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3252 DP_ERR(dev,
3253 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3254 cq->icid, qp->icid);
3255 wc_status = IB_WC_RETRY_EXC_ERR;
3256 break;
3257 default:
3258 DP_ERR(dev,
3259 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3260 cq->icid, qp->icid);
3261 wc_status = IB_WC_GENERAL_ERR;
3262 }
3263 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3264 wc_status, 1);
3265 }
3266 }
3267
3268 return cnt;
3269}
3270
3271static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3272 struct qedr_cq *cq, struct ib_wc *wc,
3273 struct rdma_cqe_responder *resp, u64 wr_id)
3274{
3275 enum ib_wc_status wc_status = IB_WC_SUCCESS;
3276 u8 flags;
3277
3278 wc->opcode = IB_WC_RECV;
3279 wc->wc_flags = 0;
3280
3281 switch (resp->status) {
3282 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3283 wc_status = IB_WC_LOC_ACCESS_ERR;
3284 break;
3285 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3286 wc_status = IB_WC_LOC_LEN_ERR;
3287 break;
3288 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3289 wc_status = IB_WC_LOC_QP_OP_ERR;
3290 break;
3291 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3292 wc_status = IB_WC_LOC_PROT_ERR;
3293 break;
3294 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3295 wc_status = IB_WC_MW_BIND_ERR;
3296 break;
3297 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3298 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
3299 break;
3300 case RDMA_CQE_RESP_STS_OK:
3301 wc_status = IB_WC_SUCCESS;
3302 wc->byte_len = le32_to_cpu(resp->length);
3303
3304 flags = resp->flags & QEDR_RESP_RDMA_IMM;
3305
3306 if (flags == QEDR_RESP_RDMA_IMM)
3307 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3308
3309 if (flags == QEDR_RESP_RDMA_IMM || flags == QEDR_RESP_IMM) {
3310 wc->ex.imm_data =
3311 le32_to_cpu(resp->imm_data_or_inv_r_Key);
3312 wc->wc_flags |= IB_WC_WITH_IMM;
3313 }
3314 break;
3315 default:
3316 wc->status = IB_WC_GENERAL_ERR;
3317 DP_ERR(dev, "Invalid CQE status detected\n");
3318 }
3319
3320 /* fill WC */
3321 wc->status = wc_status;
27035a1b 3322 wc->vendor_err = 0;
afa0e13b
RA		 3323 wc->src_qp = qp->id;
3324 wc->qp = &qp->ibqp;
3325 wc->wr_id = wr_id;
3326}
3327
3328static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3329 struct qedr_cq *cq, struct ib_wc *wc,
3330 struct rdma_cqe_responder *resp)
3331{
3332 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3333
3334 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3335
3336 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3337 qed_chain_consume(&qp->rq.pbl);
3338 qedr_inc_sw_cons(&qp->rq);
3339
3340 return 1;
3341}
3342
3343static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3344 int num_entries, struct ib_wc *wc, u16 hw_cons)
3345{
3346 u16 cnt = 0;
3347
3348 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3349 /* fill WC */
3350 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 3351 wc->vendor_err = 0;
afa0e13b
RA		 3352 wc->wc_flags = 0;
3353 wc->src_qp = qp->id;
3354 wc->byte_len = 0;
3355 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3356 wc->qp = &qp->ibqp;
3357 num_entries--;
3358 wc++;
3359 cnt++;
3360 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3361 qed_chain_consume(&qp->rq.pbl);
3362 qedr_inc_sw_cons(&qp->rq);
3363 }
3364
3365 return cnt;
3366}
3367
3368static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3369 struct rdma_cqe_responder *resp, int *update)
3370{
3371 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3372 consume_cqe(cq);
3373 *update |= 1;
3374 }
3375}
3376
3377static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3378 struct qedr_cq *cq, int num_entries,
3379 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3380 int *update)
3381{
3382 int cnt;
3383
3384 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3385 cnt = process_resp_flush(qp, cq, num_entries, wc,
3386 resp->rq_cons);
3387 try_consume_resp_cqe(cq, qp, resp, update);
3388 } else {
3389 cnt = process_resp_one(dev, qp, cq, wc, resp);
3390 consume_cqe(cq);
3391 *update |= 1;
3392 }
3393
3394 return cnt;
3395}
3396
3397static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3398 struct rdma_cqe_requester *req, int *update)
3399{
3400 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3401 consume_cqe(cq);
3402 *update |= 1;
3403 }
3404}
3405
3406int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3407{
3408 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3409 struct qedr_cq *cq = get_qedr_cq(ibcq);
3410 union rdma_cqe *cqe = cq->latest_cqe;
3411 u32 old_cons, new_cons;
3412 unsigned long flags;
3413 int update = 0;
3414 int done = 0;
3415
04886779
RA		 3416 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3417 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3418
afa0e13b
RA		 3419 spin_lock_irqsave(&cq->cq_lock, flags);
3420 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3421 while (num_entries && is_valid_cqe(cq, cqe)) {
3422 struct qedr_qp *qp;
3423 int cnt = 0;
3424
3425 /* prevent speculative reads of any field of CQE */
3426 rmb();
3427
3428 qp = cqe_get_qp(cqe);
3429 if (!qp) {
3430 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3431 break;
3432 }
3433
3434 wc->qp = &qp->ibqp;
3435
3436 switch (cqe_get_type(cqe)) {
3437 case RDMA_CQE_TYPE_REQUESTER:
3438 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3439 &cqe->req);
3440 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3441 break;
3442 case RDMA_CQE_TYPE_RESPONDER_RQ:
3443 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3444 &cqe->resp, &update);
3445 break;
3446 case RDMA_CQE_TYPE_INVALID:
3447 default:
3448 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3449 cqe_get_type(cqe));
3450 }
3451 num_entries -= cnt;
3452 wc += cnt;
3453 done += cnt;
3454
3455 cqe = get_cqe(cq);
3456 }
3457 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3458
3459 cq->cq_cons += new_cons - old_cons;
3460
3461 if (update)
3462 /* doorbell notifies abount latest VALID entry,
3463 * but chain already point to the next INVALID one
3464 */
3465 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3466
3467 spin_unlock_irqrestore(&cq->cq_lock, flags);
3468 return done;
3469}
993d1b52
RA		 3470
3471int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3472 u8 port_num,
3473 const struct ib_wc *in_wc,
3474 const struct ib_grh *in_grh,
3475 const struct ib_mad_hdr *mad_hdr,
3476 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3477 size_t *out_mad_size, u16 *out_mad_pkey_index)
3478{
3479 struct qedr_dev *dev = get_qedr_dev(ibdev);
3480
3481 DP_DEBUG(dev, QEDR_MSG_GSI,
3482 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3483 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3484 mad_hdr->class_specific, mad_hdr->class_version,
3485 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3486 return IB_MAD_RESULT_SUCCESS;
3487}
3488
3489int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3490 struct ib_port_immutable *immutable)
3491{
3492 struct ib_port_attr attr;
3493 int err;
3494
3495 err = qedr_query_port(ibdev, port_num, &attr);
3496 if (err)
3497 return err;
3498
3499 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3500 immutable->gid_tbl_len = attr.gid_tbl_len;
3501 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3502 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3503 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3504
3505 return 0;
3506}
Linux 6.x block layer and io_uring tree(s)