projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
qedr: configure the number of CQEs on CQ creation
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45
46#include "qedr_hsi.h"
47#include <linux/qed/qed_if.h>
48#include "qedr.h"
49#include "verbs.h"
50#include <rdma/qedr-abi.h>
cecbcddf 51#include "qedr_cm.h"
ac1b36e5 52
a7efd777
RA		 53#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56{
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62}
63
ac1b36e5
RA		 64int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66{
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83}
84
85int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88{
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99}
100
101int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103{
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114}
115
116int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118{
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170}
171
172#define QEDR_SPEED_SDR (1)
173#define QEDR_SPEED_DDR (2)
174#define QEDR_SPEED_QDR (4)
175#define QEDR_SPEED_FDR10 (8)
176#define QEDR_SPEED_FDR (16)
177#define QEDR_SPEED_EDR (32)
178
179static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181{
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222}
223
224int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225{
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241 memset(attr, 0, sizeof(*attr));
242
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267}
268
269int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271{
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281}
282
283static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285{
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313}
314
315static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317{
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335}
336
337struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339{
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397}
398
399int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400{
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419}
420
421int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422{
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468}
a7efd777
RA		 469
470struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472{
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
474 struct qedr_ucontext *uctx = NULL;
475 struct qedr_alloc_pd_uresp uresp;
476 struct qedr_pd *pd;
477 u16 pd_id;
478 int rc;
479
480 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
481 (udata && context) ? "User Lib" : "Kernel");
482
483 if (!dev->rdma_ctx) {
484 DP_ERR(dev, "invlaid RDMA context\n");
485 return ERR_PTR(-EINVAL);
486 }
487
488 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
489 if (!pd)
490 return ERR_PTR(-ENOMEM);
491
492 dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
493
494 uresp.pd_id = pd_id;
495 pd->pd_id = pd_id;
496
497 if (udata && context) {
498 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
499 if (rc)
500 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
501 uctx = get_qedr_ucontext(context);
502 uctx->pd = pd;
503 pd->uctx = uctx;
504 }
505
506 return &pd->ibpd;
507}
508
509int qedr_dealloc_pd(struct ib_pd *ibpd)
510{
511 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
512 struct qedr_pd *pd = get_qedr_pd(ibpd);
513
ea7ef2ac 514 if (!pd) {
a7efd777 515 pr_err("Invalid PD received in dealloc_pd\n");
ea7ef2ac
CIK		 516 return -EINVAL;
517 }
a7efd777
RA		 518
519 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
520 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
521
522 kfree(pd);
523
524 return 0;
525}
526
527static void qedr_free_pbl(struct qedr_dev *dev,
528 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
529{
530 struct pci_dev *pdev = dev->pdev;
531 int i;
532
533 for (i = 0; i < pbl_info->num_pbls; i++) {
534 if (!pbl[i].va)
535 continue;
536 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
537 pbl[i].va, pbl[i].pa);
538 }
539
540 kfree(pbl);
541}
542
543#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
544#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
545
546#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
547#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
548#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
549
550static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
551 struct qedr_pbl_info *pbl_info,
552 gfp_t flags)
553{
554 struct pci_dev *pdev = dev->pdev;
555 struct qedr_pbl *pbl_table;
556 dma_addr_t *pbl_main_tbl;
557 dma_addr_t pa;
558 void *va;
559 int i;
560
561 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
562 if (!pbl_table)
563 return ERR_PTR(-ENOMEM);
564
565 for (i = 0; i < pbl_info->num_pbls; i++) {
566 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
567 &pa, flags);
568 if (!va)
569 goto err;
570
571 memset(va, 0, pbl_info->pbl_size);
572 pbl_table[i].va = va;
573 pbl_table[i].pa = pa;
574 }
575
576 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
577 * the first one with physical pointers to all of the rest
578 */
579 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
580 for (i = 0; i < pbl_info->num_pbls - 1; i++)
581 pbl_main_tbl[i] = pbl_table[i + 1].pa;
582
583 return pbl_table;
584
585err:
586 for (i--; i >= 0; i--)
587 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
588 pbl_table[i].va, pbl_table[i].pa);
589
590 qedr_free_pbl(dev, pbl_info, pbl_table);
591
592 return ERR_PTR(-ENOMEM);
593}
594
595static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
596 struct qedr_pbl_info *pbl_info,
597 u32 num_pbes, int two_layer_capable)
598{
599 u32 pbl_capacity;
600 u32 pbl_size;
601 u32 num_pbls;
602
603 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
604 if (num_pbes > MAX_PBES_TWO_LAYER) {
605 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
606 num_pbes);
607 return -EINVAL;
608 }
609
610 /* calculate required pbl page size */
611 pbl_size = MIN_FW_PBL_PAGE_SIZE;
612 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
613 NUM_PBES_ON_PAGE(pbl_size);
614
615 while (pbl_capacity < num_pbes) {
616 pbl_size *= 2;
617 pbl_capacity = pbl_size / sizeof(u64);
618 pbl_capacity = pbl_capacity * pbl_capacity;
619 }
620
621 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
622 num_pbls++; /* One for the layer0 ( points to the pbls) */
623 pbl_info->two_layered = true;
624 } else {
625 /* One layered PBL */
626 num_pbls = 1;
627 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
628 roundup_pow_of_two((num_pbes * sizeof(u64))));
629 pbl_info->two_layered = false;
630 }
631
632 pbl_info->num_pbls = num_pbls;
633 pbl_info->pbl_size = pbl_size;
634 pbl_info->num_pbes = num_pbes;
635
636 DP_DEBUG(dev, QEDR_MSG_MR,
637 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
638 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
639
640 return 0;
641}
642
643static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
644 struct qedr_pbl *pbl,
645 struct qedr_pbl_info *pbl_info)
646{
647 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
648 struct qedr_pbl *pbl_tbl;
649 struct scatterlist *sg;
650 struct regpair *pbe;
651 int entry;
652 u32 addr;
653
654 if (!pbl_info->num_pbes)
655 return;
656
657 /* If we have a two layered pbl, the first pbl points to the rest
658 * of the pbls and the first entry lays on the second pbl in the table
659 */
660 if (pbl_info->two_layered)
661 pbl_tbl = &pbl[1];
662 else
663 pbl_tbl = pbl;
664
665 pbe = (struct regpair *)pbl_tbl->va;
666 if (!pbe) {
667 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
668 return;
669 }
670
671 pbe_cnt = 0;
672
673 shift = ilog2(umem->page_size);
674
675 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
676 pages = sg_dma_len(sg) >> shift;
677 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
678 /* store the page address in pbe */
679 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
680 umem->page_size * pg_cnt);
681 addr = upper_32_bits(sg_dma_address(sg) +
682 umem->page_size * pg_cnt);
683 pbe->hi = cpu_to_le32(addr);
684 pbe_cnt++;
685 total_num_pbes++;
686 pbe++;
687
688 if (total_num_pbes == pbl_info->num_pbes)
689 return;
690
691 /* If the given pbl is full storing the pbes,
692 * move to next pbl.
693 */
694 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
695 pbl_tbl++;
696 pbe = (struct regpair *)pbl_tbl->va;
697 pbe_cnt = 0;
698 }
699 }
700 }
701}
702
703static int qedr_copy_cq_uresp(struct qedr_dev *dev,
704 struct qedr_cq *cq, struct ib_udata *udata)
705{
706 struct qedr_create_cq_uresp uresp;
707 int rc;
708
709 memset(&uresp, 0, sizeof(uresp));
710
711 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
712 uresp.icid = cq->icid;
713
714 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
715 if (rc)
716 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
717
718 return rc;
719}
720
721static void consume_cqe(struct qedr_cq *cq)
722{
723 if (cq->latest_cqe == cq->toggle_cqe)
724 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
725
726 cq->latest_cqe = qed_chain_consume(&cq->pbl);
727}
728
729static inline int qedr_align_cq_entries(int entries)
730{
731 u64 size, aligned_size;
732
733 /* We allocate an extra entry that we don't report to the FW. */
734 size = (entries + 1) * QEDR_CQE_SIZE;
735 aligned_size = ALIGN(size, PAGE_SIZE);
736
737 return aligned_size / QEDR_CQE_SIZE;
738}
739
740static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
741 struct qedr_dev *dev,
742 struct qedr_userq *q,
743 u64 buf_addr, size_t buf_len,
744 int access, int dmasync)
745{
746 int page_cnt;
747 int rc;
748
749 q->buf_addr = buf_addr;
750 q->buf_len = buf_len;
751 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
752 if (IS_ERR(q->umem)) {
753 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
754 PTR_ERR(q->umem));
755 return PTR_ERR(q->umem);
756 }
757
758 page_cnt = ib_umem_page_count(q->umem);
759 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
760 if (rc)
761 goto err0;
762
763 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
764 if (IS_ERR_OR_NULL(q->pbl_tbl))
765 goto err0;
766
767 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
768
769 return 0;
770
771err0:
772 ib_umem_release(q->umem);
773
774 return rc;
775}
776
777static inline void qedr_init_cq_params(struct qedr_cq *cq,
778 struct qedr_ucontext *ctx,
779 struct qedr_dev *dev, int vector,
780 int chain_entries, int page_cnt,
781 u64 pbl_ptr,
782 struct qed_rdma_create_cq_in_params
783 *params)
784{
785 memset(params, 0, sizeof(*params));
786 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
787 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
788 params->cnq_id = vector;
789 params->cq_size = chain_entries - 1;
790 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
791 params->pbl_num_pages = page_cnt;
792 params->pbl_ptr = pbl_ptr;
793 params->pbl_two_level = 0;
794}
795
796static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
797{
798 /* Flush data before signalling doorbell */
799 wmb();
800 cq->db.data.agg_flags = flags;
801 cq->db.data.value = cpu_to_le32(cons);
802 writeq(cq->db.raw, cq->db_addr);
803
804 /* Make sure write would stick */
805 mmiowb();
806}
807
808int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
809{
810 struct qedr_cq *cq = get_qedr_cq(ibcq);
811 unsigned long sflags;
812
813 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
814 return 0;
815
816 spin_lock_irqsave(&cq->cq_lock, sflags);
817
818 cq->arm_flags = 0;
819
820 if (flags & IB_CQ_SOLICITED)
821 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
822
823 if (flags & IB_CQ_NEXT_COMP)
824 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
825
826 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
827
828 spin_unlock_irqrestore(&cq->cq_lock, sflags);
829
830 return 0;
831}
832
833struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
834 const struct ib_cq_init_attr *attr,
835 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
836{
837 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
838 struct qed_rdma_destroy_cq_out_params destroy_oparams;
839 struct qed_rdma_destroy_cq_in_params destroy_iparams;
840 struct qedr_dev *dev = get_qedr_dev(ibdev);
841 struct qed_rdma_create_cq_in_params params;
842 struct qedr_create_cq_ureq ureq;
843 int vector = attr->comp_vector;
844 int entries = attr->cqe;
845 struct qedr_cq *cq;
846 int chain_entries;
847 int page_cnt;
848 u64 pbl_ptr;
849 u16 icid;
850 int rc;
851
852 DP_DEBUG(dev, QEDR_MSG_INIT,
853 "create_cq: called from %s. entries=%d, vector=%d\n",
854 udata ? "User Lib" : "Kernel", entries, vector);
855
856 if (entries > QEDR_MAX_CQES) {
857 DP_ERR(dev,
858 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
859 entries, QEDR_MAX_CQES);
860 return ERR_PTR(-EINVAL);
861 }
862
863 chain_entries = qedr_align_cq_entries(entries);
864 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
865
866 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
867 if (!cq)
868 return ERR_PTR(-ENOMEM);
869
870 if (udata) {
871 memset(&ureq, 0, sizeof(ureq));
872 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
873 DP_ERR(dev,
874 "create cq: problem copying data from user space\n");
875 goto err0;
876 }
877
878 if (!ureq.len) {
879 DP_ERR(dev,
880 "create cq: cannot create a cq with 0 entries\n");
881 goto err0;
882 }
883
884 cq->cq_type = QEDR_CQ_TYPE_USER;
885
886 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
887 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
888 if (rc)
889 goto err0;
890
891 pbl_ptr = cq->q.pbl_tbl->pa;
892 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 893
894 cq->ibcq.cqe = chain_entries;
a7efd777
RA		 895 } else {
896 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
897
898 rc = dev->ops->common->chain_alloc(dev->cdev,
899 QED_CHAIN_USE_TO_CONSUME,
900 QED_CHAIN_MODE_PBL,
901 QED_CHAIN_CNT_TYPE_U32,
902 chain_entries,
903 sizeof(union rdma_cqe),
904 &cq->pbl);
905 if (rc)
906 goto err1;
907
908 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
909 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 910 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 911 }
912
913 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
914 pbl_ptr, &params);
915
916 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
917 if (rc)
918 goto err2;
919
920 cq->icid = icid;
921 cq->sig = QEDR_CQ_MAGIC_NUMBER;
922 spin_lock_init(&cq->cq_lock);
923
924 if (ib_ctx) {
925 rc = qedr_copy_cq_uresp(dev, cq, udata);
926 if (rc)
927 goto err3;
928 } else {
929 /* Generate doorbell address. */
930 cq->db_addr = dev->db_addr +
931 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
932 cq->db.data.icid = cq->icid;
933 cq->db.data.params = DB_AGG_CMD_SET <<
934 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
935
936 /* point to the very last element, passing it we will toggle */
937 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
938 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
939 cq->latest_cqe = NULL;
940 consume_cqe(cq);
941 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
942 }
943
944 DP_DEBUG(dev, QEDR_MSG_CQ,
945 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
946 cq->icid, cq, params.cq_size);
947
948 return &cq->ibcq;
949
950err3:
951 destroy_iparams.icid = cq->icid;
952 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
953 &destroy_oparams);
954err2:
955 if (udata)
956 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
957 else
958 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
959err1:
960 if (udata)
961 ib_umem_release(cq->q.umem);
962err0:
963 kfree(cq);
964 return ERR_PTR(-EINVAL);
965}
966
967int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
968{
969 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
970 struct qedr_cq *cq = get_qedr_cq(ibcq);
971
972 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
973
974 return 0;
975}
976
977int qedr_destroy_cq(struct ib_cq *ibcq)
978{
979 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
980 struct qed_rdma_destroy_cq_out_params oparams;
981 struct qed_rdma_destroy_cq_in_params iparams;
982 struct qedr_cq *cq = get_qedr_cq(ibcq);
983
984 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
985
986 /* GSIs CQs are handled by driver, so they don't exist in the FW */
987 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
988 iparams.icid = cq->icid;
989 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
990 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
991 }
992
993 if (ibcq->uobject && ibcq->uobject->context) {
994 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
995 ib_umem_release(cq->q.umem);
996 }
997
998 kfree(cq);
999
1000 return 0;
1001}
cecbcddf
RA		 1002
1003static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1004 struct ib_qp_attr *attr,
1005 int attr_mask,
1006 struct qed_rdma_modify_qp_in_params
1007 *qp_params)
1008{
1009 enum rdma_network_type nw_type;
1010 struct ib_gid_attr gid_attr;
1011 union ib_gid gid;
1012 u32 ipv4_addr;
1013 int rc = 0;
1014 int i;
1015
1016 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1017 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1018 if (rc)
1019 return rc;
1020
1021 if (!memcmp(&gid, &zgid, sizeof(gid)))
1022 return -ENOENT;
1023
1024 if (gid_attr.ndev) {
1025 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1026
1027 dev_put(gid_attr.ndev);
1028 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1029 switch (nw_type) {
1030 case RDMA_NETWORK_IPV6:
1031 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1032 sizeof(qp_params->sgid));
1033 memcpy(&qp_params->dgid.bytes[0],
1034 &attr->ah_attr.grh.dgid,
1035 sizeof(qp_params->dgid));
1036 qp_params->roce_mode = ROCE_V2_IPV6;
1037 SET_FIELD(qp_params->modify_flags,
1038 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1039 break;
1040 case RDMA_NETWORK_IB:
1041 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1042 sizeof(qp_params->sgid));
1043 memcpy(&qp_params->dgid.bytes[0],
1044 &attr->ah_attr.grh.dgid,
1045 sizeof(qp_params->dgid));
1046 qp_params->roce_mode = ROCE_V1;
1047 break;
1048 case RDMA_NETWORK_IPV4:
1049 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1050 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1051 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1052 qp_params->sgid.ipv4_addr = ipv4_addr;
1053 ipv4_addr =
1054 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1055 qp_params->dgid.ipv4_addr = ipv4_addr;
1056 SET_FIELD(qp_params->modify_flags,
1057 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1058 qp_params->roce_mode = ROCE_V2_IPV4;
1059 break;
1060 }
1061 }
1062
1063 for (i = 0; i < 4; i++) {
1064 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1065 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1066 }
1067
1068 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1069 qp_params->vlan_id = 0;
1070
1071 return 0;
1072}
1073
1074static void qedr_cleanup_user_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1075{
1076 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1077 ib_umem_release(qp->usq.umem);
1078}
1079
1080static void qedr_cleanup_user_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1081{
1082 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1083 ib_umem_release(qp->urq.umem);
1084}
1085
1086static void qedr_cleanup_kernel_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1087{
1088 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1089 kfree(qp->wqe_wr_id);
1090}
1091
1092static void qedr_cleanup_kernel_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1093{
1094 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1095 kfree(qp->rqe_wr_id);
1096}
1097
1098static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1099 struct ib_qp_init_attr *attrs)
1100{
1101 struct qedr_device_attr *qattr = &dev->attr;
1102
1103 /* QP0... attrs->qp_type == IB_QPT_GSI */
1104 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1105 DP_DEBUG(dev, QEDR_MSG_QP,
1106 "create qp: unsupported qp type=0x%x requested\n",
1107 attrs->qp_type);
1108 return -EINVAL;
1109 }
1110
1111 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1112 DP_ERR(dev,
1113 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1114 attrs->cap.max_send_wr, qattr->max_sqe);
1115 return -EINVAL;
1116 }
1117
1118 if (attrs->cap.max_inline_data > qattr->max_inline) {
1119 DP_ERR(dev,
1120 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1121 attrs->cap.max_inline_data, qattr->max_inline);
1122 return -EINVAL;
1123 }
1124
1125 if (attrs->cap.max_send_sge > qattr->max_sge) {
1126 DP_ERR(dev,
1127 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1128 attrs->cap.max_send_sge, qattr->max_sge);
1129 return -EINVAL;
1130 }
1131
1132 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1133 DP_ERR(dev,
1134 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1135 attrs->cap.max_recv_sge, qattr->max_sge);
1136 return -EINVAL;
1137 }
1138
1139 /* Unprivileged user space cannot create special QP */
1140 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1141 DP_ERR(dev,
1142 "create qp: userspace can't create special QPs of type=0x%x\n",
1143 attrs->qp_type);
1144 return -EINVAL;
1145 }
1146
1147 return 0;
1148}
1149
1150static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1151 struct qedr_qp *qp)
1152{
1153 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1154 uresp->rq_icid = qp->icid;
1155}
1156
1157static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1158 struct qedr_qp *qp)
1159{
1160 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1161 uresp->sq_icid = qp->icid + 1;
1162}
1163
1164static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1165 struct qedr_qp *qp, struct ib_udata *udata)
1166{
1167 struct qedr_create_qp_uresp uresp;
1168 int rc;
1169
1170 memset(&uresp, 0, sizeof(uresp));
1171 qedr_copy_sq_uresp(&uresp, qp);
1172 qedr_copy_rq_uresp(&uresp, qp);
1173
1174 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1175 uresp.qp_id = qp->qp_id;
1176
1177 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1178 if (rc)
1179 DP_ERR(dev,
1180 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1181 qp->icid);
1182
1183 return rc;
1184}
1185
1186static void qedr_set_qp_init_params(struct qedr_dev *dev,
1187 struct qedr_qp *qp,
1188 struct qedr_pd *pd,
1189 struct ib_qp_init_attr *attrs)
1190{
1191 qp->pd = pd;
1192
1193 spin_lock_init(&qp->q_lock);
1194
1195 qp->qp_type = attrs->qp_type;
1196 qp->max_inline_data = attrs->cap.max_inline_data;
1197 qp->sq.max_sges = attrs->cap.max_send_sge;
1198 qp->state = QED_ROCE_QP_STATE_RESET;
1199 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1200 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1201 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1202 qp->dev = dev;
1203
1204 DP_DEBUG(dev, QEDR_MSG_QP,
1205 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1206 pd->pd_id, qp->qp_type, qp->max_inline_data,
1207 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1208 DP_DEBUG(dev, QEDR_MSG_QP,
1209 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1210 qp->sq.max_sges, qp->sq_cq->icid);
1211 qp->rq.max_sges = attrs->cap.max_recv_sge;
1212 DP_DEBUG(dev, QEDR_MSG_QP,
1213 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1214 qp->rq.max_sges, qp->rq_cq->icid);
1215}
1216
1217static inline void
1218qedr_init_qp_user_params(struct qed_rdma_create_qp_in_params *params,
1219 struct qedr_create_qp_ureq *ureq)
1220{
1221 /* QP handle to be written in CQE */
1222 params->qp_handle_lo = ureq->qp_handle_lo;
1223 params->qp_handle_hi = ureq->qp_handle_hi;
1224}
1225
1226static inline void
1227qedr_init_qp_kernel_doorbell_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1228{
1229 qp->sq.db = dev->db_addr +
1230 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1231 qp->sq.db_data.data.icid = qp->icid + 1;
1232}
1233
1234static inline void
1235qedr_init_qp_kernel_doorbell_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1236{
1237 qp->rq.db = dev->db_addr +
1238 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1239 qp->rq.db_data.data.icid = qp->icid;
1240}
1241
1242static inline int
1243qedr_init_qp_kernel_params_rq(struct qedr_dev *dev,
1244 struct qedr_qp *qp, struct ib_qp_init_attr *attrs)
1245{
1246 /* Allocate driver internal RQ array */
1247 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1248 GFP_KERNEL);
1249 if (!qp->rqe_wr_id)
1250 return -ENOMEM;
1251
1252 DP_DEBUG(dev, QEDR_MSG_QP, "RQ max_wr set to %d.\n", qp->rq.max_wr);
1253
1254 return 0;
1255}
1256
1257static inline int
1258qedr_init_qp_kernel_params_sq(struct qedr_dev *dev,
1259 struct qedr_qp *qp,
1260 struct ib_qp_init_attr *attrs,
1261 struct qed_rdma_create_qp_in_params *params)
1262{
1263 u32 temp_max_wr;
1264
1265 /* Allocate driver internal SQ array */
1266 temp_max_wr = attrs->cap.max_send_wr * dev->wq_multiplier;
1267 temp_max_wr = min_t(u32, temp_max_wr, dev->attr.max_sqe);
1268
1269 /* temp_max_wr < attr->max_sqe < u16 so the casting is safe */
1270 qp->sq.max_wr = (u16)temp_max_wr;
1271 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1272 GFP_KERNEL);
1273 if (!qp->wqe_wr_id)
1274 return -ENOMEM;
1275
1276 DP_DEBUG(dev, QEDR_MSG_QP, "SQ max_wr set to %d.\n", qp->sq.max_wr);
1277
1278 /* QP handle to be written in CQE */
1279 params->qp_handle_lo = lower_32_bits((uintptr_t)qp);
1280 params->qp_handle_hi = upper_32_bits((uintptr_t)qp);
1281
1282 return 0;
1283}
1284
1285static inline int qedr_init_qp_kernel_sq(struct qedr_dev *dev,
1286 struct qedr_qp *qp,
1287 struct ib_qp_init_attr *attrs)
1288{
1289 u32 n_sq_elems, n_sq_entries;
1290 int rc;
1291
1292 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1293 * the ring. The ring should allow at least a single WR, even if the
1294 * user requested none, due to allocation issues.
1295 */
1296 n_sq_entries = attrs->cap.max_send_wr;
1297 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1298 n_sq_entries = max_t(u32, n_sq_entries, 1);
1299 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1300 rc = dev->ops->common->chain_alloc(dev->cdev,
1301 QED_CHAIN_USE_TO_PRODUCE,
1302 QED_CHAIN_MODE_PBL,
1303 QED_CHAIN_CNT_TYPE_U32,
1304 n_sq_elems,
1305 QEDR_SQE_ELEMENT_SIZE,
1306 &qp->sq.pbl);
1307 if (rc) {
1308 DP_ERR(dev, "failed to allocate QP %p SQ\n", qp);
1309 return rc;
1310 }
1311
1312 DP_DEBUG(dev, QEDR_MSG_SQ,
1313 "SQ Pbl base addr = %llx max_send_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1314 qed_chain_get_pbl_phys(&qp->sq.pbl), attrs->cap.max_send_wr,
1315 n_sq_entries, qed_chain_get_capacity(&qp->sq.pbl), rc);
1316 return 0;
1317}
1318
1319static inline int qedr_init_qp_kernel_rq(struct qedr_dev *dev,
1320 struct qedr_qp *qp,
1321 struct ib_qp_init_attr *attrs)
1322{
1323 u32 n_rq_elems, n_rq_entries;
1324 int rc;
1325
1326 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1327 * the ring. There ring should allow at least a single WR, even if the
1328 * user requested none, due to allocation issues.
1329 */
1330 n_rq_entries = max_t(u32, attrs->cap.max_recv_wr, 1);
1331 n_rq_elems = n_rq_entries * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1332 rc = dev->ops->common->chain_alloc(dev->cdev,
1333 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1334 QED_CHAIN_MODE_PBL,
1335 QED_CHAIN_CNT_TYPE_U32,
1336 n_rq_elems,
1337 QEDR_RQE_ELEMENT_SIZE,
1338 &qp->rq.pbl);
1339
1340 if (rc) {
1341 DP_ERR(dev, "failed to allocate memory for QP %p RQ\n", qp);
1342 return -ENOMEM;
1343 }
1344
1345 DP_DEBUG(dev, QEDR_MSG_RQ,
1346 "RQ Pbl base addr = %llx max_recv_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1347 qed_chain_get_pbl_phys(&qp->rq.pbl), attrs->cap.max_recv_wr,
1348 n_rq_entries, qed_chain_get_capacity(&qp->rq.pbl), rc);
1349
1350 /* n_rq_entries < u16 so the casting is safe */
1351 qp->rq.max_wr = (u16)n_rq_entries;
1352
1353 return 0;
1354}
1355
1356static inline void
1357qedr_init_qp_in_params_sq(struct qedr_dev *dev,
1358 struct qedr_pd *pd,
1359 struct qedr_qp *qp,
1360 struct ib_qp_init_attr *attrs,
1361 struct ib_udata *udata,
1362 struct qed_rdma_create_qp_in_params *params)
1363{
1364 /* QP handle to be written in an async event */
1365 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
1366 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
1367
1368 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1369 params->fmr_and_reserved_lkey = !udata;
1370 params->pd = pd->pd_id;
1371 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1372 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1373 params->max_sq_sges = 0;
1374 params->stats_queue = 0;
1375
1376 if (udata) {
1377 params->sq_num_pages = qp->usq.pbl_info.num_pbes;
1378 params->sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1379 } else {
1380 params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1381 params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1382 }
1383}
1384
1385static inline void
1386qedr_init_qp_in_params_rq(struct qedr_qp *qp,
1387 struct ib_qp_init_attr *attrs,
1388 struct ib_udata *udata,
1389 struct qed_rdma_create_qp_in_params *params)
1390{
1391 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1392 params->srq_id = 0;
1393 params->use_srq = false;
1394
1395 if (udata) {
1396 params->rq_num_pages = qp->urq.pbl_info.num_pbes;
1397 params->rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1398 } else {
1399 params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1400 params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1401 }
1402}
1403
1404static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1405{
1406 DP_DEBUG(dev, QEDR_MSG_QP,
1407 "create qp: successfully created user QP. qp=%p, sq_addr=0x%llx, sq_len=%zd, rq_addr=0x%llx, rq_len=%zd\n",
1408 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
1409 qp->urq.buf_len);
1410}
1411
1412static inline int qedr_init_user_qp(struct ib_ucontext *ib_ctx,
1413 struct qedr_dev *dev,
1414 struct qedr_qp *qp,
1415 struct qedr_create_qp_ureq *ureq)
1416{
1417 int rc;
1418
1419 /* SQ - read access only (0), dma sync not required (0) */
1420 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq->sq_addr,
1421 ureq->sq_len, 0, 0);
1422 if (rc)
1423 return rc;
1424
1425 /* RQ - read access only (0), dma sync not required (0) */
1426 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq->rq_addr,
1427 ureq->rq_len, 0, 0);
1428
1429 if (rc)
1430 qedr_cleanup_user_sq(dev, qp);
1431 return rc;
1432}
1433
1434static inline int
1435qedr_init_kernel_qp(struct qedr_dev *dev,
1436 struct qedr_qp *qp,
1437 struct ib_qp_init_attr *attrs,
1438 struct qed_rdma_create_qp_in_params *params)
1439{
1440 int rc;
1441
1442 rc = qedr_init_qp_kernel_sq(dev, qp, attrs);
1443 if (rc) {
1444 DP_ERR(dev, "failed to init kernel QP %p SQ\n", qp);
1445 return rc;
1446 }
1447
1448 rc = qedr_init_qp_kernel_params_sq(dev, qp, attrs, params);
1449 if (rc) {
1450 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1451 DP_ERR(dev, "failed to init kernel QP %p SQ params\n", qp);
1452 return rc;
1453 }
1454
1455 rc = qedr_init_qp_kernel_rq(dev, qp, attrs);
1456 if (rc) {
1457 qedr_cleanup_kernel_sq(dev, qp);
1458 DP_ERR(dev, "failed to init kernel QP %p RQ\n", qp);
1459 return rc;
1460 }
1461
1462 rc = qedr_init_qp_kernel_params_rq(dev, qp, attrs);
1463 if (rc) {
1464 DP_ERR(dev, "failed to init kernel QP %p RQ params\n", qp);
1465 qedr_cleanup_kernel_sq(dev, qp);
1466 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1467 return rc;
1468 }
1469
1470 return rc;
1471}
1472
1473struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1474 struct ib_qp_init_attr *attrs,
1475 struct ib_udata *udata)
1476{
1477 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1478 struct qed_rdma_create_qp_out_params out_params;
1479 struct qed_rdma_create_qp_in_params in_params;
1480 struct qedr_pd *pd = get_qedr_pd(ibpd);
1481 struct ib_ucontext *ib_ctx = NULL;
1482 struct qedr_ucontext *ctx = NULL;
1483 struct qedr_create_qp_ureq ureq;
1484 struct qedr_qp *qp;
181d8015 1485 struct ib_qp *ibqp;
cecbcddf
RA		 1486 int rc = 0;
1487
1488 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1489 udata ? "user library" : "kernel", pd);
1490
1491 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1492 if (rc)
1493 return ERR_PTR(rc);
1494
181d8015
WY		 1495 if (attrs->srq)
1496 return ERR_PTR(-EINVAL);
1497
cecbcddf
RA		 1498 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1499 if (!qp)
1500 return ERR_PTR(-ENOMEM);
1501
cecbcddf
RA		 1502 DP_DEBUG(dev, QEDR_MSG_QP,
1503 "create qp: sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1504 get_qedr_cq(attrs->send_cq),
1505 get_qedr_cq(attrs->send_cq)->icid,
1506 get_qedr_cq(attrs->recv_cq),
1507 get_qedr_cq(attrs->recv_cq)->icid);
1508
1509 qedr_set_qp_init_params(dev, qp, pd, attrs);
1510
04886779
RA		 1511 if (attrs->qp_type == IB_QPT_GSI) {
1512 if (udata) {
1513 DP_ERR(dev,
1514 "create qp: unexpected udata when creating GSI QP\n");
1515 goto err0;
1516 }
181d8015
WY		 1517 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1518 if (IS_ERR(ibqp))
1519 kfree(qp);
1520 return ibqp;
04886779
RA		 1521 }
1522
cecbcddf
RA		 1523 memset(&in_params, 0, sizeof(in_params));
1524
1525 if (udata) {
1526 if (!(udata && ibpd->uobject && ibpd->uobject->context))
1527 goto err0;
1528
1529 ib_ctx = ibpd->uobject->context;
1530 ctx = get_qedr_ucontext(ib_ctx);
1531
1532 memset(&ureq, 0, sizeof(ureq));
1533 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1534 DP_ERR(dev,
1535 "create qp: problem copying data from user space\n");
1536 goto err0;
1537 }
1538
1539 rc = qedr_init_user_qp(ib_ctx, dev, qp, &ureq);
1540 if (rc)
1541 goto err0;
1542
1543 qedr_init_qp_user_params(&in_params, &ureq);
1544 } else {
1545 rc = qedr_init_kernel_qp(dev, qp, attrs, &in_params);
1546 if (rc)
1547 goto err0;
1548 }
1549
1550 qedr_init_qp_in_params_sq(dev, pd, qp, attrs, udata, &in_params);
1551 qedr_init_qp_in_params_rq(qp, attrs, udata, &in_params);
1552
1553 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1554 &in_params, &out_params);
1555
1556 if (!qp->qed_qp)
1557 goto err1;
1558
1559 qp->qp_id = out_params.qp_id;
1560 qp->icid = out_params.icid;
1561 qp->ibqp.qp_num = qp->qp_id;
1562
1563 if (udata) {
1564 rc = qedr_copy_qp_uresp(dev, qp, udata);
1565 if (rc)
1566 goto err2;
1567
1568 qedr_qp_user_print(dev, qp);
1569 } else {
1570 qedr_init_qp_kernel_doorbell_sq(dev, qp);
1571 qedr_init_qp_kernel_doorbell_rq(dev, qp);
1572 }
1573
1574 DP_DEBUG(dev, QEDR_MSG_QP, "created %s space QP %p\n",
1575 udata ? "user" : "kernel", qp);
1576
1577 return &qp->ibqp;
1578
1579err2:
1580 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1581 if (rc)
1582 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1583err1:
1584 if (udata) {
1585 qedr_cleanup_user_sq(dev, qp);
1586 qedr_cleanup_user_rq(dev, qp);
1587 } else {
1588 qedr_cleanup_kernel_sq(dev, qp);
1589 qedr_cleanup_kernel_rq(dev, qp);
1590 }
1591
1592err0:
1593 kfree(qp);
1594
1595 return ERR_PTR(-EFAULT);
1596}
1597
1598enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1599{
1600 switch (qp_state) {
1601 case QED_ROCE_QP_STATE_RESET:
1602 return IB_QPS_RESET;
1603 case QED_ROCE_QP_STATE_INIT:
1604 return IB_QPS_INIT;
1605 case QED_ROCE_QP_STATE_RTR:
1606 return IB_QPS_RTR;
1607 case QED_ROCE_QP_STATE_RTS:
1608 return IB_QPS_RTS;
1609 case QED_ROCE_QP_STATE_SQD:
1610 return IB_QPS_SQD;
1611 case QED_ROCE_QP_STATE_ERR:
1612 return IB_QPS_ERR;
1613 case QED_ROCE_QP_STATE_SQE:
1614 return IB_QPS_SQE;
1615 }
1616 return IB_QPS_ERR;
1617}
1618
1619enum qed_roce_qp_state qedr_get_state_from_ibqp(enum ib_qp_state qp_state)
1620{
1621 switch (qp_state) {
1622 case IB_QPS_RESET:
1623 return QED_ROCE_QP_STATE_RESET;
1624 case IB_QPS_INIT:
1625 return QED_ROCE_QP_STATE_INIT;
1626 case IB_QPS_RTR:
1627 return QED_ROCE_QP_STATE_RTR;
1628 case IB_QPS_RTS:
1629 return QED_ROCE_QP_STATE_RTS;
1630 case IB_QPS_SQD:
1631 return QED_ROCE_QP_STATE_SQD;
1632 case IB_QPS_ERR:
1633 return QED_ROCE_QP_STATE_ERR;
1634 default:
1635 return QED_ROCE_QP_STATE_ERR;
1636 }
1637}
1638
1639static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1640{
1641 qed_chain_reset(&qph->pbl);
1642 qph->prod = 0;
1643 qph->cons = 0;
1644 qph->wqe_cons = 0;
1645 qph->db_data.data.value = cpu_to_le16(0);
1646}
1647
1648static int qedr_update_qp_state(struct qedr_dev *dev,
1649 struct qedr_qp *qp,
1650 enum qed_roce_qp_state new_state)
1651{
1652 int status = 0;
1653
1654 if (new_state == qp->state)
1655 return 1;
1656
1657 switch (qp->state) {
1658 case QED_ROCE_QP_STATE_RESET:
1659 switch (new_state) {
1660 case QED_ROCE_QP_STATE_INIT:
1661 qp->prev_wqe_size = 0;
1662 qedr_reset_qp_hwq_info(&qp->sq);
1663 qedr_reset_qp_hwq_info(&qp->rq);
1664 break;
1665 default:
1666 status = -EINVAL;
1667 break;
1668 };
1669 break;
1670 case QED_ROCE_QP_STATE_INIT:
1671 switch (new_state) {
1672 case QED_ROCE_QP_STATE_RTR:
1673 /* Update doorbell (in case post_recv was
1674 * done before move to RTR)
1675 */
1676 wmb();
1677 writel(qp->rq.db_data.raw, qp->rq.db);
1678 /* Make sure write takes effect */
1679 mmiowb();
1680 break;
1681 case QED_ROCE_QP_STATE_ERR:
1682 break;
1683 default:
1684 /* Invalid state change. */
1685 status = -EINVAL;
1686 break;
1687 };
1688 break;
1689 case QED_ROCE_QP_STATE_RTR:
1690 /* RTR->XXX */
1691 switch (new_state) {
1692 case QED_ROCE_QP_STATE_RTS:
1693 break;
1694 case QED_ROCE_QP_STATE_ERR:
1695 break;
1696 default:
1697 /* Invalid state change. */
1698 status = -EINVAL;
1699 break;
1700 };
1701 break;
1702 case QED_ROCE_QP_STATE_RTS:
1703 /* RTS->XXX */
1704 switch (new_state) {
1705 case QED_ROCE_QP_STATE_SQD:
1706 break;
1707 case QED_ROCE_QP_STATE_ERR:
1708 break;
1709 default:
1710 /* Invalid state change. */
1711 status = -EINVAL;
1712 break;
1713 };
1714 break;
1715 case QED_ROCE_QP_STATE_SQD:
1716 /* SQD->XXX */
1717 switch (new_state) {
1718 case QED_ROCE_QP_STATE_RTS:
1719 case QED_ROCE_QP_STATE_ERR:
1720 break;
1721 default:
1722 /* Invalid state change. */
1723 status = -EINVAL;
1724 break;
1725 };
1726 break;
1727 case QED_ROCE_QP_STATE_ERR:
1728 /* ERR->XXX */
1729 switch (new_state) {
1730 case QED_ROCE_QP_STATE_RESET:
1731 break;
1732 default:
1733 status = -EINVAL;
1734 break;
1735 };
1736 break;
1737 default:
1738 status = -EINVAL;
1739 break;
1740 };
1741
1742 return status;
1743}
1744
1745int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1746 int attr_mask, struct ib_udata *udata)
1747{
1748 struct qedr_qp *qp = get_qedr_qp(ibqp);
1749 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1750 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1751 enum ib_qp_state old_qp_state, new_qp_state;
1752 int rc = 0;
1753
1754 DP_DEBUG(dev, QEDR_MSG_QP,
1755 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1756 attr->qp_state);
1757
1758 old_qp_state = qedr_get_ibqp_state(qp->state);
1759 if (attr_mask & IB_QP_STATE)
1760 new_qp_state = attr->qp_state;
1761 else
1762 new_qp_state = old_qp_state;
1763
1764 if (!ib_modify_qp_is_ok
1765 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1766 IB_LINK_LAYER_ETHERNET)) {
1767 DP_ERR(dev,
1768 "modify qp: invalid attribute mask=0x%x specified for\n"
1769 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1770 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1771 new_qp_state);
1772 rc = -EINVAL;
1773 goto err;
1774 }
1775
1776 /* Translate the masks... */
1777 if (attr_mask & IB_QP_STATE) {
1778 SET_FIELD(qp_params.modify_flags,
1779 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1780 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1781 }
1782
1783 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1784 qp_params.sqd_async = true;
1785
1786 if (attr_mask & IB_QP_PKEY_INDEX) {
1787 SET_FIELD(qp_params.modify_flags,
1788 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1789 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1790 rc = -EINVAL;
1791 goto err;
1792 }
1793
1794 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1795 }
1796
1797 if (attr_mask & IB_QP_QKEY)
1798 qp->qkey = attr->qkey;
1799
1800 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1801 SET_FIELD(qp_params.modify_flags,
1802 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1803 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1804 IB_ACCESS_REMOTE_READ;
1805 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1806 IB_ACCESS_REMOTE_WRITE;
1807 qp_params.incoming_atomic_en = attr->qp_access_flags &
1808 IB_ACCESS_REMOTE_ATOMIC;
1809 }
1810
1811 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1812 if (attr_mask & IB_QP_PATH_MTU) {
1813 if (attr->path_mtu < IB_MTU_256 ||
1814 attr->path_mtu > IB_MTU_4096) {
1815 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1816 rc = -EINVAL;
1817 goto err;
1818 }
1819 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1820 ib_mtu_enum_to_int(iboe_get_mtu
1821 (dev->ndev->mtu)));
1822 }
1823
1824 if (!qp->mtu) {
1825 qp->mtu =
1826 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1827 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1828 }
1829
1830 SET_FIELD(qp_params.modify_flags,
1831 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1832
1833 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1834 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1835 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1836
1837 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1838
1839 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1840 if (rc) {
1841 DP_ERR(dev,
1842 "modify qp: problems with GID index %d (rc=%d)\n",
1843 attr->ah_attr.grh.sgid_index, rc);
1844 return rc;
1845 }
1846
1847 rc = qedr_get_dmac(dev, &attr->ah_attr,
1848 qp_params.remote_mac_addr);
1849 if (rc)
1850 return rc;
1851
1852 qp_params.use_local_mac = true;
1853 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1854
1855 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1856 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1857 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1858 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1859 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1860 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1861 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1862 qp_params.remote_mac_addr);
1863;
1864
1865 qp_params.mtu = qp->mtu;
1866 qp_params.lb_indication = false;
1867 }
1868
1869 if (!qp_params.mtu) {
1870 /* Stay with current MTU */
1871 if (qp->mtu)
1872 qp_params.mtu = qp->mtu;
1873 else
1874 qp_params.mtu =
1875 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1876 }
1877
1878 if (attr_mask & IB_QP_TIMEOUT) {
1879 SET_FIELD(qp_params.modify_flags,
1880 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1881
1882 qp_params.ack_timeout = attr->timeout;
1883 if (attr->timeout) {
1884 u32 temp;
1885
1886 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1887 /* FW requires [msec] */
1888 qp_params.ack_timeout = temp;
1889 } else {
1890 /* Infinite */
1891 qp_params.ack_timeout = 0;
1892 }
1893 }
1894 if (attr_mask & IB_QP_RETRY_CNT) {
1895 SET_FIELD(qp_params.modify_flags,
1896 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1897 qp_params.retry_cnt = attr->retry_cnt;
1898 }
1899
1900 if (attr_mask & IB_QP_RNR_RETRY) {
1901 SET_FIELD(qp_params.modify_flags,
1902 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1903 qp_params.rnr_retry_cnt = attr->rnr_retry;
1904 }
1905
1906 if (attr_mask & IB_QP_RQ_PSN) {
1907 SET_FIELD(qp_params.modify_flags,
1908 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1909 qp_params.rq_psn = attr->rq_psn;
1910 qp->rq_psn = attr->rq_psn;
1911 }
1912
1913 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1914 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1915 rc = -EINVAL;
1916 DP_ERR(dev,
1917 "unsupported max_rd_atomic=%d, supported=%d\n",
1918 attr->max_rd_atomic,
1919 dev->attr.max_qp_req_rd_atomic_resc);
1920 goto err;
1921 }
1922
1923 SET_FIELD(qp_params.modify_flags,
1924 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1925 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1926 }
1927
1928 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1929 SET_FIELD(qp_params.modify_flags,
1930 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1931 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1932 }
1933
1934 if (attr_mask & IB_QP_SQ_PSN) {
1935 SET_FIELD(qp_params.modify_flags,
1936 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1937 qp_params.sq_psn = attr->sq_psn;
1938 qp->sq_psn = attr->sq_psn;
1939 }
1940
1941 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1942 if (attr->max_dest_rd_atomic >
1943 dev->attr.max_qp_resp_rd_atomic_resc) {
1944 DP_ERR(dev,
1945 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1946 attr->max_dest_rd_atomic,
1947 dev->attr.max_qp_resp_rd_atomic_resc);
1948
1949 rc = -EINVAL;
1950 goto err;
1951 }
1952
1953 SET_FIELD(qp_params.modify_flags,
1954 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1955 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1956 }
1957
1958 if (attr_mask & IB_QP_DEST_QPN) {
1959 SET_FIELD(qp_params.modify_flags,
1960 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1961
1962 qp_params.dest_qp = attr->dest_qp_num;
1963 qp->dest_qp_num = attr->dest_qp_num;
1964 }
1965
1966 if (qp->qp_type != IB_QPT_GSI)
1967 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1968 qp->qed_qp, &qp_params);
1969
1970 if (attr_mask & IB_QP_STATE) {
1971 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1972 qedr_update_qp_state(dev, qp, qp_params.new_state);
1973 qp->state = qp_params.new_state;
1974 }
1975
1976err:
1977 return rc;
1978}
1979
1980static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1981{
1982 int ib_qp_acc_flags = 0;
1983
1984 if (params->incoming_rdma_write_en)
1985 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1986 if (params->incoming_rdma_read_en)
1987 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
1988 if (params->incoming_atomic_en)
1989 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
1990 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1991 return ib_qp_acc_flags;
1992}
1993
1994int qedr_query_qp(struct ib_qp *ibqp,
1995 struct ib_qp_attr *qp_attr,
1996 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1997{
1998 struct qed_rdma_query_qp_out_params params;
1999 struct qedr_qp *qp = get_qedr_qp(ibqp);
2000 struct qedr_dev *dev = qp->dev;
2001 int rc = 0;
2002
2003 memset(&params, 0, sizeof(params));
2004
2005 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2006 if (rc)
2007 goto err;
2008
2009 memset(qp_attr, 0, sizeof(*qp_attr));
2010 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2011
2012 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2013 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2014 qp_attr->path_mtu = iboe_get_mtu(params.mtu);
2015 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2016 qp_attr->rq_psn = params.rq_psn;
2017 qp_attr->sq_psn = params.sq_psn;
2018 qp_attr->dest_qp_num = params.dest_qp;
2019
2020 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2021
2022 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2023 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2024 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2025 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2026 qp_attr->cap.max_inline_data = qp->max_inline_data;
2027 qp_init_attr->cap = qp_attr->cap;
2028
2029 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
2030 sizeof(qp_attr->ah_attr.grh.dgid.raw));
2031
2032 qp_attr->ah_attr.grh.flow_label = params.flow_label;
2033 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
2034 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
2035 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
2036
2037 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
2038 qp_attr->ah_attr.port_num = 1;
2039 qp_attr->ah_attr.sl = 0;
2040 qp_attr->timeout = params.timeout;
2041 qp_attr->rnr_retry = params.rnr_retry;
2042 qp_attr->retry_cnt = params.retry_cnt;
2043 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2044 qp_attr->pkey_index = params.pkey_index;
2045 qp_attr->port_num = 1;
2046 qp_attr->ah_attr.src_path_bits = 0;
2047 qp_attr->ah_attr.static_rate = 0;
2048 qp_attr->alt_pkey_index = 0;
2049 qp_attr->alt_port_num = 0;
2050 qp_attr->alt_timeout = 0;
2051 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2052
2053 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2054 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2055 qp_attr->max_rd_atomic = params.max_rd_atomic;
2056 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2057
2058 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2059 qp_attr->cap.max_inline_data);
2060
2061err:
2062 return rc;
2063}
2064
2065int qedr_destroy_qp(struct ib_qp *ibqp)
2066{
2067 struct qedr_qp *qp = get_qedr_qp(ibqp);
2068 struct qedr_dev *dev = qp->dev;
2069 struct ib_qp_attr attr;
2070 int attr_mask = 0;
2071 int rc = 0;
2072
2073 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2074 qp, qp->qp_type);
2075
2076 if (qp->state != (QED_ROCE_QP_STATE_RESET | QED_ROCE_QP_STATE_ERR |
2077 QED_ROCE_QP_STATE_INIT)) {
2078 attr.qp_state = IB_QPS_ERR;
2079 attr_mask |= IB_QP_STATE;
2080
2081 /* Change the QP state to ERROR */
2082 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2083 }
2084
2085 if (qp->qp_type != IB_QPT_GSI) {
2086 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2087 if (rc)
2088 return rc;
04886779
RA		 2089 } else {
2090 qedr_destroy_gsi_qp(dev);
cecbcddf
RA		 2091 }
2092
2093 if (ibqp->uobject && ibqp->uobject->context) {
2094 qedr_cleanup_user_sq(dev, qp);
2095 qedr_cleanup_user_rq(dev, qp);
2096 } else {
2097 qedr_cleanup_kernel_sq(dev, qp);
2098 qedr_cleanup_kernel_rq(dev, qp);
2099 }
2100
2101 kfree(qp);
2102
2103 return rc;
2104}
e0290cce 2105
477864c8
MS		 2106struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
2107 struct ib_udata *udata)
04886779
RA		 2108{
2109 struct qedr_ah *ah;
2110
2111 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2112 if (!ah)
2113 return ERR_PTR(-ENOMEM);
2114
2115 ah->attr = *attr;
2116
2117 return &ah->ibah;
2118}
2119
2120int qedr_destroy_ah(struct ib_ah *ibah)
2121{
2122 struct qedr_ah *ah = get_qedr_ah(ibah);
2123
2124 kfree(ah);
2125 return 0;
2126}
2127
e0290cce
RA		 2128static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2129{
2130 struct qedr_pbl *pbl, *tmp;
2131
2132 if (info->pbl_table)
2133 list_add_tail(&info->pbl_table->list_entry,
2134 &info->free_pbl_list);
2135
2136 if (!list_empty(&info->inuse_pbl_list))
2137 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2138
2139 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2140 list_del(&pbl->list_entry);
2141 qedr_free_pbl(dev, &info->pbl_info, pbl);
2142 }
2143}
2144
2145static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2146 size_t page_list_len, bool two_layered)
2147{
2148 struct qedr_pbl *tmp;
2149 int rc;
2150
2151 INIT_LIST_HEAD(&info->free_pbl_list);
2152 INIT_LIST_HEAD(&info->inuse_pbl_list);
2153
2154 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2155 page_list_len, two_layered);
2156 if (rc)
2157 goto done;
2158
2159 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2160 if (!info->pbl_table) {
2161 rc = -ENOMEM;
2162 goto done;
2163 }
2164
2165 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2166 &info->pbl_table->pa);
2167
2168 /* in usual case we use 2 PBLs, so we add one to free
2169 * list and allocating another one
2170 */
2171 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2172 if (!tmp) {
2173 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2174 goto done;
2175 }
2176
2177 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2178
2179 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2180
2181done:
2182 if (rc)
2183 free_mr_info(dev, info);
2184
2185 return rc;
2186}
2187
2188struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2189 u64 usr_addr, int acc, struct ib_udata *udata)
2190{
2191 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2192 struct qedr_mr *mr;
2193 struct qedr_pd *pd;
2194 int rc = -ENOMEM;
2195
2196 pd = get_qedr_pd(ibpd);
2197 DP_DEBUG(dev, QEDR_MSG_MR,
2198 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2199 pd->pd_id, start, len, usr_addr, acc);
2200
2201 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2202 return ERR_PTR(-EINVAL);
2203
2204 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2205 if (!mr)
2206 return ERR_PTR(rc);
2207
2208 mr->type = QEDR_MR_USER;
2209
2210 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2211 if (IS_ERR(mr->umem)) {
2212 rc = -EFAULT;
2213 goto err0;
2214 }
2215
2216 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2217 if (rc)
2218 goto err1;
2219
2220 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2221 &mr->info.pbl_info);
2222
2223 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2224 if (rc) {
2225 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2226 goto err1;
2227 }
2228
2229 /* Index only, 18 bit long, lkey = itid << 8 | key */
2230 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2231 mr->hw_mr.key = 0;
2232 mr->hw_mr.pd = pd->pd_id;
2233 mr->hw_mr.local_read = 1;
2234 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2235 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2236 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2237 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2238 mr->hw_mr.mw_bind = false;
2239 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2240 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2241 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2242 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size);
2243 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2244 mr->hw_mr.length = len;
2245 mr->hw_mr.vaddr = usr_addr;
2246 mr->hw_mr.zbva = false;
2247 mr->hw_mr.phy_mr = false;
2248 mr->hw_mr.dma_mr = false;
2249
2250 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2251 if (rc) {
2252 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2253 goto err2;
2254 }
2255
2256 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2257 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2258 mr->hw_mr.remote_atomic)
2259 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2260
2261 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2262 mr->ibmr.lkey);
2263 return &mr->ibmr;
2264
2265err2:
2266 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2267err1:
2268 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2269err0:
2270 kfree(mr);
2271 return ERR_PTR(rc);
2272}
2273
2274int qedr_dereg_mr(struct ib_mr *ib_mr)
2275{
2276 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2277 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2278 int rc = 0;
2279
2280 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2281 if (rc)
2282 return rc;
2283
2284 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2285
2286 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2287 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2288
2289 /* it could be user registered memory. */
2290 if (mr->umem)
2291 ib_umem_release(mr->umem);
2292
2293 kfree(mr);
2294
2295 return rc;
2296}
2297
2298struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
2299{
2300 struct qedr_pd *pd = get_qedr_pd(ibpd);
2301 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2302 struct qedr_mr *mr;
2303 int rc = -ENOMEM;
2304
2305 DP_DEBUG(dev, QEDR_MSG_MR,
2306 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2307 max_page_list_len);
2308
2309 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2310 if (!mr)
2311 return ERR_PTR(rc);
2312
2313 mr->dev = dev;
2314 mr->type = QEDR_MR_FRMR;
2315
2316 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2317 if (rc)
2318 goto err0;
2319
2320 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2321 if (rc) {
2322 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2323 goto err0;
2324 }
2325
2326 /* Index only, 18 bit long, lkey = itid << 8 | key */
2327 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2328 mr->hw_mr.key = 0;
2329 mr->hw_mr.pd = pd->pd_id;
2330 mr->hw_mr.local_read = 1;
2331 mr->hw_mr.local_write = 0;
2332 mr->hw_mr.remote_read = 0;
2333 mr->hw_mr.remote_write = 0;
2334 mr->hw_mr.remote_atomic = 0;
2335 mr->hw_mr.mw_bind = false;
2336 mr->hw_mr.pbl_ptr = 0;
2337 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2338 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2339 mr->hw_mr.fbo = 0;
2340 mr->hw_mr.length = 0;
2341 mr->hw_mr.vaddr = 0;
2342 mr->hw_mr.zbva = false;
2343 mr->hw_mr.phy_mr = true;
2344 mr->hw_mr.dma_mr = false;
2345
2346 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2347 if (rc) {
2348 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2349 goto err1;
2350 }
2351
2352 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2353 mr->ibmr.rkey = mr->ibmr.lkey;
2354
2355 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2356 return mr;
2357
2358err1:
2359 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2360err0:
2361 kfree(mr);
2362 return ERR_PTR(rc);
2363}
2364
2365struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2366 enum ib_mr_type mr_type, u32 max_num_sg)
2367{
2368 struct qedr_dev *dev;
2369 struct qedr_mr *mr;
2370
2371 if (mr_type != IB_MR_TYPE_MEM_REG)
2372 return ERR_PTR(-EINVAL);
2373
2374 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2375
2376 if (IS_ERR(mr))
2377 return ERR_PTR(-EINVAL);
2378
2379 dev = mr->dev;
2380
2381 return &mr->ibmr;
2382}
2383
2384static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2385{
2386 struct qedr_mr *mr = get_qedr_mr(ibmr);
2387 struct qedr_pbl *pbl_table;
2388 struct regpair *pbe;
2389 u32 pbes_in_page;
2390
2391 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2392 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2393 return -ENOMEM;
2394 }
2395
2396 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2397 mr->npages, addr);
2398
2399 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2400 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2401 pbe = (struct regpair *)pbl_table->va;
2402 pbe += mr->npages % pbes_in_page;
2403 pbe->lo = cpu_to_le32((u32)addr);
2404 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2405
2406 mr->npages++;
2407
2408 return 0;
2409}
2410
2411static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2412{
2413 int work = info->completed - info->completed_handled - 1;
2414
2415 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2416 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2417 struct qedr_pbl *pbl;
2418
2419 /* Free all the page list that are possible to be freed
2420 * (all the ones that were invalidated), under the assumption
2421 * that if an FMR was completed successfully that means that
2422 * if there was an invalidate operation before it also ended
2423 */
2424 pbl = list_first_entry(&info->inuse_pbl_list,
2425 struct qedr_pbl, list_entry);
aafec388 2426 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 2427 info->completed_handled++;
2428 }
2429}
2430
2431int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2432 int sg_nents, unsigned int *sg_offset)
2433{
2434 struct qedr_mr *mr = get_qedr_mr(ibmr);
2435
2436 mr->npages = 0;
2437
2438 handle_completed_mrs(mr->dev, &mr->info);
2439 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2440}
2441
2442struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2443{
2444 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2445 struct qedr_pd *pd = get_qedr_pd(ibpd);
2446 struct qedr_mr *mr;
2447 int rc;
2448
2449 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2450 if (!mr)
2451 return ERR_PTR(-ENOMEM);
2452
2453 mr->type = QEDR_MR_DMA;
2454
2455 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2456 if (rc) {
2457 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2458 goto err1;
2459 }
2460
2461 /* index only, 18 bit long, lkey = itid << 8 | key */
2462 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2463 mr->hw_mr.pd = pd->pd_id;
2464 mr->hw_mr.local_read = 1;
2465 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2466 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2467 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2468 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2469 mr->hw_mr.dma_mr = true;
2470
2471 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2472 if (rc) {
2473 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2474 goto err2;
2475 }
2476
2477 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2478 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2479 mr->hw_mr.remote_atomic)
2480 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2481
2482 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2483 return &mr->ibmr;
2484
2485err2:
2486 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2487err1:
2488 kfree(mr);
2489 return ERR_PTR(rc);
2490}
afa0e13b
RA		 2491
2492static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2493{
2494 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2495}
2496
2497static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2498{
2499 int i, len = 0;
2500
2501 for (i = 0; i < num_sge; i++)
2502 len += sg_list[i].length;
2503
2504 return len;
2505}
2506
2507static void swap_wqe_data64(u64 *p)
2508{
2509 int i;
2510
2511 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2512 *p = cpu_to_be64(cpu_to_le64(*p));
2513}
2514
2515static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2516 struct qedr_qp *qp, u8 *wqe_size,
2517 struct ib_send_wr *wr,
2518 struct ib_send_wr **bad_wr, u8 *bits,
2519 u8 bit)
2520{
2521 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2522 char *seg_prt, *wqe;
2523 int i, seg_siz;
2524
2525 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2526 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2527 *bad_wr = wr;
2528 return 0;
2529 }
2530
2531 if (!data_size)
2532 return data_size;
2533
2534 *bits |= bit;
2535
2536 seg_prt = NULL;
2537 wqe = NULL;
2538 seg_siz = 0;
2539
2540 /* Copy data inline */
2541 for (i = 0; i < wr->num_sge; i++) {
2542 u32 len = wr->sg_list[i].length;
2543 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2544
2545 while (len > 0) {
2546 u32 cur;
2547
2548 /* New segment required */
2549 if (!seg_siz) {
2550 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2551 seg_prt = wqe;
2552 seg_siz = sizeof(struct rdma_sq_common_wqe);
2553 (*wqe_size)++;
2554 }
2555
2556 /* Calculate currently allowed length */
2557 cur = min_t(u32, len, seg_siz);
2558 memcpy(seg_prt, src, cur);
2559
2560 /* Update segment variables */
2561 seg_prt += cur;
2562 seg_siz -= cur;
2563
2564 /* Update sge variables */
2565 src += cur;
2566 len -= cur;
2567
2568 /* Swap fully-completed segments */
2569 if (!seg_siz)
2570 swap_wqe_data64((u64 *)wqe);
2571 }
2572 }
2573
2574 /* swap last not completed segment */
2575 if (seg_siz)
2576 swap_wqe_data64((u64 *)wqe);
2577
2578 return data_size;
2579}
2580
2581#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2582 do { \
2583 DMA_REGPAIR_LE(sge->addr, vaddr); \
2584 (sge)->length = cpu_to_le32(vlength); \
2585 (sge)->flags = cpu_to_le32(vflags); \
2586 } while (0)
2587
2588#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2589 do { \
2590 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2591 (hdr)->num_sges = num_sge; \
2592 } while (0)
2593
2594#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2595 do { \
2596 DMA_REGPAIR_LE(sge->addr, vaddr); \
2597 (sge)->length = cpu_to_le32(vlength); \
2598 (sge)->l_key = cpu_to_le32(vlkey); \
2599 } while (0)
2600
2601static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2602 struct ib_send_wr *wr)
2603{
2604 u32 data_size = 0;
2605 int i;
2606
2607 for (i = 0; i < wr->num_sge; i++) {
2608 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2609
2610 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2611 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2612 sge->length = cpu_to_le32(wr->sg_list[i].length);
2613 data_size += wr->sg_list[i].length;
2614 }
2615
2616 if (wqe_size)
2617 *wqe_size += wr->num_sge;
2618
2619 return data_size;
2620}
2621
2622static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2623 struct qedr_qp *qp,
2624 struct rdma_sq_rdma_wqe_1st *rwqe,
2625 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2626 struct ib_send_wr *wr,
2627 struct ib_send_wr **bad_wr)
2628{
2629 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2630 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2631
2632 if (wr->send_flags & IB_SEND_INLINE) {
2633 u8 flags = 0;
2634
2635 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2636 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2637 bad_wr, &rwqe->flags, flags);
2638 }
2639
2640 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2641}
2642
2643static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2644 struct qedr_qp *qp,
2645 struct rdma_sq_send_wqe_1st *swqe,
2646 struct rdma_sq_send_wqe_2st *swqe2,
2647 struct ib_send_wr *wr,
2648 struct ib_send_wr **bad_wr)
2649{
2650 memset(swqe2, 0, sizeof(*swqe2));
2651 if (wr->send_flags & IB_SEND_INLINE) {
2652 u8 flags = 0;
2653
2654 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2655 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2656 bad_wr, &swqe->flags, flags);
2657 }
2658
2659 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2660}
2661
2662static int qedr_prepare_reg(struct qedr_qp *qp,
2663 struct rdma_sq_fmr_wqe_1st *fwqe1,
2664 struct ib_reg_wr *wr)
2665{
2666 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2667 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2668
2669 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2670 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2671 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2672 fwqe1->l_key = wr->key;
2673
2674 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2675 !!(wr->access & IB_ACCESS_REMOTE_READ));
2676 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2677 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2678 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2679 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2680 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2681 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2682 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2683 fwqe2->fmr_ctrl = 0;
2684
2685 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2686 ilog2(mr->ibmr.page_size) - 12);
2687
2688 fwqe2->length_hi = 0;
2689 fwqe2->length_lo = mr->ibmr.length;
2690 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2691 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2692
2693 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2694
2695 return 0;
2696}
2697
2698enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2699{
2700 switch (opcode) {
2701 case IB_WR_RDMA_WRITE:
2702 case IB_WR_RDMA_WRITE_WITH_IMM:
2703 return IB_WC_RDMA_WRITE;
2704 case IB_WR_SEND_WITH_IMM:
2705 case IB_WR_SEND:
2706 case IB_WR_SEND_WITH_INV:
2707 return IB_WC_SEND;
2708 case IB_WR_RDMA_READ:
2709 return IB_WC_RDMA_READ;
2710 case IB_WR_ATOMIC_CMP_AND_SWP:
2711 return IB_WC_COMP_SWAP;
2712 case IB_WR_ATOMIC_FETCH_AND_ADD:
2713 return IB_WC_FETCH_ADD;
2714 case IB_WR_REG_MR:
2715 return IB_WC_REG_MR;
2716 case IB_WR_LOCAL_INV:
2717 return IB_WC_LOCAL_INV;
2718 default:
2719 return IB_WC_SEND;
2720 }
2721}
2722
2723inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2724{
2725 int wq_is_full, err_wr, pbl_is_full;
2726 struct qedr_dev *dev = qp->dev;
2727
2728 /* prevent SQ overflow and/or processing of a bad WR */
2729 err_wr = wr->num_sge > qp->sq.max_sges;
2730 wq_is_full = qedr_wq_is_full(&qp->sq);
2731 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2732 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2733 if (wq_is_full || err_wr || pbl_is_full) {
2734 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2735 DP_ERR(dev,
2736 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2737 qp);
2738 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2739 }
2740
2741 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2742 DP_ERR(dev,
2743 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2744 qp);
2745 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2746 }
2747
2748 if (pbl_is_full &&
2749 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2750 DP_ERR(dev,
2751 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2752 qp);
2753 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2754 }
2755 return false;
2756 }
2757 return true;
2758}
2759
2760int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2761 struct ib_send_wr **bad_wr)
2762{
2763 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2764 struct qedr_qp *qp = get_qedr_qp(ibqp);
2765 struct rdma_sq_atomic_wqe_1st *awqe1;
2766 struct rdma_sq_atomic_wqe_2nd *awqe2;
2767 struct rdma_sq_atomic_wqe_3rd *awqe3;
2768 struct rdma_sq_send_wqe_2st *swqe2;
2769 struct rdma_sq_local_inv_wqe *iwqe;
2770 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2771 struct rdma_sq_send_wqe_1st *swqe;
2772 struct rdma_sq_rdma_wqe_1st *rwqe;
2773 struct rdma_sq_fmr_wqe_1st *fwqe1;
2774 struct rdma_sq_common_wqe *wqe;
2775 u32 length;
2776 int rc = 0;
2777 bool comp;
2778
2779 if (!qedr_can_post_send(qp, wr)) {
2780 *bad_wr = wr;
2781 return -ENOMEM;
2782 }
2783
2784 wqe = qed_chain_produce(&qp->sq.pbl);
2785 qp->wqe_wr_id[qp->sq.prod].signaled =
2786 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2787
2788 wqe->flags = 0;
2789 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2790 !!(wr->send_flags & IB_SEND_SOLICITED));
2791 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2792 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2793 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2794 !!(wr->send_flags & IB_SEND_FENCE));
2795 wqe->prev_wqe_size = qp->prev_wqe_size;
2796
2797 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2798
2799 switch (wr->opcode) {
2800 case IB_WR_SEND_WITH_IMM:
2801 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2802 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2803 swqe->wqe_size = 2;
2804 swqe2 = qed_chain_produce(&qp->sq.pbl);
2805
2806 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2807 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2808 wr, bad_wr);
2809 swqe->length = cpu_to_le32(length);
2810 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2811 qp->prev_wqe_size = swqe->wqe_size;
2812 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2813 break;
2814 case IB_WR_SEND:
2815 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2816 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2817
2818 swqe->wqe_size = 2;
2819 swqe2 = qed_chain_produce(&qp->sq.pbl);
2820 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2821 wr, bad_wr);
2822 swqe->length = cpu_to_le32(length);
2823 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2824 qp->prev_wqe_size = swqe->wqe_size;
2825 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2826 break;
2827 case IB_WR_SEND_WITH_INV:
2828 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2829 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2830 swqe2 = qed_chain_produce(&qp->sq.pbl);
2831 swqe->wqe_size = 2;
2832 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2833 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2834 wr, bad_wr);
2835 swqe->length = cpu_to_le32(length);
2836 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2837 qp->prev_wqe_size = swqe->wqe_size;
2838 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2839 break;
2840
2841 case IB_WR_RDMA_WRITE_WITH_IMM:
2842 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2843 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2844
2845 rwqe->wqe_size = 2;
2846 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2847 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2848 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2849 wr, bad_wr);
2850 rwqe->length = cpu_to_le32(length);
2851 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2852 qp->prev_wqe_size = rwqe->wqe_size;
2853 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2854 break;
2855 case IB_WR_RDMA_WRITE:
2856 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2857 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2858
2859 rwqe->wqe_size = 2;
2860 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2861 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2862 wr, bad_wr);
2863 rwqe->length = cpu_to_le32(length);
2864 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2865 qp->prev_wqe_size = rwqe->wqe_size;
2866 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2867 break;
2868 case IB_WR_RDMA_READ_WITH_INV:
2869 DP_ERR(dev,
2870 "RDMA READ WITH INVALIDATE not supported\n");
2871 *bad_wr = wr;
2872 rc = -EINVAL;
2873 break;
2874
2875 case IB_WR_RDMA_READ:
2876 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2877 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2878
2879 rwqe->wqe_size = 2;
2880 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2881 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2882 wr, bad_wr);
2883 rwqe->length = cpu_to_le32(length);
2884 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2885 qp->prev_wqe_size = rwqe->wqe_size;
2886 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2887 break;
2888
2889 case IB_WR_ATOMIC_CMP_AND_SWP:
2890 case IB_WR_ATOMIC_FETCH_AND_ADD:
2891 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2892 awqe1->wqe_size = 4;
2893
2894 awqe2 = qed_chain_produce(&qp->sq.pbl);
2895 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2896 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2897
2898 awqe3 = qed_chain_produce(&qp->sq.pbl);
2899
2900 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2901 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2902 DMA_REGPAIR_LE(awqe3->swap_data,
2903 atomic_wr(wr)->compare_add);
2904 } else {
2905 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2906 DMA_REGPAIR_LE(awqe3->swap_data,
2907 atomic_wr(wr)->swap);
2908 DMA_REGPAIR_LE(awqe3->cmp_data,
2909 atomic_wr(wr)->compare_add);
2910 }
2911
2912 qedr_prepare_sq_sges(qp, NULL, wr);
2913
2914 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2915 qp->prev_wqe_size = awqe1->wqe_size;
2916 break;
2917
2918 case IB_WR_LOCAL_INV:
2919 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2920 iwqe->wqe_size = 1;
2921
2922 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2923 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2924 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2925 qp->prev_wqe_size = iwqe->wqe_size;
2926 break;
2927 case IB_WR_REG_MR:
2928 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2929 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2930 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2931 fwqe1->wqe_size = 2;
2932
2933 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2934 if (rc) {
2935 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2936 *bad_wr = wr;
2937 break;
2938 }
2939
2940 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2941 qp->prev_wqe_size = fwqe1->wqe_size;
2942 break;
2943 default:
2944 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2945 rc = -EINVAL;
2946 *bad_wr = wr;
2947 break;
2948 }
2949
2950 if (*bad_wr) {
2951 u16 value;
2952
2953 /* Restore prod to its position before
2954 * this WR was processed
2955 */
2956 value = le16_to_cpu(qp->sq.db_data.data.value);
2957 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2958
2959 /* Restore prev_wqe_size */
2960 qp->prev_wqe_size = wqe->prev_wqe_size;
2961 rc = -EINVAL;
2962 DP_ERR(dev, "POST SEND FAILED\n");
2963 }
2964
2965 return rc;
2966}
2967
2968int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2969 struct ib_send_wr **bad_wr)
2970{
2971 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2972 struct qedr_qp *qp = get_qedr_qp(ibqp);
2973 unsigned long flags;
2974 int rc = 0;
2975
2976 *bad_wr = NULL;
2977
04886779
RA		 2978 if (qp->qp_type == IB_QPT_GSI)
2979 return qedr_gsi_post_send(ibqp, wr, bad_wr);
2980
afa0e13b
RA		 2981 spin_lock_irqsave(&qp->q_lock, flags);
2982
2983 if ((qp->state == QED_ROCE_QP_STATE_RESET) ||
2984 (qp->state == QED_ROCE_QP_STATE_ERR)) {
2985 spin_unlock_irqrestore(&qp->q_lock, flags);
2986 *bad_wr = wr;
2987 DP_DEBUG(dev, QEDR_MSG_CQ,
2988 "QP in wrong state! QP icid=0x%x state %d\n",
2989 qp->icid, qp->state);
2990 return -EINVAL;
2991 }
2992
afa0e13b
RA		 2993 while (wr) {
2994 rc = __qedr_post_send(ibqp, wr, bad_wr);
2995 if (rc)
2996 break;
2997
2998 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
2999
3000 qedr_inc_sw_prod(&qp->sq);
3001
3002 qp->sq.db_data.data.value++;
3003
3004 wr = wr->next;
3005 }
3006
3007 /* Trigger doorbell
3008 * If there was a failure in the first WR then it will be triggered in
3009 * vane. However this is not harmful (as long as the producer value is
3010 * unchanged). For performance reasons we avoid checking for this
3011 * redundant doorbell.
3012 */
3013 wmb();
3014 writel(qp->sq.db_data.raw, qp->sq.db);
3015
3016 /* Make sure write sticks */
3017 mmiowb();
3018
3019 spin_unlock_irqrestore(&qp->q_lock, flags);
3020
3021 return rc;
3022}
3023
3024int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3025 struct ib_recv_wr **bad_wr)
3026{
3027 struct qedr_qp *qp = get_qedr_qp(ibqp);
3028 struct qedr_dev *dev = qp->dev;
3029 unsigned long flags;
3030 int status = 0;
3031
04886779
RA		 3032 if (qp->qp_type == IB_QPT_GSI)
3033 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3034
afa0e13b
RA		 3035 spin_lock_irqsave(&qp->q_lock, flags);
3036
3037 if ((qp->state == QED_ROCE_QP_STATE_RESET) ||
3038 (qp->state == QED_ROCE_QP_STATE_ERR)) {
3039 spin_unlock_irqrestore(&qp->q_lock, flags);
3040 *bad_wr = wr;
3041 return -EINVAL;
3042 }
3043
3044 while (wr) {
3045 int i;
3046
3047 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3048 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3049 wr->num_sge > qp->rq.max_sges) {
3050 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3051 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3052 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3053 qp->rq.max_sges);
3054 status = -ENOMEM;
3055 *bad_wr = wr;
3056 break;
3057 }
3058 for (i = 0; i < wr->num_sge; i++) {
3059 u32 flags = 0;
3060 struct rdma_rq_sge *rqe =
3061 qed_chain_produce(&qp->rq.pbl);
3062
3063 /* First one must include the number
3064 * of SGE in the list
3065 */
3066 if (!i)
3067 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3068 wr->num_sge);
3069
3070 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3071 wr->sg_list[i].lkey);
3072
3073 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3074 wr->sg_list[i].length, flags);
3075 }
3076
3077 /* Special case of no sges. FW requires between 1-4 sges...
3078 * in this case we need to post 1 sge with length zero. this is
3079 * because rdma write with immediate consumes an RQ.
3080 */
3081 if (!wr->num_sge) {
3082 u32 flags = 0;
3083 struct rdma_rq_sge *rqe =
3084 qed_chain_produce(&qp->rq.pbl);
3085
3086 /* First one must include the number
3087 * of SGE in the list
3088 */
3089 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3090 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3091
3092 RQ_SGE_SET(rqe, 0, 0, flags);
3093 i = 1;
3094 }
3095
3096 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3097 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3098
3099 qedr_inc_sw_prod(&qp->rq);
3100
3101 /* Flush all the writes before signalling doorbell */
3102 wmb();
3103
3104 qp->rq.db_data.data.value++;
3105
3106 writel(qp->rq.db_data.raw, qp->rq.db);
3107
3108 /* Make sure write sticks */
3109 mmiowb();
3110
3111 wr = wr->next;
3112 }
3113
3114 spin_unlock_irqrestore(&qp->q_lock, flags);
3115
3116 return status;
3117}
3118
3119static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3120{
3121 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3122
3123 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3124 cq->pbl_toggle;
3125}
3126
3127static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3128{
3129 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3130 struct qedr_qp *qp;
3131
3132 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3133 resp_cqe->qp_handle.lo,
3134 u64);
3135 return qp;
3136}
3137
3138static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3139{
3140 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3141
3142 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3143}
3144
3145/* Return latest CQE (needs processing) */
3146static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3147{
3148 return cq->latest_cqe;
3149}
3150
3151/* In fmr we need to increase the number of fmr completed counter for the fmr
3152 * algorithm determining whether we can free a pbl or not.
3153 * we need to perform this whether the work request was signaled or not. for
3154 * this purpose we call this function from the condition that checks if a wr
3155 * should be skipped, to make sure we don't miss it ( possibly this fmr
3156 * operation was not signalted)
3157 */
3158static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3159{
3160 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3161 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3162}
3163
3164static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3165 struct qedr_cq *cq, int num_entries,
3166 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3167 int force)
3168{
3169 u16 cnt = 0;
3170
3171 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3172 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3173 qedr_chk_if_fmr(qp);
3174 /* skip WC */
3175 goto next_cqe;
3176 }
3177
3178 /* fill WC */
3179 wc->status = status;
3180 wc->wc_flags = 0;
3181 wc->src_qp = qp->id;
3182 wc->qp = &qp->ibqp;
3183
3184 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3185 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3186
3187 switch (wc->opcode) {
3188 case IB_WC_RDMA_WRITE:
3189 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3190 break;
3191 case IB_WC_COMP_SWAP:
3192 case IB_WC_FETCH_ADD:
3193 wc->byte_len = 8;
3194 break;
3195 case IB_WC_REG_MR:
3196 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3197 break;
3198 default:
3199 break;
3200 }
3201
3202 num_entries--;
3203 wc++;
3204 cnt++;
3205next_cqe:
3206 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3207 qed_chain_consume(&qp->sq.pbl);
3208 qedr_inc_sw_cons(&qp->sq);
3209 }
3210
3211 return cnt;
3212}
3213
3214static int qedr_poll_cq_req(struct qedr_dev *dev,
3215 struct qedr_qp *qp, struct qedr_cq *cq,
3216 int num_entries, struct ib_wc *wc,
3217 struct rdma_cqe_requester *req)
3218{
3219 int cnt = 0;
3220
3221 switch (req->status) {
3222 case RDMA_CQE_REQ_STS_OK:
3223 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3224 IB_WC_SUCCESS, 0);
3225 break;
3226 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3227 DP_ERR(dev,
3228 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3229 cq->icid, qp->icid);
3230 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3231 IB_WC_WR_FLUSH_ERR, 0);
3232 break;
3233 default:
3234 /* process all WQE before the cosumer */
3235 qp->state = QED_ROCE_QP_STATE_ERR;
3236 cnt = process_req(dev, qp, cq, num_entries, wc,
3237 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3238 wc += cnt;
3239 /* if we have extra WC fill it with actual error info */
3240 if (cnt < num_entries) {
3241 enum ib_wc_status wc_status;
3242
3243 switch (req->status) {
3244 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3245 DP_ERR(dev,
3246 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3247 cq->icid, qp->icid);
3248 wc_status = IB_WC_BAD_RESP_ERR;
3249 break;
3250 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3251 DP_ERR(dev,
3252 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3253 cq->icid, qp->icid);
3254 wc_status = IB_WC_LOC_LEN_ERR;
3255 break;
3256 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3257 DP_ERR(dev,
3258 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3259 cq->icid, qp->icid);
3260 wc_status = IB_WC_LOC_QP_OP_ERR;
3261 break;
3262 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3263 DP_ERR(dev,
3264 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3265 cq->icid, qp->icid);
3266 wc_status = IB_WC_LOC_PROT_ERR;
3267 break;
3268 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3269 DP_ERR(dev,
3270 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3271 cq->icid, qp->icid);
3272 wc_status = IB_WC_MW_BIND_ERR;
3273 break;
3274 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3275 DP_ERR(dev,
3276 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3277 cq->icid, qp->icid);
3278 wc_status = IB_WC_REM_INV_REQ_ERR;
3279 break;
3280 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3281 DP_ERR(dev,
3282 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3283 cq->icid, qp->icid);
3284 wc_status = IB_WC_REM_ACCESS_ERR;
3285 break;
3286 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3287 DP_ERR(dev,
3288 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3289 cq->icid, qp->icid);
3290 wc_status = IB_WC_REM_OP_ERR;
3291 break;
3292 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3293 DP_ERR(dev,
3294 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3295 cq->icid, qp->icid);
3296 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3297 break;
3298 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3299 DP_ERR(dev,
3300 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3301 cq->icid, qp->icid);
3302 wc_status = IB_WC_RETRY_EXC_ERR;
3303 break;
3304 default:
3305 DP_ERR(dev,
3306 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3307 cq->icid, qp->icid);
3308 wc_status = IB_WC_GENERAL_ERR;
3309 }
3310 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3311 wc_status, 1);
3312 }
3313 }
3314
3315 return cnt;
3316}
3317
3318static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3319 struct qedr_cq *cq, struct ib_wc *wc,
3320 struct rdma_cqe_responder *resp, u64 wr_id)
3321{
3322 enum ib_wc_status wc_status = IB_WC_SUCCESS;
3323 u8 flags;
3324
3325 wc->opcode = IB_WC_RECV;
3326 wc->wc_flags = 0;
3327
3328 switch (resp->status) {
3329 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3330 wc_status = IB_WC_LOC_ACCESS_ERR;
3331 break;
3332 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3333 wc_status = IB_WC_LOC_LEN_ERR;
3334 break;
3335 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3336 wc_status = IB_WC_LOC_QP_OP_ERR;
3337 break;
3338 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3339 wc_status = IB_WC_LOC_PROT_ERR;
3340 break;
3341 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3342 wc_status = IB_WC_MW_BIND_ERR;
3343 break;
3344 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3345 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
3346 break;
3347 case RDMA_CQE_RESP_STS_OK:
3348 wc_status = IB_WC_SUCCESS;
3349 wc->byte_len = le32_to_cpu(resp->length);
3350
3351 flags = resp->flags & QEDR_RESP_RDMA_IMM;
3352
3353 if (flags == QEDR_RESP_RDMA_IMM)
3354 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3355
3356 if (flags == QEDR_RESP_RDMA_IMM || flags == QEDR_RESP_IMM) {
3357 wc->ex.imm_data =
3358 le32_to_cpu(resp->imm_data_or_inv_r_Key);
3359 wc->wc_flags |= IB_WC_WITH_IMM;
3360 }
3361 break;
3362 default:
3363 wc->status = IB_WC_GENERAL_ERR;
3364 DP_ERR(dev, "Invalid CQE status detected\n");
3365 }
3366
3367 /* fill WC */
3368 wc->status = wc_status;
3369 wc->src_qp = qp->id;
3370 wc->qp = &qp->ibqp;
3371 wc->wr_id = wr_id;
3372}
3373
3374static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3375 struct qedr_cq *cq, struct ib_wc *wc,
3376 struct rdma_cqe_responder *resp)
3377{
3378 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3379
3380 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3381
3382 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3383 qed_chain_consume(&qp->rq.pbl);
3384 qedr_inc_sw_cons(&qp->rq);
3385
3386 return 1;
3387}
3388
3389static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3390 int num_entries, struct ib_wc *wc, u16 hw_cons)
3391{
3392 u16 cnt = 0;
3393
3394 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3395 /* fill WC */
3396 wc->status = IB_WC_WR_FLUSH_ERR;
3397 wc->wc_flags = 0;
3398 wc->src_qp = qp->id;
3399 wc->byte_len = 0;
3400 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3401 wc->qp = &qp->ibqp;
3402 num_entries--;
3403 wc++;
3404 cnt++;
3405 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3406 qed_chain_consume(&qp->rq.pbl);
3407 qedr_inc_sw_cons(&qp->rq);
3408 }
3409
3410 return cnt;
3411}
3412
3413static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3414 struct rdma_cqe_responder *resp, int *update)
3415{
3416 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3417 consume_cqe(cq);
3418 *update |= 1;
3419 }
3420}
3421
3422static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3423 struct qedr_cq *cq, int num_entries,
3424 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3425 int *update)
3426{
3427 int cnt;
3428
3429 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3430 cnt = process_resp_flush(qp, cq, num_entries, wc,
3431 resp->rq_cons);
3432 try_consume_resp_cqe(cq, qp, resp, update);
3433 } else {
3434 cnt = process_resp_one(dev, qp, cq, wc, resp);
3435 consume_cqe(cq);
3436 *update |= 1;
3437 }
3438
3439 return cnt;
3440}
3441
3442static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3443 struct rdma_cqe_requester *req, int *update)
3444{
3445 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3446 consume_cqe(cq);
3447 *update |= 1;
3448 }
3449}
3450
3451int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3452{
3453 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3454 struct qedr_cq *cq = get_qedr_cq(ibcq);
3455 union rdma_cqe *cqe = cq->latest_cqe;
3456 u32 old_cons, new_cons;
3457 unsigned long flags;
3458 int update = 0;
3459 int done = 0;
3460
04886779
RA		 3461 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3462 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3463
afa0e13b
RA		 3464 spin_lock_irqsave(&cq->cq_lock, flags);
3465 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3466 while (num_entries && is_valid_cqe(cq, cqe)) {
3467 struct qedr_qp *qp;
3468 int cnt = 0;
3469
3470 /* prevent speculative reads of any field of CQE */
3471 rmb();
3472
3473 qp = cqe_get_qp(cqe);
3474 if (!qp) {
3475 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3476 break;
3477 }
3478
3479 wc->qp = &qp->ibqp;
3480
3481 switch (cqe_get_type(cqe)) {
3482 case RDMA_CQE_TYPE_REQUESTER:
3483 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3484 &cqe->req);
3485 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3486 break;
3487 case RDMA_CQE_TYPE_RESPONDER_RQ:
3488 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3489 &cqe->resp, &update);
3490 break;
3491 case RDMA_CQE_TYPE_INVALID:
3492 default:
3493 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3494 cqe_get_type(cqe));
3495 }
3496 num_entries -= cnt;
3497 wc += cnt;
3498 done += cnt;
3499
3500 cqe = get_cqe(cq);
3501 }
3502 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3503
3504 cq->cq_cons += new_cons - old_cons;
3505
3506 if (update)
3507 /* doorbell notifies abount latest VALID entry,
3508 * but chain already point to the next INVALID one
3509 */
3510 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3511
3512 spin_unlock_irqrestore(&cq->cq_lock, flags);
3513 return done;
3514}
993d1b52
RA		 3515
3516int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3517 u8 port_num,
3518 const struct ib_wc *in_wc,
3519 const struct ib_grh *in_grh,
3520 const struct ib_mad_hdr *mad_hdr,
3521 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3522 size_t *out_mad_size, u16 *out_mad_pkey_index)
3523{
3524 struct qedr_dev *dev = get_qedr_dev(ibdev);
3525
3526 DP_DEBUG(dev, QEDR_MSG_GSI,
3527 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3528 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3529 mad_hdr->class_specific, mad_hdr->class_version,
3530 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3531 return IB_MAD_RESULT_SUCCESS;
3532}
3533
3534int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3535 struct ib_port_immutable *immutable)
3536{
3537 struct ib_port_attr attr;
3538 int err;
3539
3540 err = qedr_query_port(ibdev, port_num, &attr);
3541 if (err)
3542 return err;
3543
3544 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3545 immutable->gid_tbl_len = attr.gid_tbl_len;
3546 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3547 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3548 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3549
3550 return 0;
3551}
Linux 6.x block layer and io_uring tree(s)