projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/qedr: Add iWARP connection management qp related callbacks
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45
be086e7c
MY		 46#include <linux/qed/common_hsi.h>
47#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 48#include <linux/qed/qed_if.h>
49#include "qedr.h"
50#include "verbs.h"
51#include <rdma/qedr-abi.h>
99d195cc 52#include "qedr_roce_cm.h"
ac1b36e5 53
a7efd777
RA		 54#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
55
c75d3ec8
AR		 56static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
57 size_t len)
58{
59 size_t min_len = min_t(size_t, len, udata->outlen);
60
61 return ib_copy_to_udata(udata, src, min_len);
62}
63
a7efd777
RA		 64int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
65{
66 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
67 return -EINVAL;
68
69 *pkey = QEDR_ROCE_PKEY_DEFAULT;
70 return 0;
71}
72
e6a38c54
KM		 73int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
74 int index, union ib_gid *sgid)
75{
76 struct qedr_dev *dev = get_qedr_dev(ibdev);
77
78 memset(sgid->raw, 0, sizeof(sgid->raw));
79 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
80
81 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
82 sgid->global.interface_id, sgid->global.subnet_prefix);
83
84 return 0;
85}
86
ac1b36e5
RA		 87int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
88 union ib_gid *sgid)
89{
90 struct qedr_dev *dev = get_qedr_dev(ibdev);
91 int rc = 0;
92
93 if (!rdma_cap_roce_gid_table(ibdev, port))
94 return -ENODEV;
95
96 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
97 if (rc == -EAGAIN) {
98 memcpy(sgid, &zgid, sizeof(*sgid));
99 return 0;
100 }
101
102 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
103 sgid->global.interface_id, sgid->global.subnet_prefix);
104
105 return rc;
106}
107
108int qedr_add_gid(struct ib_device *device, u8 port_num,
109 unsigned int index, const union ib_gid *gid,
110 const struct ib_gid_attr *attr, void **context)
111{
112 if (!rdma_cap_roce_gid_table(device, port_num))
113 return -EINVAL;
114
115 if (port_num > QEDR_MAX_PORT)
116 return -EINVAL;
117
118 if (!context)
119 return -EINVAL;
120
121 return 0;
122}
123
124int qedr_del_gid(struct ib_device *device, u8 port_num,
125 unsigned int index, void **context)
126{
127 if (!rdma_cap_roce_gid_table(device, port_num))
128 return -EINVAL;
129
130 if (port_num > QEDR_MAX_PORT)
131 return -EINVAL;
132
133 if (!context)
134 return -EINVAL;
135
136 return 0;
137}
138
139int qedr_query_device(struct ib_device *ibdev,
140 struct ib_device_attr *attr, struct ib_udata *udata)
141{
142 struct qedr_dev *dev = get_qedr_dev(ibdev);
143 struct qedr_device_attr *qattr = &dev->attr;
144
145 if (!dev->rdma_ctx) {
146 DP_ERR(dev,
147 "qedr_query_device called with invalid params rdma_ctx=%p\n",
148 dev->rdma_ctx);
149 return -EINVAL;
150 }
151
152 memset(attr, 0, sizeof(*attr));
153
154 attr->fw_ver = qattr->fw_ver;
155 attr->sys_image_guid = qattr->sys_image_guid;
156 attr->max_mr_size = qattr->max_mr_size;
157 attr->page_size_cap = qattr->page_size_caps;
158 attr->vendor_id = qattr->vendor_id;
159 attr->vendor_part_id = qattr->vendor_part_id;
160 attr->hw_ver = qattr->hw_ver;
161 attr->max_qp = qattr->max_qp;
162 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
163 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
164 IB_DEVICE_RC_RNR_NAK_GEN |
165 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
166
167 attr->max_sge = qattr->max_sge;
168 attr->max_sge_rd = qattr->max_sge;
169 attr->max_cq = qattr->max_cq;
170 attr->max_cqe = qattr->max_cqe;
171 attr->max_mr = qattr->max_mr;
172 attr->max_mw = qattr->max_mw;
173 attr->max_pd = qattr->max_pd;
174 attr->atomic_cap = dev->atomic_cap;
175 attr->max_fmr = qattr->max_fmr;
176 attr->max_map_per_fmr = 16;
177 attr->max_qp_init_rd_atom =
178 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
179 attr->max_qp_rd_atom =
180 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
181 attr->max_qp_init_rd_atom);
182
183 attr->max_srq = qattr->max_srq;
184 attr->max_srq_sge = qattr->max_srq_sge;
185 attr->max_srq_wr = qattr->max_srq_wr;
186
187 attr->local_ca_ack_delay = qattr->dev_ack_delay;
188 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
189 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
190 attr->max_ah = qattr->max_ah;
191
192 return 0;
193}
194
195#define QEDR_SPEED_SDR (1)
196#define QEDR_SPEED_DDR (2)
197#define QEDR_SPEED_QDR (4)
198#define QEDR_SPEED_FDR10 (8)
199#define QEDR_SPEED_FDR (16)
200#define QEDR_SPEED_EDR (32)
201
202static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
203 u8 *ib_width)
204{
205 switch (speed) {
206 case 1000:
207 *ib_speed = QEDR_SPEED_SDR;
208 *ib_width = IB_WIDTH_1X;
209 break;
210 case 10000:
211 *ib_speed = QEDR_SPEED_QDR;
212 *ib_width = IB_WIDTH_1X;
213 break;
214
215 case 20000:
216 *ib_speed = QEDR_SPEED_DDR;
217 *ib_width = IB_WIDTH_4X;
218 break;
219
220 case 25000:
221 *ib_speed = QEDR_SPEED_EDR;
222 *ib_width = IB_WIDTH_1X;
223 break;
224
225 case 40000:
226 *ib_speed = QEDR_SPEED_QDR;
227 *ib_width = IB_WIDTH_4X;
228 break;
229
230 case 50000:
231 *ib_speed = QEDR_SPEED_QDR;
232 *ib_width = IB_WIDTH_4X;
233 break;
234
235 case 100000:
236 *ib_speed = QEDR_SPEED_EDR;
237 *ib_width = IB_WIDTH_4X;
238 break;
239
240 default:
241 /* Unsupported */
242 *ib_speed = QEDR_SPEED_SDR;
243 *ib_width = IB_WIDTH_1X;
244 }
245}
246
247int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
248{
249 struct qedr_dev *dev;
250 struct qed_rdma_port *rdma_port;
251
252 dev = get_qedr_dev(ibdev);
253 if (port > 1) {
254 DP_ERR(dev, "invalid_port=0x%x\n", port);
255 return -EINVAL;
256 }
257
258 if (!dev->rdma_ctx) {
259 DP_ERR(dev, "rdma_ctx is NULL\n");
260 return -EINVAL;
261 }
262
263 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 264
c4550c63 265 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 266 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
267 attr->state = IB_PORT_ACTIVE;
268 attr->phys_state = 5;
269 } else {
270 attr->state = IB_PORT_DOWN;
271 attr->phys_state = 3;
272 }
273 attr->max_mtu = IB_MTU_4096;
274 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
275 attr->lid = 0;
276 attr->lmc = 0;
277 attr->sm_lid = 0;
278 attr->sm_sl = 0;
279 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
f5b1b177
KM		 280 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
281 attr->gid_tbl_len = 1;
282 attr->pkey_tbl_len = 1;
283 } else {
284 attr->gid_tbl_len = QEDR_MAX_SGID;
285 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
286 }
ac1b36e5
RA		 287 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
288 attr->qkey_viol_cntr = 0;
289 get_link_speed_and_width(rdma_port->link_speed,
290 &attr->active_speed, &attr->active_width);
291 attr->max_msg_sz = rdma_port->max_msg_size;
292 attr->max_vl_num = 4;
293
294 return 0;
295}
296
297int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
298 struct ib_port_modify *props)
299{
300 struct qedr_dev *dev;
301
302 dev = get_qedr_dev(ibdev);
303 if (port > 1) {
304 DP_ERR(dev, "invalid_port=0x%x\n", port);
305 return -EINVAL;
306 }
307
308 return 0;
309}
310
311static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
312 unsigned long len)
313{
314 struct qedr_mm *mm;
315
316 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
317 if (!mm)
318 return -ENOMEM;
319
320 mm->key.phy_addr = phy_addr;
321 /* This function might be called with a length which is not a multiple
322 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
323 * forces this granularity by increasing the requested size if needed.
324 * When qedr_mmap is called, it will search the list with the updated
325 * length as a key. To prevent search failures, the length is rounded up
326 * in advance to PAGE_SIZE.
327 */
328 mm->key.len = roundup(len, PAGE_SIZE);
329 INIT_LIST_HEAD(&mm->entry);
330
331 mutex_lock(&uctx->mm_list_lock);
332 list_add(&mm->entry, &uctx->mm_head);
333 mutex_unlock(&uctx->mm_list_lock);
334
335 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
336 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
337 (unsigned long long)mm->key.phy_addr,
338 (unsigned long)mm->key.len, uctx);
339
340 return 0;
341}
342
343static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
344 unsigned long len)
345{
346 bool found = false;
347 struct qedr_mm *mm;
348
349 mutex_lock(&uctx->mm_list_lock);
350 list_for_each_entry(mm, &uctx->mm_head, entry) {
351 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
352 continue;
353
354 found = true;
355 break;
356 }
357 mutex_unlock(&uctx->mm_list_lock);
358 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
359 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
360 mm->key.phy_addr, mm->key.len, uctx, found);
361
362 return found;
363}
364
365struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
366 struct ib_udata *udata)
367{
368 int rc;
369 struct qedr_ucontext *ctx;
370 struct qedr_alloc_ucontext_resp uresp;
371 struct qedr_dev *dev = get_qedr_dev(ibdev);
372 struct qed_rdma_add_user_out_params oparams;
373
374 if (!udata)
375 return ERR_PTR(-EFAULT);
376
377 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
378 if (!ctx)
379 return ERR_PTR(-ENOMEM);
380
381 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
382 if (rc) {
383 DP_ERR(dev,
384 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
385 rc);
386 goto err;
387 }
388
389 ctx->dpi = oparams.dpi;
390 ctx->dpi_addr = oparams.dpi_addr;
391 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
392 ctx->dpi_size = oparams.dpi_size;
393 INIT_LIST_HEAD(&ctx->mm_head);
394 mutex_init(&ctx->mm_list_lock);
395
396 memset(&uresp, 0, sizeof(uresp));
397
398 uresp.db_pa = ctx->dpi_phys_addr;
399 uresp.db_size = ctx->dpi_size;
400 uresp.max_send_wr = dev->attr.max_sqe;
401 uresp.max_recv_wr = dev->attr.max_rqe;
402 uresp.max_srq_wr = dev->attr.max_srq_wr;
403 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
404 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
405 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
406 uresp.max_cqes = QEDR_MAX_CQES;
407
c75d3ec8 408 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5
RA		 409 if (rc)
410 goto err;
411
412 ctx->dev = dev;
413
414 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
415 if (rc)
416 goto err;
417
418 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
419 &ctx->ibucontext);
420 return &ctx->ibucontext;
421
422err:
423 kfree(ctx);
424 return ERR_PTR(rc);
425}
426
427int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
428{
429 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
430 struct qedr_mm *mm, *tmp;
431 int status = 0;
432
433 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
434 uctx);
435 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
436
437 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
438 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
439 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
440 mm->key.phy_addr, mm->key.len, uctx);
441 list_del(&mm->entry);
442 kfree(mm);
443 }
444
445 kfree(uctx);
446 return status;
447}
448
449int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
450{
451 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
452 struct qedr_dev *dev = get_qedr_dev(context->device);
453 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
454 u64 unmapped_db = dev->db_phys_addr;
455 unsigned long len = (vma->vm_end - vma->vm_start);
456 int rc = 0;
457 bool found;
458
459 DP_DEBUG(dev, QEDR_MSG_INIT,
460 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
461 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
462 if (vma->vm_start & (PAGE_SIZE - 1)) {
463 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
464 vma->vm_start);
465 return -EINVAL;
466 }
467
468 found = qedr_search_mmap(ucontext, vm_page, len);
469 if (!found) {
470 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
471 vma->vm_pgoff);
472 return -EINVAL;
473 }
474
475 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
476
477 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
478 dev->db_size))) {
479 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
480 if (vma->vm_flags & VM_READ) {
481 DP_ERR(dev, "Trying to map doorbell bar for read\n");
482 return -EPERM;
483 }
484
485 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
486
487 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
488 PAGE_SIZE, vma->vm_page_prot);
489 } else {
490 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
491 rc = remap_pfn_range(vma, vma->vm_start,
492 vma->vm_pgoff, len, vma->vm_page_prot);
493 }
494 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
495 return rc;
496}
a7efd777
RA		 497
498struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
499 struct ib_ucontext *context, struct ib_udata *udata)
500{
501 struct qedr_dev *dev = get_qedr_dev(ibdev);
a7efd777
RA		 502 struct qedr_pd *pd;
503 u16 pd_id;
504 int rc;
505
506 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
507 (udata && context) ? "User Lib" : "Kernel");
508
509 if (!dev->rdma_ctx) {
510 DP_ERR(dev, "invlaid RDMA context\n");
511 return ERR_PTR(-EINVAL);
512 }
513
514 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
515 if (!pd)
516 return ERR_PTR(-ENOMEM);
517
9c1e0228
RA		 518 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
519 if (rc)
520 goto err;
a7efd777 521
a7efd777
RA		 522 pd->pd_id = pd_id;
523
524 if (udata && context) {
9c1e0228
RA		 525 struct qedr_alloc_pd_uresp uresp;
526
527 uresp.pd_id = pd_id;
528
c75d3ec8 529 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 530 if (rc) {
a7efd777 531 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228
RA		 532 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
533 goto err;
534 }
535
536 pd->uctx = get_qedr_ucontext(context);
537 pd->uctx->pd = pd;
a7efd777
RA		 538 }
539
540 return &pd->ibpd;
9c1e0228
RA		 541
542err:
543 kfree(pd);
544 return ERR_PTR(rc);
a7efd777
RA		 545}
546
547int qedr_dealloc_pd(struct ib_pd *ibpd)
548{
549 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
550 struct qedr_pd *pd = get_qedr_pd(ibpd);
551
ea7ef2ac 552 if (!pd) {
a7efd777 553 pr_err("Invalid PD received in dealloc_pd\n");
ea7ef2ac
CIK		 554 return -EINVAL;
555 }
a7efd777
RA		 556
557 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
558 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
559
560 kfree(pd);
561
562 return 0;
563}
564
565static void qedr_free_pbl(struct qedr_dev *dev,
566 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
567{
568 struct pci_dev *pdev = dev->pdev;
569 int i;
570
571 for (i = 0; i < pbl_info->num_pbls; i++) {
572 if (!pbl[i].va)
573 continue;
574 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
575 pbl[i].va, pbl[i].pa);
576 }
577
578 kfree(pbl);
579}
580
581#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
582#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
583
584#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
585#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
586#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
587
588static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
589 struct qedr_pbl_info *pbl_info,
590 gfp_t flags)
591{
592 struct pci_dev *pdev = dev->pdev;
593 struct qedr_pbl *pbl_table;
594 dma_addr_t *pbl_main_tbl;
595 dma_addr_t pa;
596 void *va;
597 int i;
598
599 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
600 if (!pbl_table)
601 return ERR_PTR(-ENOMEM);
602
603 for (i = 0; i < pbl_info->num_pbls; i++) {
604 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
605 &pa, flags);
606 if (!va)
607 goto err;
608
609 memset(va, 0, pbl_info->pbl_size);
610 pbl_table[i].va = va;
611 pbl_table[i].pa = pa;
612 }
613
614 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
615 * the first one with physical pointers to all of the rest
616 */
617 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
618 for (i = 0; i < pbl_info->num_pbls - 1; i++)
619 pbl_main_tbl[i] = pbl_table[i + 1].pa;
620
621 return pbl_table;
622
623err:
624 for (i--; i >= 0; i--)
625 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
626 pbl_table[i].va, pbl_table[i].pa);
627
628 qedr_free_pbl(dev, pbl_info, pbl_table);
629
630 return ERR_PTR(-ENOMEM);
631}
632
633static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
634 struct qedr_pbl_info *pbl_info,
635 u32 num_pbes, int two_layer_capable)
636{
637 u32 pbl_capacity;
638 u32 pbl_size;
639 u32 num_pbls;
640
641 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
642 if (num_pbes > MAX_PBES_TWO_LAYER) {
643 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
644 num_pbes);
645 return -EINVAL;
646 }
647
648 /* calculate required pbl page size */
649 pbl_size = MIN_FW_PBL_PAGE_SIZE;
650 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
651 NUM_PBES_ON_PAGE(pbl_size);
652
653 while (pbl_capacity < num_pbes) {
654 pbl_size *= 2;
655 pbl_capacity = pbl_size / sizeof(u64);
656 pbl_capacity = pbl_capacity * pbl_capacity;
657 }
658
659 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
660 num_pbls++; /* One for the layer0 ( points to the pbls) */
661 pbl_info->two_layered = true;
662 } else {
663 /* One layered PBL */
664 num_pbls = 1;
665 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
666 roundup_pow_of_two((num_pbes * sizeof(u64))));
667 pbl_info->two_layered = false;
668 }
669
670 pbl_info->num_pbls = num_pbls;
671 pbl_info->pbl_size = pbl_size;
672 pbl_info->num_pbes = num_pbes;
673
674 DP_DEBUG(dev, QEDR_MSG_MR,
675 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
676 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
677
678 return 0;
679}
680
681static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
682 struct qedr_pbl *pbl,
e57bb6be 683 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777
RA		 684{
685 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
e57bb6be 686 u32 fw_pg_cnt, fw_pg_per_umem_pg;
a7efd777
RA		 687 struct qedr_pbl *pbl_tbl;
688 struct scatterlist *sg;
689 struct regpair *pbe;
e57bb6be 690 u64 pg_addr;
a7efd777 691 int entry;
a7efd777
RA		 692
693 if (!pbl_info->num_pbes)
694 return;
695
696 /* If we have a two layered pbl, the first pbl points to the rest
697 * of the pbls and the first entry lays on the second pbl in the table
698 */
699 if (pbl_info->two_layered)
700 pbl_tbl = &pbl[1];
701 else
702 pbl_tbl = pbl;
703
704 pbe = (struct regpair *)pbl_tbl->va;
705 if (!pbe) {
706 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
707 return;
708 }
709
710 pbe_cnt = 0;
711
3e7e1193 712 shift = umem->page_shift;
a7efd777 713
e57bb6be
RA		 714 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
715
a7efd777
RA		 716 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
717 pages = sg_dma_len(sg) >> shift;
e57bb6be 718 pg_addr = sg_dma_address(sg);
a7efd777 719 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
e57bb6be
RA		 720 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
721 pbe->lo = cpu_to_le32(pg_addr);
722 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
723
724 pg_addr += BIT(pg_shift);
725 pbe_cnt++;
726 total_num_pbes++;
727 pbe++;
728
729 if (total_num_pbes == pbl_info->num_pbes)
730 return;
731
732 /* If the given pbl is full storing the pbes,
733 * move to next pbl.
734 */
735 if (pbe_cnt ==
736 (pbl_info->pbl_size / sizeof(u64))) {
737 pbl_tbl++;
738 pbe = (struct regpair *)pbl_tbl->va;
739 pbe_cnt = 0;
740 }
741
742 fw_pg_cnt++;
a7efd777
RA		 743 }
744 }
745 }
746}
747
748static int qedr_copy_cq_uresp(struct qedr_dev *dev,
749 struct qedr_cq *cq, struct ib_udata *udata)
750{
751 struct qedr_create_cq_uresp uresp;
752 int rc;
753
754 memset(&uresp, 0, sizeof(uresp));
755
756 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
757 uresp.icid = cq->icid;
758
c75d3ec8 759 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 760 if (rc)
761 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
762
763 return rc;
764}
765
766static void consume_cqe(struct qedr_cq *cq)
767{
768 if (cq->latest_cqe == cq->toggle_cqe)
769 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
770
771 cq->latest_cqe = qed_chain_consume(&cq->pbl);
772}
773
774static inline int qedr_align_cq_entries(int entries)
775{
776 u64 size, aligned_size;
777
778 /* We allocate an extra entry that we don't report to the FW. */
779 size = (entries + 1) * QEDR_CQE_SIZE;
780 aligned_size = ALIGN(size, PAGE_SIZE);
781
782 return aligned_size / QEDR_CQE_SIZE;
783}
784
785static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
786 struct qedr_dev *dev,
787 struct qedr_userq *q,
788 u64 buf_addr, size_t buf_len,
789 int access, int dmasync)
790{
e57bb6be 791 u32 fw_pages;
a7efd777
RA		 792 int rc;
793
794 q->buf_addr = buf_addr;
795 q->buf_len = buf_len;
796 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
797 if (IS_ERR(q->umem)) {
798 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
799 PTR_ERR(q->umem));
800 return PTR_ERR(q->umem);
801 }
802
e57bb6be
RA		 803 fw_pages = ib_umem_page_count(q->umem) <<
804 (q->umem->page_shift - FW_PAGE_SHIFT);
805
806 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 807 if (rc)
808 goto err0;
809
810 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 811 if (IS_ERR(q->pbl_tbl)) {
812 rc = PTR_ERR(q->pbl_tbl);
a7efd777 813 goto err0;
4cd33aaf 814 }
a7efd777 815
e57bb6be
RA		 816 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
817 FW_PAGE_SHIFT);
a7efd777
RA		 818
819 return 0;
820
821err0:
822 ib_umem_release(q->umem);
823
824 return rc;
825}
826
827static inline void qedr_init_cq_params(struct qedr_cq *cq,
828 struct qedr_ucontext *ctx,
829 struct qedr_dev *dev, int vector,
830 int chain_entries, int page_cnt,
831 u64 pbl_ptr,
832 struct qed_rdma_create_cq_in_params
833 *params)
834{
835 memset(params, 0, sizeof(*params));
836 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
837 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
838 params->cnq_id = vector;
839 params->cq_size = chain_entries - 1;
840 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
841 params->pbl_num_pages = page_cnt;
842 params->pbl_ptr = pbl_ptr;
843 params->pbl_two_level = 0;
844}
845
846static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
847{
848 /* Flush data before signalling doorbell */
849 wmb();
850 cq->db.data.agg_flags = flags;
851 cq->db.data.value = cpu_to_le32(cons);
852 writeq(cq->db.raw, cq->db_addr);
853
854 /* Make sure write would stick */
855 mmiowb();
856}
857
858int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
859{
860 struct qedr_cq *cq = get_qedr_cq(ibcq);
861 unsigned long sflags;
4dd72636
AR		 862 struct qedr_dev *dev;
863
864 dev = get_qedr_dev(ibcq->device);
865
866 if (cq->destroyed) {
867 DP_ERR(dev,
868 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
869 cq, cq->icid);
870 return -EINVAL;
871 }
872
a7efd777
RA		 873
874 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
875 return 0;
876
877 spin_lock_irqsave(&cq->cq_lock, sflags);
878
879 cq->arm_flags = 0;
880
881 if (flags & IB_CQ_SOLICITED)
882 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
883
884 if (flags & IB_CQ_NEXT_COMP)
885 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
886
887 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
888
889 spin_unlock_irqrestore(&cq->cq_lock, sflags);
890
891 return 0;
892}
893
894struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
895 const struct ib_cq_init_attr *attr,
896 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
897{
898 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
899 struct qed_rdma_destroy_cq_out_params destroy_oparams;
900 struct qed_rdma_destroy_cq_in_params destroy_iparams;
901 struct qedr_dev *dev = get_qedr_dev(ibdev);
902 struct qed_rdma_create_cq_in_params params;
903 struct qedr_create_cq_ureq ureq;
904 int vector = attr->comp_vector;
905 int entries = attr->cqe;
906 struct qedr_cq *cq;
907 int chain_entries;
908 int page_cnt;
909 u64 pbl_ptr;
910 u16 icid;
911 int rc;
912
913 DP_DEBUG(dev, QEDR_MSG_INIT,
914 "create_cq: called from %s. entries=%d, vector=%d\n",
915 udata ? "User Lib" : "Kernel", entries, vector);
916
917 if (entries > QEDR_MAX_CQES) {
918 DP_ERR(dev,
919 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
920 entries, QEDR_MAX_CQES);
921 return ERR_PTR(-EINVAL);
922 }
923
924 chain_entries = qedr_align_cq_entries(entries);
925 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
926
927 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
928 if (!cq)
929 return ERR_PTR(-ENOMEM);
930
931 if (udata) {
932 memset(&ureq, 0, sizeof(ureq));
933 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
934 DP_ERR(dev,
935 "create cq: problem copying data from user space\n");
936 goto err0;
937 }
938
939 if (!ureq.len) {
940 DP_ERR(dev,
941 "create cq: cannot create a cq with 0 entries\n");
942 goto err0;
943 }
944
945 cq->cq_type = QEDR_CQ_TYPE_USER;
946
947 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
948 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
949 if (rc)
950 goto err0;
951
952 pbl_ptr = cq->q.pbl_tbl->pa;
953 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 954
955 cq->ibcq.cqe = chain_entries;
a7efd777
RA		 956 } else {
957 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
958
959 rc = dev->ops->common->chain_alloc(dev->cdev,
960 QED_CHAIN_USE_TO_CONSUME,
961 QED_CHAIN_MODE_PBL,
962 QED_CHAIN_CNT_TYPE_U32,
963 chain_entries,
964 sizeof(union rdma_cqe),
1a4a6975 965 &cq->pbl, NULL);
a7efd777
RA		 966 if (rc)
967 goto err1;
968
969 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
970 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 971 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 972 }
973
974 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
975 pbl_ptr, &params);
976
977 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
978 if (rc)
979 goto err2;
980
981 cq->icid = icid;
982 cq->sig = QEDR_CQ_MAGIC_NUMBER;
983 spin_lock_init(&cq->cq_lock);
984
985 if (ib_ctx) {
986 rc = qedr_copy_cq_uresp(dev, cq, udata);
987 if (rc)
988 goto err3;
989 } else {
990 /* Generate doorbell address. */
991 cq->db_addr = dev->db_addr +
992 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
993 cq->db.data.icid = cq->icid;
994 cq->db.data.params = DB_AGG_CMD_SET <<
995 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
996
997 /* point to the very last element, passing it we will toggle */
998 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
999 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1000 cq->latest_cqe = NULL;
1001 consume_cqe(cq);
1002 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1003 }
1004
1005 DP_DEBUG(dev, QEDR_MSG_CQ,
1006 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1007 cq->icid, cq, params.cq_size);
1008
1009 return &cq->ibcq;
1010
1011err3:
1012 destroy_iparams.icid = cq->icid;
1013 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1014 &destroy_oparams);
1015err2:
1016 if (udata)
1017 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1018 else
1019 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1020err1:
1021 if (udata)
1022 ib_umem_release(cq->q.umem);
1023err0:
1024 kfree(cq);
1025 return ERR_PTR(-EINVAL);
1026}
1027
1028int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1029{
1030 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1031 struct qedr_cq *cq = get_qedr_cq(ibcq);
1032
1033 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1034
1035 return 0;
1036}
1037
4dd72636
AR		 1038#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1039#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1040
a7efd777
RA		 1041int qedr_destroy_cq(struct ib_cq *ibcq)
1042{
1043 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1044 struct qed_rdma_destroy_cq_out_params oparams;
1045 struct qed_rdma_destroy_cq_in_params iparams;
1046 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 1047 int iter;
942b3b2c 1048 int rc;
a7efd777 1049
942b3b2c 1050 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 1051
4dd72636
AR		 1052 cq->destroyed = 1;
1053
a7efd777 1054 /* GSIs CQs are handled by driver, so they don't exist in the FW */
942b3b2c
AR		 1055 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1056 goto done;
a1211359 1057
942b3b2c
AR		 1058 iparams.icid = cq->icid;
1059 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1060 if (rc)
1061 return rc;
1062
1063 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777
RA		 1064
1065 if (ibcq->uobject && ibcq->uobject->context) {
1066 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1067 ib_umem_release(cq->q.umem);
1068 }
1069
4dd72636
AR		 1070 /* We don't want the IRQ handler to handle a non-existing CQ so we
1071 * wait until all CNQ interrupts, if any, are received. This will always
1072 * happen and will always happen very fast. If not, then a serious error
1073 * has occured. That is why we can use a long delay.
1074 * We spin for a short time so we don’t lose time on context switching
1075 * in case all the completions are handled in that span. Otherwise
1076 * we sleep for a while and check again. Since the CNQ may be
1077 * associated with (only) the current CPU we use msleep to allow the
1078 * current CPU to be freed.
1079 * The CNQ notification is increased in qedr_irq_handler().
1080 */
1081 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1082 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1083 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1084 iter--;
1085 }
1086
1087 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1088 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1089 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1090 iter--;
1091 }
1092
1093 if (oparams.num_cq_notif != cq->cnq_notif)
1094 goto err;
1095
1096 /* Note that we don't need to have explicit code to wait for the
1097 * completion of the event handler because it is invoked from the EQ.
1098 * Since the destroy CQ ramrod has also been received on the EQ we can
1099 * be certain that there's no event handler in process.
1100 */
942b3b2c 1101done:
4dd72636
AR		 1102 cq->sig = ~cq->sig;
1103
a7efd777
RA		 1104 kfree(cq);
1105
1106 return 0;
4dd72636
AR		 1107
1108err:
1109 DP_ERR(dev,
1110 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1111 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1112
1113 return -EINVAL;
a7efd777 1114}
cecbcddf
RA		 1115
1116static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1117 struct ib_qp_attr *attr,
1118 int attr_mask,
1119 struct qed_rdma_modify_qp_in_params
1120 *qp_params)
1121{
1122 enum rdma_network_type nw_type;
1123 struct ib_gid_attr gid_attr;
d8966fcd 1124 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf
RA		 1125 union ib_gid gid;
1126 u32 ipv4_addr;
1127 int rc = 0;
1128 int i;
1129
d8966fcd
DC		 1130 rc = ib_get_cached_gid(ibqp->device,
1131 rdma_ah_get_port_num(&attr->ah_attr),
1132 grh->sgid_index, &gid, &gid_attr);
cecbcddf
RA		 1133 if (rc)
1134 return rc;
1135
1136 if (!memcmp(&gid, &zgid, sizeof(gid)))
1137 return -ENOENT;
1138
1139 if (gid_attr.ndev) {
1140 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1141
1142 dev_put(gid_attr.ndev);
1143 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1144 switch (nw_type) {
1145 case RDMA_NETWORK_IPV6:
1146 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1147 sizeof(qp_params->sgid));
1148 memcpy(&qp_params->dgid.bytes[0],
d8966fcd 1149 &grh->dgid,
cecbcddf
RA		 1150 sizeof(qp_params->dgid));
1151 qp_params->roce_mode = ROCE_V2_IPV6;
1152 SET_FIELD(qp_params->modify_flags,
1153 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154 break;
1155 case RDMA_NETWORK_IB:
1156 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1157 sizeof(qp_params->sgid));
1158 memcpy(&qp_params->dgid.bytes[0],
d8966fcd 1159 &grh->dgid,
cecbcddf
RA		 1160 sizeof(qp_params->dgid));
1161 qp_params->roce_mode = ROCE_V1;
1162 break;
1163 case RDMA_NETWORK_IPV4:
1164 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1167 qp_params->sgid.ipv4_addr = ipv4_addr;
1168 ipv4_addr =
d8966fcd 1169 qedr_get_ipv4_from_gid(grh->dgid.raw);
cecbcddf
RA		 1170 qp_params->dgid.ipv4_addr = ipv4_addr;
1171 SET_FIELD(qp_params->modify_flags,
1172 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173 qp_params->roce_mode = ROCE_V2_IPV4;
1174 break;
1175 }
1176 }
1177
1178 for (i = 0; i < 4; i++) {
1179 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1180 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1181 }
1182
1183 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1184 qp_params->vlan_id = 0;
1185
1186 return 0;
1187}
1188
cecbcddf
RA		 1189static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1190 struct ib_qp_init_attr *attrs)
1191{
1192 struct qedr_device_attr *qattr = &dev->attr;
1193
1194 /* QP0... attrs->qp_type == IB_QPT_GSI */
1195 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1196 DP_DEBUG(dev, QEDR_MSG_QP,
1197 "create qp: unsupported qp type=0x%x requested\n",
1198 attrs->qp_type);
1199 return -EINVAL;
1200 }
1201
1202 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1203 DP_ERR(dev,
1204 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1205 attrs->cap.max_send_wr, qattr->max_sqe);
1206 return -EINVAL;
1207 }
1208
1209 if (attrs->cap.max_inline_data > qattr->max_inline) {
1210 DP_ERR(dev,
1211 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1212 attrs->cap.max_inline_data, qattr->max_inline);
1213 return -EINVAL;
1214 }
1215
1216 if (attrs->cap.max_send_sge > qattr->max_sge) {
1217 DP_ERR(dev,
1218 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1219 attrs->cap.max_send_sge, qattr->max_sge);
1220 return -EINVAL;
1221 }
1222
1223 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1224 DP_ERR(dev,
1225 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1226 attrs->cap.max_recv_sge, qattr->max_sge);
1227 return -EINVAL;
1228 }
1229
1230 /* Unprivileged user space cannot create special QP */
1231 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1232 DP_ERR(dev,
1233 "create qp: userspace can't create special QPs of type=0x%x\n",
1234 attrs->qp_type);
1235 return -EINVAL;
1236 }
1237
1238 return 0;
1239}
1240
1241static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1242 struct qedr_qp *qp)
1243{
1244 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1245 uresp->rq_icid = qp->icid;
1246}
1247
1248static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1249 struct qedr_qp *qp)
1250{
1251 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1252 uresp->sq_icid = qp->icid + 1;
1253}
1254
1255static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1256 struct qedr_qp *qp, struct ib_udata *udata)
1257{
1258 struct qedr_create_qp_uresp uresp;
1259 int rc;
1260
1261 memset(&uresp, 0, sizeof(uresp));
1262 qedr_copy_sq_uresp(&uresp, qp);
1263 qedr_copy_rq_uresp(&uresp, qp);
1264
1265 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1266 uresp.qp_id = qp->qp_id;
1267
c75d3ec8 1268 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
cecbcddf
RA		 1269 if (rc)
1270 DP_ERR(dev,
1271 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1272 qp->icid);
1273
1274 return rc;
1275}
1276
df158561
AR		 1277static void qedr_set_common_qp_params(struct qedr_dev *dev,
1278 struct qedr_qp *qp,
1279 struct qedr_pd *pd,
1280 struct ib_qp_init_attr *attrs)
cecbcddf 1281{
cecbcddf 1282 spin_lock_init(&qp->q_lock);
de0089e6 1283 atomic_set(&qp->refcnt, 1);
df158561 1284 qp->pd = pd;
cecbcddf
RA		 1285 qp->qp_type = attrs->qp_type;
1286 qp->max_inline_data = attrs->cap.max_inline_data;
1287 qp->sq.max_sges = attrs->cap.max_send_sge;
1288 qp->state = QED_ROCE_QP_STATE_RESET;
1289 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1290 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1291 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1292 qp->dev = dev;
df158561 1293 qp->rq.max_sges = attrs->cap.max_recv_sge;
cecbcddf 1294
df158561
AR		 1295 DP_DEBUG(dev, QEDR_MSG_QP,
1296 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1297 qp->rq.max_sges, qp->rq_cq->icid);
cecbcddf
RA		 1298 DP_DEBUG(dev, QEDR_MSG_QP,
1299 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1300 pd->pd_id, qp->qp_type, qp->max_inline_data,
1301 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1302 DP_DEBUG(dev, QEDR_MSG_QP,
1303 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1304 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1305}
1306
df158561 1307static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf
RA		 1308{
1309 qp->sq.db = dev->db_addr +
1310 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1311 qp->sq.db_data.data.icid = qp->icid + 1;
df158561
AR		 1312 qp->rq.db = dev->db_addr +
1313 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1314 qp->rq.db_data.data.icid = qp->icid;
cecbcddf
RA		 1315}
1316
1317static inline void
df158561
AR		 1318qedr_init_common_qp_in_params(struct qedr_dev *dev,
1319 struct qedr_pd *pd,
1320 struct qedr_qp *qp,
1321 struct ib_qp_init_attr *attrs,
1322 bool fmr_and_reserved_lkey,
1323 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1324{
df158561
AR		 1325 /* QP handle to be written in an async event */
1326 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1327 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1328
1329 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1330 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1331 params->pd = pd->pd_id;
1332 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1333 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1334 params->stats_queue = 0;
1335 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1336 params->srq_id = 0;
1337 params->use_srq = false;
cecbcddf
RA		 1338}
1339
df158561 1340static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1341{
df158561
AR		 1342 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1343 "qp=%p. "
1344 "sq_addr=0x%llx, "
1345 "sq_len=%zd, "
1346 "rq_addr=0x%llx, "
1347 "rq_len=%zd"
1348 "\n",
1349 qp,
1350 qp->usq.buf_addr,
1351 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1352}
cecbcddf 1353
de0089e6
KM		 1354static int qedr_idr_add(struct qedr_dev *dev, void *ptr, u32 id)
1355{
1356 int rc;
1357
1358 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
1359 return 0;
1360
1361 idr_preload(GFP_KERNEL);
1362 spin_lock_irq(&dev->idr_lock);
1363
1364 rc = idr_alloc(&dev->qpidr, ptr, id, id + 1, GFP_ATOMIC);
1365
1366 spin_unlock_irq(&dev->idr_lock);
1367 idr_preload_end();
1368
1369 return rc < 0 ? rc : 0;
1370}
1371
1372static void qedr_idr_remove(struct qedr_dev *dev, u32 id)
1373{
1374 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
1375 return;
1376
1377 spin_lock_irq(&dev->idr_lock);
1378 idr_remove(&dev->qpidr, id);
1379 spin_unlock_irq(&dev->idr_lock);
1380}
df158561
AR		 1381static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1382{
1383 if (qp->usq.umem)
1384 ib_umem_release(qp->usq.umem);
1385 qp->usq.umem = NULL;
cecbcddf 1386
df158561
AR		 1387 if (qp->urq.umem)
1388 ib_umem_release(qp->urq.umem);
1389 qp->urq.umem = NULL;
cecbcddf
RA		 1390}
1391
df158561
AR		 1392static int qedr_create_user_qp(struct qedr_dev *dev,
1393 struct qedr_qp *qp,
1394 struct ib_pd *ibpd,
1395 struct ib_udata *udata,
1396 struct ib_qp_init_attr *attrs)
cecbcddf 1397{
df158561
AR		 1398 struct qed_rdma_create_qp_in_params in_params;
1399 struct qed_rdma_create_qp_out_params out_params;
1400 struct qedr_pd *pd = get_qedr_pd(ibpd);
1401 struct ib_ucontext *ib_ctx = NULL;
1402 struct qedr_ucontext *ctx = NULL;
1403 struct qedr_create_qp_ureq ureq;
1404 int rc = -EINVAL;
cecbcddf 1405
df158561
AR		 1406 ib_ctx = ibpd->uobject->context;
1407 ctx = get_qedr_ucontext(ib_ctx);
cecbcddf 1408
df158561
AR		 1409 memset(&ureq, 0, sizeof(ureq));
1410 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1411 if (rc) {
1412 DP_ERR(dev, "Problem copying data from user space\n");
1413 return rc;
1414 }
cecbcddf 1415
df158561
AR		 1416 /* SQ - read access only (0), dma sync not required (0) */
1417 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1418 ureq.sq_len, 0, 0);
1419 if (rc)
1420 return rc;
cecbcddf 1421
df158561
AR		 1422 /* RQ - read access only (0), dma sync not required (0) */
1423 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1424 ureq.rq_len, 0, 0);
1425
1426 if (rc)
1427 return rc;
1428
1429 memset(&in_params, 0, sizeof(in_params));
1430 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1431 in_params.qp_handle_lo = ureq.qp_handle_lo;
1432 in_params.qp_handle_hi = ureq.qp_handle_hi;
1433 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1434 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1435 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1436 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1437
1438 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1439 &in_params, &out_params);
1440
1441 if (!qp->qed_qp) {
1442 rc = -ENOMEM;
1443 goto err1;
1444 }
1445
1446 qp->qp_id = out_params.qp_id;
1447 qp->icid = out_params.icid;
1448
1449 rc = qedr_copy_qp_uresp(dev, qp, udata);
1450 if (rc)
1451 goto err;
1452
1453 qedr_qp_user_print(dev, qp);
cecbcddf
RA		 1454
1455 return 0;
df158561
AR		 1456err:
1457 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1458 if (rc)
1459 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1460
1461err1:
1462 qedr_cleanup_user(dev, qp);
1463 return rc;
cecbcddf
RA		 1464}
1465
f5b1b177
KM		 1466static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1467{
1468 qp->sq.db = dev->db_addr +
1469 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1470 qp->sq.db_data.data.icid = qp->icid;
1471
1472 qp->rq.db = dev->db_addr +
1473 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1474 qp->rq.db_data.data.icid = qp->icid;
1475 qp->rq.iwarp_db2 = dev->db_addr +
1476 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1477 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1478 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1479}
1480
df158561
AR		 1481static int
1482qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1483 struct qedr_qp *qp,
1484 struct qed_rdma_create_qp_in_params *in_params,
1485 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1486{
df158561 1487 struct qed_rdma_create_qp_out_params out_params;
cecbcddf
RA		 1488 int rc;
1489
cecbcddf
RA		 1490 rc = dev->ops->common->chain_alloc(dev->cdev,
1491 QED_CHAIN_USE_TO_PRODUCE,
1492 QED_CHAIN_MODE_PBL,
1493 QED_CHAIN_CNT_TYPE_U32,
1494 n_sq_elems,
1495 QEDR_SQE_ELEMENT_SIZE,
1a4a6975 1496 &qp->sq.pbl, NULL);
cecbcddf 1497
df158561
AR		 1498 if (rc)
1499 return rc;
cecbcddf 1500
df158561
AR		 1501 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1502 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1503
cecbcddf
RA		 1504 rc = dev->ops->common->chain_alloc(dev->cdev,
1505 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1506 QED_CHAIN_MODE_PBL,
1507 QED_CHAIN_CNT_TYPE_U32,
1508 n_rq_elems,
1509 QEDR_RQE_ELEMENT_SIZE,
1a4a6975 1510 &qp->rq.pbl, NULL);
df158561
AR		 1511 if (rc)
1512 return rc;
cecbcddf 1513
df158561
AR		 1514 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1515 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1516
df158561
AR		 1517 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1518 in_params, &out_params);
cecbcddf 1519
df158561
AR		 1520 if (!qp->qed_qp)
1521 return -EINVAL;
cecbcddf 1522
df158561
AR		 1523 qp->qp_id = out_params.qp_id;
1524 qp->icid = out_params.icid;
cecbcddf 1525
df158561 1526 qedr_set_roce_db_info(dev, qp);
f5b1b177
KM		 1527 return rc;
1528}
cecbcddf 1529
f5b1b177
KM		 1530static int
1531qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1532 struct qedr_qp *qp,
1533 struct qed_rdma_create_qp_in_params *in_params,
1534 u32 n_sq_elems, u32 n_rq_elems)
1535{
1536 struct qed_rdma_create_qp_out_params out_params;
1537 struct qed_chain_ext_pbl ext_pbl;
1538 int rc;
1539
1540 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1541 QEDR_SQE_ELEMENT_SIZE,
1542 QED_CHAIN_MODE_PBL);
1543 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1544 QEDR_RQE_ELEMENT_SIZE,
1545 QED_CHAIN_MODE_PBL);
1546
1547 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1548 in_params, &out_params);
1549
1550 if (!qp->qed_qp)
1551 return -EINVAL;
1552
1553 /* Now we allocate the chain */
1554 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1555 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1556
1557 rc = dev->ops->common->chain_alloc(dev->cdev,
1558 QED_CHAIN_USE_TO_PRODUCE,
1559 QED_CHAIN_MODE_PBL,
1560 QED_CHAIN_CNT_TYPE_U32,
1561 n_sq_elems,
1562 QEDR_SQE_ELEMENT_SIZE,
1563 &qp->sq.pbl, &ext_pbl);
1564
1565 if (rc)
1566 goto err;
1567
1568 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1569 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1570
1571 rc = dev->ops->common->chain_alloc(dev->cdev,
1572 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1573 QED_CHAIN_MODE_PBL,
1574 QED_CHAIN_CNT_TYPE_U32,
1575 n_rq_elems,
1576 QEDR_RQE_ELEMENT_SIZE,
1577 &qp->rq.pbl, &ext_pbl);
1578
1579 if (rc)
1580 goto err;
1581
1582 qp->qp_id = out_params.qp_id;
1583 qp->icid = out_params.icid;
1584
1585 qedr_set_iwarp_db_info(dev, qp);
1586 return rc;
1587
1588err:
1589 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1590
1591 return rc;
cecbcddf
RA		 1592}
1593
df158561 1594static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1595{
df158561
AR		 1596 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1597 kfree(qp->wqe_wr_id);
cecbcddf 1598
df158561
AR		 1599 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1600 kfree(qp->rqe_wr_id);
cecbcddf
RA		 1601}
1602
df158561
AR		 1603static int qedr_create_kernel_qp(struct qedr_dev *dev,
1604 struct qedr_qp *qp,
1605 struct ib_pd *ibpd,
1606 struct ib_qp_init_attr *attrs)
cecbcddf 1607{
df158561
AR		 1608 struct qed_rdma_create_qp_in_params in_params;
1609 struct qedr_pd *pd = get_qedr_pd(ibpd);
1610 int rc = -EINVAL;
1611 u32 n_rq_elems;
1612 u32 n_sq_elems;
1613 u32 n_sq_entries;
cecbcddf 1614
df158561 1615 memset(&in_params, 0, sizeof(in_params));
cecbcddf 1616
df158561
AR		 1617 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1618 * the ring. The ring should allow at least a single WR, even if the
1619 * user requested none, due to allocation issues.
1620 * We should add an extra WR since the prod and cons indices of
1621 * wqe_wr_id are managed in such a way that the WQ is considered full
1622 * when (prod+1)%max_wr==cons. We currently don't do that because we
1623 * double the number of entries due an iSER issue that pushes far more
1624 * WRs than indicated. If we decline its ib_post_send() then we get
1625 * error prints in the dmesg we'd like to avoid.
1626 */
1627 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1628 dev->attr.max_sqe);
cecbcddf 1629
df158561
AR		 1630 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1631 GFP_KERNEL);
1632 if (!qp->wqe_wr_id) {
1633 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1634 return -ENOMEM;
1635 }
cecbcddf 1636
df158561
AR		 1637 /* QP handle to be written in CQE */
1638 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1639 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 1640
df158561
AR		 1641 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1642 * the ring. There ring should allow at least a single WR, even if the
1643 * user requested none, due to allocation issues.
1644 */
1645 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 1646
df158561
AR		 1647 /* Allocate driver internal RQ array */
1648 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1649 GFP_KERNEL);
1650 if (!qp->rqe_wr_id) {
1651 DP_ERR(dev,
1652 "create qp: failed RQ shadow memory allocation\n");
1653 kfree(qp->wqe_wr_id);
1654 return -ENOMEM;
cecbcddf
RA		 1655 }
1656
df158561 1657 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 1658
df158561
AR		 1659 n_sq_entries = attrs->cap.max_send_wr;
1660 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1661 n_sq_entries = max_t(u32, n_sq_entries, 1);
1662 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 1663
df158561
AR		 1664 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1665
f5b1b177
KM		 1666 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1667 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1668 n_sq_elems, n_rq_elems);
1669 else
1670 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1671 n_sq_elems, n_rq_elems);
df158561
AR		 1672 if (rc)
1673 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 1674
1675 return rc;
1676}
1677
1678struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1679 struct ib_qp_init_attr *attrs,
1680 struct ib_udata *udata)
1681{
1682 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 1683 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 1684 struct qedr_qp *qp;
181d8015 1685 struct ib_qp *ibqp;
cecbcddf
RA		 1686 int rc = 0;
1687
1688 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1689 udata ? "user library" : "kernel", pd);
1690
1691 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1692 if (rc)
1693 return ERR_PTR(rc);
1694
181d8015
WY		 1695 if (attrs->srq)
1696 return ERR_PTR(-EINVAL);
1697
cecbcddf 1698 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 1699 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1700 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 1701 get_qedr_cq(attrs->send_cq),
1702 get_qedr_cq(attrs->send_cq)->icid,
1703 get_qedr_cq(attrs->recv_cq),
1704 get_qedr_cq(attrs->recv_cq)->icid);
1705
df158561
AR		 1706 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1707 if (!qp) {
1708 DP_ERR(dev, "create qp: failed allocating memory\n");
1709 return ERR_PTR(-ENOMEM);
1710 }
1711
1712 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 1713
04886779 1714 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 1715 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1716 if (IS_ERR(ibqp))
1717 kfree(qp);
1718 return ibqp;
04886779
RA		 1719 }
1720
df158561
AR		 1721 if (udata)
1722 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1723 else
1724 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 1725
df158561
AR		 1726 if (rc)
1727 goto err;
cecbcddf 1728
cecbcddf
RA		 1729 qp->ibqp.qp_num = qp->qp_id;
1730
de0089e6
KM		 1731 rc = qedr_idr_add(dev, qp, qp->qp_id);
1732 if (rc)
1733 goto err;
1734
cecbcddf
RA		 1735 return &qp->ibqp;
1736
df158561 1737err:
cecbcddf
RA		 1738 kfree(qp);
1739
1740 return ERR_PTR(-EFAULT);
1741}
1742
27a4b1a6 1743static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 1744{
1745 switch (qp_state) {
1746 case QED_ROCE_QP_STATE_RESET:
1747 return IB_QPS_RESET;
1748 case QED_ROCE_QP_STATE_INIT:
1749 return IB_QPS_INIT;
1750 case QED_ROCE_QP_STATE_RTR:
1751 return IB_QPS_RTR;
1752 case QED_ROCE_QP_STATE_RTS:
1753 return IB_QPS_RTS;
1754 case QED_ROCE_QP_STATE_SQD:
1755 return IB_QPS_SQD;
1756 case QED_ROCE_QP_STATE_ERR:
1757 return IB_QPS_ERR;
1758 case QED_ROCE_QP_STATE_SQE:
1759 return IB_QPS_SQE;
1760 }
1761 return IB_QPS_ERR;
1762}
1763
27a4b1a6
RA		 1764static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1765 enum ib_qp_state qp_state)
cecbcddf
RA		 1766{
1767 switch (qp_state) {
1768 case IB_QPS_RESET:
1769 return QED_ROCE_QP_STATE_RESET;
1770 case IB_QPS_INIT:
1771 return QED_ROCE_QP_STATE_INIT;
1772 case IB_QPS_RTR:
1773 return QED_ROCE_QP_STATE_RTR;
1774 case IB_QPS_RTS:
1775 return QED_ROCE_QP_STATE_RTS;
1776 case IB_QPS_SQD:
1777 return QED_ROCE_QP_STATE_SQD;
1778 case IB_QPS_ERR:
1779 return QED_ROCE_QP_STATE_ERR;
1780 default:
1781 return QED_ROCE_QP_STATE_ERR;
1782 }
1783}
1784
1785static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1786{
1787 qed_chain_reset(&qph->pbl);
1788 qph->prod = 0;
1789 qph->cons = 0;
1790 qph->wqe_cons = 0;
1791 qph->db_data.data.value = cpu_to_le16(0);
1792}
1793
1794static int qedr_update_qp_state(struct qedr_dev *dev,
1795 struct qedr_qp *qp,
1796 enum qed_roce_qp_state new_state)
1797{
1798 int status = 0;
1799
1800 if (new_state == qp->state)
865cea40 1801 return 0;
cecbcddf
RA		 1802
1803 switch (qp->state) {
1804 case QED_ROCE_QP_STATE_RESET:
1805 switch (new_state) {
1806 case QED_ROCE_QP_STATE_INIT:
1807 qp->prev_wqe_size = 0;
1808 qedr_reset_qp_hwq_info(&qp->sq);
1809 qedr_reset_qp_hwq_info(&qp->rq);
1810 break;
1811 default:
1812 status = -EINVAL;
1813 break;
1814 };
1815 break;
1816 case QED_ROCE_QP_STATE_INIT:
1817 switch (new_state) {
1818 case QED_ROCE_QP_STATE_RTR:
1819 /* Update doorbell (in case post_recv was
1820 * done before move to RTR)
1821 */
f5b1b177
KM		 1822
1823 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1824 wmb();
1825 writel(qp->rq.db_data.raw, qp->rq.db);
1826 /* Make sure write takes effect */
1827 mmiowb();
1828 }
cecbcddf
RA		 1829 break;
1830 case QED_ROCE_QP_STATE_ERR:
1831 break;
1832 default:
1833 /* Invalid state change. */
1834 status = -EINVAL;
1835 break;
1836 };
1837 break;
1838 case QED_ROCE_QP_STATE_RTR:
1839 /* RTR->XXX */
1840 switch (new_state) {
1841 case QED_ROCE_QP_STATE_RTS:
1842 break;
1843 case QED_ROCE_QP_STATE_ERR:
1844 break;
1845 default:
1846 /* Invalid state change. */
1847 status = -EINVAL;
1848 break;
1849 };
1850 break;
1851 case QED_ROCE_QP_STATE_RTS:
1852 /* RTS->XXX */
1853 switch (new_state) {
1854 case QED_ROCE_QP_STATE_SQD:
1855 break;
1856 case QED_ROCE_QP_STATE_ERR:
1857 break;
1858 default:
1859 /* Invalid state change. */
1860 status = -EINVAL;
1861 break;
1862 };
1863 break;
1864 case QED_ROCE_QP_STATE_SQD:
1865 /* SQD->XXX */
1866 switch (new_state) {
1867 case QED_ROCE_QP_STATE_RTS:
1868 case QED_ROCE_QP_STATE_ERR:
1869 break;
1870 default:
1871 /* Invalid state change. */
1872 status = -EINVAL;
1873 break;
1874 };
1875 break;
1876 case QED_ROCE_QP_STATE_ERR:
1877 /* ERR->XXX */
1878 switch (new_state) {
1879 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 1880 if ((qp->rq.prod != qp->rq.cons) ||
1881 (qp->sq.prod != qp->sq.cons)) {
1882 DP_NOTICE(dev,
1883 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1884 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1885 qp->sq.cons);
1886 status = -EINVAL;
1887 }
cecbcddf
RA		 1888 break;
1889 default:
1890 status = -EINVAL;
1891 break;
1892 };
1893 break;
1894 default:
1895 status = -EINVAL;
1896 break;
1897 };
1898
1899 return status;
1900}
1901
1902int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1903 int attr_mask, struct ib_udata *udata)
1904{
1905 struct qedr_qp *qp = get_qedr_qp(ibqp);
1906 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1907 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 1908 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf
RA		 1909 enum ib_qp_state old_qp_state, new_qp_state;
1910 int rc = 0;
1911
1912 DP_DEBUG(dev, QEDR_MSG_QP,
1913 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1914 attr->qp_state);
1915
1916 old_qp_state = qedr_get_ibqp_state(qp->state);
1917 if (attr_mask & IB_QP_STATE)
1918 new_qp_state = attr->qp_state;
1919 else
1920 new_qp_state = old_qp_state;
1921
f5b1b177
KM		 1922 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1923 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
1924 ibqp->qp_type, attr_mask,
1925 IB_LINK_LAYER_ETHERNET)) {
1926 DP_ERR(dev,
1927 "modify qp: invalid attribute mask=0x%x specified for\n"
1928 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1929 attr_mask, qp->qp_id, ibqp->qp_type,
1930 old_qp_state, new_qp_state);
1931 rc = -EINVAL;
1932 goto err;
1933 }
cecbcddf
RA		 1934 }
1935
1936 /* Translate the masks... */
1937 if (attr_mask & IB_QP_STATE) {
1938 SET_FIELD(qp_params.modify_flags,
1939 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1940 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1941 }
1942
1943 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1944 qp_params.sqd_async = true;
1945
1946 if (attr_mask & IB_QP_PKEY_INDEX) {
1947 SET_FIELD(qp_params.modify_flags,
1948 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1949 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1950 rc = -EINVAL;
1951 goto err;
1952 }
1953
1954 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1955 }
1956
1957 if (attr_mask & IB_QP_QKEY)
1958 qp->qkey = attr->qkey;
1959
1960 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1961 SET_FIELD(qp_params.modify_flags,
1962 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1963 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1964 IB_ACCESS_REMOTE_READ;
1965 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1966 IB_ACCESS_REMOTE_WRITE;
1967 qp_params.incoming_atomic_en = attr->qp_access_flags &
1968 IB_ACCESS_REMOTE_ATOMIC;
1969 }
1970
1971 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1972 if (attr_mask & IB_QP_PATH_MTU) {
1973 if (attr->path_mtu < IB_MTU_256 ||
1974 attr->path_mtu > IB_MTU_4096) {
1975 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1976 rc = -EINVAL;
1977 goto err;
1978 }
1979 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1980 ib_mtu_enum_to_int(iboe_get_mtu
1981 (dev->ndev->mtu)));
1982 }
1983
1984 if (!qp->mtu) {
1985 qp->mtu =
1986 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1987 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1988 }
1989
1990 SET_FIELD(qp_params.modify_flags,
1991 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1992
d8966fcd
DC		 1993 qp_params.traffic_class_tos = grh->traffic_class;
1994 qp_params.flow_label = grh->flow_label;
1995 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 1996
d8966fcd 1997 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 1998
1999 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2000 if (rc) {
2001 DP_ERR(dev,
2002 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2003 grh->sgid_index, rc);
cecbcddf
RA		 2004 return rc;
2005 }
2006
2007 rc = qedr_get_dmac(dev, &attr->ah_attr,
2008 qp_params.remote_mac_addr);
2009 if (rc)
2010 return rc;
2011
2012 qp_params.use_local_mac = true;
2013 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2014
2015 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2016 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2017 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2018 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2019 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2020 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2021 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2022 qp_params.remote_mac_addr);
cecbcddf
RA		 2023
2024 qp_params.mtu = qp->mtu;
2025 qp_params.lb_indication = false;
2026 }
2027
2028 if (!qp_params.mtu) {
2029 /* Stay with current MTU */
2030 if (qp->mtu)
2031 qp_params.mtu = qp->mtu;
2032 else
2033 qp_params.mtu =
2034 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2035 }
2036
2037 if (attr_mask & IB_QP_TIMEOUT) {
2038 SET_FIELD(qp_params.modify_flags,
2039 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2040
2041 qp_params.ack_timeout = attr->timeout;
2042 if (attr->timeout) {
2043 u32 temp;
2044
2045 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
2046 /* FW requires [msec] */
2047 qp_params.ack_timeout = temp;
2048 } else {
2049 /* Infinite */
2050 qp_params.ack_timeout = 0;
2051 }
2052 }
2053 if (attr_mask & IB_QP_RETRY_CNT) {
2054 SET_FIELD(qp_params.modify_flags,
2055 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2056 qp_params.retry_cnt = attr->retry_cnt;
2057 }
2058
2059 if (attr_mask & IB_QP_RNR_RETRY) {
2060 SET_FIELD(qp_params.modify_flags,
2061 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2062 qp_params.rnr_retry_cnt = attr->rnr_retry;
2063 }
2064
2065 if (attr_mask & IB_QP_RQ_PSN) {
2066 SET_FIELD(qp_params.modify_flags,
2067 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2068 qp_params.rq_psn = attr->rq_psn;
2069 qp->rq_psn = attr->rq_psn;
2070 }
2071
2072 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2073 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2074 rc = -EINVAL;
2075 DP_ERR(dev,
2076 "unsupported max_rd_atomic=%d, supported=%d\n",
2077 attr->max_rd_atomic,
2078 dev->attr.max_qp_req_rd_atomic_resc);
2079 goto err;
2080 }
2081
2082 SET_FIELD(qp_params.modify_flags,
2083 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2084 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2085 }
2086
2087 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2088 SET_FIELD(qp_params.modify_flags,
2089 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2090 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2091 }
2092
2093 if (attr_mask & IB_QP_SQ_PSN) {
2094 SET_FIELD(qp_params.modify_flags,
2095 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2096 qp_params.sq_psn = attr->sq_psn;
2097 qp->sq_psn = attr->sq_psn;
2098 }
2099
2100 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2101 if (attr->max_dest_rd_atomic >
2102 dev->attr.max_qp_resp_rd_atomic_resc) {
2103 DP_ERR(dev,
2104 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2105 attr->max_dest_rd_atomic,
2106 dev->attr.max_qp_resp_rd_atomic_resc);
2107
2108 rc = -EINVAL;
2109 goto err;
2110 }
2111
2112 SET_FIELD(qp_params.modify_flags,
2113 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2114 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2115 }
2116
2117 if (attr_mask & IB_QP_DEST_QPN) {
2118 SET_FIELD(qp_params.modify_flags,
2119 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2120
2121 qp_params.dest_qp = attr->dest_qp_num;
2122 qp->dest_qp_num = attr->dest_qp_num;
2123 }
2124
2125 if (qp->qp_type != IB_QPT_GSI)
2126 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2127 qp->qed_qp, &qp_params);
2128
2129 if (attr_mask & IB_QP_STATE) {
2130 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
d6ebbf29 2131 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
cecbcddf
RA		 2132 qp->state = qp_params.new_state;
2133 }
2134
2135err:
2136 return rc;
2137}
2138
2139static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2140{
2141 int ib_qp_acc_flags = 0;
2142
2143 if (params->incoming_rdma_write_en)
2144 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2145 if (params->incoming_rdma_read_en)
2146 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2147 if (params->incoming_atomic_en)
2148 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2149 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2150 return ib_qp_acc_flags;
2151}
2152
2153int qedr_query_qp(struct ib_qp *ibqp,
2154 struct ib_qp_attr *qp_attr,
2155 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2156{
2157 struct qed_rdma_query_qp_out_params params;
2158 struct qedr_qp *qp = get_qedr_qp(ibqp);
2159 struct qedr_dev *dev = qp->dev;
2160 int rc = 0;
2161
2162 memset(&params, 0, sizeof(params));
2163
2164 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2165 if (rc)
2166 goto err;
2167
2168 memset(qp_attr, 0, sizeof(*qp_attr));
2169 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2170
2171 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2172 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2173 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2174 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2175 qp_attr->rq_psn = params.rq_psn;
2176 qp_attr->sq_psn = params.sq_psn;
2177 qp_attr->dest_qp_num = params.dest_qp;
2178
2179 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2180
2181 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2182 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2183 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2184 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 2185 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 2186 qp_init_attr->cap = qp_attr->cap;
2187
44c58487 2188 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2189 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2190 params.flow_label, qp->sgid_idx,
2191 params.hop_limit_ttl, params.traffic_class_tos);
2192 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2193 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2194 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2195 qp_attr->timeout = params.timeout;
2196 qp_attr->rnr_retry = params.rnr_retry;
2197 qp_attr->retry_cnt = params.retry_cnt;
2198 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2199 qp_attr->pkey_index = params.pkey_index;
2200 qp_attr->port_num = 1;
d8966fcd
DC		 2201 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2202 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2203 qp_attr->alt_pkey_index = 0;
2204 qp_attr->alt_port_num = 0;
2205 qp_attr->alt_timeout = 0;
2206 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2207
2208 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2209 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2210 qp_attr->max_rd_atomic = params.max_rd_atomic;
2211 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2212
2213 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2214 qp_attr->cap.max_inline_data);
2215
2216err:
2217 return rc;
2218}
2219
df158561
AR		 2220int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2221{
2222 int rc = 0;
2223
2224 if (qp->qp_type != IB_QPT_GSI) {
2225 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2226 if (rc)
2227 return rc;
2228 }
2229
2230 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2231 qedr_cleanup_user(dev, qp);
2232 else
2233 qedr_cleanup_kernel(dev, qp);
2234
2235 return 0;
2236}
2237
cecbcddf
RA		 2238int qedr_destroy_qp(struct ib_qp *ibqp)
2239{
2240 struct qedr_qp *qp = get_qedr_qp(ibqp);
2241 struct qedr_dev *dev = qp->dev;
2242 struct ib_qp_attr attr;
2243 int attr_mask = 0;
2244 int rc = 0;
2245
2246 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2247 qp, qp->qp_type);
2248
f5b1b177
KM		 2249 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2250 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2251 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2252 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2253
f5b1b177
KM		 2254 attr.qp_state = IB_QPS_ERR;
2255 attr_mask |= IB_QP_STATE;
cecbcddf 2256
f5b1b177
KM		 2257 /* Change the QP state to ERROR */
2258 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2259 }
cecbcddf
RA		 2260 }
2261
df158561 2262 if (qp->qp_type == IB_QPT_GSI)
04886779 2263 qedr_destroy_gsi_qp(dev);
cecbcddf 2264
df158561 2265 qedr_free_qp_resources(dev, qp);
cecbcddf 2266
de0089e6
KM		 2267 if (atomic_dec_and_test(&qp->refcnt)) {
2268 qedr_idr_remove(dev, qp->qp_id);
2269 kfree(qp);
2270 }
cecbcddf
RA		 2271 return rc;
2272}
e0290cce 2273
90898850 2274struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
477864c8 2275 struct ib_udata *udata)
04886779
RA		 2276{
2277 struct qedr_ah *ah;
2278
2279 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2280 if (!ah)
2281 return ERR_PTR(-ENOMEM);
2282
2283 ah->attr = *attr;
2284
2285 return &ah->ibah;
2286}
2287
2288int qedr_destroy_ah(struct ib_ah *ibah)
2289{
2290 struct qedr_ah *ah = get_qedr_ah(ibah);
2291
2292 kfree(ah);
2293 return 0;
2294}
2295
e0290cce
RA		 2296static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2297{
2298 struct qedr_pbl *pbl, *tmp;
2299
2300 if (info->pbl_table)
2301 list_add_tail(&info->pbl_table->list_entry,
2302 &info->free_pbl_list);
2303
2304 if (!list_empty(&info->inuse_pbl_list))
2305 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2306
2307 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2308 list_del(&pbl->list_entry);
2309 qedr_free_pbl(dev, &info->pbl_info, pbl);
2310 }
2311}
2312
2313static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2314 size_t page_list_len, bool two_layered)
2315{
2316 struct qedr_pbl *tmp;
2317 int rc;
2318
2319 INIT_LIST_HEAD(&info->free_pbl_list);
2320 INIT_LIST_HEAD(&info->inuse_pbl_list);
2321
2322 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2323 page_list_len, two_layered);
2324 if (rc)
2325 goto done;
2326
2327 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2328 if (IS_ERR(info->pbl_table)) {
2329 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2330 goto done;
2331 }
2332
2333 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2334 &info->pbl_table->pa);
2335
2336 /* in usual case we use 2 PBLs, so we add one to free
2337 * list and allocating another one
2338 */
2339 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2340 if (IS_ERR(tmp)) {
e0290cce
RA		 2341 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2342 goto done;
2343 }
2344
2345 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2346
2347 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2348
2349done:
2350 if (rc)
2351 free_mr_info(dev, info);
2352
2353 return rc;
2354}
2355
2356struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2357 u64 usr_addr, int acc, struct ib_udata *udata)
2358{
2359 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2360 struct qedr_mr *mr;
2361 struct qedr_pd *pd;
2362 int rc = -ENOMEM;
2363
2364 pd = get_qedr_pd(ibpd);
2365 DP_DEBUG(dev, QEDR_MSG_MR,
2366 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2367 pd->pd_id, start, len, usr_addr, acc);
2368
2369 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2370 return ERR_PTR(-EINVAL);
2371
2372 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2373 if (!mr)
2374 return ERR_PTR(rc);
2375
2376 mr->type = QEDR_MR_USER;
2377
2378 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2379 if (IS_ERR(mr->umem)) {
2380 rc = -EFAULT;
2381 goto err0;
2382 }
2383
2384 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2385 if (rc)
2386 goto err1;
2387
2388 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
e57bb6be 2389 &mr->info.pbl_info, mr->umem->page_shift);
e0290cce
RA		 2390
2391 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2392 if (rc) {
2393 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2394 goto err1;
2395 }
2396
2397 /* Index only, 18 bit long, lkey = itid << 8 | key */
2398 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2399 mr->hw_mr.key = 0;
2400 mr->hw_mr.pd = pd->pd_id;
2401 mr->hw_mr.local_read = 1;
2402 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2403 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2404 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2405 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2406 mr->hw_mr.mw_bind = false;
2407 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2408 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2409 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3e7e1193 2410 mr->hw_mr.page_size_log = mr->umem->page_shift;
e0290cce
RA		 2411 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2412 mr->hw_mr.length = len;
2413 mr->hw_mr.vaddr = usr_addr;
2414 mr->hw_mr.zbva = false;
2415 mr->hw_mr.phy_mr = false;
2416 mr->hw_mr.dma_mr = false;
2417
2418 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2419 if (rc) {
2420 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2421 goto err2;
2422 }
2423
2424 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2425 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2426 mr->hw_mr.remote_atomic)
2427 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2428
2429 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2430 mr->ibmr.lkey);
2431 return &mr->ibmr;
2432
2433err2:
2434 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2435err1:
2436 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2437err0:
2438 kfree(mr);
2439 return ERR_PTR(rc);
2440}
2441
2442int qedr_dereg_mr(struct ib_mr *ib_mr)
2443{
2444 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2445 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2446 int rc = 0;
2447
2448 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2449 if (rc)
2450 return rc;
2451
2452 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2453
2454 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2455 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2456
2457 /* it could be user registered memory. */
2458 if (mr->umem)
2459 ib_umem_release(mr->umem);
2460
2461 kfree(mr);
2462
2463 return rc;
2464}
2465
27a4b1a6
RA		 2466static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2467 int max_page_list_len)
e0290cce
RA		 2468{
2469 struct qedr_pd *pd = get_qedr_pd(ibpd);
2470 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2471 struct qedr_mr *mr;
2472 int rc = -ENOMEM;
2473
2474 DP_DEBUG(dev, QEDR_MSG_MR,
2475 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2476 max_page_list_len);
2477
2478 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2479 if (!mr)
2480 return ERR_PTR(rc);
2481
2482 mr->dev = dev;
2483 mr->type = QEDR_MR_FRMR;
2484
2485 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2486 if (rc)
2487 goto err0;
2488
2489 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2490 if (rc) {
2491 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2492 goto err0;
2493 }
2494
2495 /* Index only, 18 bit long, lkey = itid << 8 | key */
2496 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2497 mr->hw_mr.key = 0;
2498 mr->hw_mr.pd = pd->pd_id;
2499 mr->hw_mr.local_read = 1;
2500 mr->hw_mr.local_write = 0;
2501 mr->hw_mr.remote_read = 0;
2502 mr->hw_mr.remote_write = 0;
2503 mr->hw_mr.remote_atomic = 0;
2504 mr->hw_mr.mw_bind = false;
2505 mr->hw_mr.pbl_ptr = 0;
2506 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2507 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2508 mr->hw_mr.fbo = 0;
2509 mr->hw_mr.length = 0;
2510 mr->hw_mr.vaddr = 0;
2511 mr->hw_mr.zbva = false;
2512 mr->hw_mr.phy_mr = true;
2513 mr->hw_mr.dma_mr = false;
2514
2515 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2516 if (rc) {
2517 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2518 goto err1;
2519 }
2520
2521 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2522 mr->ibmr.rkey = mr->ibmr.lkey;
2523
2524 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2525 return mr;
2526
2527err1:
2528 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2529err0:
2530 kfree(mr);
2531 return ERR_PTR(rc);
2532}
2533
2534struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2535 enum ib_mr_type mr_type, u32 max_num_sg)
2536{
2537 struct qedr_dev *dev;
2538 struct qedr_mr *mr;
2539
2540 if (mr_type != IB_MR_TYPE_MEM_REG)
2541 return ERR_PTR(-EINVAL);
2542
2543 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2544
2545 if (IS_ERR(mr))
2546 return ERR_PTR(-EINVAL);
2547
2548 dev = mr->dev;
2549
2550 return &mr->ibmr;
2551}
2552
2553static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2554{
2555 struct qedr_mr *mr = get_qedr_mr(ibmr);
2556 struct qedr_pbl *pbl_table;
2557 struct regpair *pbe;
2558 u32 pbes_in_page;
2559
2560 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2561 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2562 return -ENOMEM;
2563 }
2564
2565 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2566 mr->npages, addr);
2567
2568 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2569 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2570 pbe = (struct regpair *)pbl_table->va;
2571 pbe += mr->npages % pbes_in_page;
2572 pbe->lo = cpu_to_le32((u32)addr);
2573 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2574
2575 mr->npages++;
2576
2577 return 0;
2578}
2579
2580static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2581{
2582 int work = info->completed - info->completed_handled - 1;
2583
2584 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2585 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2586 struct qedr_pbl *pbl;
2587
2588 /* Free all the page list that are possible to be freed
2589 * (all the ones that were invalidated), under the assumption
2590 * that if an FMR was completed successfully that means that
2591 * if there was an invalidate operation before it also ended
2592 */
2593 pbl = list_first_entry(&info->inuse_pbl_list,
2594 struct qedr_pbl, list_entry);
aafec388 2595 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 2596 info->completed_handled++;
2597 }
2598}
2599
2600int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2601 int sg_nents, unsigned int *sg_offset)
2602{
2603 struct qedr_mr *mr = get_qedr_mr(ibmr);
2604
2605 mr->npages = 0;
2606
2607 handle_completed_mrs(mr->dev, &mr->info);
2608 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2609}
2610
2611struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2612{
2613 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2614 struct qedr_pd *pd = get_qedr_pd(ibpd);
2615 struct qedr_mr *mr;
2616 int rc;
2617
2618 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2619 if (!mr)
2620 return ERR_PTR(-ENOMEM);
2621
2622 mr->type = QEDR_MR_DMA;
2623
2624 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2625 if (rc) {
2626 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2627 goto err1;
2628 }
2629
2630 /* index only, 18 bit long, lkey = itid << 8 | key */
2631 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2632 mr->hw_mr.pd = pd->pd_id;
2633 mr->hw_mr.local_read = 1;
2634 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2635 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2636 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2637 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2638 mr->hw_mr.dma_mr = true;
2639
2640 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2641 if (rc) {
2642 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2643 goto err2;
2644 }
2645
2646 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2647 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2648 mr->hw_mr.remote_atomic)
2649 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2650
2651 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2652 return &mr->ibmr;
2653
2654err2:
2655 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2656err1:
2657 kfree(mr);
2658 return ERR_PTR(rc);
2659}
afa0e13b
RA		 2660
2661static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2662{
2663 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2664}
2665
2666static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2667{
2668 int i, len = 0;
2669
2670 for (i = 0; i < num_sge; i++)
2671 len += sg_list[i].length;
2672
2673 return len;
2674}
2675
2676static void swap_wqe_data64(u64 *p)
2677{
2678 int i;
2679
2680 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2681 *p = cpu_to_be64(cpu_to_le64(*p));
2682}
2683
2684static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2685 struct qedr_qp *qp, u8 *wqe_size,
2686 struct ib_send_wr *wr,
2687 struct ib_send_wr **bad_wr, u8 *bits,
2688 u8 bit)
2689{
2690 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2691 char *seg_prt, *wqe;
2692 int i, seg_siz;
2693
2694 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2695 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2696 *bad_wr = wr;
2697 return 0;
2698 }
2699
2700 if (!data_size)
2701 return data_size;
2702
2703 *bits |= bit;
2704
2705 seg_prt = NULL;
2706 wqe = NULL;
2707 seg_siz = 0;
2708
2709 /* Copy data inline */
2710 for (i = 0; i < wr->num_sge; i++) {
2711 u32 len = wr->sg_list[i].length;
2712 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2713
2714 while (len > 0) {
2715 u32 cur;
2716
2717 /* New segment required */
2718 if (!seg_siz) {
2719 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2720 seg_prt = wqe;
2721 seg_siz = sizeof(struct rdma_sq_common_wqe);
2722 (*wqe_size)++;
2723 }
2724
2725 /* Calculate currently allowed length */
2726 cur = min_t(u32, len, seg_siz);
2727 memcpy(seg_prt, src, cur);
2728
2729 /* Update segment variables */
2730 seg_prt += cur;
2731 seg_siz -= cur;
2732
2733 /* Update sge variables */
2734 src += cur;
2735 len -= cur;
2736
2737 /* Swap fully-completed segments */
2738 if (!seg_siz)
2739 swap_wqe_data64((u64 *)wqe);
2740 }
2741 }
2742
2743 /* swap last not completed segment */
2744 if (seg_siz)
2745 swap_wqe_data64((u64 *)wqe);
2746
2747 return data_size;
2748}
2749
2750#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2751 do { \
2752 DMA_REGPAIR_LE(sge->addr, vaddr); \
2753 (sge)->length = cpu_to_le32(vlength); \
2754 (sge)->flags = cpu_to_le32(vflags); \
2755 } while (0)
2756
2757#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2758 do { \
2759 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2760 (hdr)->num_sges = num_sge; \
2761 } while (0)
2762
2763#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2764 do { \
2765 DMA_REGPAIR_LE(sge->addr, vaddr); \
2766 (sge)->length = cpu_to_le32(vlength); \
2767 (sge)->l_key = cpu_to_le32(vlkey); \
2768 } while (0)
2769
2770static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2771 struct ib_send_wr *wr)
2772{
2773 u32 data_size = 0;
2774 int i;
2775
2776 for (i = 0; i < wr->num_sge; i++) {
2777 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2778
2779 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2780 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2781 sge->length = cpu_to_le32(wr->sg_list[i].length);
2782 data_size += wr->sg_list[i].length;
2783 }
2784
2785 if (wqe_size)
2786 *wqe_size += wr->num_sge;
2787
2788 return data_size;
2789}
2790
2791static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2792 struct qedr_qp *qp,
2793 struct rdma_sq_rdma_wqe_1st *rwqe,
2794 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2795 struct ib_send_wr *wr,
2796 struct ib_send_wr **bad_wr)
2797{
2798 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2799 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2800
8b0cabc6
AR		 2801 if (wr->send_flags & IB_SEND_INLINE &&
2802 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2803 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 2804 u8 flags = 0;
2805
2806 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2807 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2808 bad_wr, &rwqe->flags, flags);
2809 }
2810
2811 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2812}
2813
2814static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2815 struct qedr_qp *qp,
2816 struct rdma_sq_send_wqe_1st *swqe,
2817 struct rdma_sq_send_wqe_2st *swqe2,
2818 struct ib_send_wr *wr,
2819 struct ib_send_wr **bad_wr)
2820{
2821 memset(swqe2, 0, sizeof(*swqe2));
2822 if (wr->send_flags & IB_SEND_INLINE) {
2823 u8 flags = 0;
2824
2825 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2826 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2827 bad_wr, &swqe->flags, flags);
2828 }
2829
2830 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2831}
2832
2833static int qedr_prepare_reg(struct qedr_qp *qp,
2834 struct rdma_sq_fmr_wqe_1st *fwqe1,
2835 struct ib_reg_wr *wr)
2836{
2837 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2838 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2839
2840 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2841 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2842 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2843 fwqe1->l_key = wr->key;
2844
08c4cf51
AR		 2845 fwqe2->access_ctrl = 0;
2846
afa0e13b
RA		 2847 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2848 !!(wr->access & IB_ACCESS_REMOTE_READ));
2849 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2850 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2851 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2852 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2853 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2854 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2855 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2856 fwqe2->fmr_ctrl = 0;
2857
2858 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2859 ilog2(mr->ibmr.page_size) - 12);
2860
2861 fwqe2->length_hi = 0;
2862 fwqe2->length_lo = mr->ibmr.length;
2863 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2864 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2865
2866 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2867
2868 return 0;
2869}
2870
27a4b1a6 2871static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 2872{
2873 switch (opcode) {
2874 case IB_WR_RDMA_WRITE:
2875 case IB_WR_RDMA_WRITE_WITH_IMM:
2876 return IB_WC_RDMA_WRITE;
2877 case IB_WR_SEND_WITH_IMM:
2878 case IB_WR_SEND:
2879 case IB_WR_SEND_WITH_INV:
2880 return IB_WC_SEND;
2881 case IB_WR_RDMA_READ:
fb1a22be 2882 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 2883 return IB_WC_RDMA_READ;
2884 case IB_WR_ATOMIC_CMP_AND_SWP:
2885 return IB_WC_COMP_SWAP;
2886 case IB_WR_ATOMIC_FETCH_AND_ADD:
2887 return IB_WC_FETCH_ADD;
2888 case IB_WR_REG_MR:
2889 return IB_WC_REG_MR;
2890 case IB_WR_LOCAL_INV:
2891 return IB_WC_LOCAL_INV;
2892 default:
2893 return IB_WC_SEND;
2894 }
2895}
2896
27a4b1a6 2897static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
afa0e13b
RA		 2898{
2899 int wq_is_full, err_wr, pbl_is_full;
2900 struct qedr_dev *dev = qp->dev;
2901
2902 /* prevent SQ overflow and/or processing of a bad WR */
2903 err_wr = wr->num_sge > qp->sq.max_sges;
2904 wq_is_full = qedr_wq_is_full(&qp->sq);
2905 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2906 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2907 if (wq_is_full || err_wr || pbl_is_full) {
2908 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2909 DP_ERR(dev,
2910 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2911 qp);
2912 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2913 }
2914
2915 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2916 DP_ERR(dev,
2917 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2918 qp);
2919 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2920 }
2921
2922 if (pbl_is_full &&
2923 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2924 DP_ERR(dev,
2925 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2926 qp);
2927 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2928 }
2929 return false;
2930 }
2931 return true;
2932}
2933
27a4b1a6 2934static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
afa0e13b
RA		 2935 struct ib_send_wr **bad_wr)
2936{
2937 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2938 struct qedr_qp *qp = get_qedr_qp(ibqp);
2939 struct rdma_sq_atomic_wqe_1st *awqe1;
2940 struct rdma_sq_atomic_wqe_2nd *awqe2;
2941 struct rdma_sq_atomic_wqe_3rd *awqe3;
2942 struct rdma_sq_send_wqe_2st *swqe2;
2943 struct rdma_sq_local_inv_wqe *iwqe;
2944 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2945 struct rdma_sq_send_wqe_1st *swqe;
2946 struct rdma_sq_rdma_wqe_1st *rwqe;
2947 struct rdma_sq_fmr_wqe_1st *fwqe1;
2948 struct rdma_sq_common_wqe *wqe;
2949 u32 length;
2950 int rc = 0;
2951 bool comp;
2952
2953 if (!qedr_can_post_send(qp, wr)) {
2954 *bad_wr = wr;
2955 return -ENOMEM;
2956 }
2957
2958 wqe = qed_chain_produce(&qp->sq.pbl);
2959 qp->wqe_wr_id[qp->sq.prod].signaled =
2960 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2961
2962 wqe->flags = 0;
2963 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2964 !!(wr->send_flags & IB_SEND_SOLICITED));
2965 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2966 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2967 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2968 !!(wr->send_flags & IB_SEND_FENCE));
2969 wqe->prev_wqe_size = qp->prev_wqe_size;
2970
2971 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2972
2973 switch (wr->opcode) {
2974 case IB_WR_SEND_WITH_IMM:
2975 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2976 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2977 swqe->wqe_size = 2;
2978 swqe2 = qed_chain_produce(&qp->sq.pbl);
2979
2980 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2981 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2982 wr, bad_wr);
2983 swqe->length = cpu_to_le32(length);
2984 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2985 qp->prev_wqe_size = swqe->wqe_size;
2986 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2987 break;
2988 case IB_WR_SEND:
2989 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2990 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2991
2992 swqe->wqe_size = 2;
2993 swqe2 = qed_chain_produce(&qp->sq.pbl);
2994 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2995 wr, bad_wr);
2996 swqe->length = cpu_to_le32(length);
2997 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2998 qp->prev_wqe_size = swqe->wqe_size;
2999 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3000 break;
3001 case IB_WR_SEND_WITH_INV:
3002 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3003 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3004 swqe2 = qed_chain_produce(&qp->sq.pbl);
3005 swqe->wqe_size = 2;
3006 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3007 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3008 wr, bad_wr);
3009 swqe->length = cpu_to_le32(length);
3010 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3011 qp->prev_wqe_size = swqe->wqe_size;
3012 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3013 break;
3014
3015 case IB_WR_RDMA_WRITE_WITH_IMM:
3016 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3017 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3018
3019 rwqe->wqe_size = 2;
3020 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3021 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3022 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3023 wr, bad_wr);
3024 rwqe->length = cpu_to_le32(length);
3025 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3026 qp->prev_wqe_size = rwqe->wqe_size;
3027 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3028 break;
3029 case IB_WR_RDMA_WRITE:
3030 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3031 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3032
3033 rwqe->wqe_size = 2;
3034 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3035 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3036 wr, bad_wr);
3037 rwqe->length = cpu_to_le32(length);
3038 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3039 qp->prev_wqe_size = rwqe->wqe_size;
3040 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3041 break;
3042 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be
KM		 3043 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3044 /* fallthrough... same is identical to RDMA READ */
afa0e13b
RA		 3045
3046 case IB_WR_RDMA_READ:
3047 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3048 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3049
3050 rwqe->wqe_size = 2;
3051 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3052 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3053 wr, bad_wr);
3054 rwqe->length = cpu_to_le32(length);
3055 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3056 qp->prev_wqe_size = rwqe->wqe_size;
3057 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3058 break;
3059
3060 case IB_WR_ATOMIC_CMP_AND_SWP:
3061 case IB_WR_ATOMIC_FETCH_AND_ADD:
3062 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3063 awqe1->wqe_size = 4;
3064
3065 awqe2 = qed_chain_produce(&qp->sq.pbl);
3066 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3067 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3068
3069 awqe3 = qed_chain_produce(&qp->sq.pbl);
3070
3071 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3072 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3073 DMA_REGPAIR_LE(awqe3->swap_data,
3074 atomic_wr(wr)->compare_add);
3075 } else {
3076 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3077 DMA_REGPAIR_LE(awqe3->swap_data,
3078 atomic_wr(wr)->swap);
3079 DMA_REGPAIR_LE(awqe3->cmp_data,
3080 atomic_wr(wr)->compare_add);
3081 }
3082
3083 qedr_prepare_sq_sges(qp, NULL, wr);
3084
3085 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3086 qp->prev_wqe_size = awqe1->wqe_size;
3087 break;
3088
3089 case IB_WR_LOCAL_INV:
3090 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3091 iwqe->wqe_size = 1;
3092
3093 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3094 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3095 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3096 qp->prev_wqe_size = iwqe->wqe_size;
3097 break;
3098 case IB_WR_REG_MR:
3099 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3100 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3101 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3102 fwqe1->wqe_size = 2;
3103
3104 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3105 if (rc) {
3106 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3107 *bad_wr = wr;
3108 break;
3109 }
3110
3111 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3112 qp->prev_wqe_size = fwqe1->wqe_size;
3113 break;
3114 default:
3115 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3116 rc = -EINVAL;
3117 *bad_wr = wr;
3118 break;
3119 }
3120
3121 if (*bad_wr) {
3122 u16 value;
3123
3124 /* Restore prod to its position before
3125 * this WR was processed
3126 */
3127 value = le16_to_cpu(qp->sq.db_data.data.value);
3128 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3129
3130 /* Restore prev_wqe_size */
3131 qp->prev_wqe_size = wqe->prev_wqe_size;
3132 rc = -EINVAL;
3133 DP_ERR(dev, "POST SEND FAILED\n");
3134 }
3135
3136 return rc;
3137}
3138
3139int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3140 struct ib_send_wr **bad_wr)
3141{
3142 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3143 struct qedr_qp *qp = get_qedr_qp(ibqp);
3144 unsigned long flags;
3145 int rc = 0;
3146
3147 *bad_wr = NULL;
3148
04886779
RA		 3149 if (qp->qp_type == IB_QPT_GSI)
3150 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3151
afa0e13b
RA		 3152 spin_lock_irqsave(&qp->q_lock, flags);
3153
f5b1b177
KM		 3154 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3155 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3156 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3157 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3158 spin_unlock_irqrestore(&qp->q_lock, flags);
3159 *bad_wr = wr;
3160 DP_DEBUG(dev, QEDR_MSG_CQ,
3161 "QP in wrong state! QP icid=0x%x state %d\n",
3162 qp->icid, qp->state);
3163 return -EINVAL;
3164 }
afa0e13b
RA		 3165 }
3166
afa0e13b
RA		 3167 while (wr) {
3168 rc = __qedr_post_send(ibqp, wr, bad_wr);
3169 if (rc)
3170 break;
3171
3172 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3173
3174 qedr_inc_sw_prod(&qp->sq);
3175
3176 qp->sq.db_data.data.value++;
3177
3178 wr = wr->next;
3179 }
3180
3181 /* Trigger doorbell
3182 * If there was a failure in the first WR then it will be triggered in
3183 * vane. However this is not harmful (as long as the producer value is
3184 * unchanged). For performance reasons we avoid checking for this
3185 * redundant doorbell.
3186 */
3187 wmb();
3188 writel(qp->sq.db_data.raw, qp->sq.db);
3189
3190 /* Make sure write sticks */
3191 mmiowb();
3192
3193 spin_unlock_irqrestore(&qp->q_lock, flags);
3194
3195 return rc;
3196}
3197
3198int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3199 struct ib_recv_wr **bad_wr)
3200{
3201 struct qedr_qp *qp = get_qedr_qp(ibqp);
3202 struct qedr_dev *dev = qp->dev;
3203 unsigned long flags;
3204 int status = 0;
3205
04886779
RA		 3206 if (qp->qp_type == IB_QPT_GSI)
3207 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3208
afa0e13b
RA		 3209 spin_lock_irqsave(&qp->q_lock, flags);
3210
922d9a40 3211 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3212 spin_unlock_irqrestore(&qp->q_lock, flags);
3213 *bad_wr = wr;
3214 return -EINVAL;
3215 }
3216
3217 while (wr) {
3218 int i;
3219
3220 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3221 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3222 wr->num_sge > qp->rq.max_sges) {
3223 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3224 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3225 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3226 qp->rq.max_sges);
3227 status = -ENOMEM;
3228 *bad_wr = wr;
3229 break;
3230 }
3231 for (i = 0; i < wr->num_sge; i++) {
3232 u32 flags = 0;
3233 struct rdma_rq_sge *rqe =
3234 qed_chain_produce(&qp->rq.pbl);
3235
3236 /* First one must include the number
3237 * of SGE in the list
3238 */
3239 if (!i)
3240 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3241 wr->num_sge);
3242
3243 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3244 wr->sg_list[i].lkey);
3245
3246 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3247 wr->sg_list[i].length, flags);
3248 }
3249
3250 /* Special case of no sges. FW requires between 1-4 sges...
3251 * in this case we need to post 1 sge with length zero. this is
3252 * because rdma write with immediate consumes an RQ.
3253 */
3254 if (!wr->num_sge) {
3255 u32 flags = 0;
3256 struct rdma_rq_sge *rqe =
3257 qed_chain_produce(&qp->rq.pbl);
3258
3259 /* First one must include the number
3260 * of SGE in the list
3261 */
3262 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3263 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3264
3265 RQ_SGE_SET(rqe, 0, 0, flags);
3266 i = 1;
3267 }
3268
3269 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3270 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3271
3272 qedr_inc_sw_prod(&qp->rq);
3273
3274 /* Flush all the writes before signalling doorbell */
3275 wmb();
3276
3277 qp->rq.db_data.data.value++;
3278
3279 writel(qp->rq.db_data.raw, qp->rq.db);
3280
3281 /* Make sure write sticks */
3282 mmiowb();
3283
f5b1b177
KM		 3284 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3285 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3286 mmiowb(); /* for second doorbell */
3287 }
3288
afa0e13b
RA		 3289 wr = wr->next;
3290 }
3291
3292 spin_unlock_irqrestore(&qp->q_lock, flags);
3293
3294 return status;
3295}
3296
3297static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3298{
3299 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3300
3301 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3302 cq->pbl_toggle;
3303}
3304
3305static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3306{
3307 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3308 struct qedr_qp *qp;
3309
3310 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3311 resp_cqe->qp_handle.lo,
3312 u64);
3313 return qp;
3314}
3315
3316static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3317{
3318 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3319
3320 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3321}
3322
3323/* Return latest CQE (needs processing) */
3324static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3325{
3326 return cq->latest_cqe;
3327}
3328
3329/* In fmr we need to increase the number of fmr completed counter for the fmr
3330 * algorithm determining whether we can free a pbl or not.
3331 * we need to perform this whether the work request was signaled or not. for
3332 * this purpose we call this function from the condition that checks if a wr
3333 * should be skipped, to make sure we don't miss it ( possibly this fmr
3334 * operation was not signalted)
3335 */
3336static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3337{
3338 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3339 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3340}
3341
3342static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3343 struct qedr_cq *cq, int num_entries,
3344 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3345 int force)
3346{
3347 u16 cnt = 0;
3348
3349 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3350 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3351 qedr_chk_if_fmr(qp);
3352 /* skip WC */
3353 goto next_cqe;
3354 }
3355
3356 /* fill WC */
3357 wc->status = status;
27035a1b 3358 wc->vendor_err = 0;
afa0e13b
RA		 3359 wc->wc_flags = 0;
3360 wc->src_qp = qp->id;
3361 wc->qp = &qp->ibqp;
3362
3363 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3364 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3365
3366 switch (wc->opcode) {
3367 case IB_WC_RDMA_WRITE:
3368 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3369 break;
3370 case IB_WC_COMP_SWAP:
3371 case IB_WC_FETCH_ADD:
3372 wc->byte_len = 8;
3373 break;
3374 case IB_WC_REG_MR:
3375 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3376 break;
dac27386
MK		 3377 case IB_WC_RDMA_READ:
3378 case IB_WC_SEND:
3379 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3380 break;
afa0e13b
RA		 3381 default:
3382 break;
3383 }
3384
3385 num_entries--;
3386 wc++;
3387 cnt++;
3388next_cqe:
3389 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3390 qed_chain_consume(&qp->sq.pbl);
3391 qedr_inc_sw_cons(&qp->sq);
3392 }
3393
3394 return cnt;
3395}
3396
3397static int qedr_poll_cq_req(struct qedr_dev *dev,
3398 struct qedr_qp *qp, struct qedr_cq *cq,
3399 int num_entries, struct ib_wc *wc,
3400 struct rdma_cqe_requester *req)
3401{
3402 int cnt = 0;
3403
3404 switch (req->status) {
3405 case RDMA_CQE_REQ_STS_OK:
3406 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3407 IB_WC_SUCCESS, 0);
3408 break;
3409 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149
RA		 3410 if (qp->state != QED_ROCE_QP_STATE_ERR)
3411 DP_ERR(dev,
3412 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3413 cq->icid, qp->icid);
afa0e13b 3414 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3415 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 3416 break;
3417 default:
3418 /* process all WQE before the cosumer */
3419 qp->state = QED_ROCE_QP_STATE_ERR;
3420 cnt = process_req(dev, qp, cq, num_entries, wc,
3421 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3422 wc += cnt;
3423 /* if we have extra WC fill it with actual error info */
3424 if (cnt < num_entries) {
3425 enum ib_wc_status wc_status;
3426
3427 switch (req->status) {
3428 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3429 DP_ERR(dev,
3430 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3431 cq->icid, qp->icid);
3432 wc_status = IB_WC_BAD_RESP_ERR;
3433 break;
3434 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3435 DP_ERR(dev,
3436 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3437 cq->icid, qp->icid);
3438 wc_status = IB_WC_LOC_LEN_ERR;
3439 break;
3440 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3441 DP_ERR(dev,
3442 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3443 cq->icid, qp->icid);
3444 wc_status = IB_WC_LOC_QP_OP_ERR;
3445 break;
3446 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3447 DP_ERR(dev,
3448 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3449 cq->icid, qp->icid);
3450 wc_status = IB_WC_LOC_PROT_ERR;
3451 break;
3452 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3453 DP_ERR(dev,
3454 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3455 cq->icid, qp->icid);
3456 wc_status = IB_WC_MW_BIND_ERR;
3457 break;
3458 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3459 DP_ERR(dev,
3460 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3461 cq->icid, qp->icid);
3462 wc_status = IB_WC_REM_INV_REQ_ERR;
3463 break;
3464 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3465 DP_ERR(dev,
3466 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3467 cq->icid, qp->icid);
3468 wc_status = IB_WC_REM_ACCESS_ERR;
3469 break;
3470 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3471 DP_ERR(dev,
3472 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3473 cq->icid, qp->icid);
3474 wc_status = IB_WC_REM_OP_ERR;
3475 break;
3476 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3477 DP_ERR(dev,
3478 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3479 cq->icid, qp->icid);
3480 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3481 break;
3482 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3483 DP_ERR(dev,
3484 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3485 cq->icid, qp->icid);
3486 wc_status = IB_WC_RETRY_EXC_ERR;
3487 break;
3488 default:
3489 DP_ERR(dev,
3490 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3491 cq->icid, qp->icid);
3492 wc_status = IB_WC_GENERAL_ERR;
3493 }
3494 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3495 wc_status, 1);
3496 }
3497 }
3498
3499 return cnt;
3500}
3501
b6acd71f 3502static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 3503{
b6acd71f 3504 switch (status) {
afa0e13b 3505 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 3506 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 3507 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 3508 return IB_WC_LOC_LEN_ERR;
afa0e13b 3509 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 3510 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 3511 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 3512 return IB_WC_LOC_PROT_ERR;
afa0e13b 3513 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 3514 return IB_WC_MW_BIND_ERR;
afa0e13b 3515 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 3516 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 3517 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 3518 return IB_WC_SUCCESS;
3519 default:
3520 return IB_WC_GENERAL_ERR;
3521 }
3522}
afa0e13b 3523
b6acd71f
AR		 3524static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3525 struct ib_wc *wc)
3526{
3527 wc->status = IB_WC_SUCCESS;
3528 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 3529
b6acd71f
AR		 3530 if (resp->flags & QEDR_RESP_IMM) {
3531 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3532 wc->wc_flags |= IB_WC_WITH_IMM;
3533
3534 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 3535 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3536
b6acd71f
AR		 3537 if (resp->flags & QEDR_RESP_INV)
3538 return -EINVAL;
3539
3540 } else if (resp->flags & QEDR_RESP_INV) {
3541 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3542 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3543
3544 if (resp->flags & QEDR_RESP_RDMA)
3545 return -EINVAL;
3546
3547 } else if (resp->flags & QEDR_RESP_RDMA) {
3548 return -EINVAL;
3549 }
3550
3551 return 0;
3552}
3553
3554static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3555 struct qedr_cq *cq, struct ib_wc *wc,
3556 struct rdma_cqe_responder *resp, u64 wr_id)
3557{
3558 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3559 wc->opcode = IB_WC_RECV;
3560 wc->wc_flags = 0;
3561
3562 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3563 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3564 DP_ERR(dev,
3565 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3566 cq, cq->icid, resp->flags);
3567
3568 } else {
3569 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3570 if (wc->status == IB_WC_GENERAL_ERR)
3571 DP_ERR(dev,
3572 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3573 cq, cq->icid, resp->status);
afa0e13b
RA		 3574 }
3575
b6acd71f 3576 /* Fill the rest of the WC */
27035a1b 3577 wc->vendor_err = 0;
afa0e13b
RA		 3578 wc->src_qp = qp->id;
3579 wc->qp = &qp->ibqp;
3580 wc->wr_id = wr_id;
3581}
3582
3583static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3584 struct qedr_cq *cq, struct ib_wc *wc,
3585 struct rdma_cqe_responder *resp)
3586{
3587 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3588
3589 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3590
3591 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3592 qed_chain_consume(&qp->rq.pbl);
3593 qedr_inc_sw_cons(&qp->rq);
3594
3595 return 1;
3596}
3597
3598static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3599 int num_entries, struct ib_wc *wc, u16 hw_cons)
3600{
3601 u16 cnt = 0;
3602
3603 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3604 /* fill WC */
3605 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 3606 wc->vendor_err = 0;
afa0e13b
RA		 3607 wc->wc_flags = 0;
3608 wc->src_qp = qp->id;
3609 wc->byte_len = 0;
3610 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3611 wc->qp = &qp->ibqp;
3612 num_entries--;
3613 wc++;
3614 cnt++;
3615 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3616 qed_chain_consume(&qp->rq.pbl);
3617 qedr_inc_sw_cons(&qp->rq);
3618 }
3619
3620 return cnt;
3621}
3622
3623static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3624 struct rdma_cqe_responder *resp, int *update)
3625{
3626 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3627 consume_cqe(cq);
3628 *update |= 1;
3629 }
3630}
3631
3632static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3633 struct qedr_cq *cq, int num_entries,
3634 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3635 int *update)
3636{
3637 int cnt;
3638
3639 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3640 cnt = process_resp_flush(qp, cq, num_entries, wc,
3641 resp->rq_cons);
3642 try_consume_resp_cqe(cq, qp, resp, update);
3643 } else {
3644 cnt = process_resp_one(dev, qp, cq, wc, resp);
3645 consume_cqe(cq);
3646 *update |= 1;
3647 }
3648
3649 return cnt;
3650}
3651
3652static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3653 struct rdma_cqe_requester *req, int *update)
3654{
3655 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3656 consume_cqe(cq);
3657 *update |= 1;
3658 }
3659}
3660
3661int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3662{
3663 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3664 struct qedr_cq *cq = get_qedr_cq(ibcq);
3665 union rdma_cqe *cqe = cq->latest_cqe;
3666 u32 old_cons, new_cons;
3667 unsigned long flags;
3668 int update = 0;
3669 int done = 0;
3670
4dd72636
AR		 3671 if (cq->destroyed) {
3672 DP_ERR(dev,
3673 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
3674 cq, cq->icid);
3675 return 0;
3676 }
3677
04886779
RA		 3678 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3679 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3680
afa0e13b
RA		 3681 spin_lock_irqsave(&cq->cq_lock, flags);
3682 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3683 while (num_entries && is_valid_cqe(cq, cqe)) {
3684 struct qedr_qp *qp;
3685 int cnt = 0;
3686
3687 /* prevent speculative reads of any field of CQE */
3688 rmb();
3689
3690 qp = cqe_get_qp(cqe);
3691 if (!qp) {
3692 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3693 break;
3694 }
3695
3696 wc->qp = &qp->ibqp;
3697
3698 switch (cqe_get_type(cqe)) {
3699 case RDMA_CQE_TYPE_REQUESTER:
3700 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3701 &cqe->req);
3702 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3703 break;
3704 case RDMA_CQE_TYPE_RESPONDER_RQ:
3705 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3706 &cqe->resp, &update);
3707 break;
3708 case RDMA_CQE_TYPE_INVALID:
3709 default:
3710 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3711 cqe_get_type(cqe));
3712 }
3713 num_entries -= cnt;
3714 wc += cnt;
3715 done += cnt;
3716
3717 cqe = get_cqe(cq);
3718 }
3719 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3720
3721 cq->cq_cons += new_cons - old_cons;
3722
3723 if (update)
3724 /* doorbell notifies abount latest VALID entry,
3725 * but chain already point to the next INVALID one
3726 */
3727 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3728
3729 spin_unlock_irqrestore(&cq->cq_lock, flags);
3730 return done;
3731}
993d1b52
RA		 3732
3733int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3734 u8 port_num,
3735 const struct ib_wc *in_wc,
3736 const struct ib_grh *in_grh,
3737 const struct ib_mad_hdr *mad_hdr,
3738 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3739 size_t *out_mad_size, u16 *out_mad_pkey_index)
3740{
3741 struct qedr_dev *dev = get_qedr_dev(ibdev);
3742
3743 DP_DEBUG(dev, QEDR_MSG_GSI,
3744 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3745 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3746 mad_hdr->class_specific, mad_hdr->class_version,
3747 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3748 return IB_MAD_RESULT_SUCCESS;
3749}
Linux 6.x block layer and io_uring tree(s)