projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
qedr: Add support for QP verbs
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45
46#include "qedr_hsi.h"
47#include <linux/qed/qed_if.h>
48#include "qedr.h"
49#include "verbs.h"
50#include <rdma/qedr-abi.h>
cecbcddf 51#include "qedr_cm.h"
ac1b36e5 52
a7efd777
RA		 53#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56{
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62}
63
ac1b36e5
RA		 64int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66{
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83}
84
85int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88{
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99}
100
101int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103{
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114}
115
116int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118{
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170}
171
172#define QEDR_SPEED_SDR (1)
173#define QEDR_SPEED_DDR (2)
174#define QEDR_SPEED_QDR (4)
175#define QEDR_SPEED_FDR10 (8)
176#define QEDR_SPEED_FDR (16)
177#define QEDR_SPEED_EDR (32)
178
179static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181{
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222}
223
224int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225{
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241 memset(attr, 0, sizeof(*attr));
242
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267}
268
269int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271{
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281}
282
283static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285{
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313}
314
315static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317{
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335}
336
337struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339{
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397}
398
399int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400{
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419}
420
421int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422{
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468}
a7efd777
RA		 469
470struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472{
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
474 struct qedr_ucontext *uctx = NULL;
475 struct qedr_alloc_pd_uresp uresp;
476 struct qedr_pd *pd;
477 u16 pd_id;
478 int rc;
479
480 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
481 (udata && context) ? "User Lib" : "Kernel");
482
483 if (!dev->rdma_ctx) {
484 DP_ERR(dev, "invlaid RDMA context\n");
485 return ERR_PTR(-EINVAL);
486 }
487
488 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
489 if (!pd)
490 return ERR_PTR(-ENOMEM);
491
492 dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
493
494 uresp.pd_id = pd_id;
495 pd->pd_id = pd_id;
496
497 if (udata && context) {
498 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
499 if (rc)
500 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
501 uctx = get_qedr_ucontext(context);
502 uctx->pd = pd;
503 pd->uctx = uctx;
504 }
505
506 return &pd->ibpd;
507}
508
509int qedr_dealloc_pd(struct ib_pd *ibpd)
510{
511 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
512 struct qedr_pd *pd = get_qedr_pd(ibpd);
513
514 if (!pd)
515 pr_err("Invalid PD received in dealloc_pd\n");
516
517 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
518 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
519
520 kfree(pd);
521
522 return 0;
523}
524
525static void qedr_free_pbl(struct qedr_dev *dev,
526 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
527{
528 struct pci_dev *pdev = dev->pdev;
529 int i;
530
531 for (i = 0; i < pbl_info->num_pbls; i++) {
532 if (!pbl[i].va)
533 continue;
534 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
535 pbl[i].va, pbl[i].pa);
536 }
537
538 kfree(pbl);
539}
540
541#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
542#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
543
544#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
545#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
546#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
547
548static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
549 struct qedr_pbl_info *pbl_info,
550 gfp_t flags)
551{
552 struct pci_dev *pdev = dev->pdev;
553 struct qedr_pbl *pbl_table;
554 dma_addr_t *pbl_main_tbl;
555 dma_addr_t pa;
556 void *va;
557 int i;
558
559 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
560 if (!pbl_table)
561 return ERR_PTR(-ENOMEM);
562
563 for (i = 0; i < pbl_info->num_pbls; i++) {
564 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
565 &pa, flags);
566 if (!va)
567 goto err;
568
569 memset(va, 0, pbl_info->pbl_size);
570 pbl_table[i].va = va;
571 pbl_table[i].pa = pa;
572 }
573
574 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
575 * the first one with physical pointers to all of the rest
576 */
577 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
578 for (i = 0; i < pbl_info->num_pbls - 1; i++)
579 pbl_main_tbl[i] = pbl_table[i + 1].pa;
580
581 return pbl_table;
582
583err:
584 for (i--; i >= 0; i--)
585 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
586 pbl_table[i].va, pbl_table[i].pa);
587
588 qedr_free_pbl(dev, pbl_info, pbl_table);
589
590 return ERR_PTR(-ENOMEM);
591}
592
593static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
594 struct qedr_pbl_info *pbl_info,
595 u32 num_pbes, int two_layer_capable)
596{
597 u32 pbl_capacity;
598 u32 pbl_size;
599 u32 num_pbls;
600
601 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
602 if (num_pbes > MAX_PBES_TWO_LAYER) {
603 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
604 num_pbes);
605 return -EINVAL;
606 }
607
608 /* calculate required pbl page size */
609 pbl_size = MIN_FW_PBL_PAGE_SIZE;
610 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
611 NUM_PBES_ON_PAGE(pbl_size);
612
613 while (pbl_capacity < num_pbes) {
614 pbl_size *= 2;
615 pbl_capacity = pbl_size / sizeof(u64);
616 pbl_capacity = pbl_capacity * pbl_capacity;
617 }
618
619 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
620 num_pbls++; /* One for the layer0 ( points to the pbls) */
621 pbl_info->two_layered = true;
622 } else {
623 /* One layered PBL */
624 num_pbls = 1;
625 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
626 roundup_pow_of_two((num_pbes * sizeof(u64))));
627 pbl_info->two_layered = false;
628 }
629
630 pbl_info->num_pbls = num_pbls;
631 pbl_info->pbl_size = pbl_size;
632 pbl_info->num_pbes = num_pbes;
633
634 DP_DEBUG(dev, QEDR_MSG_MR,
635 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
636 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
637
638 return 0;
639}
640
641static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
642 struct qedr_pbl *pbl,
643 struct qedr_pbl_info *pbl_info)
644{
645 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
646 struct qedr_pbl *pbl_tbl;
647 struct scatterlist *sg;
648 struct regpair *pbe;
649 int entry;
650 u32 addr;
651
652 if (!pbl_info->num_pbes)
653 return;
654
655 /* If we have a two layered pbl, the first pbl points to the rest
656 * of the pbls and the first entry lays on the second pbl in the table
657 */
658 if (pbl_info->two_layered)
659 pbl_tbl = &pbl[1];
660 else
661 pbl_tbl = pbl;
662
663 pbe = (struct regpair *)pbl_tbl->va;
664 if (!pbe) {
665 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
666 return;
667 }
668
669 pbe_cnt = 0;
670
671 shift = ilog2(umem->page_size);
672
673 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
674 pages = sg_dma_len(sg) >> shift;
675 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
676 /* store the page address in pbe */
677 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
678 umem->page_size * pg_cnt);
679 addr = upper_32_bits(sg_dma_address(sg) +
680 umem->page_size * pg_cnt);
681 pbe->hi = cpu_to_le32(addr);
682 pbe_cnt++;
683 total_num_pbes++;
684 pbe++;
685
686 if (total_num_pbes == pbl_info->num_pbes)
687 return;
688
689 /* If the given pbl is full storing the pbes,
690 * move to next pbl.
691 */
692 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
693 pbl_tbl++;
694 pbe = (struct regpair *)pbl_tbl->va;
695 pbe_cnt = 0;
696 }
697 }
698 }
699}
700
701static int qedr_copy_cq_uresp(struct qedr_dev *dev,
702 struct qedr_cq *cq, struct ib_udata *udata)
703{
704 struct qedr_create_cq_uresp uresp;
705 int rc;
706
707 memset(&uresp, 0, sizeof(uresp));
708
709 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
710 uresp.icid = cq->icid;
711
712 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
713 if (rc)
714 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
715
716 return rc;
717}
718
719static void consume_cqe(struct qedr_cq *cq)
720{
721 if (cq->latest_cqe == cq->toggle_cqe)
722 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
723
724 cq->latest_cqe = qed_chain_consume(&cq->pbl);
725}
726
727static inline int qedr_align_cq_entries(int entries)
728{
729 u64 size, aligned_size;
730
731 /* We allocate an extra entry that we don't report to the FW. */
732 size = (entries + 1) * QEDR_CQE_SIZE;
733 aligned_size = ALIGN(size, PAGE_SIZE);
734
735 return aligned_size / QEDR_CQE_SIZE;
736}
737
738static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
739 struct qedr_dev *dev,
740 struct qedr_userq *q,
741 u64 buf_addr, size_t buf_len,
742 int access, int dmasync)
743{
744 int page_cnt;
745 int rc;
746
747 q->buf_addr = buf_addr;
748 q->buf_len = buf_len;
749 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
750 if (IS_ERR(q->umem)) {
751 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
752 PTR_ERR(q->umem));
753 return PTR_ERR(q->umem);
754 }
755
756 page_cnt = ib_umem_page_count(q->umem);
757 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
758 if (rc)
759 goto err0;
760
761 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
762 if (IS_ERR_OR_NULL(q->pbl_tbl))
763 goto err0;
764
765 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
766
767 return 0;
768
769err0:
770 ib_umem_release(q->umem);
771
772 return rc;
773}
774
775static inline void qedr_init_cq_params(struct qedr_cq *cq,
776 struct qedr_ucontext *ctx,
777 struct qedr_dev *dev, int vector,
778 int chain_entries, int page_cnt,
779 u64 pbl_ptr,
780 struct qed_rdma_create_cq_in_params
781 *params)
782{
783 memset(params, 0, sizeof(*params));
784 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
785 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
786 params->cnq_id = vector;
787 params->cq_size = chain_entries - 1;
788 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
789 params->pbl_num_pages = page_cnt;
790 params->pbl_ptr = pbl_ptr;
791 params->pbl_two_level = 0;
792}
793
794static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
795{
796 /* Flush data before signalling doorbell */
797 wmb();
798 cq->db.data.agg_flags = flags;
799 cq->db.data.value = cpu_to_le32(cons);
800 writeq(cq->db.raw, cq->db_addr);
801
802 /* Make sure write would stick */
803 mmiowb();
804}
805
806int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
807{
808 struct qedr_cq *cq = get_qedr_cq(ibcq);
809 unsigned long sflags;
810
811 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
812 return 0;
813
814 spin_lock_irqsave(&cq->cq_lock, sflags);
815
816 cq->arm_flags = 0;
817
818 if (flags & IB_CQ_SOLICITED)
819 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
820
821 if (flags & IB_CQ_NEXT_COMP)
822 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
823
824 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
825
826 spin_unlock_irqrestore(&cq->cq_lock, sflags);
827
828 return 0;
829}
830
831struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
832 const struct ib_cq_init_attr *attr,
833 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
834{
835 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
836 struct qed_rdma_destroy_cq_out_params destroy_oparams;
837 struct qed_rdma_destroy_cq_in_params destroy_iparams;
838 struct qedr_dev *dev = get_qedr_dev(ibdev);
839 struct qed_rdma_create_cq_in_params params;
840 struct qedr_create_cq_ureq ureq;
841 int vector = attr->comp_vector;
842 int entries = attr->cqe;
843 struct qedr_cq *cq;
844 int chain_entries;
845 int page_cnt;
846 u64 pbl_ptr;
847 u16 icid;
848 int rc;
849
850 DP_DEBUG(dev, QEDR_MSG_INIT,
851 "create_cq: called from %s. entries=%d, vector=%d\n",
852 udata ? "User Lib" : "Kernel", entries, vector);
853
854 if (entries > QEDR_MAX_CQES) {
855 DP_ERR(dev,
856 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
857 entries, QEDR_MAX_CQES);
858 return ERR_PTR(-EINVAL);
859 }
860
861 chain_entries = qedr_align_cq_entries(entries);
862 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
863
864 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
865 if (!cq)
866 return ERR_PTR(-ENOMEM);
867
868 if (udata) {
869 memset(&ureq, 0, sizeof(ureq));
870 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
871 DP_ERR(dev,
872 "create cq: problem copying data from user space\n");
873 goto err0;
874 }
875
876 if (!ureq.len) {
877 DP_ERR(dev,
878 "create cq: cannot create a cq with 0 entries\n");
879 goto err0;
880 }
881
882 cq->cq_type = QEDR_CQ_TYPE_USER;
883
884 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
885 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
886 if (rc)
887 goto err0;
888
889 pbl_ptr = cq->q.pbl_tbl->pa;
890 page_cnt = cq->q.pbl_info.num_pbes;
891 } else {
892 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
893
894 rc = dev->ops->common->chain_alloc(dev->cdev,
895 QED_CHAIN_USE_TO_CONSUME,
896 QED_CHAIN_MODE_PBL,
897 QED_CHAIN_CNT_TYPE_U32,
898 chain_entries,
899 sizeof(union rdma_cqe),
900 &cq->pbl);
901 if (rc)
902 goto err1;
903
904 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
905 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
906 }
907
908 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
909 pbl_ptr, &params);
910
911 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
912 if (rc)
913 goto err2;
914
915 cq->icid = icid;
916 cq->sig = QEDR_CQ_MAGIC_NUMBER;
917 spin_lock_init(&cq->cq_lock);
918
919 if (ib_ctx) {
920 rc = qedr_copy_cq_uresp(dev, cq, udata);
921 if (rc)
922 goto err3;
923 } else {
924 /* Generate doorbell address. */
925 cq->db_addr = dev->db_addr +
926 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
927 cq->db.data.icid = cq->icid;
928 cq->db.data.params = DB_AGG_CMD_SET <<
929 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
930
931 /* point to the very last element, passing it we will toggle */
932 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
933 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
934 cq->latest_cqe = NULL;
935 consume_cqe(cq);
936 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
937 }
938
939 DP_DEBUG(dev, QEDR_MSG_CQ,
940 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
941 cq->icid, cq, params.cq_size);
942
943 return &cq->ibcq;
944
945err3:
946 destroy_iparams.icid = cq->icid;
947 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
948 &destroy_oparams);
949err2:
950 if (udata)
951 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
952 else
953 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
954err1:
955 if (udata)
956 ib_umem_release(cq->q.umem);
957err0:
958 kfree(cq);
959 return ERR_PTR(-EINVAL);
960}
961
962int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
963{
964 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
965 struct qedr_cq *cq = get_qedr_cq(ibcq);
966
967 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
968
969 return 0;
970}
971
972int qedr_destroy_cq(struct ib_cq *ibcq)
973{
974 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
975 struct qed_rdma_destroy_cq_out_params oparams;
976 struct qed_rdma_destroy_cq_in_params iparams;
977 struct qedr_cq *cq = get_qedr_cq(ibcq);
978
979 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
980
981 /* GSIs CQs are handled by driver, so they don't exist in the FW */
982 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
983 iparams.icid = cq->icid;
984 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
985 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
986 }
987
988 if (ibcq->uobject && ibcq->uobject->context) {
989 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
990 ib_umem_release(cq->q.umem);
991 }
992
993 kfree(cq);
994
995 return 0;
996}
cecbcddf
RA		 997
998static inline int get_gid_info_from_table(struct ib_qp *ibqp,
999 struct ib_qp_attr *attr,
1000 int attr_mask,
1001 struct qed_rdma_modify_qp_in_params
1002 *qp_params)
1003{
1004 enum rdma_network_type nw_type;
1005 struct ib_gid_attr gid_attr;
1006 union ib_gid gid;
1007 u32 ipv4_addr;
1008 int rc = 0;
1009 int i;
1010
1011 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1012 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1013 if (rc)
1014 return rc;
1015
1016 if (!memcmp(&gid, &zgid, sizeof(gid)))
1017 return -ENOENT;
1018
1019 if (gid_attr.ndev) {
1020 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1021
1022 dev_put(gid_attr.ndev);
1023 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1024 switch (nw_type) {
1025 case RDMA_NETWORK_IPV6:
1026 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1027 sizeof(qp_params->sgid));
1028 memcpy(&qp_params->dgid.bytes[0],
1029 &attr->ah_attr.grh.dgid,
1030 sizeof(qp_params->dgid));
1031 qp_params->roce_mode = ROCE_V2_IPV6;
1032 SET_FIELD(qp_params->modify_flags,
1033 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1034 break;
1035 case RDMA_NETWORK_IB:
1036 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1037 sizeof(qp_params->sgid));
1038 memcpy(&qp_params->dgid.bytes[0],
1039 &attr->ah_attr.grh.dgid,
1040 sizeof(qp_params->dgid));
1041 qp_params->roce_mode = ROCE_V1;
1042 break;
1043 case RDMA_NETWORK_IPV4:
1044 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1045 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1046 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1047 qp_params->sgid.ipv4_addr = ipv4_addr;
1048 ipv4_addr =
1049 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1050 qp_params->dgid.ipv4_addr = ipv4_addr;
1051 SET_FIELD(qp_params->modify_flags,
1052 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1053 qp_params->roce_mode = ROCE_V2_IPV4;
1054 break;
1055 }
1056 }
1057
1058 for (i = 0; i < 4; i++) {
1059 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1060 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1061 }
1062
1063 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1064 qp_params->vlan_id = 0;
1065
1066 return 0;
1067}
1068
1069static void qedr_cleanup_user_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1070{
1071 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1072 ib_umem_release(qp->usq.umem);
1073}
1074
1075static void qedr_cleanup_user_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1076{
1077 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1078 ib_umem_release(qp->urq.umem);
1079}
1080
1081static void qedr_cleanup_kernel_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1082{
1083 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1084 kfree(qp->wqe_wr_id);
1085}
1086
1087static void qedr_cleanup_kernel_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1088{
1089 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1090 kfree(qp->rqe_wr_id);
1091}
1092
1093static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1094 struct ib_qp_init_attr *attrs)
1095{
1096 struct qedr_device_attr *qattr = &dev->attr;
1097
1098 /* QP0... attrs->qp_type == IB_QPT_GSI */
1099 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1100 DP_DEBUG(dev, QEDR_MSG_QP,
1101 "create qp: unsupported qp type=0x%x requested\n",
1102 attrs->qp_type);
1103 return -EINVAL;
1104 }
1105
1106 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1107 DP_ERR(dev,
1108 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1109 attrs->cap.max_send_wr, qattr->max_sqe);
1110 return -EINVAL;
1111 }
1112
1113 if (attrs->cap.max_inline_data > qattr->max_inline) {
1114 DP_ERR(dev,
1115 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1116 attrs->cap.max_inline_data, qattr->max_inline);
1117 return -EINVAL;
1118 }
1119
1120 if (attrs->cap.max_send_sge > qattr->max_sge) {
1121 DP_ERR(dev,
1122 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1123 attrs->cap.max_send_sge, qattr->max_sge);
1124 return -EINVAL;
1125 }
1126
1127 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1128 DP_ERR(dev,
1129 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1130 attrs->cap.max_recv_sge, qattr->max_sge);
1131 return -EINVAL;
1132 }
1133
1134 /* Unprivileged user space cannot create special QP */
1135 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1136 DP_ERR(dev,
1137 "create qp: userspace can't create special QPs of type=0x%x\n",
1138 attrs->qp_type);
1139 return -EINVAL;
1140 }
1141
1142 return 0;
1143}
1144
1145static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1146 struct qedr_qp *qp)
1147{
1148 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1149 uresp->rq_icid = qp->icid;
1150}
1151
1152static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1153 struct qedr_qp *qp)
1154{
1155 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1156 uresp->sq_icid = qp->icid + 1;
1157}
1158
1159static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1160 struct qedr_qp *qp, struct ib_udata *udata)
1161{
1162 struct qedr_create_qp_uresp uresp;
1163 int rc;
1164
1165 memset(&uresp, 0, sizeof(uresp));
1166 qedr_copy_sq_uresp(&uresp, qp);
1167 qedr_copy_rq_uresp(&uresp, qp);
1168
1169 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1170 uresp.qp_id = qp->qp_id;
1171
1172 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1173 if (rc)
1174 DP_ERR(dev,
1175 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1176 qp->icid);
1177
1178 return rc;
1179}
1180
1181static void qedr_set_qp_init_params(struct qedr_dev *dev,
1182 struct qedr_qp *qp,
1183 struct qedr_pd *pd,
1184 struct ib_qp_init_attr *attrs)
1185{
1186 qp->pd = pd;
1187
1188 spin_lock_init(&qp->q_lock);
1189
1190 qp->qp_type = attrs->qp_type;
1191 qp->max_inline_data = attrs->cap.max_inline_data;
1192 qp->sq.max_sges = attrs->cap.max_send_sge;
1193 qp->state = QED_ROCE_QP_STATE_RESET;
1194 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1195 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1196 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1197 qp->dev = dev;
1198
1199 DP_DEBUG(dev, QEDR_MSG_QP,
1200 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1201 pd->pd_id, qp->qp_type, qp->max_inline_data,
1202 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1203 DP_DEBUG(dev, QEDR_MSG_QP,
1204 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1205 qp->sq.max_sges, qp->sq_cq->icid);
1206 qp->rq.max_sges = attrs->cap.max_recv_sge;
1207 DP_DEBUG(dev, QEDR_MSG_QP,
1208 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1209 qp->rq.max_sges, qp->rq_cq->icid);
1210}
1211
1212static inline void
1213qedr_init_qp_user_params(struct qed_rdma_create_qp_in_params *params,
1214 struct qedr_create_qp_ureq *ureq)
1215{
1216 /* QP handle to be written in CQE */
1217 params->qp_handle_lo = ureq->qp_handle_lo;
1218 params->qp_handle_hi = ureq->qp_handle_hi;
1219}
1220
1221static inline void
1222qedr_init_qp_kernel_doorbell_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1223{
1224 qp->sq.db = dev->db_addr +
1225 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1226 qp->sq.db_data.data.icid = qp->icid + 1;
1227}
1228
1229static inline void
1230qedr_init_qp_kernel_doorbell_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1231{
1232 qp->rq.db = dev->db_addr +
1233 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1234 qp->rq.db_data.data.icid = qp->icid;
1235}
1236
1237static inline int
1238qedr_init_qp_kernel_params_rq(struct qedr_dev *dev,
1239 struct qedr_qp *qp, struct ib_qp_init_attr *attrs)
1240{
1241 /* Allocate driver internal RQ array */
1242 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1243 GFP_KERNEL);
1244 if (!qp->rqe_wr_id)
1245 return -ENOMEM;
1246
1247 DP_DEBUG(dev, QEDR_MSG_QP, "RQ max_wr set to %d.\n", qp->rq.max_wr);
1248
1249 return 0;
1250}
1251
1252static inline int
1253qedr_init_qp_kernel_params_sq(struct qedr_dev *dev,
1254 struct qedr_qp *qp,
1255 struct ib_qp_init_attr *attrs,
1256 struct qed_rdma_create_qp_in_params *params)
1257{
1258 u32 temp_max_wr;
1259
1260 /* Allocate driver internal SQ array */
1261 temp_max_wr = attrs->cap.max_send_wr * dev->wq_multiplier;
1262 temp_max_wr = min_t(u32, temp_max_wr, dev->attr.max_sqe);
1263
1264 /* temp_max_wr < attr->max_sqe < u16 so the casting is safe */
1265 qp->sq.max_wr = (u16)temp_max_wr;
1266 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1267 GFP_KERNEL);
1268 if (!qp->wqe_wr_id)
1269 return -ENOMEM;
1270
1271 DP_DEBUG(dev, QEDR_MSG_QP, "SQ max_wr set to %d.\n", qp->sq.max_wr);
1272
1273 /* QP handle to be written in CQE */
1274 params->qp_handle_lo = lower_32_bits((uintptr_t)qp);
1275 params->qp_handle_hi = upper_32_bits((uintptr_t)qp);
1276
1277 return 0;
1278}
1279
1280static inline int qedr_init_qp_kernel_sq(struct qedr_dev *dev,
1281 struct qedr_qp *qp,
1282 struct ib_qp_init_attr *attrs)
1283{
1284 u32 n_sq_elems, n_sq_entries;
1285 int rc;
1286
1287 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1288 * the ring. The ring should allow at least a single WR, even if the
1289 * user requested none, due to allocation issues.
1290 */
1291 n_sq_entries = attrs->cap.max_send_wr;
1292 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1293 n_sq_entries = max_t(u32, n_sq_entries, 1);
1294 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1295 rc = dev->ops->common->chain_alloc(dev->cdev,
1296 QED_CHAIN_USE_TO_PRODUCE,
1297 QED_CHAIN_MODE_PBL,
1298 QED_CHAIN_CNT_TYPE_U32,
1299 n_sq_elems,
1300 QEDR_SQE_ELEMENT_SIZE,
1301 &qp->sq.pbl);
1302 if (rc) {
1303 DP_ERR(dev, "failed to allocate QP %p SQ\n", qp);
1304 return rc;
1305 }
1306
1307 DP_DEBUG(dev, QEDR_MSG_SQ,
1308 "SQ Pbl base addr = %llx max_send_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1309 qed_chain_get_pbl_phys(&qp->sq.pbl), attrs->cap.max_send_wr,
1310 n_sq_entries, qed_chain_get_capacity(&qp->sq.pbl), rc);
1311 return 0;
1312}
1313
1314static inline int qedr_init_qp_kernel_rq(struct qedr_dev *dev,
1315 struct qedr_qp *qp,
1316 struct ib_qp_init_attr *attrs)
1317{
1318 u32 n_rq_elems, n_rq_entries;
1319 int rc;
1320
1321 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1322 * the ring. There ring should allow at least a single WR, even if the
1323 * user requested none, due to allocation issues.
1324 */
1325 n_rq_entries = max_t(u32, attrs->cap.max_recv_wr, 1);
1326 n_rq_elems = n_rq_entries * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1327 rc = dev->ops->common->chain_alloc(dev->cdev,
1328 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1329 QED_CHAIN_MODE_PBL,
1330 QED_CHAIN_CNT_TYPE_U32,
1331 n_rq_elems,
1332 QEDR_RQE_ELEMENT_SIZE,
1333 &qp->rq.pbl);
1334
1335 if (rc) {
1336 DP_ERR(dev, "failed to allocate memory for QP %p RQ\n", qp);
1337 return -ENOMEM;
1338 }
1339
1340 DP_DEBUG(dev, QEDR_MSG_RQ,
1341 "RQ Pbl base addr = %llx max_recv_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1342 qed_chain_get_pbl_phys(&qp->rq.pbl), attrs->cap.max_recv_wr,
1343 n_rq_entries, qed_chain_get_capacity(&qp->rq.pbl), rc);
1344
1345 /* n_rq_entries < u16 so the casting is safe */
1346 qp->rq.max_wr = (u16)n_rq_entries;
1347
1348 return 0;
1349}
1350
1351static inline void
1352qedr_init_qp_in_params_sq(struct qedr_dev *dev,
1353 struct qedr_pd *pd,
1354 struct qedr_qp *qp,
1355 struct ib_qp_init_attr *attrs,
1356 struct ib_udata *udata,
1357 struct qed_rdma_create_qp_in_params *params)
1358{
1359 /* QP handle to be written in an async event */
1360 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
1361 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
1362
1363 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1364 params->fmr_and_reserved_lkey = !udata;
1365 params->pd = pd->pd_id;
1366 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1367 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1368 params->max_sq_sges = 0;
1369 params->stats_queue = 0;
1370
1371 if (udata) {
1372 params->sq_num_pages = qp->usq.pbl_info.num_pbes;
1373 params->sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1374 } else {
1375 params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1376 params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1377 }
1378}
1379
1380static inline void
1381qedr_init_qp_in_params_rq(struct qedr_qp *qp,
1382 struct ib_qp_init_attr *attrs,
1383 struct ib_udata *udata,
1384 struct qed_rdma_create_qp_in_params *params)
1385{
1386 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1387 params->srq_id = 0;
1388 params->use_srq = false;
1389
1390 if (udata) {
1391 params->rq_num_pages = qp->urq.pbl_info.num_pbes;
1392 params->rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1393 } else {
1394 params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1395 params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1396 }
1397}
1398
1399static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1400{
1401 DP_DEBUG(dev, QEDR_MSG_QP,
1402 "create qp: successfully created user QP. qp=%p, sq_addr=0x%llx, sq_len=%zd, rq_addr=0x%llx, rq_len=%zd\n",
1403 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
1404 qp->urq.buf_len);
1405}
1406
1407static inline int qedr_init_user_qp(struct ib_ucontext *ib_ctx,
1408 struct qedr_dev *dev,
1409 struct qedr_qp *qp,
1410 struct qedr_create_qp_ureq *ureq)
1411{
1412 int rc;
1413
1414 /* SQ - read access only (0), dma sync not required (0) */
1415 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq->sq_addr,
1416 ureq->sq_len, 0, 0);
1417 if (rc)
1418 return rc;
1419
1420 /* RQ - read access only (0), dma sync not required (0) */
1421 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq->rq_addr,
1422 ureq->rq_len, 0, 0);
1423
1424 if (rc)
1425 qedr_cleanup_user_sq(dev, qp);
1426 return rc;
1427}
1428
1429static inline int
1430qedr_init_kernel_qp(struct qedr_dev *dev,
1431 struct qedr_qp *qp,
1432 struct ib_qp_init_attr *attrs,
1433 struct qed_rdma_create_qp_in_params *params)
1434{
1435 int rc;
1436
1437 rc = qedr_init_qp_kernel_sq(dev, qp, attrs);
1438 if (rc) {
1439 DP_ERR(dev, "failed to init kernel QP %p SQ\n", qp);
1440 return rc;
1441 }
1442
1443 rc = qedr_init_qp_kernel_params_sq(dev, qp, attrs, params);
1444 if (rc) {
1445 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1446 DP_ERR(dev, "failed to init kernel QP %p SQ params\n", qp);
1447 return rc;
1448 }
1449
1450 rc = qedr_init_qp_kernel_rq(dev, qp, attrs);
1451 if (rc) {
1452 qedr_cleanup_kernel_sq(dev, qp);
1453 DP_ERR(dev, "failed to init kernel QP %p RQ\n", qp);
1454 return rc;
1455 }
1456
1457 rc = qedr_init_qp_kernel_params_rq(dev, qp, attrs);
1458 if (rc) {
1459 DP_ERR(dev, "failed to init kernel QP %p RQ params\n", qp);
1460 qedr_cleanup_kernel_sq(dev, qp);
1461 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1462 return rc;
1463 }
1464
1465 return rc;
1466}
1467
1468struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1469 struct ib_qp_init_attr *attrs,
1470 struct ib_udata *udata)
1471{
1472 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1473 struct qed_rdma_create_qp_out_params out_params;
1474 struct qed_rdma_create_qp_in_params in_params;
1475 struct qedr_pd *pd = get_qedr_pd(ibpd);
1476 struct ib_ucontext *ib_ctx = NULL;
1477 struct qedr_ucontext *ctx = NULL;
1478 struct qedr_create_qp_ureq ureq;
1479 struct qedr_qp *qp;
1480 int rc = 0;
1481
1482 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1483 udata ? "user library" : "kernel", pd);
1484
1485 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1486 if (rc)
1487 return ERR_PTR(rc);
1488
1489 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1490 if (!qp)
1491 return ERR_PTR(-ENOMEM);
1492
1493 if (attrs->srq)
1494 return ERR_PTR(-EINVAL);
1495
1496 DP_DEBUG(dev, QEDR_MSG_QP,
1497 "create qp: sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1498 get_qedr_cq(attrs->send_cq),
1499 get_qedr_cq(attrs->send_cq)->icid,
1500 get_qedr_cq(attrs->recv_cq),
1501 get_qedr_cq(attrs->recv_cq)->icid);
1502
1503 qedr_set_qp_init_params(dev, qp, pd, attrs);
1504
1505 memset(&in_params, 0, sizeof(in_params));
1506
1507 if (udata) {
1508 if (!(udata && ibpd->uobject && ibpd->uobject->context))
1509 goto err0;
1510
1511 ib_ctx = ibpd->uobject->context;
1512 ctx = get_qedr_ucontext(ib_ctx);
1513
1514 memset(&ureq, 0, sizeof(ureq));
1515 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1516 DP_ERR(dev,
1517 "create qp: problem copying data from user space\n");
1518 goto err0;
1519 }
1520
1521 rc = qedr_init_user_qp(ib_ctx, dev, qp, &ureq);
1522 if (rc)
1523 goto err0;
1524
1525 qedr_init_qp_user_params(&in_params, &ureq);
1526 } else {
1527 rc = qedr_init_kernel_qp(dev, qp, attrs, &in_params);
1528 if (rc)
1529 goto err0;
1530 }
1531
1532 qedr_init_qp_in_params_sq(dev, pd, qp, attrs, udata, &in_params);
1533 qedr_init_qp_in_params_rq(qp, attrs, udata, &in_params);
1534
1535 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1536 &in_params, &out_params);
1537
1538 if (!qp->qed_qp)
1539 goto err1;
1540
1541 qp->qp_id = out_params.qp_id;
1542 qp->icid = out_params.icid;
1543 qp->ibqp.qp_num = qp->qp_id;
1544
1545 if (udata) {
1546 rc = qedr_copy_qp_uresp(dev, qp, udata);
1547 if (rc)
1548 goto err2;
1549
1550 qedr_qp_user_print(dev, qp);
1551 } else {
1552 qedr_init_qp_kernel_doorbell_sq(dev, qp);
1553 qedr_init_qp_kernel_doorbell_rq(dev, qp);
1554 }
1555
1556 DP_DEBUG(dev, QEDR_MSG_QP, "created %s space QP %p\n",
1557 udata ? "user" : "kernel", qp);
1558
1559 return &qp->ibqp;
1560
1561err2:
1562 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1563 if (rc)
1564 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1565err1:
1566 if (udata) {
1567 qedr_cleanup_user_sq(dev, qp);
1568 qedr_cleanup_user_rq(dev, qp);
1569 } else {
1570 qedr_cleanup_kernel_sq(dev, qp);
1571 qedr_cleanup_kernel_rq(dev, qp);
1572 }
1573
1574err0:
1575 kfree(qp);
1576
1577 return ERR_PTR(-EFAULT);
1578}
1579
1580enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1581{
1582 switch (qp_state) {
1583 case QED_ROCE_QP_STATE_RESET:
1584 return IB_QPS_RESET;
1585 case QED_ROCE_QP_STATE_INIT:
1586 return IB_QPS_INIT;
1587 case QED_ROCE_QP_STATE_RTR:
1588 return IB_QPS_RTR;
1589 case QED_ROCE_QP_STATE_RTS:
1590 return IB_QPS_RTS;
1591 case QED_ROCE_QP_STATE_SQD:
1592 return IB_QPS_SQD;
1593 case QED_ROCE_QP_STATE_ERR:
1594 return IB_QPS_ERR;
1595 case QED_ROCE_QP_STATE_SQE:
1596 return IB_QPS_SQE;
1597 }
1598 return IB_QPS_ERR;
1599}
1600
1601enum qed_roce_qp_state qedr_get_state_from_ibqp(enum ib_qp_state qp_state)
1602{
1603 switch (qp_state) {
1604 case IB_QPS_RESET:
1605 return QED_ROCE_QP_STATE_RESET;
1606 case IB_QPS_INIT:
1607 return QED_ROCE_QP_STATE_INIT;
1608 case IB_QPS_RTR:
1609 return QED_ROCE_QP_STATE_RTR;
1610 case IB_QPS_RTS:
1611 return QED_ROCE_QP_STATE_RTS;
1612 case IB_QPS_SQD:
1613 return QED_ROCE_QP_STATE_SQD;
1614 case IB_QPS_ERR:
1615 return QED_ROCE_QP_STATE_ERR;
1616 default:
1617 return QED_ROCE_QP_STATE_ERR;
1618 }
1619}
1620
1621static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1622{
1623 qed_chain_reset(&qph->pbl);
1624 qph->prod = 0;
1625 qph->cons = 0;
1626 qph->wqe_cons = 0;
1627 qph->db_data.data.value = cpu_to_le16(0);
1628}
1629
1630static int qedr_update_qp_state(struct qedr_dev *dev,
1631 struct qedr_qp *qp,
1632 enum qed_roce_qp_state new_state)
1633{
1634 int status = 0;
1635
1636 if (new_state == qp->state)
1637 return 1;
1638
1639 switch (qp->state) {
1640 case QED_ROCE_QP_STATE_RESET:
1641 switch (new_state) {
1642 case QED_ROCE_QP_STATE_INIT:
1643 qp->prev_wqe_size = 0;
1644 qedr_reset_qp_hwq_info(&qp->sq);
1645 qedr_reset_qp_hwq_info(&qp->rq);
1646 break;
1647 default:
1648 status = -EINVAL;
1649 break;
1650 };
1651 break;
1652 case QED_ROCE_QP_STATE_INIT:
1653 switch (new_state) {
1654 case QED_ROCE_QP_STATE_RTR:
1655 /* Update doorbell (in case post_recv was
1656 * done before move to RTR)
1657 */
1658 wmb();
1659 writel(qp->rq.db_data.raw, qp->rq.db);
1660 /* Make sure write takes effect */
1661 mmiowb();
1662 break;
1663 case QED_ROCE_QP_STATE_ERR:
1664 break;
1665 default:
1666 /* Invalid state change. */
1667 status = -EINVAL;
1668 break;
1669 };
1670 break;
1671 case QED_ROCE_QP_STATE_RTR:
1672 /* RTR->XXX */
1673 switch (new_state) {
1674 case QED_ROCE_QP_STATE_RTS:
1675 break;
1676 case QED_ROCE_QP_STATE_ERR:
1677 break;
1678 default:
1679 /* Invalid state change. */
1680 status = -EINVAL;
1681 break;
1682 };
1683 break;
1684 case QED_ROCE_QP_STATE_RTS:
1685 /* RTS->XXX */
1686 switch (new_state) {
1687 case QED_ROCE_QP_STATE_SQD:
1688 break;
1689 case QED_ROCE_QP_STATE_ERR:
1690 break;
1691 default:
1692 /* Invalid state change. */
1693 status = -EINVAL;
1694 break;
1695 };
1696 break;
1697 case QED_ROCE_QP_STATE_SQD:
1698 /* SQD->XXX */
1699 switch (new_state) {
1700 case QED_ROCE_QP_STATE_RTS:
1701 case QED_ROCE_QP_STATE_ERR:
1702 break;
1703 default:
1704 /* Invalid state change. */
1705 status = -EINVAL;
1706 break;
1707 };
1708 break;
1709 case QED_ROCE_QP_STATE_ERR:
1710 /* ERR->XXX */
1711 switch (new_state) {
1712 case QED_ROCE_QP_STATE_RESET:
1713 break;
1714 default:
1715 status = -EINVAL;
1716 break;
1717 };
1718 break;
1719 default:
1720 status = -EINVAL;
1721 break;
1722 };
1723
1724 return status;
1725}
1726
1727int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1728 int attr_mask, struct ib_udata *udata)
1729{
1730 struct qedr_qp *qp = get_qedr_qp(ibqp);
1731 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1732 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1733 enum ib_qp_state old_qp_state, new_qp_state;
1734 int rc = 0;
1735
1736 DP_DEBUG(dev, QEDR_MSG_QP,
1737 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1738 attr->qp_state);
1739
1740 old_qp_state = qedr_get_ibqp_state(qp->state);
1741 if (attr_mask & IB_QP_STATE)
1742 new_qp_state = attr->qp_state;
1743 else
1744 new_qp_state = old_qp_state;
1745
1746 if (!ib_modify_qp_is_ok
1747 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1748 IB_LINK_LAYER_ETHERNET)) {
1749 DP_ERR(dev,
1750 "modify qp: invalid attribute mask=0x%x specified for\n"
1751 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1752 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1753 new_qp_state);
1754 rc = -EINVAL;
1755 goto err;
1756 }
1757
1758 /* Translate the masks... */
1759 if (attr_mask & IB_QP_STATE) {
1760 SET_FIELD(qp_params.modify_flags,
1761 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1762 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1763 }
1764
1765 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1766 qp_params.sqd_async = true;
1767
1768 if (attr_mask & IB_QP_PKEY_INDEX) {
1769 SET_FIELD(qp_params.modify_flags,
1770 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1771 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1772 rc = -EINVAL;
1773 goto err;
1774 }
1775
1776 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1777 }
1778
1779 if (attr_mask & IB_QP_QKEY)
1780 qp->qkey = attr->qkey;
1781
1782 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1783 SET_FIELD(qp_params.modify_flags,
1784 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1785 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1786 IB_ACCESS_REMOTE_READ;
1787 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1788 IB_ACCESS_REMOTE_WRITE;
1789 qp_params.incoming_atomic_en = attr->qp_access_flags &
1790 IB_ACCESS_REMOTE_ATOMIC;
1791 }
1792
1793 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1794 if (attr_mask & IB_QP_PATH_MTU) {
1795 if (attr->path_mtu < IB_MTU_256 ||
1796 attr->path_mtu > IB_MTU_4096) {
1797 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1798 rc = -EINVAL;
1799 goto err;
1800 }
1801 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1802 ib_mtu_enum_to_int(iboe_get_mtu
1803 (dev->ndev->mtu)));
1804 }
1805
1806 if (!qp->mtu) {
1807 qp->mtu =
1808 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1809 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1810 }
1811
1812 SET_FIELD(qp_params.modify_flags,
1813 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1814
1815 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1816 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1817 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1818
1819 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1820
1821 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1822 if (rc) {
1823 DP_ERR(dev,
1824 "modify qp: problems with GID index %d (rc=%d)\n",
1825 attr->ah_attr.grh.sgid_index, rc);
1826 return rc;
1827 }
1828
1829 rc = qedr_get_dmac(dev, &attr->ah_attr,
1830 qp_params.remote_mac_addr);
1831 if (rc)
1832 return rc;
1833
1834 qp_params.use_local_mac = true;
1835 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1836
1837 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1838 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1839 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1840 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1841 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1842 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1843 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1844 qp_params.remote_mac_addr);
1845;
1846
1847 qp_params.mtu = qp->mtu;
1848 qp_params.lb_indication = false;
1849 }
1850
1851 if (!qp_params.mtu) {
1852 /* Stay with current MTU */
1853 if (qp->mtu)
1854 qp_params.mtu = qp->mtu;
1855 else
1856 qp_params.mtu =
1857 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1858 }
1859
1860 if (attr_mask & IB_QP_TIMEOUT) {
1861 SET_FIELD(qp_params.modify_flags,
1862 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1863
1864 qp_params.ack_timeout = attr->timeout;
1865 if (attr->timeout) {
1866 u32 temp;
1867
1868 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1869 /* FW requires [msec] */
1870 qp_params.ack_timeout = temp;
1871 } else {
1872 /* Infinite */
1873 qp_params.ack_timeout = 0;
1874 }
1875 }
1876 if (attr_mask & IB_QP_RETRY_CNT) {
1877 SET_FIELD(qp_params.modify_flags,
1878 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1879 qp_params.retry_cnt = attr->retry_cnt;
1880 }
1881
1882 if (attr_mask & IB_QP_RNR_RETRY) {
1883 SET_FIELD(qp_params.modify_flags,
1884 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1885 qp_params.rnr_retry_cnt = attr->rnr_retry;
1886 }
1887
1888 if (attr_mask & IB_QP_RQ_PSN) {
1889 SET_FIELD(qp_params.modify_flags,
1890 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1891 qp_params.rq_psn = attr->rq_psn;
1892 qp->rq_psn = attr->rq_psn;
1893 }
1894
1895 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1896 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1897 rc = -EINVAL;
1898 DP_ERR(dev,
1899 "unsupported max_rd_atomic=%d, supported=%d\n",
1900 attr->max_rd_atomic,
1901 dev->attr.max_qp_req_rd_atomic_resc);
1902 goto err;
1903 }
1904
1905 SET_FIELD(qp_params.modify_flags,
1906 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1907 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1908 }
1909
1910 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1911 SET_FIELD(qp_params.modify_flags,
1912 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1913 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1914 }
1915
1916 if (attr_mask & IB_QP_SQ_PSN) {
1917 SET_FIELD(qp_params.modify_flags,
1918 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1919 qp_params.sq_psn = attr->sq_psn;
1920 qp->sq_psn = attr->sq_psn;
1921 }
1922
1923 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1924 if (attr->max_dest_rd_atomic >
1925 dev->attr.max_qp_resp_rd_atomic_resc) {
1926 DP_ERR(dev,
1927 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1928 attr->max_dest_rd_atomic,
1929 dev->attr.max_qp_resp_rd_atomic_resc);
1930
1931 rc = -EINVAL;
1932 goto err;
1933 }
1934
1935 SET_FIELD(qp_params.modify_flags,
1936 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1937 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1938 }
1939
1940 if (attr_mask & IB_QP_DEST_QPN) {
1941 SET_FIELD(qp_params.modify_flags,
1942 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1943
1944 qp_params.dest_qp = attr->dest_qp_num;
1945 qp->dest_qp_num = attr->dest_qp_num;
1946 }
1947
1948 if (qp->qp_type != IB_QPT_GSI)
1949 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1950 qp->qed_qp, &qp_params);
1951
1952 if (attr_mask & IB_QP_STATE) {
1953 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1954 qedr_update_qp_state(dev, qp, qp_params.new_state);
1955 qp->state = qp_params.new_state;
1956 }
1957
1958err:
1959 return rc;
1960}
1961
1962static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1963{
1964 int ib_qp_acc_flags = 0;
1965
1966 if (params->incoming_rdma_write_en)
1967 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1968 if (params->incoming_rdma_read_en)
1969 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
1970 if (params->incoming_atomic_en)
1971 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
1972 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1973 return ib_qp_acc_flags;
1974}
1975
1976int qedr_query_qp(struct ib_qp *ibqp,
1977 struct ib_qp_attr *qp_attr,
1978 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1979{
1980 struct qed_rdma_query_qp_out_params params;
1981 struct qedr_qp *qp = get_qedr_qp(ibqp);
1982 struct qedr_dev *dev = qp->dev;
1983 int rc = 0;
1984
1985 memset(&params, 0, sizeof(params));
1986
1987 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
1988 if (rc)
1989 goto err;
1990
1991 memset(qp_attr, 0, sizeof(*qp_attr));
1992 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
1993
1994 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
1995 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
1996 qp_attr->path_mtu = iboe_get_mtu(params.mtu);
1997 qp_attr->path_mig_state = IB_MIG_MIGRATED;
1998 qp_attr->rq_psn = params.rq_psn;
1999 qp_attr->sq_psn = params.sq_psn;
2000 qp_attr->dest_qp_num = params.dest_qp;
2001
2002 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2003
2004 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2005 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2006 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2007 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2008 qp_attr->cap.max_inline_data = qp->max_inline_data;
2009 qp_init_attr->cap = qp_attr->cap;
2010
2011 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
2012 sizeof(qp_attr->ah_attr.grh.dgid.raw));
2013
2014 qp_attr->ah_attr.grh.flow_label = params.flow_label;
2015 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
2016 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
2017 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
2018
2019 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
2020 qp_attr->ah_attr.port_num = 1;
2021 qp_attr->ah_attr.sl = 0;
2022 qp_attr->timeout = params.timeout;
2023 qp_attr->rnr_retry = params.rnr_retry;
2024 qp_attr->retry_cnt = params.retry_cnt;
2025 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2026 qp_attr->pkey_index = params.pkey_index;
2027 qp_attr->port_num = 1;
2028 qp_attr->ah_attr.src_path_bits = 0;
2029 qp_attr->ah_attr.static_rate = 0;
2030 qp_attr->alt_pkey_index = 0;
2031 qp_attr->alt_port_num = 0;
2032 qp_attr->alt_timeout = 0;
2033 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2034
2035 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2036 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2037 qp_attr->max_rd_atomic = params.max_rd_atomic;
2038 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2039
2040 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2041 qp_attr->cap.max_inline_data);
2042
2043err:
2044 return rc;
2045}
2046
2047int qedr_destroy_qp(struct ib_qp *ibqp)
2048{
2049 struct qedr_qp *qp = get_qedr_qp(ibqp);
2050 struct qedr_dev *dev = qp->dev;
2051 struct ib_qp_attr attr;
2052 int attr_mask = 0;
2053 int rc = 0;
2054
2055 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2056 qp, qp->qp_type);
2057
2058 if (qp->state != (QED_ROCE_QP_STATE_RESET | QED_ROCE_QP_STATE_ERR |
2059 QED_ROCE_QP_STATE_INIT)) {
2060 attr.qp_state = IB_QPS_ERR;
2061 attr_mask |= IB_QP_STATE;
2062
2063 /* Change the QP state to ERROR */
2064 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2065 }
2066
2067 if (qp->qp_type != IB_QPT_GSI) {
2068 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2069 if (rc)
2070 return rc;
2071 }
2072
2073 if (ibqp->uobject && ibqp->uobject->context) {
2074 qedr_cleanup_user_sq(dev, qp);
2075 qedr_cleanup_user_rq(dev, qp);
2076 } else {
2077 qedr_cleanup_kernel_sq(dev, qp);
2078 qedr_cleanup_kernel_rq(dev, qp);
2079 }
2080
2081 kfree(qp);
2082
2083 return rc;
2084}
Linux 6.x block layer and io_uring tree(s)