projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/qedr: Add EDPM mode type for user-fw compatibility
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
ff23dfa1 45#include <rdma/uverbs_ioctl.h>
ac1b36e5 46
be086e7c
MY		 47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
99d195cc 53#include "qedr_roce_cm.h"
82af6d19 54#include "qedr_iw_cm.h"
ac1b36e5 55
3491c9e7
YB		 56#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57#define RDMA_MAX_SGE_PER_SRQ (4)
58#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
a7efd777
RA		 60#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
4c6bb02d
MK		 62enum {
63 QEDR_USER_MMAP_IO_WC = 0,
97f61250 64 QEDR_USER_MMAP_PHYS_PAGE,
4c6bb02d
MK		 65};
66
c75d3ec8
AR		 67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 size_t len)
69{
70 size_t min_len = min_t(size_t, len, udata->outlen);
71
72 return ib_copy_to_udata(udata, src, min_len);
73}
74
a7efd777
RA		 75int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
76{
dbe30dae 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 78 return -EINVAL;
79
80 *pkey = QEDR_ROCE_PKEY_DEFAULT;
81 return 0;
82}
83
e6a38c54
KM		 84int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
85 int index, union ib_gid *sgid)
86{
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89 memset(sgid->raw, 0, sizeof(sgid->raw));
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 sgid->global.interface_id, sgid->global.subnet_prefix);
94
95 return 0;
96}
97
3491c9e7
YB		 98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99{
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
102 struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104 srq_attr->srq_limit = srq->srq_limit;
105 srq_attr->max_wr = qattr->max_srq_wr;
106 srq_attr->max_sge = qattr->max_sge;
107
108 return 0;
109}
110
ac1b36e5
RA		 111int qedr_query_device(struct ib_device *ibdev,
112 struct ib_device_attr *attr, struct ib_udata *udata)
113{
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
116
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
119 "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 dev->rdma_ctx);
121 return -EINVAL;
122 }
123
124 memset(attr, 0, sizeof(*attr));
125
126 attr->fw_ver = qattr->fw_ver;
127 attr->sys_image_guid = qattr->sys_image_guid;
128 attr->max_mr_size = qattr->max_mr_size;
129 attr->page_size_cap = qattr->page_size_caps;
130 attr->vendor_id = qattr->vendor_id;
131 attr->vendor_part_id = qattr->vendor_part_id;
132 attr->hw_ver = qattr->hw_ver;
133 attr->max_qp = qattr->max_qp;
134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 IB_DEVICE_RC_RNR_NAK_GEN |
137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
138
33023fb8
SW		 139 attr->max_send_sge = qattr->max_sge;
140 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 141 attr->max_sge_rd = qattr->max_sge;
142 attr->max_cq = qattr->max_cq;
143 attr->max_cqe = qattr->max_cqe;
144 attr->max_mr = qattr->max_mr;
145 attr->max_mw = qattr->max_mw;
146 attr->max_pd = qattr->max_pd;
147 attr->atomic_cap = dev->atomic_cap;
ac1b36e5
RA		 148 attr->max_qp_init_rd_atom =
149 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
150 attr->max_qp_rd_atom =
151 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
152 attr->max_qp_init_rd_atom);
153
154 attr->max_srq = qattr->max_srq;
155 attr->max_srq_sge = qattr->max_srq_sge;
156 attr->max_srq_wr = qattr->max_srq_wr;
157
158 attr->local_ca_ack_delay = qattr->dev_ack_delay;
159 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
160 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
161 attr->max_ah = qattr->max_ah;
162
163 return 0;
164}
165
ac1b36e5
RA		 166static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
167 u8 *ib_width)
168{
169 switch (speed) {
170 case 1000:
69054666 171 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 172 *ib_width = IB_WIDTH_1X;
173 break;
174 case 10000:
69054666 175 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 176 *ib_width = IB_WIDTH_1X;
177 break;
178
179 case 20000:
69054666 180 *ib_speed = IB_SPEED_DDR;
ac1b36e5
RA		 181 *ib_width = IB_WIDTH_4X;
182 break;
183
184 case 25000:
69054666 185 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 186 *ib_width = IB_WIDTH_1X;
187 break;
188
189 case 40000:
69054666 190 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 191 *ib_width = IB_WIDTH_4X;
192 break;
193
194 case 50000:
69054666
SO		 195 *ib_speed = IB_SPEED_HDR;
196 *ib_width = IB_WIDTH_1X;
ac1b36e5
RA		 197 break;
198
199 case 100000:
69054666 200 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 201 *ib_width = IB_WIDTH_4X;
202 break;
203
204 default:
205 /* Unsupported */
69054666 206 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 207 *ib_width = IB_WIDTH_1X;
208 }
209}
210
211int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
212{
213 struct qedr_dev *dev;
214 struct qed_rdma_port *rdma_port;
215
216 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 217
218 if (!dev->rdma_ctx) {
219 DP_ERR(dev, "rdma_ctx is NULL\n");
220 return -EINVAL;
221 }
222
223 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 224
c4550c63 225 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 226 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
227 attr->state = IB_PORT_ACTIVE;
72a7720f 228 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
ac1b36e5
RA		 229 } else {
230 attr->state = IB_PORT_DOWN;
72a7720f 231 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
ac1b36e5
RA		 232 }
233 attr->max_mtu = IB_MTU_4096;
234 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
235 attr->lid = 0;
236 attr->lmc = 0;
237 attr->sm_lid = 0;
238 attr->sm_sl = 0;
2f944c0f 239 attr->ip_gids = true;
f5b1b177
KM		 240 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
241 attr->gid_tbl_len = 1;
242 attr->pkey_tbl_len = 1;
243 } else {
244 attr->gid_tbl_len = QEDR_MAX_SGID;
245 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
246 }
ac1b36e5
RA		 247 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
248 attr->qkey_viol_cntr = 0;
249 get_link_speed_and_width(rdma_port->link_speed,
250 &attr->active_speed, &attr->active_width);
251 attr->max_msg_sz = rdma_port->max_msg_size;
252 attr->max_vl_num = 4;
253
254 return 0;
255}
256
a2a074ef 257int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 258{
a2a074ef 259 struct ib_device *ibdev = uctx->device;
ac1b36e5 260 int rc;
a2a074ef
LR		 261 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
262 struct qedr_alloc_ucontext_resp uresp = {};
97f61250 263 struct qedr_alloc_ucontext_req ureq = {};
ac1b36e5
RA		 264 struct qedr_dev *dev = get_qedr_dev(ibdev);
265 struct qed_rdma_add_user_out_params oparams;
4c6bb02d 266 struct qedr_user_mmap_entry *entry;
ac1b36e5
RA		 267
268 if (!udata)
a2a074ef 269 return -EFAULT;
ac1b36e5 270
97f61250
MK		 271 if (udata->inlen) {
272 rc = ib_copy_from_udata(&ureq, udata,
273 min(sizeof(ureq), udata->inlen));
274 if (rc) {
275 DP_ERR(dev, "Problem copying data from user space\n");
276 return -EFAULT;
277 }
bbe4f424
MK		 278 ctx->edpm_mode = !!(ureq.context_flags &
279 QEDR_ALLOC_UCTX_EDPM_MODE);
97f61250
MK		 280 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
281 }
282
ac1b36e5
RA		 283 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
284 if (rc) {
285 DP_ERR(dev,
286 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
287 rc);
a2a074ef 288 return rc;
ac1b36e5
RA		 289 }
290
291 ctx->dpi = oparams.dpi;
292 ctx->dpi_addr = oparams.dpi_addr;
293 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
294 ctx->dpi_size = oparams.dpi_size;
4c6bb02d
MK		 295 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
296 if (!entry) {
297 rc = -ENOMEM;
298 goto err;
299 }
300
301 entry->io_address = ctx->dpi_phys_addr;
302 entry->length = ctx->dpi_size;
303 entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
304 entry->dpi = ctx->dpi;
305 entry->dev = dev;
306 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
307 ctx->dpi_size);
308 if (rc) {
309 kfree(entry);
310 goto err;
311 }
312 ctx->db_mmap_entry = &entry->rdma_entry;
ac1b36e5 313
93a3d05f
MK		 314 if (!dev->user_dpm_enabled)
315 uresp.dpm_flags = 0;
316 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
317 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
318 else
319 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
bbe4f424
MK		 320 QEDR_DPM_TYPE_ROCE_LEGACY |
321 QEDR_DPM_TYPE_ROCE_EDPM_MODE;
93a3d05f
MK		 322
323 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
324 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
325 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
326
67cbe353
AR		 327 uresp.wids_enabled = 1;
328 uresp.wid_count = oparams.wid_count;
4c6bb02d 329 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
ac1b36e5
RA		 330 uresp.db_size = ctx->dpi_size;
331 uresp.max_send_wr = dev->attr.max_sqe;
332 uresp.max_recv_wr = dev->attr.max_rqe;
333 uresp.max_srq_wr = dev->attr.max_srq_wr;
334 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
335 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
336 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
337 uresp.max_cqes = QEDR_MAX_CQES;
338
c75d3ec8 339 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 340 if (rc)
4c6bb02d 341 goto err;
ac1b36e5
RA		 342
343 ctx->dev = dev;
344
ac1b36e5
RA		 345 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
346 &ctx->ibucontext);
a2a074ef 347 return 0;
4c6bb02d
MK		 348
349err:
350 if (!ctx->db_mmap_entry)
351 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
352 else
353 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
354
355 return rc;
ac1b36e5
RA		 356}
357
a2a074ef 358void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 359{
360 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
ac1b36e5
RA		 361
362 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
363 uctx);
4c6bb02d
MK		 364
365 rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
ac1b36e5
RA		 366}
367
4c6bb02d 368void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
ac1b36e5 369{
4c6bb02d
MK		 370 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
371 struct qedr_dev *dev = entry->dev;
ac1b36e5 372
97f61250
MK		 373 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
374 free_page((unsigned long)entry->address);
375 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
4c6bb02d 376 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
30bf066c 377
4c6bb02d
MK		 378 kfree(entry);
379}
ac1b36e5 380
4c6bb02d
MK		 381int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
382{
383 struct ib_device *dev = ucontext->device;
384 size_t length = vma->vm_end - vma->vm_start;
385 struct rdma_user_mmap_entry *rdma_entry;
386 struct qedr_user_mmap_entry *entry;
387 int rc = 0;
388 u64 pfn;
389
390 ibdev_dbg(dev,
391 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
392 vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
393
394 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
395 if (!rdma_entry) {
396 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
397 vma->vm_pgoff);
ac1b36e5
RA		 398 return -EINVAL;
399 }
4c6bb02d
MK		 400 entry = get_qedr_mmap_entry(rdma_entry);
401 ibdev_dbg(dev,
402 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
403 entry->io_address, length, entry->mmap_flag);
404
405 switch (entry->mmap_flag) {
406 case QEDR_USER_MMAP_IO_WC:
407 pfn = entry->io_address >> PAGE_SHIFT;
408 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
409 pgprot_writecombine(vma->vm_page_prot),
410 rdma_entry);
411 break;
97f61250
MK		 412 case QEDR_USER_MMAP_PHYS_PAGE:
413 rc = vm_insert_page(vma, vma->vm_start,
414 virt_to_page(entry->address));
415 break;
4c6bb02d
MK		 416 default:
417 rc = -EINVAL;
30bf066c 418 }
ac1b36e5 419
4c6bb02d
MK		 420 if (rc)
421 ibdev_dbg(dev,
422 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
423 entry->io_address, length, entry->mmap_flag, rc);
30bf066c 424
4c6bb02d
MK		 425 rdma_user_mmap_entry_put(rdma_entry);
426 return rc;
ac1b36e5 427}
a7efd777 428
ff23dfa1 429int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777 430{
21a428a0 431 struct ib_device *ibdev = ibpd->device;
a7efd777 432 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 433 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 434 u16 pd_id;
435 int rc;
436
437 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
ff23dfa1 438 udata ? "User Lib" : "Kernel");
a7efd777
RA		 439
440 if (!dev->rdma_ctx) {
847cb1a3 441 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 442 return -EINVAL;
a7efd777
RA		 443 }
444
9c1e0228
RA		 445 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
446 if (rc)
21a428a0 447 return rc;
a7efd777 448
a7efd777
RA		 449 pd->pd_id = pd_id;
450
ff23dfa1 451 if (udata) {
57939021
JG		 452 struct qedr_alloc_pd_uresp uresp = {
453 .pd_id = pd_id,
454 };
ff23dfa1
SR		 455 struct qedr_ucontext *context = rdma_udata_to_drv_context(
456 udata, struct qedr_ucontext, ibucontext);
9c1e0228 457
c75d3ec8 458 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 459 if (rc) {
a7efd777 460 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 461 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 462 return rc;
9c1e0228
RA		 463 }
464
ff23dfa1 465 pd->uctx = context;
9c1e0228 466 pd->uctx->pd = pd;
a7efd777
RA		 467 }
468
21a428a0 469 return 0;
a7efd777
RA		 470}
471
c4367a26 472void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 473{
474 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
475 struct qedr_pd *pd = get_qedr_pd(ibpd);
476
a7efd777
RA		 477 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
478 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
a7efd777
RA		 479}
480
481static void qedr_free_pbl(struct qedr_dev *dev,
482 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
483{
484 struct pci_dev *pdev = dev->pdev;
485 int i;
486
487 for (i = 0; i < pbl_info->num_pbls; i++) {
488 if (!pbl[i].va)
489 continue;
490 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
491 pbl[i].va, pbl[i].pa);
492 }
493
494 kfree(pbl);
495}
496
497#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
498#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
499
500#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
501#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
502#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
503
504static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
505 struct qedr_pbl_info *pbl_info,
506 gfp_t flags)
507{
508 struct pci_dev *pdev = dev->pdev;
509 struct qedr_pbl *pbl_table;
510 dma_addr_t *pbl_main_tbl;
511 dma_addr_t pa;
512 void *va;
513 int i;
514
515 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
516 if (!pbl_table)
517 return ERR_PTR(-ENOMEM);
518
519 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 520 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
521 flags);
a7efd777
RA		 522 if (!va)
523 goto err;
524
a7efd777
RA		 525 pbl_table[i].va = va;
526 pbl_table[i].pa = pa;
527 }
528
529 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
530 * the first one with physical pointers to all of the rest
531 */
532 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
533 for (i = 0; i < pbl_info->num_pbls - 1; i++)
534 pbl_main_tbl[i] = pbl_table[i + 1].pa;
535
536 return pbl_table;
537
538err:
539 for (i--; i >= 0; i--)
540 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
541 pbl_table[i].va, pbl_table[i].pa);
542
543 qedr_free_pbl(dev, pbl_info, pbl_table);
544
545 return ERR_PTR(-ENOMEM);
546}
547
548static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
549 struct qedr_pbl_info *pbl_info,
550 u32 num_pbes, int two_layer_capable)
551{
552 u32 pbl_capacity;
553 u32 pbl_size;
554 u32 num_pbls;
555
556 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
557 if (num_pbes > MAX_PBES_TWO_LAYER) {
558 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
559 num_pbes);
560 return -EINVAL;
561 }
562
563 /* calculate required pbl page size */
564 pbl_size = MIN_FW_PBL_PAGE_SIZE;
565 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
566 NUM_PBES_ON_PAGE(pbl_size);
567
568 while (pbl_capacity < num_pbes) {
569 pbl_size *= 2;
570 pbl_capacity = pbl_size / sizeof(u64);
571 pbl_capacity = pbl_capacity * pbl_capacity;
572 }
573
574 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
575 num_pbls++; /* One for the layer0 ( points to the pbls) */
576 pbl_info->two_layered = true;
577 } else {
578 /* One layered PBL */
579 num_pbls = 1;
580 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
581 roundup_pow_of_two((num_pbes * sizeof(u64))));
582 pbl_info->two_layered = false;
583 }
584
585 pbl_info->num_pbls = num_pbls;
586 pbl_info->pbl_size = pbl_size;
587 pbl_info->num_pbes = num_pbes;
588
589 DP_DEBUG(dev, QEDR_MSG_MR,
590 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
591 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
592
593 return 0;
594}
595
596static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
597 struct qedr_pbl *pbl,
e57bb6be 598 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 599{
95ad233f 600 int pbe_cnt, total_num_pbes = 0;
e57bb6be 601 u32 fw_pg_cnt, fw_pg_per_umem_pg;
a7efd777 602 struct qedr_pbl *pbl_tbl;
95ad233f 603 struct sg_dma_page_iter sg_iter;
a7efd777 604 struct regpair *pbe;
e57bb6be 605 u64 pg_addr;
a7efd777
RA		 606
607 if (!pbl_info->num_pbes)
608 return;
609
610 /* If we have a two layered pbl, the first pbl points to the rest
611 * of the pbls and the first entry lays on the second pbl in the table
612 */
613 if (pbl_info->two_layered)
614 pbl_tbl = &pbl[1];
615 else
616 pbl_tbl = pbl;
617
618 pbe = (struct regpair *)pbl_tbl->va;
619 if (!pbe) {
620 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
621 return;
622 }
623
624 pbe_cnt = 0;
625
95ad233f
SS		 626 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
627
628 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
629 pg_addr = sg_page_iter_dma_address(&sg_iter);
630 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
631 pbe->lo = cpu_to_le32(pg_addr);
632 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
633
634 pg_addr += BIT(pg_shift);
635 pbe_cnt++;
636 total_num_pbes++;
637 pbe++;
e57bb6be 638
95ad233f
SS		 639 if (total_num_pbes == pbl_info->num_pbes)
640 return;
641
642 /* If the given pbl is full storing the pbes,
643 * move to next pbl.
644 */
645 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
646 pbl_tbl++;
647 pbe = (struct regpair *)pbl_tbl->va;
648 pbe_cnt = 0;
a7efd777 649 }
95ad233f
SS		 650
651 fw_pg_cnt++;
a7efd777
RA		 652 }
653 }
654}
655
97f61250
MK		 656static int qedr_db_recovery_add(struct qedr_dev *dev,
657 void __iomem *db_addr,
658 void *db_data,
659 enum qed_db_rec_width db_width,
660 enum qed_db_rec_space db_space)
661{
662 if (!db_data) {
663 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
664 return 0;
665 }
666
667 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
668 db_width, db_space);
669}
670
671static void qedr_db_recovery_del(struct qedr_dev *dev,
672 void __iomem *db_addr,
673 void *db_data)
674{
675 if (!db_data) {
676 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
677 return;
678 }
679
680 /* Ignore return code as there is not much we can do about it. Error
681 * log will be printed inside.
682 */
683 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
684}
685
a7efd777 686static int qedr_copy_cq_uresp(struct qedr_dev *dev,
97f61250
MK		 687 struct qedr_cq *cq, struct ib_udata *udata,
688 u32 db_offset)
a7efd777
RA		 689{
690 struct qedr_create_cq_uresp uresp;
691 int rc;
692
693 memset(&uresp, 0, sizeof(uresp));
694
97f61250 695 uresp.db_offset = db_offset;
a7efd777 696 uresp.icid = cq->icid;
a25984f3
MK		 697 if (cq->q.db_mmap_entry)
698 uresp.db_rec_addr =
699 rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
a7efd777 700
c75d3ec8 701 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 702 if (rc)
703 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
704
705 return rc;
706}
707
708static void consume_cqe(struct qedr_cq *cq)
709{
710 if (cq->latest_cqe == cq->toggle_cqe)
711 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
712
713 cq->latest_cqe = qed_chain_consume(&cq->pbl);
714}
715
716static inline int qedr_align_cq_entries(int entries)
717{
718 u64 size, aligned_size;
719
720 /* We allocate an extra entry that we don't report to the FW. */
721 size = (entries + 1) * QEDR_CQE_SIZE;
722 aligned_size = ALIGN(size, PAGE_SIZE);
723
724 return aligned_size / QEDR_CQE_SIZE;
725}
726
97f61250
MK		 727static int qedr_init_user_db_rec(struct ib_udata *udata,
728 struct qedr_dev *dev, struct qedr_userq *q,
729 bool requires_db_rec)
730{
731 struct qedr_ucontext *uctx =
732 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
733 ibucontext);
734 struct qedr_user_mmap_entry *entry;
735 int rc;
736
737 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
738 if (requires_db_rec == 0 || !uctx->db_rec)
739 return 0;
740
741 /* Allocate a page for doorbell recovery, add to mmap */
742 q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
743 if (!q->db_rec_data) {
744 DP_ERR(dev, "get_zeroed_page failed\n");
745 return -ENOMEM;
746 }
747
748 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
749 if (!entry)
750 goto err_free_db_data;
751
752 entry->address = q->db_rec_data;
753 entry->length = PAGE_SIZE;
754 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
755 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
756 &entry->rdma_entry,
757 PAGE_SIZE);
758 if (rc)
759 goto err_free_entry;
760
761 q->db_mmap_entry = &entry->rdma_entry;
762
763 return 0;
764
765err_free_entry:
766 kfree(entry);
767
768err_free_db_data:
769 free_page((unsigned long)q->db_rec_data);
770 q->db_rec_data = NULL;
771 return -ENOMEM;
772}
773
b0ea0fa5 774static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 775 struct qedr_dev *dev,
b0ea0fa5 776 struct qedr_userq *q, u64 buf_addr,
97f61250 777 size_t buf_len, bool requires_db_rec,
72b894b0 778 int access,
69ad0e7f 779 int alloc_and_init)
a7efd777 780{
e57bb6be 781 u32 fw_pages;
a7efd777
RA		 782 int rc;
783
784 q->buf_addr = buf_addr;
785 q->buf_len = buf_len;
c320e527 786 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
a7efd777
RA		 787 if (IS_ERR(q->umem)) {
788 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
789 PTR_ERR(q->umem));
790 return PTR_ERR(q->umem);
791 }
792
e57bb6be 793 fw_pages = ib_umem_page_count(q->umem) <<
95ad233f 794 (PAGE_SHIFT - FW_PAGE_SHIFT);
e57bb6be
RA		 795
796 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 797 if (rc)
798 goto err0;
799
69ad0e7f
KM		 800 if (alloc_and_init) {
801 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
802 if (IS_ERR(q->pbl_tbl)) {
803 rc = PTR_ERR(q->pbl_tbl);
804 goto err0;
805 }
e57bb6be
RA		 806 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
807 FW_PAGE_SHIFT);
69ad0e7f
KM		 808 } else {
809 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 810 if (!q->pbl_tbl) {
811 rc = -ENOMEM;
69ad0e7f 812 goto err0;
89fd2576 813 }
69ad0e7f 814 }
a7efd777 815
97f61250
MK		 816 /* mmap the user address used to store doorbell data for recovery */
817 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
a7efd777
RA		 818
819err0:
820 ib_umem_release(q->umem);
69ad0e7f 821 q->umem = NULL;
a7efd777
RA		 822
823 return rc;
824}
825
826static inline void qedr_init_cq_params(struct qedr_cq *cq,
827 struct qedr_ucontext *ctx,
828 struct qedr_dev *dev, int vector,
829 int chain_entries, int page_cnt,
830 u64 pbl_ptr,
831 struct qed_rdma_create_cq_in_params
832 *params)
833{
834 memset(params, 0, sizeof(*params));
835 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
836 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
837 params->cnq_id = vector;
838 params->cq_size = chain_entries - 1;
839 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
840 params->pbl_num_pages = page_cnt;
841 params->pbl_ptr = pbl_ptr;
842 params->pbl_two_level = 0;
843}
844
845static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
846{
a7efd777
RA		 847 cq->db.data.agg_flags = flags;
848 cq->db.data.value = cpu_to_le32(cons);
849 writeq(cq->db.raw, cq->db_addr);
a7efd777
RA		 850}
851
852int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
853{
854 struct qedr_cq *cq = get_qedr_cq(ibcq);
855 unsigned long sflags;
4dd72636
AR		 856 struct qedr_dev *dev;
857
858 dev = get_qedr_dev(ibcq->device);
859
860 if (cq->destroyed) {
861 DP_ERR(dev,
862 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
863 cq, cq->icid);
864 return -EINVAL;
865 }
866
a7efd777
RA		 867
868 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
869 return 0;
870
871 spin_lock_irqsave(&cq->cq_lock, sflags);
872
873 cq->arm_flags = 0;
874
875 if (flags & IB_CQ_SOLICITED)
876 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
877
878 if (flags & IB_CQ_NEXT_COMP)
879 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
880
881 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
882
883 spin_unlock_irqrestore(&cq->cq_lock, sflags);
884
885 return 0;
886}
887
e39afe3d
LR		 888int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
889 struct ib_udata *udata)
a7efd777 890{
e39afe3d 891 struct ib_device *ibdev = ibcq->device;
ff23dfa1
SR		 892 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
893 udata, struct qedr_ucontext, ibucontext);
a7efd777
RA		 894 struct qed_rdma_destroy_cq_out_params destroy_oparams;
895 struct qed_rdma_destroy_cq_in_params destroy_iparams;
896 struct qedr_dev *dev = get_qedr_dev(ibdev);
897 struct qed_rdma_create_cq_in_params params;
e39afe3d 898 struct qedr_create_cq_ureq ureq = {};
a7efd777
RA		 899 int vector = attr->comp_vector;
900 int entries = attr->cqe;
e39afe3d 901 struct qedr_cq *cq = get_qedr_cq(ibcq);
a7efd777 902 int chain_entries;
97f61250 903 u32 db_offset;
a7efd777
RA		 904 int page_cnt;
905 u64 pbl_ptr;
906 u16 icid;
907 int rc;
908
909 DP_DEBUG(dev, QEDR_MSG_INIT,
910 "create_cq: called from %s. entries=%d, vector=%d\n",
911 udata ? "User Lib" : "Kernel", entries, vector);
912
913 if (entries > QEDR_MAX_CQES) {
914 DP_ERR(dev,
915 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
916 entries, QEDR_MAX_CQES);
e39afe3d 917 return -EINVAL;
a7efd777
RA		 918 }
919
920 chain_entries = qedr_align_cq_entries(entries);
921 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
922
97f61250
MK		 923 /* calc db offset. user will add DPI base, kernel will add db addr */
924 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
925
a7efd777 926 if (udata) {
97f61250
MK		 927 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
928 udata->inlen))) {
a7efd777
RA		 929 DP_ERR(dev,
930 "create cq: problem copying data from user space\n");
931 goto err0;
932 }
933
934 if (!ureq.len) {
935 DP_ERR(dev,
936 "create cq: cannot create a cq with 0 entries\n");
937 goto err0;
938 }
939
940 cq->cq_type = QEDR_CQ_TYPE_USER;
941
b0ea0fa5 942 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
72b894b0
CH		 943 ureq.len, true, IB_ACCESS_LOCAL_WRITE,
944 1);
a7efd777
RA		 945 if (rc)
946 goto err0;
947
948 pbl_ptr = cq->q.pbl_tbl->pa;
949 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 950
951 cq->ibcq.cqe = chain_entries;
97f61250 952 cq->q.db_addr = ctx->dpi_addr + db_offset;
a7efd777
RA		 953 } else {
954 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
955
956 rc = dev->ops->common->chain_alloc(dev->cdev,
957 QED_CHAIN_USE_TO_CONSUME,
958 QED_CHAIN_MODE_PBL,
959 QED_CHAIN_CNT_TYPE_U32,
960 chain_entries,
961 sizeof(union rdma_cqe),
1a4a6975 962 &cq->pbl, NULL);
a7efd777 963 if (rc)
97f61250 964 goto err0;
a7efd777
RA		 965
966 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
967 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 968 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 969 }
970
971 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
972 pbl_ptr, &params);
973
974 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
975 if (rc)
97f61250 976 goto err1;
a7efd777
RA		 977
978 cq->icid = icid;
979 cq->sig = QEDR_CQ_MAGIC_NUMBER;
980 spin_lock_init(&cq->cq_lock);
981
ff23dfa1 982 if (udata) {
97f61250
MK		 983 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
984 if (rc)
985 goto err2;
986
987 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
988 &cq->q.db_rec_data->db_data,
989 DB_REC_WIDTH_64B,
990 DB_REC_USER);
a7efd777 991 if (rc)
97f61250
MK		 992 goto err2;
993
a7efd777
RA		 994 } else {
995 /* Generate doorbell address. */
a7efd777 996 cq->db.data.icid = cq->icid;
97f61250 997 cq->db_addr = dev->db_addr + db_offset;
a7efd777
RA		 998 cq->db.data.params = DB_AGG_CMD_SET <<
999 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1000
1001 /* point to the very last element, passing it we will toggle */
1002 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1003 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1004 cq->latest_cqe = NULL;
1005 consume_cqe(cq);
1006 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
97f61250
MK		 1007
1008 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1009 DB_REC_WIDTH_64B, DB_REC_KERNEL);
1010 if (rc)
1011 goto err2;
a7efd777
RA		 1012 }
1013
1014 DP_DEBUG(dev, QEDR_MSG_CQ,
1015 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1016 cq->icid, cq, params.cq_size);
1017
e39afe3d 1018 return 0;
a7efd777 1019
97f61250 1020err2:
a7efd777
RA		 1021 destroy_iparams.icid = cq->icid;
1022 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1023 &destroy_oparams);
a7efd777 1024err1:
97f61250
MK		 1025 if (udata) {
1026 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
a7efd777 1027 ib_umem_release(cq->q.umem);
a25984f3 1028 if (cq->q.db_mmap_entry)
97f61250
MK		 1029 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1030 } else {
1031 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1032 }
a7efd777 1033err0:
e39afe3d 1034 return -EINVAL;
a7efd777
RA		 1035}
1036
1037int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1038{
1039 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1040 struct qedr_cq *cq = get_qedr_cq(ibcq);
1041
1042 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1043
1044 return 0;
1045}
1046
4dd72636
AR		 1047#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1048#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1049
a52c8e24 1050void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 1051{
1052 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1053 struct qed_rdma_destroy_cq_out_params oparams;
1054 struct qed_rdma_destroy_cq_in_params iparams;
1055 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 1056 int iter;
a7efd777 1057
942b3b2c 1058 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 1059
4dd72636
AR		 1060 cq->destroyed = 1;
1061
a7efd777 1062 /* GSIs CQs are handled by driver, so they don't exist in the FW */
97f61250
MK		 1063 if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1064 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
e39afe3d 1065 return;
97f61250 1066 }
a1211359 1067
942b3b2c 1068 iparams.icid = cq->icid;
a52c8e24 1069 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
942b3b2c 1070 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777 1071
bdeacabd 1072 if (udata) {
a7efd777
RA		 1073 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1074 ib_umem_release(cq->q.umem);
97f61250
MK		 1075
1076 if (cq->q.db_rec_data) {
1077 qedr_db_recovery_del(dev, cq->q.db_addr,
1078 &cq->q.db_rec_data->db_data);
1079 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1080 }
1081 } else {
1082 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
a7efd777
RA		 1083 }
1084
4dd72636
AR		 1085 /* We don't want the IRQ handler to handle a non-existing CQ so we
1086 * wait until all CNQ interrupts, if any, are received. This will always
1087 * happen and will always happen very fast. If not, then a serious error
1088 * has occured. That is why we can use a long delay.
1089 * We spin for a short time so we don’t lose time on context switching
1090 * in case all the completions are handled in that span. Otherwise
1091 * we sleep for a while and check again. Since the CNQ may be
1092 * associated with (only) the current CPU we use msleep to allow the
1093 * current CPU to be freed.
1094 * The CNQ notification is increased in qedr_irq_handler().
1095 */
1096 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1097 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1098 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1099 iter--;
1100 }
1101
1102 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1103 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1104 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1105 iter--;
1106 }
1107
4dd72636
AR		 1108 /* Note that we don't need to have explicit code to wait for the
1109 * completion of the event handler because it is invoked from the EQ.
1110 * Since the destroy CQ ramrod has also been received on the EQ we can
1111 * be certain that there's no event handler in process.
1112 */
a7efd777 1113}
cecbcddf
RA		 1114
1115static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1116 struct ib_qp_attr *attr,
1117 int attr_mask,
1118 struct qed_rdma_modify_qp_in_params
1119 *qp_params)
1120{
47ec3866 1121 const struct ib_gid_attr *gid_attr;
cecbcddf 1122 enum rdma_network_type nw_type;
d8966fcd 1123 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1124 u32 ipv4_addr;
a70c0739 1125 int ret;
cecbcddf
RA		 1126 int i;
1127
47ec3866 1128 gid_attr = grh->sgid_attr;
a70c0739
PP		 1129 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1130 if (ret)
1131 return ret;
3e44e0ee 1132
47ec3866 1133 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1134 switch (nw_type) {
1135 case RDMA_NETWORK_IPV6:
47ec3866 1136 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1137 sizeof(qp_params->sgid));
1138 memcpy(&qp_params->dgid.bytes[0],
1139 &grh->dgid,
1140 sizeof(qp_params->dgid));
1141 qp_params->roce_mode = ROCE_V2_IPV6;
1142 SET_FIELD(qp_params->modify_flags,
1143 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1144 break;
1145 case RDMA_NETWORK_IB:
47ec3866 1146 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1147 sizeof(qp_params->sgid));
1148 memcpy(&qp_params->dgid.bytes[0],
1149 &grh->dgid,
1150 sizeof(qp_params->dgid));
1151 qp_params->roce_mode = ROCE_V1;
1152 break;
1153 case RDMA_NETWORK_IPV4:
1154 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1155 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1156 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1157 qp_params->sgid.ipv4_addr = ipv4_addr;
1158 ipv4_addr =
1159 qedr_get_ipv4_from_gid(grh->dgid.raw);
1160 qp_params->dgid.ipv4_addr = ipv4_addr;
1161 SET_FIELD(qp_params->modify_flags,
1162 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1163 qp_params->roce_mode = ROCE_V2_IPV4;
1164 break;
cecbcddf
RA		 1165 }
1166
1167 for (i = 0; i < 4; i++) {
1168 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1169 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1170 }
1171
1172 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1173 qp_params->vlan_id = 0;
1174
1175 return 0;
1176}
1177
cecbcddf 1178static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1179 struct ib_qp_init_attr *attrs,
1180 struct ib_udata *udata)
cecbcddf
RA		 1181{
1182 struct qedr_device_attr *qattr = &dev->attr;
1183
1184 /* QP0... attrs->qp_type == IB_QPT_GSI */
1185 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1186 DP_DEBUG(dev, QEDR_MSG_QP,
1187 "create qp: unsupported qp type=0x%x requested\n",
1188 attrs->qp_type);
bb8865f4 1189 return -EOPNOTSUPP;
cecbcddf
RA		 1190 }
1191
1192 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1193 DP_ERR(dev,
1194 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1195 attrs->cap.max_send_wr, qattr->max_sqe);
1196 return -EINVAL;
1197 }
1198
1199 if (attrs->cap.max_inline_data > qattr->max_inline) {
1200 DP_ERR(dev,
1201 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1202 attrs->cap.max_inline_data, qattr->max_inline);
1203 return -EINVAL;
1204 }
1205
1206 if (attrs->cap.max_send_sge > qattr->max_sge) {
1207 DP_ERR(dev,
1208 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1209 attrs->cap.max_send_sge, qattr->max_sge);
1210 return -EINVAL;
1211 }
1212
1213 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1214 DP_ERR(dev,
1215 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1216 attrs->cap.max_recv_sge, qattr->max_sge);
1217 return -EINVAL;
1218 }
1219
1220 /* Unprivileged user space cannot create special QP */
e00b64f7 1221 if (udata && attrs->qp_type == IB_QPT_GSI) {
cecbcddf
RA		 1222 DP_ERR(dev,
1223 "create qp: userspace can't create special QPs of type=0x%x\n",
1224 attrs->qp_type);
1225 return -EINVAL;
1226 }
1227
1228 return 0;
1229}
1230
40b173dd
YB		 1231static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1232 struct qedr_srq *srq, struct ib_udata *udata)
1233{
1234 struct qedr_create_srq_uresp uresp = {};
1235 int rc;
1236
1237 uresp.srq_id = srq->srq_id;
1238
1239 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1240 if (rc)
1241 DP_ERR(dev, "create srq: problem copying data to user space\n");
1242
1243 return rc;
1244}
1245
69ad0e7f 1246static void qedr_copy_rq_uresp(struct qedr_dev *dev,
97f61250
MK		 1247 struct qedr_create_qp_uresp *uresp,
1248 struct qedr_qp *qp)
cecbcddf 1249{
69ad0e7f
KM		 1250 /* iWARP requires two doorbells per RQ. */
1251 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1252 uresp->rq_db_offset =
1253 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1254 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1255 } else {
1256 uresp->rq_db_offset =
1257 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1258 }
1259
cecbcddf 1260 uresp->rq_icid = qp->icid;
a25984f3
MK		 1261 if (qp->urq.db_mmap_entry)
1262 uresp->rq_db_rec_addr =
1263 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
cecbcddf
RA		 1264}
1265
69ad0e7f
KM		 1266static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1267 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1268 struct qedr_qp *qp)
1269{
1270 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1271
1272 /* iWARP uses the same cid for rq and sq */
1273 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1274 uresp->sq_icid = qp->icid;
1275 else
1276 uresp->sq_icid = qp->icid + 1;
97f61250 1277
a25984f3
MK		 1278 if (qp->usq.db_mmap_entry)
1279 uresp->sq_db_rec_addr =
1280 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
cecbcddf
RA		 1281}
1282
1283static int qedr_copy_qp_uresp(struct qedr_dev *dev,
97f61250
MK		 1284 struct qedr_qp *qp, struct ib_udata *udata,
1285 struct qedr_create_qp_uresp *uresp)
cecbcddf 1286{
cecbcddf
RA		 1287 int rc;
1288
97f61250
MK		 1289 memset(uresp, 0, sizeof(*uresp));
1290 qedr_copy_sq_uresp(dev, uresp, qp);
1291 qedr_copy_rq_uresp(dev, uresp, qp);
cecbcddf 1292
97f61250
MK		 1293 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1294 uresp->qp_id = qp->qp_id;
cecbcddf 1295
97f61250 1296 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
cecbcddf
RA		 1297 if (rc)
1298 DP_ERR(dev,
1299 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1300 qp->icid);
1301
1302 return rc;
1303}
1304
df158561
AR		 1305static void qedr_set_common_qp_params(struct qedr_dev *dev,
1306 struct qedr_qp *qp,
1307 struct qedr_pd *pd,
1308 struct ib_qp_init_attr *attrs)
cecbcddf 1309{
cecbcddf 1310 spin_lock_init(&qp->q_lock);
82af6d19
MK		 1311 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1312 kref_init(&qp->refcnt);
1313 init_completion(&qp->iwarp_cm_comp);
1314 }
df158561 1315 qp->pd = pd;
cecbcddf
RA		 1316 qp->qp_type = attrs->qp_type;
1317 qp->max_inline_data = attrs->cap.max_inline_data;
1318 qp->sq.max_sges = attrs->cap.max_send_sge;
1319 qp->state = QED_ROCE_QP_STATE_RESET;
1320 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1321 qp->sq_cq = get_qedr_cq(attrs->send_cq);
cecbcddf
RA		 1322 qp->dev = dev;
1323
3491c9e7
YB		 1324 if (attrs->srq) {
1325 qp->srq = get_qedr_srq(attrs->srq);
1326 } else {
1327 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1328 qp->rq.max_sges = attrs->cap.max_recv_sge;
1329 DP_DEBUG(dev, QEDR_MSG_QP,
1330 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1331 qp->rq.max_sges, qp->rq_cq->icid);
1332 }
1333
cecbcddf
RA		 1334 DP_DEBUG(dev, QEDR_MSG_QP,
1335 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1336 pd->pd_id, qp->qp_type, qp->max_inline_data,
1337 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1338 DP_DEBUG(dev, QEDR_MSG_QP,
1339 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1340 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1341}
1342
97f61250 1343static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1344{
97f61250
MK		 1345 int rc;
1346
cecbcddf
RA		 1347 qp->sq.db = dev->db_addr +
1348 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1349 qp->sq.db_data.data.icid = qp->icid + 1;
97f61250
MK		 1350 rc = qedr_db_recovery_add(dev, qp->sq.db,
1351 &qp->sq.db_data,
1352 DB_REC_WIDTH_32B,
1353 DB_REC_KERNEL);
1354 if (rc)
1355 return rc;
1356
3491c9e7
YB		 1357 if (!qp->srq) {
1358 qp->rq.db = dev->db_addr +
1359 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1360 qp->rq.db_data.data.icid = qp->icid;
97f61250
MK		 1361
1362 rc = qedr_db_recovery_add(dev, qp->rq.db,
1363 &qp->rq.db_data,
1364 DB_REC_WIDTH_32B,
1365 DB_REC_KERNEL);
1366 if (rc)
1367 qedr_db_recovery_del(dev, qp->sq.db,
1368 &qp->sq.db_data);
3491c9e7 1369 }
97f61250
MK		 1370
1371 return rc;
3491c9e7
YB		 1372}
1373
68e326de 1374static int qedr_check_srq_params(struct qedr_dev *dev,
3491c9e7
YB		 1375 struct ib_srq_init_attr *attrs,
1376 struct ib_udata *udata)
1377{
1378 struct qedr_device_attr *qattr = &dev->attr;
1379
1380 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1381 DP_ERR(dev,
1382 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1383 attrs->attr.max_wr, qattr->max_srq_wr);
1384 return -EINVAL;
1385 }
1386
1387 if (attrs->attr.max_sge > qattr->max_sge) {
1388 DP_ERR(dev,
1389 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1390 attrs->attr.max_sge, qattr->max_sge);
1391 return -EINVAL;
1392 }
1393
1394 return 0;
1395}
1396
40b173dd
YB		 1397static void qedr_free_srq_user_params(struct qedr_srq *srq)
1398{
1399 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1400 ib_umem_release(srq->usrq.umem);
1401 ib_umem_release(srq->prod_umem);
1402}
1403
3491c9e7
YB		 1404static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1405{
1406 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1407 struct qedr_dev *dev = srq->dev;
1408
1409 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1410
1411 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1412 hw_srq->virt_prod_pair_addr,
1413 hw_srq->phy_prod_pair_addr);
1414}
1415
b0ea0fa5 1416static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1417 struct qedr_srq *srq,
1418 struct qedr_create_srq_ureq *ureq,
72b894b0 1419 int access)
40b173dd
YB		 1420{
1421 struct scatterlist *sg;
1422 int rc;
1423
b0ea0fa5 1424 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
72b894b0 1425 ureq->srq_len, false, access, 1);
40b173dd
YB		 1426 if (rc)
1427 return rc;
1428
c320e527
MS		 1429 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1430 sizeof(struct rdma_srq_producers), access);
40b173dd
YB		 1431 if (IS_ERR(srq->prod_umem)) {
1432 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1433 ib_umem_release(srq->usrq.umem);
1434 DP_ERR(srq->dev,
1435 "create srq: failed ib_umem_get for producer, got %ld\n",
1436 PTR_ERR(srq->prod_umem));
1437 return PTR_ERR(srq->prod_umem);
1438 }
1439
1440 sg = srq->prod_umem->sg_head.sgl;
1441 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1442
1443 return 0;
1444}
1445
3491c9e7
YB		 1446static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1447 struct qedr_dev *dev,
1448 struct ib_srq_init_attr *init_attr)
1449{
1450 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1451 dma_addr_t phy_prod_pair_addr;
1452 u32 num_elems;
1453 void *va;
1454 int rc;
1455
1456 va = dma_alloc_coherent(&dev->pdev->dev,
1457 sizeof(struct rdma_srq_producers),
1458 &phy_prod_pair_addr, GFP_KERNEL);
1459 if (!va) {
1460 DP_ERR(dev,
1461 "create srq: failed to allocate dma memory for producer\n");
1462 return -ENOMEM;
1463 }
1464
1465 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1466 hw_srq->virt_prod_pair_addr = va;
1467
1468 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1469 rc = dev->ops->common->chain_alloc(dev->cdev,
1470 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1471 QED_CHAIN_MODE_PBL,
1472 QED_CHAIN_CNT_TYPE_U32,
1473 num_elems,
1474 QEDR_SRQ_WQE_ELEM_SIZE,
1475 &hw_srq->pbl, NULL);
1476 if (rc)
1477 goto err0;
1478
1479 hw_srq->num_elems = num_elems;
1480
1481 return 0;
1482
1483err0:
1484 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1485 va, phy_prod_pair_addr);
1486 return rc;
1487}
1488
68e326de
LR		 1489int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1490 struct ib_udata *udata)
3491c9e7
YB		 1491{
1492 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1493 struct qed_rdma_create_srq_in_params in_params = {};
68e326de 1494 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
3491c9e7 1495 struct qed_rdma_create_srq_out_params out_params;
68e326de 1496 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
40b173dd 1497 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1498 u64 pbl_base_addr, phy_prod_pair_addr;
1499 struct qedr_srq_hwq_info *hw_srq;
1500 u32 page_cnt, page_size;
68e326de 1501 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3491c9e7
YB		 1502 int rc = 0;
1503
1504 DP_DEBUG(dev, QEDR_MSG_QP,
1505 "create SRQ called from %s (pd %p)\n",
1506 (udata) ? "User lib" : "kernel", pd);
1507
68e326de 1508 rc = qedr_check_srq_params(dev, init_attr, udata);
3491c9e7 1509 if (rc)
68e326de 1510 return -EINVAL;
3491c9e7
YB		 1511
1512 srq->dev = dev;
1513 hw_srq = &srq->hw_srq;
1514 spin_lock_init(&srq->lock);
1515
1516 hw_srq->max_wr = init_attr->attr.max_wr;
1517 hw_srq->max_sges = init_attr->attr.max_sge;
1518
89944450 1519 if (udata) {
97f61250
MK		 1520 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1521 udata->inlen))) {
40b173dd
YB		 1522 DP_ERR(dev,
1523 "create srq: problem copying data from user space\n");
1524 goto err0;
1525 }
1526
72b894b0 1527 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
40b173dd
YB		 1528 if (rc)
1529 goto err0;
1530
1531 page_cnt = srq->usrq.pbl_info.num_pbes;
1532 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1533 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1534 page_size = PAGE_SIZE;
40b173dd
YB		 1535 } else {
1536 struct qed_chain *pbl;
1537
1538 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1539 if (rc)
1540 goto err0;
1541
1542 pbl = &hw_srq->pbl;
1543 page_cnt = qed_chain_get_page_cnt(pbl);
1544 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1545 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1546 page_size = QED_CHAIN_PAGE_SIZE;
1547 }
3491c9e7 1548
3491c9e7
YB		 1549 in_params.pd_id = pd->pd_id;
1550 in_params.pbl_base_addr = pbl_base_addr;
1551 in_params.prod_pair_addr = phy_prod_pair_addr;
1552 in_params.num_pages = page_cnt;
1553 in_params.page_size = page_size;
1554
1555 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1556 if (rc)
1557 goto err1;
1558
1559 srq->srq_id = out_params.srq_id;
1560
40b173dd
YB		 1561 if (udata) {
1562 rc = qedr_copy_srq_uresp(dev, srq, udata);
1563 if (rc)
1564 goto err2;
1565 }
1566
9fd15987 1567 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1568 if (rc)
1569 goto err2;
1570
1571 DP_DEBUG(dev, QEDR_MSG_SRQ,
1572 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
68e326de 1573 return 0;
3491c9e7
YB		 1574
1575err2:
1576 destroy_in_params.srq_id = srq->srq_id;
1577
1578 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1579err1:
40b173dd
YB		 1580 if (udata)
1581 qedr_free_srq_user_params(srq);
1582 else
1583 qedr_free_srq_kernel_params(srq);
3491c9e7 1584err0:
68e326de 1585 return -EFAULT;
3491c9e7
YB		 1586}
1587
68e326de 1588void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1589{
1590 struct qed_rdma_destroy_srq_in_params in_params = {};
1591 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1592 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1593
9fd15987 1594 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7
YB		 1595 in_params.srq_id = srq->srq_id;
1596 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1597
e00b64f7 1598 if (ibsrq->uobject)
40b173dd
YB		 1599 qedr_free_srq_user_params(srq);
1600 else
1601 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1602
1603 DP_DEBUG(dev, QEDR_MSG_SRQ,
1604 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1605 srq->srq_id);
3491c9e7
YB		 1606}
1607
1608int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1609 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1610{
1611 struct qed_rdma_modify_srq_in_params in_params = {};
1612 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1613 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1614 int rc;
1615
1616 if (attr_mask & IB_SRQ_MAX_WR) {
1617 DP_ERR(dev,
1618 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1619 attr_mask, srq);
1620 return -EINVAL;
1621 }
1622
1623 if (attr_mask & IB_SRQ_LIMIT) {
1624 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1625 DP_ERR(dev,
1626 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1627 attr->srq_limit, srq->hw_srq.max_wr);
1628 return -EINVAL;
1629 }
1630
1631 in_params.srq_id = srq->srq_id;
1632 in_params.wqe_limit = attr->srq_limit;
1633 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1634 if (rc)
1635 return rc;
1636 }
1637
1638 srq->srq_limit = attr->srq_limit;
1639
1640 DP_DEBUG(dev, QEDR_MSG_SRQ,
1641 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1642
1643 return 0;
cecbcddf
RA		 1644}
1645
1646static inline void
df158561
AR		 1647qedr_init_common_qp_in_params(struct qedr_dev *dev,
1648 struct qedr_pd *pd,
1649 struct qedr_qp *qp,
1650 struct ib_qp_init_attr *attrs,
1651 bool fmr_and_reserved_lkey,
1652 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1653{
df158561
AR		 1654 /* QP handle to be written in an async event */
1655 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1656 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1657
1658 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1659 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1660 params->pd = pd->pd_id;
1661 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1662 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1663 params->stats_queue = 0;
df158561
AR		 1664 params->srq_id = 0;
1665 params->use_srq = false;
3491c9e7
YB		 1666
1667 if (!qp->srq) {
1668 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1669
1670 } else {
1671 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1672 params->srq_id = qp->srq->srq_id;
1673 params->use_srq = true;
1674 }
cecbcddf
RA		 1675}
1676
df158561 1677static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1678{
df158561
AR		 1679 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1680 "qp=%p. "
1681 "sq_addr=0x%llx, "
1682 "sq_len=%zd, "
1683 "rq_addr=0x%llx, "
1684 "rq_len=%zd"
1685 "\n",
1686 qp,
1687 qp->usq.buf_addr,
1688 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1689}
cecbcddf 1690
69ad0e7f
KM		 1691static inline void
1692qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1693 struct qedr_qp *qp,
1694 struct qed_rdma_create_qp_out_params *out_params)
1695{
1696 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1697 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1698
1699 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1700 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1701 if (!qp->srq) {
1702 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1703 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1704 }
69ad0e7f
KM		 1705
1706 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1707 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1708}
1709
97f61250
MK		 1710static void qedr_cleanup_user(struct qedr_dev *dev,
1711 struct qedr_ucontext *ctx,
1712 struct qedr_qp *qp)
df158561 1713{
836a0fbb 1714 ib_umem_release(qp->usq.umem);
df158561 1715 qp->usq.umem = NULL;
cecbcddf 1716
836a0fbb 1717 ib_umem_release(qp->urq.umem);
df158561 1718 qp->urq.umem = NULL;
24e412c1
MK		 1719
1720 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1721 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1722 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1723 } else {
1724 kfree(qp->usq.pbl_tbl);
1725 kfree(qp->urq.pbl_tbl);
1726 }
97f61250
MK		 1727
1728 if (qp->usq.db_rec_data) {
1729 qedr_db_recovery_del(dev, qp->usq.db_addr,
1730 &qp->usq.db_rec_data->db_data);
1731 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1732 }
1733
1734 if (qp->urq.db_rec_data) {
1735 qedr_db_recovery_del(dev, qp->urq.db_addr,
1736 &qp->urq.db_rec_data->db_data);
1737 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1738 }
b4bc7660
MK		 1739
1740 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1741 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1742 &qp->urq.db_rec_db2_data);
cecbcddf
RA		 1743}
1744
df158561
AR		 1745static int qedr_create_user_qp(struct qedr_dev *dev,
1746 struct qedr_qp *qp,
1747 struct ib_pd *ibpd,
1748 struct ib_udata *udata,
1749 struct ib_qp_init_attr *attrs)
cecbcddf 1750{
df158561
AR		 1751 struct qed_rdma_create_qp_in_params in_params;
1752 struct qed_rdma_create_qp_out_params out_params;
1753 struct qedr_pd *pd = get_qedr_pd(ibpd);
97f61250 1754 struct qedr_create_qp_uresp uresp;
bbe4f424 1755 struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
df158561 1756 struct qedr_create_qp_ureq ureq;
69ad0e7f 1757 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
df158561 1758 int rc = -EINVAL;
cecbcddf 1759
82af6d19 1760 qp->create_type = QEDR_QP_CREATE_USER;
df158561 1761 memset(&ureq, 0, sizeof(ureq));
97f61250 1762 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
df158561
AR		 1763 if (rc) {
1764 DP_ERR(dev, "Problem copying data from user space\n");
1765 return rc;
1766 }
cecbcddf 1767
72b894b0 1768 /* SQ - read access only (0) */
b0ea0fa5 1769 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
72b894b0 1770 ureq.sq_len, true, 0, alloc_and_init);
df158561
AR		 1771 if (rc)
1772 return rc;
cecbcddf 1773
40b173dd 1774 if (!qp->srq) {
72b894b0 1775 /* RQ - read access only (0) */
b0ea0fa5 1776 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
72b894b0 1777 ureq.rq_len, true, 0, alloc_and_init);
40b173dd
YB		 1778 if (rc)
1779 return rc;
1780 }
df158561
AR		 1781
1782 memset(&in_params, 0, sizeof(in_params));
1783 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1784 in_params.qp_handle_lo = ureq.qp_handle_lo;
1785 in_params.qp_handle_hi = ureq.qp_handle_hi;
1786 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1787 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
40b173dd
YB		 1788 if (!qp->srq) {
1789 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1790 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1791 }
df158561 1792
bbe4f424
MK		 1793 if (ctx)
1794 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1795
df158561
AR		 1796 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1797 &in_params, &out_params);
1798
1799 if (!qp->qed_qp) {
1800 rc = -ENOMEM;
1801 goto err1;
1802 }
1803
69ad0e7f
KM		 1804 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1805 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1806
df158561
AR		 1807 qp->qp_id = out_params.qp_id;
1808 qp->icid = out_params.icid;
1809
97f61250
MK		 1810 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1811 if (rc)
1812 goto err;
1813
1814 /* db offset was calculated in copy_qp_uresp, now set in the user q */
1815 ctx = pd->uctx;
1816 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1817 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1818
b4bc7660
MK		 1819 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1820 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1821
1822 /* calculate the db_rec_db2 data since it is constant so no
1823 * need to reflect from user
1824 */
1825 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1826 qp->urq.db_rec_db2_data.data.value =
1827 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1828 }
1829
97f61250
MK		 1830 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1831 &qp->usq.db_rec_data->db_data,
1832 DB_REC_WIDTH_32B,
1833 DB_REC_USER);
1834 if (rc)
1835 goto err;
1836
1837 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1838 &qp->urq.db_rec_data->db_data,
1839 DB_REC_WIDTH_32B,
1840 DB_REC_USER);
df158561
AR		 1841 if (rc)
1842 goto err;
1843
b4bc7660
MK		 1844 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1845 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1846 &qp->urq.db_rec_db2_data,
1847 DB_REC_WIDTH_32B,
1848 DB_REC_USER);
1849 if (rc)
1850 goto err;
1851 }
df158561 1852 qedr_qp_user_print(dev, qp);
cecbcddf 1853
97f61250 1854 return rc;
df158561
AR		 1855err:
1856 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1857 if (rc)
1858 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1859
1860err1:
97f61250 1861 qedr_cleanup_user(dev, ctx, qp);
df158561 1862 return rc;
cecbcddf
RA		 1863}
1864
97f61250 1865static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
f5b1b177 1866{
97f61250
MK		 1867 int rc;
1868
f5b1b177
KM		 1869 qp->sq.db = dev->db_addr +
1870 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1871 qp->sq.db_data.data.icid = qp->icid;
1872
97f61250
MK		 1873 rc = qedr_db_recovery_add(dev, qp->sq.db,
1874 &qp->sq.db_data,
1875 DB_REC_WIDTH_32B,
1876 DB_REC_KERNEL);
1877 if (rc)
1878 return rc;
1879
f5b1b177
KM		 1880 qp->rq.db = dev->db_addr +
1881 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1882 qp->rq.db_data.data.icid = qp->icid;
1883 qp->rq.iwarp_db2 = dev->db_addr +
1884 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1885 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1886 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
97f61250
MK		 1887
1888 rc = qedr_db_recovery_add(dev, qp->rq.db,
1889 &qp->rq.db_data,
1890 DB_REC_WIDTH_32B,
1891 DB_REC_KERNEL);
b4bc7660
MK		 1892 if (rc)
1893 return rc;
1894
1895 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1896 &qp->rq.iwarp_db2_data,
1897 DB_REC_WIDTH_32B,
1898 DB_REC_KERNEL);
97f61250 1899 return rc;
f5b1b177
KM		 1900}
1901
df158561
AR		 1902static int
1903qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1904 struct qedr_qp *qp,
1905 struct qed_rdma_create_qp_in_params *in_params,
1906 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1907{
df158561 1908 struct qed_rdma_create_qp_out_params out_params;
cecbcddf
RA		 1909 int rc;
1910
cecbcddf
RA		 1911 rc = dev->ops->common->chain_alloc(dev->cdev,
1912 QED_CHAIN_USE_TO_PRODUCE,
1913 QED_CHAIN_MODE_PBL,
1914 QED_CHAIN_CNT_TYPE_U32,
1915 n_sq_elems,
1916 QEDR_SQE_ELEMENT_SIZE,
1a4a6975 1917 &qp->sq.pbl, NULL);
cecbcddf 1918
df158561
AR		 1919 if (rc)
1920 return rc;
cecbcddf 1921
df158561
AR		 1922 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1923 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1924
cecbcddf
RA		 1925 rc = dev->ops->common->chain_alloc(dev->cdev,
1926 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1927 QED_CHAIN_MODE_PBL,
1928 QED_CHAIN_CNT_TYPE_U32,
1929 n_rq_elems,
1930 QEDR_RQE_ELEMENT_SIZE,
1a4a6975 1931 &qp->rq.pbl, NULL);
df158561
AR		 1932 if (rc)
1933 return rc;
cecbcddf 1934
df158561
AR		 1935 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1936 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1937
df158561
AR		 1938 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1939 in_params, &out_params);
cecbcddf 1940
df158561
AR		 1941 if (!qp->qed_qp)
1942 return -EINVAL;
cecbcddf 1943
df158561
AR		 1944 qp->qp_id = out_params.qp_id;
1945 qp->icid = out_params.icid;
cecbcddf 1946
97f61250 1947 return qedr_set_roce_db_info(dev, qp);
f5b1b177 1948}
cecbcddf 1949
f5b1b177
KM		 1950static int
1951qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1952 struct qedr_qp *qp,
1953 struct qed_rdma_create_qp_in_params *in_params,
1954 u32 n_sq_elems, u32 n_rq_elems)
1955{
1956 struct qed_rdma_create_qp_out_params out_params;
1957 struct qed_chain_ext_pbl ext_pbl;
1958 int rc;
1959
1960 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1961 QEDR_SQE_ELEMENT_SIZE,
1962 QED_CHAIN_MODE_PBL);
1963 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1964 QEDR_RQE_ELEMENT_SIZE,
1965 QED_CHAIN_MODE_PBL);
1966
1967 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1968 in_params, &out_params);
1969
1970 if (!qp->qed_qp)
1971 return -EINVAL;
1972
1973 /* Now we allocate the chain */
1974 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1975 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1976
1977 rc = dev->ops->common->chain_alloc(dev->cdev,
1978 QED_CHAIN_USE_TO_PRODUCE,
1979 QED_CHAIN_MODE_PBL,
1980 QED_CHAIN_CNT_TYPE_U32,
1981 n_sq_elems,
1982 QEDR_SQE_ELEMENT_SIZE,
1983 &qp->sq.pbl, &ext_pbl);
1984
1985 if (rc)
1986 goto err;
1987
1988 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1989 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1990
1991 rc = dev->ops->common->chain_alloc(dev->cdev,
1992 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1993 QED_CHAIN_MODE_PBL,
1994 QED_CHAIN_CNT_TYPE_U32,
1995 n_rq_elems,
1996 QEDR_RQE_ELEMENT_SIZE,
1997 &qp->rq.pbl, &ext_pbl);
1998
1999 if (rc)
2000 goto err;
2001
2002 qp->qp_id = out_params.qp_id;
2003 qp->icid = out_params.icid;
2004
97f61250 2005 return qedr_set_iwarp_db_info(dev, qp);
f5b1b177
KM		 2006
2007err:
2008 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2009
2010 return rc;
cecbcddf
RA		 2011}
2012
df158561 2013static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 2014{
df158561
AR		 2015 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2016 kfree(qp->wqe_wr_id);
cecbcddf 2017
df158561
AR		 2018 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2019 kfree(qp->rqe_wr_id);
97f61250
MK		 2020
2021 /* GSI qp is not registered to db mechanism so no need to delete */
2022 if (qp->qp_type == IB_QPT_GSI)
2023 return;
2024
2025 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2026
b4bc7660 2027 if (!qp->srq) {
97f61250 2028 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
b4bc7660
MK		 2029
2030 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2031 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2032 &qp->rq.iwarp_db2_data);
2033 }
cecbcddf
RA		 2034}
2035
df158561
AR		 2036static int qedr_create_kernel_qp(struct qedr_dev *dev,
2037 struct qedr_qp *qp,
2038 struct ib_pd *ibpd,
2039 struct ib_qp_init_attr *attrs)
cecbcddf 2040{
df158561
AR		 2041 struct qed_rdma_create_qp_in_params in_params;
2042 struct qedr_pd *pd = get_qedr_pd(ibpd);
2043 int rc = -EINVAL;
2044 u32 n_rq_elems;
2045 u32 n_sq_elems;
2046 u32 n_sq_entries;
cecbcddf 2047
df158561 2048 memset(&in_params, 0, sizeof(in_params));
82af6d19 2049 qp->create_type = QEDR_QP_CREATE_KERNEL;
cecbcddf 2050
df158561
AR		 2051 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2052 * the ring. The ring should allow at least a single WR, even if the
2053 * user requested none, due to allocation issues.
2054 * We should add an extra WR since the prod and cons indices of
2055 * wqe_wr_id are managed in such a way that the WQ is considered full
2056 * when (prod+1)%max_wr==cons. We currently don't do that because we
2057 * double the number of entries due an iSER issue that pushes far more
2058 * WRs than indicated. If we decline its ib_post_send() then we get
2059 * error prints in the dmesg we'd like to avoid.
2060 */
2061 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2062 dev->attr.max_sqe);
cecbcddf 2063
6396bb22 2064 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 2065 GFP_KERNEL);
2066 if (!qp->wqe_wr_id) {
2067 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2068 return -ENOMEM;
2069 }
cecbcddf 2070
df158561
AR		 2071 /* QP handle to be written in CQE */
2072 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2073 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 2074
df158561
AR		 2075 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2076 * the ring. There ring should allow at least a single WR, even if the
2077 * user requested none, due to allocation issues.
2078 */
2079 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 2080
df158561 2081 /* Allocate driver internal RQ array */
6396bb22 2082 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 2083 GFP_KERNEL);
2084 if (!qp->rqe_wr_id) {
2085 DP_ERR(dev,
2086 "create qp: failed RQ shadow memory allocation\n");
2087 kfree(qp->wqe_wr_id);
2088 return -ENOMEM;
cecbcddf
RA		 2089 }
2090
df158561 2091 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 2092
df158561
AR		 2093 n_sq_entries = attrs->cap.max_send_wr;
2094 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2095 n_sq_entries = max_t(u32, n_sq_entries, 1);
2096 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 2097
df158561
AR		 2098 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2099
f5b1b177
KM		 2100 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2101 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2102 n_sq_elems, n_rq_elems);
2103 else
2104 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2105 n_sq_elems, n_rq_elems);
df158561
AR		 2106 if (rc)
2107 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 2108
2109 return rc;
2110}
2111
2112struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2113 struct ib_qp_init_attr *attrs,
2114 struct ib_udata *udata)
2115{
2116 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 2117 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 2118 struct qedr_qp *qp;
181d8015 2119 struct ib_qp *ibqp;
cecbcddf
RA		 2120 int rc = 0;
2121
2122 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2123 udata ? "user library" : "kernel", pd);
2124
e00b64f7 2125 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf
RA		 2126 if (rc)
2127 return ERR_PTR(rc);
2128
cecbcddf 2129 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 2130 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2131 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 2132 get_qedr_cq(attrs->send_cq),
2133 get_qedr_cq(attrs->send_cq)->icid,
2134 get_qedr_cq(attrs->recv_cq),
3491c9e7 2135 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 2136
df158561
AR		 2137 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2138 if (!qp) {
2139 DP_ERR(dev, "create qp: failed allocating memory\n");
2140 return ERR_PTR(-ENOMEM);
2141 }
2142
2143 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 2144
04886779 2145 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 2146 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2147 if (IS_ERR(ibqp))
2148 kfree(qp);
2149 return ibqp;
04886779
RA		 2150 }
2151
df158561
AR		 2152 if (udata)
2153 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2154 else
2155 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 2156
df158561
AR		 2157 if (rc)
2158 goto err;
cecbcddf 2159
cecbcddf
RA		 2160 qp->ibqp.qp_num = qp->qp_id;
2161
1212767e 2162 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 2163 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e
YB		 2164 if (rc)
2165 goto err;
2166 }
de0089e6 2167
cecbcddf
RA		 2168 return &qp->ibqp;
2169
df158561 2170err:
cecbcddf
RA		 2171 kfree(qp);
2172
2173 return ERR_PTR(-EFAULT);
2174}
2175
27a4b1a6 2176static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 2177{
2178 switch (qp_state) {
2179 case QED_ROCE_QP_STATE_RESET:
2180 return IB_QPS_RESET;
2181 case QED_ROCE_QP_STATE_INIT:
2182 return IB_QPS_INIT;
2183 case QED_ROCE_QP_STATE_RTR:
2184 return IB_QPS_RTR;
2185 case QED_ROCE_QP_STATE_RTS:
2186 return IB_QPS_RTS;
2187 case QED_ROCE_QP_STATE_SQD:
2188 return IB_QPS_SQD;
2189 case QED_ROCE_QP_STATE_ERR:
2190 return IB_QPS_ERR;
2191 case QED_ROCE_QP_STATE_SQE:
2192 return IB_QPS_SQE;
2193 }
2194 return IB_QPS_ERR;
2195}
2196
27a4b1a6
RA		 2197static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2198 enum ib_qp_state qp_state)
cecbcddf
RA		 2199{
2200 switch (qp_state) {
2201 case IB_QPS_RESET:
2202 return QED_ROCE_QP_STATE_RESET;
2203 case IB_QPS_INIT:
2204 return QED_ROCE_QP_STATE_INIT;
2205 case IB_QPS_RTR:
2206 return QED_ROCE_QP_STATE_RTR;
2207 case IB_QPS_RTS:
2208 return QED_ROCE_QP_STATE_RTS;
2209 case IB_QPS_SQD:
2210 return QED_ROCE_QP_STATE_SQD;
2211 case IB_QPS_ERR:
2212 return QED_ROCE_QP_STATE_ERR;
2213 default:
2214 return QED_ROCE_QP_STATE_ERR;
2215 }
2216}
2217
2218static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2219{
2220 qed_chain_reset(&qph->pbl);
2221 qph->prod = 0;
2222 qph->cons = 0;
2223 qph->wqe_cons = 0;
2224 qph->db_data.data.value = cpu_to_le16(0);
2225}
2226
2227static int qedr_update_qp_state(struct qedr_dev *dev,
2228 struct qedr_qp *qp,
caf61b1b 2229 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 2230 enum qed_roce_qp_state new_state)
2231{
2232 int status = 0;
2233
caf61b1b 2234 if (new_state == cur_state)
865cea40 2235 return 0;
cecbcddf 2236
caf61b1b 2237 switch (cur_state) {
cecbcddf
RA		 2238 case QED_ROCE_QP_STATE_RESET:
2239 switch (new_state) {
2240 case QED_ROCE_QP_STATE_INIT:
2241 qp->prev_wqe_size = 0;
2242 qedr_reset_qp_hwq_info(&qp->sq);
2243 qedr_reset_qp_hwq_info(&qp->rq);
2244 break;
2245 default:
2246 status = -EINVAL;
2247 break;
790b57f6 2248 }
cecbcddf
RA		 2249 break;
2250 case QED_ROCE_QP_STATE_INIT:
2251 switch (new_state) {
2252 case QED_ROCE_QP_STATE_RTR:
2253 /* Update doorbell (in case post_recv was
2254 * done before move to RTR)
2255 */
f5b1b177
KM		 2256
2257 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177 2258 writel(qp->rq.db_data.raw, qp->rq.db);
f5b1b177 2259 }
cecbcddf
RA		 2260 break;
2261 case QED_ROCE_QP_STATE_ERR:
2262 break;
2263 default:
2264 /* Invalid state change. */
2265 status = -EINVAL;
2266 break;
790b57f6 2267 }
cecbcddf
RA		 2268 break;
2269 case QED_ROCE_QP_STATE_RTR:
2270 /* RTR->XXX */
2271 switch (new_state) {
2272 case QED_ROCE_QP_STATE_RTS:
2273 break;
2274 case QED_ROCE_QP_STATE_ERR:
2275 break;
2276 default:
2277 /* Invalid state change. */
2278 status = -EINVAL;
2279 break;
790b57f6 2280 }
cecbcddf
RA		 2281 break;
2282 case QED_ROCE_QP_STATE_RTS:
2283 /* RTS->XXX */
2284 switch (new_state) {
2285 case QED_ROCE_QP_STATE_SQD:
2286 break;
2287 case QED_ROCE_QP_STATE_ERR:
2288 break;
2289 default:
2290 /* Invalid state change. */
2291 status = -EINVAL;
2292 break;
790b57f6 2293 }
cecbcddf
RA		 2294 break;
2295 case QED_ROCE_QP_STATE_SQD:
2296 /* SQD->XXX */
2297 switch (new_state) {
2298 case QED_ROCE_QP_STATE_RTS:
2299 case QED_ROCE_QP_STATE_ERR:
2300 break;
2301 default:
2302 /* Invalid state change. */
2303 status = -EINVAL;
2304 break;
790b57f6 2305 }
cecbcddf
RA		 2306 break;
2307 case QED_ROCE_QP_STATE_ERR:
2308 /* ERR->XXX */
2309 switch (new_state) {
2310 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2311 if ((qp->rq.prod != qp->rq.cons) ||
2312 (qp->sq.prod != qp->sq.cons)) {
2313 DP_NOTICE(dev,
2314 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2315 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2316 qp->sq.cons);
2317 status = -EINVAL;
2318 }
cecbcddf
RA		 2319 break;
2320 default:
2321 status = -EINVAL;
2322 break;
790b57f6 2323 }
cecbcddf
RA		 2324 break;
2325 default:
2326 status = -EINVAL;
2327 break;
790b57f6 2328 }
cecbcddf
RA		 2329
2330 return status;
2331}
2332
2333int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2334 int attr_mask, struct ib_udata *udata)
2335{
2336 struct qedr_qp *qp = get_qedr_qp(ibqp);
2337 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2338 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2339 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2340 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2341 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2342 int rc = 0;
2343
2344 DP_DEBUG(dev, QEDR_MSG_QP,
2345 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2346 attr->qp_state);
2347
2348 old_qp_state = qedr_get_ibqp_state(qp->state);
2349 if (attr_mask & IB_QP_STATE)
2350 new_qp_state = attr->qp_state;
2351 else
2352 new_qp_state = old_qp_state;
2353
f5b1b177
KM		 2354 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2355 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2356 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2357 DP_ERR(dev,
2358 "modify qp: invalid attribute mask=0x%x specified for\n"
2359 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2360 attr_mask, qp->qp_id, ibqp->qp_type,
2361 old_qp_state, new_qp_state);
2362 rc = -EINVAL;
2363 goto err;
2364 }
cecbcddf
RA		 2365 }
2366
2367 /* Translate the masks... */
2368 if (attr_mask & IB_QP_STATE) {
2369 SET_FIELD(qp_params.modify_flags,
2370 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2371 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2372 }
2373
2374 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2375 qp_params.sqd_async = true;
2376
2377 if (attr_mask & IB_QP_PKEY_INDEX) {
2378 SET_FIELD(qp_params.modify_flags,
2379 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2380 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2381 rc = -EINVAL;
2382 goto err;
2383 }
2384
2385 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2386 }
2387
2388 if (attr_mask & IB_QP_QKEY)
2389 qp->qkey = attr->qkey;
2390
2391 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2392 SET_FIELD(qp_params.modify_flags,
2393 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2394 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2395 IB_ACCESS_REMOTE_READ;
2396 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2397 IB_ACCESS_REMOTE_WRITE;
2398 qp_params.incoming_atomic_en = attr->qp_access_flags &
2399 IB_ACCESS_REMOTE_ATOMIC;
2400 }
2401
2402 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2403 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2404 return -EINVAL;
2405
cecbcddf
RA		 2406 if (attr_mask & IB_QP_PATH_MTU) {
2407 if (attr->path_mtu < IB_MTU_256 ||
2408 attr->path_mtu > IB_MTU_4096) {
2409 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2410 rc = -EINVAL;
2411 goto err;
2412 }
2413 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2414 ib_mtu_enum_to_int(iboe_get_mtu
2415 (dev->ndev->mtu)));
2416 }
2417
2418 if (!qp->mtu) {
2419 qp->mtu =
2420 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2421 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2422 }
2423
2424 SET_FIELD(qp_params.modify_flags,
2425 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2426
d8966fcd
DC		 2427 qp_params.traffic_class_tos = grh->traffic_class;
2428 qp_params.flow_label = grh->flow_label;
2429 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2430
d8966fcd 2431 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2432
2433 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2434 if (rc) {
2435 DP_ERR(dev,
2436 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2437 grh->sgid_index, rc);
cecbcddf
RA		 2438 return rc;
2439 }
2440
2441 rc = qedr_get_dmac(dev, &attr->ah_attr,
2442 qp_params.remote_mac_addr);
2443 if (rc)
2444 return rc;
2445
2446 qp_params.use_local_mac = true;
2447 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2448
2449 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2450 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2451 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2452 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2453 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2454 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2455 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2456 qp_params.remote_mac_addr);
cecbcddf
RA		 2457
2458 qp_params.mtu = qp->mtu;
2459 qp_params.lb_indication = false;
2460 }
2461
2462 if (!qp_params.mtu) {
2463 /* Stay with current MTU */
2464 if (qp->mtu)
2465 qp_params.mtu = qp->mtu;
2466 else
2467 qp_params.mtu =
2468 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2469 }
2470
2471 if (attr_mask & IB_QP_TIMEOUT) {
2472 SET_FIELD(qp_params.modify_flags,
2473 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2474
c3594f22
KM		 2475 /* The received timeout value is an exponent used like this:
2476 * "12.7.34 LOCAL ACK TIMEOUT
2477 * Value representing the transport (ACK) timeout for use by
2478 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2479 * The FW expects timeout in msec so we need to divide the usec
2480 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2481 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2482 * The value of zero means infinite so we use a 'max_t' to make
2483 * sure that sub 1 msec values will be configured as 1 msec.
2484 */
2485 if (attr->timeout)
2486 qp_params.ack_timeout =
2487 1 << max_t(int, attr->timeout - 8, 0);
2488 else
cecbcddf 2489 qp_params.ack_timeout = 0;
cecbcddf 2490 }
c3594f22 2491
cecbcddf
RA		 2492 if (attr_mask & IB_QP_RETRY_CNT) {
2493 SET_FIELD(qp_params.modify_flags,
2494 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2495 qp_params.retry_cnt = attr->retry_cnt;
2496 }
2497
2498 if (attr_mask & IB_QP_RNR_RETRY) {
2499 SET_FIELD(qp_params.modify_flags,
2500 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2501 qp_params.rnr_retry_cnt = attr->rnr_retry;
2502 }
2503
2504 if (attr_mask & IB_QP_RQ_PSN) {
2505 SET_FIELD(qp_params.modify_flags,
2506 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2507 qp_params.rq_psn = attr->rq_psn;
2508 qp->rq_psn = attr->rq_psn;
2509 }
2510
2511 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2512 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2513 rc = -EINVAL;
2514 DP_ERR(dev,
2515 "unsupported max_rd_atomic=%d, supported=%d\n",
2516 attr->max_rd_atomic,
2517 dev->attr.max_qp_req_rd_atomic_resc);
2518 goto err;
2519 }
2520
2521 SET_FIELD(qp_params.modify_flags,
2522 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2523 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2524 }
2525
2526 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2527 SET_FIELD(qp_params.modify_flags,
2528 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2529 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2530 }
2531
2532 if (attr_mask & IB_QP_SQ_PSN) {
2533 SET_FIELD(qp_params.modify_flags,
2534 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2535 qp_params.sq_psn = attr->sq_psn;
2536 qp->sq_psn = attr->sq_psn;
2537 }
2538
2539 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2540 if (attr->max_dest_rd_atomic >
2541 dev->attr.max_qp_resp_rd_atomic_resc) {
2542 DP_ERR(dev,
2543 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2544 attr->max_dest_rd_atomic,
2545 dev->attr.max_qp_resp_rd_atomic_resc);
2546
2547 rc = -EINVAL;
2548 goto err;
2549 }
2550
2551 SET_FIELD(qp_params.modify_flags,
2552 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2553 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2554 }
2555
2556 if (attr_mask & IB_QP_DEST_QPN) {
2557 SET_FIELD(qp_params.modify_flags,
2558 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2559
2560 qp_params.dest_qp = attr->dest_qp_num;
2561 qp->dest_qp_num = attr->dest_qp_num;
2562 }
2563
caf61b1b
KM		 2564 cur_state = qp->state;
2565
2566 /* Update the QP state before the actual ramrod to prevent a race with
2567 * fast path. Modifying the QP state to error will cause the device to
2568 * flush the CQEs and while polling the flushed CQEs will considered as
2569 * a potential issue if the QP isn't in error state.
2570 */
2571 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2572 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2573 qp->state = QED_ROCE_QP_STATE_ERR;
2574
cecbcddf
RA		 2575 if (qp->qp_type != IB_QPT_GSI)
2576 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2577 qp->qed_qp, &qp_params);
2578
2579 if (attr_mask & IB_QP_STATE) {
2580 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2581 rc = qedr_update_qp_state(dev, qp, cur_state,
2582 qp_params.new_state);
cecbcddf
RA		 2583 qp->state = qp_params.new_state;
2584 }
2585
2586err:
2587 return rc;
2588}
2589
2590static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2591{
2592 int ib_qp_acc_flags = 0;
2593
2594 if (params->incoming_rdma_write_en)
2595 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2596 if (params->incoming_rdma_read_en)
2597 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2598 if (params->incoming_atomic_en)
2599 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2600 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2601 return ib_qp_acc_flags;
2602}
2603
2604int qedr_query_qp(struct ib_qp *ibqp,
2605 struct ib_qp_attr *qp_attr,
2606 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2607{
2608 struct qed_rdma_query_qp_out_params params;
2609 struct qedr_qp *qp = get_qedr_qp(ibqp);
2610 struct qedr_dev *dev = qp->dev;
2611 int rc = 0;
2612
2613 memset(&params, 0, sizeof(params));
2614
2615 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2616 if (rc)
2617 goto err;
2618
2619 memset(qp_attr, 0, sizeof(*qp_attr));
2620 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2621
2622 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2623 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2624 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2625 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2626 qp_attr->rq_psn = params.rq_psn;
2627 qp_attr->sq_psn = params.sq_psn;
2628 qp_attr->dest_qp_num = params.dest_qp;
2629
2630 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2631
2632 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2633 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2634 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2635 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
59e8970b 2636 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
cecbcddf
RA		 2637 qp_init_attr->cap = qp_attr->cap;
2638
44c58487 2639 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2640 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2641 params.flow_label, qp->sgid_idx,
2642 params.hop_limit_ttl, params.traffic_class_tos);
2643 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2644 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2645 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2646 qp_attr->timeout = params.timeout;
2647 qp_attr->rnr_retry = params.rnr_retry;
2648 qp_attr->retry_cnt = params.retry_cnt;
2649 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2650 qp_attr->pkey_index = params.pkey_index;
2651 qp_attr->port_num = 1;
d8966fcd
DC		 2652 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2653 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2654 qp_attr->alt_pkey_index = 0;
2655 qp_attr->alt_port_num = 0;
2656 qp_attr->alt_timeout = 0;
2657 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2658
2659 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2660 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2661 qp_attr->max_rd_atomic = params.max_rd_atomic;
2662 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2663
2664 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2665 qp_attr->cap.max_inline_data);
2666
2667err:
2668 return rc;
2669}
2670
bdeacabd
SR		 2671static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2672 struct ib_udata *udata)
df158561 2673{
97f61250
MK		 2674 struct qedr_ucontext *ctx =
2675 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2676 ibucontext);
2677 int rc;
df158561
AR		 2678
2679 if (qp->qp_type != IB_QPT_GSI) {
2680 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2681 if (rc)
2682 return rc;
2683 }
2684
82af6d19 2685 if (qp->create_type == QEDR_QP_CREATE_USER)
97f61250 2686 qedr_cleanup_user(dev, ctx, qp);
df158561
AR		 2687 else
2688 qedr_cleanup_kernel(dev, qp);
2689
2690 return 0;
2691}
2692
c4367a26 2693int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2694{
2695 struct qedr_qp *qp = get_qedr_qp(ibqp);
2696 struct qedr_dev *dev = qp->dev;
2697 struct ib_qp_attr attr;
2698 int attr_mask = 0;
cecbcddf
RA		 2699
2700 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2701 qp, qp->qp_type);
2702
f5b1b177
KM		 2703 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2704 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2705 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2706 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2707
f5b1b177
KM		 2708 attr.qp_state = IB_QPS_ERR;
2709 attr_mask |= IB_QP_STATE;
cecbcddf 2710
f5b1b177
KM		 2711 /* Change the QP state to ERROR */
2712 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2713 }
e411e058 2714 } else {
82af6d19
MK		 2715 /* If connection establishment started the WAIT_FOR_CONNECT
2716 * bit will be on and we need to Wait for the establishment
2717 * to complete before destroying the qp.
2718 */
2719 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2720 &qp->iwarp_cm_flags))
2721 wait_for_completion(&qp->iwarp_cm_comp);
2722
2723 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2724 * bit will be on, and we need to wait for the disconnect to
2725 * complete before continuing. We can use the same completion,
2726 * iwarp_cm_comp, since this is the only place that waits for
2727 * this completion and it is sequential. In addition,
2728 * disconnect can't occur before the connection is fully
2729 * established, therefore if WAIT_FOR_DISCONNECT is on it
2730 * means WAIT_FOR_CONNECT is also on and the completion for
2731 * CONNECT already occurred.
2732 */
2733 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2734 &qp->iwarp_cm_flags))
2735 wait_for_completion(&qp->iwarp_cm_comp);
cecbcddf
RA		 2736 }
2737
df158561 2738 if (qp->qp_type == IB_QPT_GSI)
04886779 2739 qedr_destroy_gsi_qp(dev);
cecbcddf 2740
82af6d19
MK		 2741 /* We need to remove the entry from the xarray before we release the
2742 * qp_id to avoid a race of the qp_id being reallocated and failing
2743 * on xa_insert
2744 */
2745 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2746 xa_erase(&dev->qps, qp->qp_id);
2747
bdeacabd 2748 qedr_free_qp_resources(dev, qp, udata);
cecbcddf 2749
82af6d19
MK		 2750 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2751 qedr_iw_qp_rem_ref(&qp->ibqp);
2752
cf167e5e 2753 return 0;
cecbcddf 2754}
e0290cce 2755
fa5d010c 2756int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
d3456914 2757 struct ib_udata *udata)
04886779 2758{
d3456914 2759 struct qedr_ah *ah = get_qedr_ah(ibah);
04886779 2760
fa5d010c 2761 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
04886779 2762
d3456914 2763 return 0;
04886779
RA		 2764}
2765
d3456914 2766void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
04886779
RA		 2767{
2768 struct qedr_ah *ah = get_qedr_ah(ibah);
2769
d97099fe 2770 rdma_destroy_ah_attr(&ah->attr);
04886779
RA		 2771}
2772
e0290cce
RA		 2773static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2774{
2775 struct qedr_pbl *pbl, *tmp;
2776
2777 if (info->pbl_table)
2778 list_add_tail(&info->pbl_table->list_entry,
2779 &info->free_pbl_list);
2780
2781 if (!list_empty(&info->inuse_pbl_list))
2782 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2783
2784 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2785 list_del(&pbl->list_entry);
2786 qedr_free_pbl(dev, &info->pbl_info, pbl);
2787 }
2788}
2789
2790static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2791 size_t page_list_len, bool two_layered)
2792{
2793 struct qedr_pbl *tmp;
2794 int rc;
2795
2796 INIT_LIST_HEAD(&info->free_pbl_list);
2797 INIT_LIST_HEAD(&info->inuse_pbl_list);
2798
2799 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2800 page_list_len, two_layered);
2801 if (rc)
2802 goto done;
2803
2804 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2805 if (IS_ERR(info->pbl_table)) {
2806 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2807 goto done;
2808 }
2809
2810 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2811 &info->pbl_table->pa);
2812
2813 /* in usual case we use 2 PBLs, so we add one to free
2814 * list and allocating another one
2815 */
2816 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2817 if (IS_ERR(tmp)) {
e0290cce
RA		 2818 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2819 goto done;
2820 }
2821
2822 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2823
2824 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2825
2826done:
2827 if (rc)
2828 free_mr_info(dev, info);
2829
2830 return rc;
2831}
2832
2833struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2834 u64 usr_addr, int acc, struct ib_udata *udata)
2835{
2836 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2837 struct qedr_mr *mr;
2838 struct qedr_pd *pd;
2839 int rc = -ENOMEM;
2840
2841 pd = get_qedr_pd(ibpd);
2842 DP_DEBUG(dev, QEDR_MSG_MR,
2843 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2844 pd->pd_id, start, len, usr_addr, acc);
2845
2846 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2847 return ERR_PTR(-EINVAL);
2848
2849 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2850 if (!mr)
2851 return ERR_PTR(rc);
2852
2853 mr->type = QEDR_MR_USER;
2854
c320e527 2855 mr->umem = ib_umem_get(ibpd->device, start, len, acc);
e0290cce
RA		 2856 if (IS_ERR(mr->umem)) {
2857 rc = -EFAULT;
2858 goto err0;
2859 }
2860
2861 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2862 if (rc)
2863 goto err1;
2864
2865 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2866 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2867
2868 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2869 if (rc) {
2870 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2871 goto err1;
2872 }
2873
2874 /* Index only, 18 bit long, lkey = itid << 8 | key */
2875 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2876 mr->hw_mr.key = 0;
2877 mr->hw_mr.pd = pd->pd_id;
2878 mr->hw_mr.local_read = 1;
2879 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2880 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2881 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2882 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2883 mr->hw_mr.mw_bind = false;
2884 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2885 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2886 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 2887 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 2888 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2889 mr->hw_mr.length = len;
2890 mr->hw_mr.vaddr = usr_addr;
2891 mr->hw_mr.zbva = false;
2892 mr->hw_mr.phy_mr = false;
2893 mr->hw_mr.dma_mr = false;
2894
2895 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2896 if (rc) {
2897 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2898 goto err2;
2899 }
2900
2901 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2902 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2903 mr->hw_mr.remote_atomic)
2904 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2905
2906 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2907 mr->ibmr.lkey);
2908 return &mr->ibmr;
2909
2910err2:
2911 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2912err1:
2913 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2914err0:
2915 kfree(mr);
2916 return ERR_PTR(rc);
2917}
2918
c4367a26 2919int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 2920{
2921 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2922 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2923 int rc = 0;
2924
2925 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2926 if (rc)
2927 return rc;
2928
2929 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2930
24e412c1
MK		 2931 if (mr->type != QEDR_MR_DMA)
2932 free_mr_info(dev, &mr->info);
e0290cce
RA		 2933
2934 /* it could be user registered memory. */
836a0fbb 2935 ib_umem_release(mr->umem);
e0290cce
RA		 2936
2937 kfree(mr);
2938
2939 return rc;
2940}
2941
27a4b1a6
RA		 2942static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2943 int max_page_list_len)
e0290cce
RA		 2944{
2945 struct qedr_pd *pd = get_qedr_pd(ibpd);
2946 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2947 struct qedr_mr *mr;
2948 int rc = -ENOMEM;
2949
2950 DP_DEBUG(dev, QEDR_MSG_MR,
2951 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2952 max_page_list_len);
2953
2954 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2955 if (!mr)
2956 return ERR_PTR(rc);
2957
2958 mr->dev = dev;
2959 mr->type = QEDR_MR_FRMR;
2960
2961 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2962 if (rc)
2963 goto err0;
2964
2965 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2966 if (rc) {
2967 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2968 goto err0;
2969 }
2970
2971 /* Index only, 18 bit long, lkey = itid << 8 | key */
2972 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2973 mr->hw_mr.key = 0;
2974 mr->hw_mr.pd = pd->pd_id;
2975 mr->hw_mr.local_read = 1;
2976 mr->hw_mr.local_write = 0;
2977 mr->hw_mr.remote_read = 0;
2978 mr->hw_mr.remote_write = 0;
2979 mr->hw_mr.remote_atomic = 0;
2980 mr->hw_mr.mw_bind = false;
2981 mr->hw_mr.pbl_ptr = 0;
2982 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2983 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2984 mr->hw_mr.fbo = 0;
2985 mr->hw_mr.length = 0;
2986 mr->hw_mr.vaddr = 0;
2987 mr->hw_mr.zbva = false;
2988 mr->hw_mr.phy_mr = true;
2989 mr->hw_mr.dma_mr = false;
2990
2991 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2992 if (rc) {
2993 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2994 goto err1;
2995 }
2996
2997 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2998 mr->ibmr.rkey = mr->ibmr.lkey;
2999
3000 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3001 return mr;
3002
3003err1:
3004 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3005err0:
3006 kfree(mr);
3007 return ERR_PTR(rc);
3008}
3009
c4367a26 3010struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
42a3b153 3011 u32 max_num_sg)
e0290cce 3012{
e0290cce
RA		 3013 struct qedr_mr *mr;
3014
3015 if (mr_type != IB_MR_TYPE_MEM_REG)
3016 return ERR_PTR(-EINVAL);
3017
3018 mr = __qedr_alloc_mr(ibpd, max_num_sg);
3019
3020 if (IS_ERR(mr))
3021 return ERR_PTR(-EINVAL);
3022
e0290cce
RA		 3023 return &mr->ibmr;
3024}
3025
3026static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3027{
3028 struct qedr_mr *mr = get_qedr_mr(ibmr);
3029 struct qedr_pbl *pbl_table;
3030 struct regpair *pbe;
3031 u32 pbes_in_page;
3032
3033 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 3034 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 3035 return -ENOMEM;
3036 }
3037
3038 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3039 mr->npages, addr);
3040
3041 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3042 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3043 pbe = (struct regpair *)pbl_table->va;
3044 pbe += mr->npages % pbes_in_page;
3045 pbe->lo = cpu_to_le32((u32)addr);
3046 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3047
3048 mr->npages++;
3049
3050 return 0;
3051}
3052
3053static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3054{
3055 int work = info->completed - info->completed_handled - 1;
3056
3057 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3058 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3059 struct qedr_pbl *pbl;
3060
3061 /* Free all the page list that are possible to be freed
3062 * (all the ones that were invalidated), under the assumption
3063 * that if an FMR was completed successfully that means that
3064 * if there was an invalidate operation before it also ended
3065 */
3066 pbl = list_first_entry(&info->inuse_pbl_list,
3067 struct qedr_pbl, list_entry);
aafec388 3068 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 3069 info->completed_handled++;
3070 }
3071}
3072
3073int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3074 int sg_nents, unsigned int *sg_offset)
3075{
3076 struct qedr_mr *mr = get_qedr_mr(ibmr);
3077
3078 mr->npages = 0;
3079
3080 handle_completed_mrs(mr->dev, &mr->info);
3081 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3082}
3083
3084struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3085{
3086 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3087 struct qedr_pd *pd = get_qedr_pd(ibpd);
3088 struct qedr_mr *mr;
3089 int rc;
3090
3091 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3092 if (!mr)
3093 return ERR_PTR(-ENOMEM);
3094
3095 mr->type = QEDR_MR_DMA;
3096
3097 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3098 if (rc) {
3099 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3100 goto err1;
3101 }
3102
3103 /* index only, 18 bit long, lkey = itid << 8 | key */
3104 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3105 mr->hw_mr.pd = pd->pd_id;
3106 mr->hw_mr.local_read = 1;
3107 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3108 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3109 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3110 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3111 mr->hw_mr.dma_mr = true;
3112
3113 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3114 if (rc) {
3115 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3116 goto err2;
3117 }
3118
3119 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3120 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3121 mr->hw_mr.remote_atomic)
3122 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3123
3124 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3125 return &mr->ibmr;
3126
3127err2:
3128 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3129err1:
3130 kfree(mr);
3131 return ERR_PTR(rc);
3132}
afa0e13b
RA		 3133
3134static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3135{
3136 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3137}
3138
3139static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3140{
3141 int i, len = 0;
3142
3143 for (i = 0; i < num_sge; i++)
3144 len += sg_list[i].length;
3145
3146 return len;
3147}
3148
3149static void swap_wqe_data64(u64 *p)
3150{
3151 int i;
3152
3153 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3154 *p = cpu_to_be64(cpu_to_le64(*p));
3155}
3156
3157static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3158 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 3159 const struct ib_send_wr *wr,
3160 const struct ib_send_wr **bad_wr,
3161 u8 *bits, u8 bit)
afa0e13b
RA		 3162{
3163 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3164 char *seg_prt, *wqe;
3165 int i, seg_siz;
3166
3167 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3168 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3169 *bad_wr = wr;
3170 return 0;
3171 }
3172
3173 if (!data_size)
3174 return data_size;
3175
3176 *bits |= bit;
3177
3178 seg_prt = NULL;
3179 wqe = NULL;
3180 seg_siz = 0;
3181
3182 /* Copy data inline */
3183 for (i = 0; i < wr->num_sge; i++) {
3184 u32 len = wr->sg_list[i].length;
3185 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3186
3187 while (len > 0) {
3188 u32 cur;
3189
3190 /* New segment required */
3191 if (!seg_siz) {
3192 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3193 seg_prt = wqe;
3194 seg_siz = sizeof(struct rdma_sq_common_wqe);
3195 (*wqe_size)++;
3196 }
3197
3198 /* Calculate currently allowed length */
3199 cur = min_t(u32, len, seg_siz);
3200 memcpy(seg_prt, src, cur);
3201
3202 /* Update segment variables */
3203 seg_prt += cur;
3204 seg_siz -= cur;
3205
3206 /* Update sge variables */
3207 src += cur;
3208 len -= cur;
3209
3210 /* Swap fully-completed segments */
3211 if (!seg_siz)
3212 swap_wqe_data64((u64 *)wqe);
3213 }
3214 }
3215
3216 /* swap last not completed segment */
3217 if (seg_siz)
3218 swap_wqe_data64((u64 *)wqe);
3219
3220 return data_size;
3221}
3222
3223#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3224 do { \
3225 DMA_REGPAIR_LE(sge->addr, vaddr); \
3226 (sge)->length = cpu_to_le32(vlength); \
3227 (sge)->flags = cpu_to_le32(vflags); \
3228 } while (0)
3229
3230#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3231 do { \
3232 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3233 (hdr)->num_sges = num_sge; \
3234 } while (0)
3235
3236#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3237 do { \
3238 DMA_REGPAIR_LE(sge->addr, vaddr); \
3239 (sge)->length = cpu_to_le32(vlength); \
3240 (sge)->l_key = cpu_to_le32(vlkey); \
3241 } while (0)
3242
3243static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 3244 const struct ib_send_wr *wr)
afa0e13b
RA		 3245{
3246 u32 data_size = 0;
3247 int i;
3248
3249 for (i = 0; i < wr->num_sge; i++) {
3250 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3251
3252 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3253 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3254 sge->length = cpu_to_le32(wr->sg_list[i].length);
3255 data_size += wr->sg_list[i].length;
3256 }
3257
3258 if (wqe_size)
3259 *wqe_size += wr->num_sge;
3260
3261 return data_size;
3262}
3263
3264static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3265 struct qedr_qp *qp,
3266 struct rdma_sq_rdma_wqe_1st *rwqe,
3267 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3268 const struct ib_send_wr *wr,
3269 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3270{
3271 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3272 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3273
8b0cabc6
AR		 3274 if (wr->send_flags & IB_SEND_INLINE &&
3275 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3276 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3277 u8 flags = 0;
3278
3279 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3280 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3281 bad_wr, &rwqe->flags, flags);
3282 }
3283
3284 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3285}
3286
3287static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3288 struct qedr_qp *qp,
3289 struct rdma_sq_send_wqe_1st *swqe,
3290 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3291 const struct ib_send_wr *wr,
3292 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3293{
3294 memset(swqe2, 0, sizeof(*swqe2));
3295 if (wr->send_flags & IB_SEND_INLINE) {
3296 u8 flags = 0;
3297
3298 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3299 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3300 bad_wr, &swqe->flags, flags);
3301 }
3302
3303 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3304}
3305
3306static int qedr_prepare_reg(struct qedr_qp *qp,
3307 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3308 const struct ib_reg_wr *wr)
afa0e13b
RA		 3309{
3310 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3311 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3312
3313 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3314 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3315 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3316 fwqe1->l_key = wr->key;
3317
08c4cf51
AR		 3318 fwqe2->access_ctrl = 0;
3319
afa0e13b
RA		 3320 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3321 !!(wr->access & IB_ACCESS_REMOTE_READ));
3322 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3323 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3324 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3325 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3326 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3327 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3328 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3329 fwqe2->fmr_ctrl = 0;
3330
3331 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3332 ilog2(mr->ibmr.page_size) - 12);
3333
3334 fwqe2->length_hi = 0;
3335 fwqe2->length_lo = mr->ibmr.length;
3336 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3337 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3338
3339 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3340
3341 return 0;
3342}
3343
27a4b1a6 3344static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3345{
3346 switch (opcode) {
3347 case IB_WR_RDMA_WRITE:
3348 case IB_WR_RDMA_WRITE_WITH_IMM:
3349 return IB_WC_RDMA_WRITE;
3350 case IB_WR_SEND_WITH_IMM:
3351 case IB_WR_SEND:
3352 case IB_WR_SEND_WITH_INV:
3353 return IB_WC_SEND;
3354 case IB_WR_RDMA_READ:
fb1a22be 3355 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3356 return IB_WC_RDMA_READ;
3357 case IB_WR_ATOMIC_CMP_AND_SWP:
3358 return IB_WC_COMP_SWAP;
3359 case IB_WR_ATOMIC_FETCH_AND_ADD:
3360 return IB_WC_FETCH_ADD;
3361 case IB_WR_REG_MR:
3362 return IB_WC_REG_MR;
3363 case IB_WR_LOCAL_INV:
3364 return IB_WC_LOCAL_INV;
3365 default:
3366 return IB_WC_SEND;
3367 }
3368}
3369
f696bf6d
BVA		 3370static inline bool qedr_can_post_send(struct qedr_qp *qp,
3371 const struct ib_send_wr *wr)
afa0e13b
RA		 3372{
3373 int wq_is_full, err_wr, pbl_is_full;
3374 struct qedr_dev *dev = qp->dev;
3375
3376 /* prevent SQ overflow and/or processing of a bad WR */
3377 err_wr = wr->num_sge > qp->sq.max_sges;
3378 wq_is_full = qedr_wq_is_full(&qp->sq);
3379 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3380 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3381 if (wq_is_full || err_wr || pbl_is_full) {
3382 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3383 DP_ERR(dev,
3384 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3385 qp);
3386 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3387 }
3388
3389 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3390 DP_ERR(dev,
3391 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3392 qp);
3393 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3394 }
3395
3396 if (pbl_is_full &&
3397 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3398 DP_ERR(dev,
3399 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3400 qp);
3401 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3402 }
3403 return false;
3404 }
3405 return true;
3406}
3407
d34ac5cd
BVA		 3408static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3409 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3410{
3411 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3412 struct qedr_qp *qp = get_qedr_qp(ibqp);
3413 struct rdma_sq_atomic_wqe_1st *awqe1;
3414 struct rdma_sq_atomic_wqe_2nd *awqe2;
3415 struct rdma_sq_atomic_wqe_3rd *awqe3;
3416 struct rdma_sq_send_wqe_2st *swqe2;
3417 struct rdma_sq_local_inv_wqe *iwqe;
3418 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3419 struct rdma_sq_send_wqe_1st *swqe;
3420 struct rdma_sq_rdma_wqe_1st *rwqe;
3421 struct rdma_sq_fmr_wqe_1st *fwqe1;
3422 struct rdma_sq_common_wqe *wqe;
3423 u32 length;
3424 int rc = 0;
3425 bool comp;
3426
3427 if (!qedr_can_post_send(qp, wr)) {
3428 *bad_wr = wr;
3429 return -ENOMEM;
3430 }
3431
3432 wqe = qed_chain_produce(&qp->sq.pbl);
3433 qp->wqe_wr_id[qp->sq.prod].signaled =
3434 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3435
3436 wqe->flags = 0;
3437 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3438 !!(wr->send_flags & IB_SEND_SOLICITED));
3439 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3440 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3441 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3442 !!(wr->send_flags & IB_SEND_FENCE));
3443 wqe->prev_wqe_size = qp->prev_wqe_size;
3444
3445 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3446
3447 switch (wr->opcode) {
3448 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3449 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3450 rc = -EINVAL;
3451 *bad_wr = wr;
3452 break;
3453 }
afa0e13b
RA		 3454 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3455 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3456 swqe->wqe_size = 2;
3457 swqe2 = qed_chain_produce(&qp->sq.pbl);
3458
7bed7ebc 3459 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3460 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3461 wr, bad_wr);
3462 swqe->length = cpu_to_le32(length);
3463 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3464 qp->prev_wqe_size = swqe->wqe_size;
3465 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3466 break;
3467 case IB_WR_SEND:
3468 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3469 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3470
3471 swqe->wqe_size = 2;
3472 swqe2 = qed_chain_produce(&qp->sq.pbl);
3473 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3474 wr, bad_wr);
3475 swqe->length = cpu_to_le32(length);
3476 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3477 qp->prev_wqe_size = swqe->wqe_size;
3478 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3479 break;
3480 case IB_WR_SEND_WITH_INV:
3481 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3482 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3483 swqe2 = qed_chain_produce(&qp->sq.pbl);
3484 swqe->wqe_size = 2;
3485 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3486 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3487 wr, bad_wr);
3488 swqe->length = cpu_to_le32(length);
3489 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3490 qp->prev_wqe_size = swqe->wqe_size;
3491 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3492 break;
3493
3494 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3495 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3496 rc = -EINVAL;
3497 *bad_wr = wr;
3498 break;
3499 }
afa0e13b
RA		 3500 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3501 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3502
3503 rwqe->wqe_size = 2;
3504 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3505 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3506 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3507 wr, bad_wr);
3508 rwqe->length = cpu_to_le32(length);
3509 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3510 qp->prev_wqe_size = rwqe->wqe_size;
3511 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3512 break;
3513 case IB_WR_RDMA_WRITE:
3514 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3515 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3516
3517 rwqe->wqe_size = 2;
3518 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3519 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3520 wr, bad_wr);
3521 rwqe->length = cpu_to_le32(length);
3522 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3523 qp->prev_wqe_size = rwqe->wqe_size;
3524 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3525 break;
3526 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3527 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
1b8a708b 3528 /* fallthrough -- same is identical to RDMA READ */
afa0e13b
RA		 3529
3530 case IB_WR_RDMA_READ:
3531 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3532 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3533
3534 rwqe->wqe_size = 2;
3535 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3536 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3537 wr, bad_wr);
3538 rwqe->length = cpu_to_le32(length);
3539 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3540 qp->prev_wqe_size = rwqe->wqe_size;
3541 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3542 break;
3543
3544 case IB_WR_ATOMIC_CMP_AND_SWP:
3545 case IB_WR_ATOMIC_FETCH_AND_ADD:
3546 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3547 awqe1->wqe_size = 4;
3548
3549 awqe2 = qed_chain_produce(&qp->sq.pbl);
3550 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3551 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3552
3553 awqe3 = qed_chain_produce(&qp->sq.pbl);
3554
3555 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3556 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3557 DMA_REGPAIR_LE(awqe3->swap_data,
3558 atomic_wr(wr)->compare_add);
3559 } else {
3560 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3561 DMA_REGPAIR_LE(awqe3->swap_data,
3562 atomic_wr(wr)->swap);
3563 DMA_REGPAIR_LE(awqe3->cmp_data,
3564 atomic_wr(wr)->compare_add);
3565 }
3566
3567 qedr_prepare_sq_sges(qp, NULL, wr);
3568
3569 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3570 qp->prev_wqe_size = awqe1->wqe_size;
3571 break;
3572
3573 case IB_WR_LOCAL_INV:
3574 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3575 iwqe->wqe_size = 1;
3576
3577 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3578 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3579 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3580 qp->prev_wqe_size = iwqe->wqe_size;
3581 break;
3582 case IB_WR_REG_MR:
3583 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3584 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3585 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3586 fwqe1->wqe_size = 2;
3587
3588 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3589 if (rc) {
3590 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3591 *bad_wr = wr;
3592 break;
3593 }
3594
3595 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3596 qp->prev_wqe_size = fwqe1->wqe_size;
3597 break;
3598 default:
3599 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3600 rc = -EINVAL;
3601 *bad_wr = wr;
3602 break;
3603 }
3604
3605 if (*bad_wr) {
3606 u16 value;
3607
3608 /* Restore prod to its position before
3609 * this WR was processed
3610 */
3611 value = le16_to_cpu(qp->sq.db_data.data.value);
3612 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3613
3614 /* Restore prev_wqe_size */
3615 qp->prev_wqe_size = wqe->prev_wqe_size;
3616 rc = -EINVAL;
3617 DP_ERR(dev, "POST SEND FAILED\n");
3618 }
3619
3620 return rc;
3621}
3622
d34ac5cd
BVA		 3623int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3624 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3625{
3626 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3627 struct qedr_qp *qp = get_qedr_qp(ibqp);
3628 unsigned long flags;
3629 int rc = 0;
3630
3631 *bad_wr = NULL;
3632
04886779
RA		 3633 if (qp->qp_type == IB_QPT_GSI)
3634 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3635
afa0e13b
RA		 3636 spin_lock_irqsave(&qp->q_lock, flags);
3637
f5b1b177
KM		 3638 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3639 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3640 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3641 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3642 spin_unlock_irqrestore(&qp->q_lock, flags);
3643 *bad_wr = wr;
3644 DP_DEBUG(dev, QEDR_MSG_CQ,
3645 "QP in wrong state! QP icid=0x%x state %d\n",
3646 qp->icid, qp->state);
3647 return -EINVAL;
3648 }
afa0e13b
RA		 3649 }
3650
afa0e13b
RA		 3651 while (wr) {
3652 rc = __qedr_post_send(ibqp, wr, bad_wr);
3653 if (rc)
3654 break;
3655
3656 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3657
3658 qedr_inc_sw_prod(&qp->sq);
3659
3660 qp->sq.db_data.data.value++;
3661
3662 wr = wr->next;
3663 }
3664
3665 /* Trigger doorbell
3666 * If there was a failure in the first WR then it will be triggered in
3667 * vane. However this is not harmful (as long as the producer value is
3668 * unchanged). For performance reasons we avoid checking for this
3669 * redundant doorbell.
09c4854f
KM		 3670 *
3671 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3672 * soon as we give the doorbell, we could get a completion
3673 * for this wr, therefore we need to make sure that the
3674 * memory is updated before giving the doorbell.
3675 * During qedr_poll_cq, rmb is called before accessing the
3676 * cqe. This covers for the smp_rmb as well.
afa0e13b 3677 */
09c4854f 3678 smp_wmb();
afa0e13b
RA		 3679 writel(qp->sq.db_data.raw, qp->sq.db);
3680
afa0e13b
RA		 3681 spin_unlock_irqrestore(&qp->q_lock, flags);
3682
3683 return rc;
3684}
3685
3491c9e7
YB		 3686static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3687{
3688 u32 used;
3689
3690 /* Calculate number of elements used based on producer
3691 * count and consumer count and subtract it from max
3692 * work request supported so that we get elements left.
3693 */
acca72e2 3694 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3491c9e7
YB		 3695
3696 return hw_srq->max_wr - used;
3697}
3698
3699int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3700 const struct ib_recv_wr **bad_wr)
3701{
3702 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3703 struct qedr_srq_hwq_info *hw_srq;
3704 struct qedr_dev *dev = srq->dev;
3705 struct qed_chain *pbl;
3706 unsigned long flags;
3707 int status = 0;
3708 u32 num_sge;
3491c9e7
YB		 3709
3710 spin_lock_irqsave(&srq->lock, flags);
3711
3712 hw_srq = &srq->hw_srq;
3713 pbl = &srq->hw_srq.pbl;
3714 while (wr) {
3715 struct rdma_srq_wqe_header *hdr;
3716 int i;
3717
3718 if (!qedr_srq_elem_left(hw_srq) ||
3719 wr->num_sge > srq->hw_srq.max_sges) {
3720 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
acca72e2
YB		 3721 hw_srq->wr_prod_cnt,
3722 atomic_read(&hw_srq->wr_cons_cnt),
3491c9e7
YB		 3723 wr->num_sge, srq->hw_srq.max_sges);
3724 status = -ENOMEM;
3725 *bad_wr = wr;
3726 break;
3727 }
3728
3729 hdr = qed_chain_produce(pbl);
3730 num_sge = wr->num_sge;
3731 /* Set number of sge and work request id in header */
3732 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3733
3734 srq->hw_srq.wr_prod_cnt++;
3735 hw_srq->wqe_prod++;
3736 hw_srq->sge_prod++;
3737
3738 DP_DEBUG(dev, QEDR_MSG_SRQ,
3739 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3740 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3741
3742 for (i = 0; i < wr->num_sge; i++) {
3743 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3744
3745 /* Set SGE length, lkey and address */
3746 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3747 wr->sg_list[i].length, wr->sg_list[i].lkey);
3748
3749 DP_DEBUG(dev, QEDR_MSG_SRQ,
3750 "[%d]: len %d key %x addr %x:%x\n",
3751 i, srq_sge->length, srq_sge->l_key,
3752 srq_sge->addr.hi, srq_sge->addr.lo);
3753 hw_srq->sge_prod++;
3754 }
3755
acca72e2 3756 /* Update WQE and SGE information before
3491c9e7
YB		 3757 * updating producer.
3758 */
acca72e2 3759 dma_wmb();
3491c9e7
YB		 3760
3761 /* SRQ producer is 8 bytes. Need to update SGE producer index
3762 * in first 4 bytes and need to update WQE producer in
3763 * next 4 bytes.
3764 */
acca72e2
YB		 3765 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3766 /* Make sure sge producer is updated first */
3767 dma_wmb();
3768 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3491c9e7 3769
3491c9e7
YB		 3770 wr = wr->next;
3771 }
3772
3773 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3774 qed_chain_get_elem_left(pbl));
3775 spin_unlock_irqrestore(&srq->lock, flags);
3776
3777 return status;
3778}
3779
d34ac5cd
BVA		 3780int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3781 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3782{
3783 struct qedr_qp *qp = get_qedr_qp(ibqp);
3784 struct qedr_dev *dev = qp->dev;
3785 unsigned long flags;
3786 int status = 0;
3787
04886779
RA		 3788 if (qp->qp_type == IB_QPT_GSI)
3789 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3790
afa0e13b
RA		 3791 spin_lock_irqsave(&qp->q_lock, flags);
3792
922d9a40 3793 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3794 spin_unlock_irqrestore(&qp->q_lock, flags);
3795 *bad_wr = wr;
3796 return -EINVAL;
3797 }
3798
3799 while (wr) {
3800 int i;
3801
3802 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3803 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3804 wr->num_sge > qp->rq.max_sges) {
3805 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3806 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3807 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3808 qp->rq.max_sges);
3809 status = -ENOMEM;
3810 *bad_wr = wr;
3811 break;
3812 }
3813 for (i = 0; i < wr->num_sge; i++) {
3814 u32 flags = 0;
3815 struct rdma_rq_sge *rqe =
3816 qed_chain_produce(&qp->rq.pbl);
3817
3818 /* First one must include the number
3819 * of SGE in the list
3820 */
3821 if (!i)
3822 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3823 wr->num_sge);
3824
d52c89f1 3825 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3826 wr->sg_list[i].lkey);
3827
3828 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3829 wr->sg_list[i].length, flags);
3830 }
3831
3832 /* Special case of no sges. FW requires between 1-4 sges...
3833 * in this case we need to post 1 sge with length zero. this is
3834 * because rdma write with immediate consumes an RQ.
3835 */
3836 if (!wr->num_sge) {
3837 u32 flags = 0;
3838 struct rdma_rq_sge *rqe =
3839 qed_chain_produce(&qp->rq.pbl);
3840
3841 /* First one must include the number
3842 * of SGE in the list
3843 */
d52c89f1 3844 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3845 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3846
3847 RQ_SGE_SET(rqe, 0, 0, flags);
3848 i = 1;
3849 }
3850
3851 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3852 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3853
3854 qedr_inc_sw_prod(&qp->rq);
3855
09c4854f
KM		 3856 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3857 * soon as we give the doorbell, we could get a completion
3858 * for this wr, therefore we need to make sure that the
3859 * memory is update before giving the doorbell.
3860 * During qedr_poll_cq, rmb is called before accessing the
3861 * cqe. This covers for the smp_rmb as well.
3862 */
3863 smp_wmb();
afa0e13b
RA		 3864
3865 qp->rq.db_data.data.value++;
3866
3867 writel(qp->rq.db_data.raw, qp->rq.db);
3868
f5b1b177
KM		 3869 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3870 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
f5b1b177
KM		 3871 }
3872
afa0e13b
RA		 3873 wr = wr->next;
3874 }
3875
3876 spin_unlock_irqrestore(&qp->q_lock, flags);
3877
3878 return status;
3879}
3880
3881static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3882{
3883 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3884
3885 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3886 cq->pbl_toggle;
3887}
3888
3889static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3890{
3891 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3892 struct qedr_qp *qp;
3893
3894 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3895 resp_cqe->qp_handle.lo,
3896 u64);
3897 return qp;
3898}
3899
3900static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3901{
3902 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3903
3904 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3905}
3906
3907/* Return latest CQE (needs processing) */
3908static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3909{
3910 return cq->latest_cqe;
3911}
3912
3913/* In fmr we need to increase the number of fmr completed counter for the fmr
3914 * algorithm determining whether we can free a pbl or not.
3915 * we need to perform this whether the work request was signaled or not. for
3916 * this purpose we call this function from the condition that checks if a wr
3917 * should be skipped, to make sure we don't miss it ( possibly this fmr
3918 * operation was not signalted)
3919 */
3920static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3921{
3922 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3923 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3924}
3925
3926static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3927 struct qedr_cq *cq, int num_entries,
3928 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3929 int force)
3930{
3931 u16 cnt = 0;
3932
3933 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3934 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3935 qedr_chk_if_fmr(qp);
3936 /* skip WC */
3937 goto next_cqe;
3938 }
3939
3940 /* fill WC */
3941 wc->status = status;
27035a1b 3942 wc->vendor_err = 0;
afa0e13b
RA		 3943 wc->wc_flags = 0;
3944 wc->src_qp = qp->id;
3945 wc->qp = &qp->ibqp;
3946
3947 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3948 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3949
3950 switch (wc->opcode) {
3951 case IB_WC_RDMA_WRITE:
3952 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3953 break;
3954 case IB_WC_COMP_SWAP:
3955 case IB_WC_FETCH_ADD:
3956 wc->byte_len = 8;
3957 break;
3958 case IB_WC_REG_MR:
3959 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3960 break;
dac27386
MK		 3961 case IB_WC_RDMA_READ:
3962 case IB_WC_SEND:
3963 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3964 break;
afa0e13b
RA		 3965 default:
3966 break;
3967 }
3968
3969 num_entries--;
3970 wc++;
3971 cnt++;
3972next_cqe:
3973 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3974 qed_chain_consume(&qp->sq.pbl);
3975 qedr_inc_sw_cons(&qp->sq);
3976 }
3977
3978 return cnt;
3979}
3980
3981static int qedr_poll_cq_req(struct qedr_dev *dev,
3982 struct qedr_qp *qp, struct qedr_cq *cq,
3983 int num_entries, struct ib_wc *wc,
3984 struct rdma_cqe_requester *req)
3985{
3986 int cnt = 0;
3987
3988 switch (req->status) {
3989 case RDMA_CQE_REQ_STS_OK:
3990 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3991 IB_WC_SUCCESS, 0);
3992 break;
3993 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 3994 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 3995 DP_DEBUG(dev, QEDR_MSG_CQ,
3996 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3997 cq->icid, qp->icid);
afa0e13b 3998 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3999 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 4000 break;
4001 default:
4002 /* process all WQE before the cosumer */
4003 qp->state = QED_ROCE_QP_STATE_ERR;
4004 cnt = process_req(dev, qp, cq, num_entries, wc,
4005 req->sq_cons - 1, IB_WC_SUCCESS, 0);
4006 wc += cnt;
4007 /* if we have extra WC fill it with actual error info */
4008 if (cnt < num_entries) {
4009 enum ib_wc_status wc_status;
4010
4011 switch (req->status) {
4012 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4013 DP_ERR(dev,
4014 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4015 cq->icid, qp->icid);
4016 wc_status = IB_WC_BAD_RESP_ERR;
4017 break;
4018 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4019 DP_ERR(dev,
4020 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4021 cq->icid, qp->icid);
4022 wc_status = IB_WC_LOC_LEN_ERR;
4023 break;
4024 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4025 DP_ERR(dev,
4026 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4027 cq->icid, qp->icid);
4028 wc_status = IB_WC_LOC_QP_OP_ERR;
4029 break;
4030 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4031 DP_ERR(dev,
4032 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4033 cq->icid, qp->icid);
4034 wc_status = IB_WC_LOC_PROT_ERR;
4035 break;
4036 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4037 DP_ERR(dev,
4038 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4039 cq->icid, qp->icid);
4040 wc_status = IB_WC_MW_BIND_ERR;
4041 break;
4042 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4043 DP_ERR(dev,
4044 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4045 cq->icid, qp->icid);
4046 wc_status = IB_WC_REM_INV_REQ_ERR;
4047 break;
4048 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4049 DP_ERR(dev,
4050 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4051 cq->icid, qp->icid);
4052 wc_status = IB_WC_REM_ACCESS_ERR;
4053 break;
4054 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4055 DP_ERR(dev,
4056 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4057 cq->icid, qp->icid);
4058 wc_status = IB_WC_REM_OP_ERR;
4059 break;
4060 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4061 DP_ERR(dev,
4062 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4063 cq->icid, qp->icid);
4064 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4065 break;
4066 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4067 DP_ERR(dev,
4068 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4069 cq->icid, qp->icid);
4070 wc_status = IB_WC_RETRY_EXC_ERR;
4071 break;
4072 default:
4073 DP_ERR(dev,
4074 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4075 cq->icid, qp->icid);
4076 wc_status = IB_WC_GENERAL_ERR;
4077 }
4078 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4079 wc_status, 1);
4080 }
4081 }
4082
4083 return cnt;
4084}
4085
b6acd71f 4086static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 4087{
b6acd71f 4088 switch (status) {
afa0e13b 4089 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 4090 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 4091 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 4092 return IB_WC_LOC_LEN_ERR;
afa0e13b 4093 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 4094 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 4095 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 4096 return IB_WC_LOC_PROT_ERR;
afa0e13b 4097 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 4098 return IB_WC_MW_BIND_ERR;
afa0e13b 4099 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 4100 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 4101 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 4102 return IB_WC_SUCCESS;
4103 default:
4104 return IB_WC_GENERAL_ERR;
4105 }
4106}
afa0e13b 4107
b6acd71f
AR		 4108static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4109 struct ib_wc *wc)
4110{
4111 wc->status = IB_WC_SUCCESS;
4112 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 4113
b6acd71f 4114 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 4115 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 4116 wc->wc_flags |= IB_WC_WITH_IMM;
4117
4118 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 4119 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4120
b6acd71f
AR		 4121 if (resp->flags & QEDR_RESP_INV)
4122 return -EINVAL;
4123
4124 } else if (resp->flags & QEDR_RESP_INV) {
4125 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4126 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4127
4128 if (resp->flags & QEDR_RESP_RDMA)
4129 return -EINVAL;
4130
4131 } else if (resp->flags & QEDR_RESP_RDMA) {
4132 return -EINVAL;
4133 }
4134
4135 return 0;
4136}
4137
4138static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4139 struct qedr_cq *cq, struct ib_wc *wc,
4140 struct rdma_cqe_responder *resp, u64 wr_id)
4141{
4142 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4143 wc->opcode = IB_WC_RECV;
4144 wc->wc_flags = 0;
4145
4146 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4147 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4148 DP_ERR(dev,
4149 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4150 cq, cq->icid, resp->flags);
4151
4152 } else {
4153 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4154 if (wc->status == IB_WC_GENERAL_ERR)
4155 DP_ERR(dev,
4156 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4157 cq, cq->icid, resp->status);
afa0e13b
RA		 4158 }
4159
b6acd71f 4160 /* Fill the rest of the WC */
27035a1b 4161 wc->vendor_err = 0;
afa0e13b
RA		 4162 wc->src_qp = qp->id;
4163 wc->qp = &qp->ibqp;
4164 wc->wr_id = wr_id;
4165}
4166
3491c9e7
YB		 4167static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4168 struct qedr_cq *cq, struct ib_wc *wc,
4169 struct rdma_cqe_responder *resp)
4170{
4171 struct qedr_srq *srq = qp->srq;
4172 u64 wr_id;
4173
4174 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4175 le32_to_cpu(resp->srq_wr_id.lo), u64);
4176
4177 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4178 wc->status = IB_WC_WR_FLUSH_ERR;
4179 wc->vendor_err = 0;
4180 wc->wr_id = wr_id;
4181 wc->byte_len = 0;
4182 wc->src_qp = qp->id;
4183 wc->qp = &qp->ibqp;
4184 wc->wr_id = wr_id;
4185 } else {
4186 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4187 }
acca72e2 4188 atomic_inc(&srq->hw_srq.wr_cons_cnt);
3491c9e7
YB		 4189
4190 return 1;
4191}
afa0e13b
RA		 4192static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4193 struct qedr_cq *cq, struct ib_wc *wc,
4194 struct rdma_cqe_responder *resp)
4195{
4196 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4197
4198 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4199
4200 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4201 qed_chain_consume(&qp->rq.pbl);
4202 qedr_inc_sw_cons(&qp->rq);
4203
4204 return 1;
4205}
4206
4207static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4208 int num_entries, struct ib_wc *wc, u16 hw_cons)
4209{
4210 u16 cnt = 0;
4211
4212 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4213 /* fill WC */
4214 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 4215 wc->vendor_err = 0;
afa0e13b
RA		 4216 wc->wc_flags = 0;
4217 wc->src_qp = qp->id;
4218 wc->byte_len = 0;
4219 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4220 wc->qp = &qp->ibqp;
4221 num_entries--;
4222 wc++;
4223 cnt++;
4224 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4225 qed_chain_consume(&qp->rq.pbl);
4226 qedr_inc_sw_cons(&qp->rq);
4227 }
4228
4229 return cnt;
4230}
4231
4232static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4233 struct rdma_cqe_responder *resp, int *update)
4234{
50bc60cb 4235 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 4236 consume_cqe(cq);
4237 *update |= 1;
4238 }
4239}
4240
3491c9e7
YB		 4241static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4242 struct qedr_cq *cq, int num_entries,
4243 struct ib_wc *wc,
4244 struct rdma_cqe_responder *resp)
4245{
4246 int cnt;
4247
4248 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4249 consume_cqe(cq);
4250
4251 return cnt;
4252}
4253
afa0e13b
RA		 4254static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4255 struct qedr_cq *cq, int num_entries,
4256 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4257 int *update)
4258{
4259 int cnt;
4260
4261 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4262 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4263 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4264 try_consume_resp_cqe(cq, qp, resp, update);
4265 } else {
4266 cnt = process_resp_one(dev, qp, cq, wc, resp);
4267 consume_cqe(cq);
4268 *update |= 1;
4269 }
4270
4271 return cnt;
4272}
4273
4274static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4275 struct rdma_cqe_requester *req, int *update)
4276{
4277 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4278 consume_cqe(cq);
4279 *update |= 1;
4280 }
4281}
4282
4283int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4284{
4285 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4286 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4287 union rdma_cqe *cqe;
afa0e13b
RA		 4288 u32 old_cons, new_cons;
4289 unsigned long flags;
4290 int update = 0;
4291 int done = 0;
4292
4dd72636
AR		 4293 if (cq->destroyed) {
4294 DP_ERR(dev,
4295 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4296 cq, cq->icid);
4297 return 0;
4298 }
4299
04886779
RA		 4300 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4301 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4302
afa0e13b 4303 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4304 cqe = cq->latest_cqe;
afa0e13b
RA		 4305 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4306 while (num_entries && is_valid_cqe(cq, cqe)) {
4307 struct qedr_qp *qp;
4308 int cnt = 0;
4309
4310 /* prevent speculative reads of any field of CQE */
4311 rmb();
4312
4313 qp = cqe_get_qp(cqe);
4314 if (!qp) {
4315 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4316 break;
4317 }
4318
4319 wc->qp = &qp->ibqp;
4320
4321 switch (cqe_get_type(cqe)) {
4322 case RDMA_CQE_TYPE_REQUESTER:
4323 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4324 &cqe->req);
4325 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4326 break;
4327 case RDMA_CQE_TYPE_RESPONDER_RQ:
4328 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4329 &cqe->resp, &update);
4330 break;
3491c9e7
YB		 4331 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4332 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4333 wc, &cqe->resp);
4334 update = 1;
4335 break;
afa0e13b
RA		 4336 case RDMA_CQE_TYPE_INVALID:
4337 default:
4338 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4339 cqe_get_type(cqe));
4340 }
4341 num_entries -= cnt;
4342 wc += cnt;
4343 done += cnt;
4344
4345 cqe = get_cqe(cq);
4346 }
4347 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4348
4349 cq->cq_cons += new_cons - old_cons;
4350
4351 if (update)
4352 /* doorbell notifies abount latest VALID entry,
4353 * but chain already point to the next INVALID one
4354 */
4355 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4356
4357 spin_unlock_irqrestore(&cq->cq_lock, flags);
4358 return done;
4359}
993d1b52
RA		 4360
4361int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
e26e7b88
LR		 4362 u8 port_num, const struct ib_wc *in_wc,
4363 const struct ib_grh *in_grh, const struct ib_mad *in,
4364 struct ib_mad *out_mad, size_t *out_mad_size,
4365 u16 *out_mad_pkey_index)
993d1b52 4366{
993d1b52
RA		 4367 return IB_MAD_RESULT_SUCCESS;
4368}
Linux 6.x block layer and io_uring tree(s)