projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'efi-efivars-removal-for-v5.20' of git://git.kernel.org/pub/scm/linux/kerne...
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
ff23dfa1 45#include <rdma/uverbs_ioctl.h>
ac1b36e5 46
be086e7c
MY		 47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
99d195cc 53#include "qedr_roce_cm.h"
82af6d19 54#include "qedr_iw_cm.h"
ac1b36e5 55
3491c9e7
YB		 56#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57#define RDMA_MAX_SGE_PER_SRQ (4)
58#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
a7efd777
RA		 60#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
4c6bb02d
MK		 62enum {
63 QEDR_USER_MMAP_IO_WC = 0,
97f61250 64 QEDR_USER_MMAP_PHYS_PAGE,
4c6bb02d
MK		 65};
66
c75d3ec8
AR		 67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 size_t len)
69{
70 size_t min_len = min_t(size_t, len, udata->outlen);
71
72 return ib_copy_to_udata(udata, src, min_len);
73}
74
1fb7f897 75int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
a7efd777 76{
dbe30dae 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 78 return -EINVAL;
79
80 *pkey = QEDR_ROCE_PKEY_DEFAULT;
81 return 0;
82}
83
1fb7f897 84int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
e6a38c54
KM		 85 int index, union ib_gid *sgid)
86{
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89 memset(sgid->raw, 0, sizeof(sgid->raw));
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 sgid->global.interface_id, sgid->global.subnet_prefix);
94
95 return 0;
96}
97
3491c9e7
YB		 98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99{
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
102 struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104 srq_attr->srq_limit = srq->srq_limit;
105 srq_attr->max_wr = qattr->max_srq_wr;
106 srq_attr->max_sge = qattr->max_sge;
107
108 return 0;
109}
110
ac1b36e5
RA		 111int qedr_query_device(struct ib_device *ibdev,
112 struct ib_device_attr *attr, struct ib_udata *udata)
113{
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
116
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
119 "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 dev->rdma_ctx);
121 return -EINVAL;
122 }
123
124 memset(attr, 0, sizeof(*attr));
125
126 attr->fw_ver = qattr->fw_ver;
127 attr->sys_image_guid = qattr->sys_image_guid;
128 attr->max_mr_size = qattr->max_mr_size;
129 attr->page_size_cap = qattr->page_size_caps;
130 attr->vendor_id = qattr->vendor_id;
131 attr->vendor_part_id = qattr->vendor_part_id;
132 attr->hw_ver = qattr->hw_ver;
133 attr->max_qp = qattr->max_qp;
134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 IB_DEVICE_RC_RNR_NAK_GEN |
e945c653
JG		 137 IB_DEVICE_MEM_MGT_EXTENSIONS;
138 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
ac1b36e5 139
06e8d1df
YB		 140 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
141 attr->device_cap_flags |= IB_DEVICE_XRC;
33023fb8
SW		 142 attr->max_send_sge = qattr->max_sge;
143 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 144 attr->max_sge_rd = qattr->max_sge;
145 attr->max_cq = qattr->max_cq;
146 attr->max_cqe = qattr->max_cqe;
147 attr->max_mr = qattr->max_mr;
148 attr->max_mw = qattr->max_mw;
149 attr->max_pd = qattr->max_pd;
150 attr->atomic_cap = dev->atomic_cap;
ac1b36e5
RA		 151 attr->max_qp_init_rd_atom =
152 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
153 attr->max_qp_rd_atom =
154 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
155 attr->max_qp_init_rd_atom);
156
157 attr->max_srq = qattr->max_srq;
158 attr->max_srq_sge = qattr->max_srq_sge;
159 attr->max_srq_wr = qattr->max_srq_wr;
160
161 attr->local_ca_ack_delay = qattr->dev_ack_delay;
162 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
7d11b478 163 attr->max_pkeys = qattr->max_pkey;
ac1b36e5
RA		 164 attr->max_ah = qattr->max_ah;
165
166 return 0;
167}
168
376ceb31 169static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
ac1b36e5
RA		 170 u8 *ib_width)
171{
172 switch (speed) {
173 case 1000:
69054666 174 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 175 *ib_width = IB_WIDTH_1X;
176 break;
177 case 10000:
69054666 178 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 179 *ib_width = IB_WIDTH_1X;
180 break;
181
182 case 20000:
69054666 183 *ib_speed = IB_SPEED_DDR;
ac1b36e5
RA		 184 *ib_width = IB_WIDTH_4X;
185 break;
186
187 case 25000:
69054666 188 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 40000:
69054666 193 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 50000:
69054666
SO		 198 *ib_speed = IB_SPEED_HDR;
199 *ib_width = IB_WIDTH_1X;
ac1b36e5
RA		 200 break;
201
202 case 100000:
69054666 203 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 default:
208 /* Unsupported */
69054666 209 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 210 *ib_width = IB_WIDTH_1X;
211 }
212}
213
1fb7f897
MB		 214int qedr_query_port(struct ib_device *ibdev, u32 port,
215 struct ib_port_attr *attr)
ac1b36e5
RA		 216{
217 struct qedr_dev *dev;
218 struct qed_rdma_port *rdma_port;
219
220 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 221
222 if (!dev->rdma_ctx) {
223 DP_ERR(dev, "rdma_ctx is NULL\n");
224 return -EINVAL;
225 }
226
227 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 228
c4550c63 229 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 230 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
231 attr->state = IB_PORT_ACTIVE;
72a7720f 232 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
ac1b36e5
RA		 233 } else {
234 attr->state = IB_PORT_DOWN;
72a7720f 235 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
ac1b36e5
RA		 236 }
237 attr->max_mtu = IB_MTU_4096;
ac1b36e5
RA		 238 attr->lid = 0;
239 attr->lmc = 0;
240 attr->sm_lid = 0;
241 attr->sm_sl = 0;
2f944c0f 242 attr->ip_gids = true;
f5b1b177 243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
cc293f54 244 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
f5b1b177 245 attr->gid_tbl_len = 1;
f5b1b177 246 } else {
cc293f54 247 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
f5b1b177
KM		 248 attr->gid_tbl_len = QEDR_MAX_SGID;
249 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
250 }
ac1b36e5
RA		 251 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
252 attr->qkey_viol_cntr = 0;
253 get_link_speed_and_width(rdma_port->link_speed,
254 &attr->active_speed, &attr->active_width);
255 attr->max_msg_sz = rdma_port->max_msg_size;
256 attr->max_vl_num = 4;
257
258 return 0;
259}
260
a2a074ef 261int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 262{
a2a074ef 263 struct ib_device *ibdev = uctx->device;
ac1b36e5 264 int rc;
a2a074ef
LR		 265 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
266 struct qedr_alloc_ucontext_resp uresp = {};
97f61250 267 struct qedr_alloc_ucontext_req ureq = {};
ac1b36e5
RA		 268 struct qedr_dev *dev = get_qedr_dev(ibdev);
269 struct qed_rdma_add_user_out_params oparams;
4c6bb02d 270 struct qedr_user_mmap_entry *entry;
ac1b36e5
RA		 271
272 if (!udata)
a2a074ef 273 return -EFAULT;
ac1b36e5 274
97f61250
MK		 275 if (udata->inlen) {
276 rc = ib_copy_from_udata(&ureq, udata,
277 min(sizeof(ureq), udata->inlen));
278 if (rc) {
279 DP_ERR(dev, "Problem copying data from user space\n");
280 return -EFAULT;
281 }
bbe4f424
MK		 282 ctx->edpm_mode = !!(ureq.context_flags &
283 QEDR_ALLOC_UCTX_EDPM_MODE);
97f61250
MK		 284 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
285 }
286
ac1b36e5
RA		 287 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
288 if (rc) {
289 DP_ERR(dev,
290 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
291 rc);
a2a074ef 292 return rc;
ac1b36e5
RA		 293 }
294
295 ctx->dpi = oparams.dpi;
296 ctx->dpi_addr = oparams.dpi_addr;
297 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
298 ctx->dpi_size = oparams.dpi_size;
4c6bb02d
MK		 299 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
300 if (!entry) {
301 rc = -ENOMEM;
302 goto err;
303 }
304
305 entry->io_address = ctx->dpi_phys_addr;
306 entry->length = ctx->dpi_size;
307 entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
308 entry->dpi = ctx->dpi;
309 entry->dev = dev;
310 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
311 ctx->dpi_size);
312 if (rc) {
313 kfree(entry);
314 goto err;
315 }
316 ctx->db_mmap_entry = &entry->rdma_entry;
ac1b36e5 317
93a3d05f
MK		 318 if (!dev->user_dpm_enabled)
319 uresp.dpm_flags = 0;
320 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
321 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
322 else
323 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
bbe4f424
MK		 324 QEDR_DPM_TYPE_ROCE_LEGACY |
325 QEDR_DPM_TYPE_ROCE_EDPM_MODE;
93a3d05f 326
eb7f84e3
MK		 327 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
328 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
329 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
330 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
331 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
332 }
93a3d05f 333
67cbe353
AR		 334 uresp.wids_enabled = 1;
335 uresp.wid_count = oparams.wid_count;
4c6bb02d 336 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
ac1b36e5
RA		 337 uresp.db_size = ctx->dpi_size;
338 uresp.max_send_wr = dev->attr.max_sqe;
339 uresp.max_recv_wr = dev->attr.max_rqe;
340 uresp.max_srq_wr = dev->attr.max_srq_wr;
341 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
342 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
343 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
344 uresp.max_cqes = QEDR_MAX_CQES;
345
c75d3ec8 346 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 347 if (rc)
4c6bb02d 348 goto err;
ac1b36e5
RA		 349
350 ctx->dev = dev;
351
ac1b36e5
RA		 352 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
353 &ctx->ibucontext);
a2a074ef 354 return 0;
4c6bb02d
MK		 355
356err:
357 if (!ctx->db_mmap_entry)
358 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
359 else
360 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
361
362 return rc;
ac1b36e5
RA		 363}
364
a2a074ef 365void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 366{
367 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
ac1b36e5
RA		 368
369 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
370 uctx);
4c6bb02d
MK		 371
372 rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
ac1b36e5
RA		 373}
374
4c6bb02d 375void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
ac1b36e5 376{
4c6bb02d
MK		 377 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
378 struct qedr_dev *dev = entry->dev;
ac1b36e5 379
97f61250
MK		 380 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
381 free_page((unsigned long)entry->address);
382 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
4c6bb02d 383 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
30bf066c 384
4c6bb02d
MK		 385 kfree(entry);
386}
ac1b36e5 387
4c6bb02d
MK		 388int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
389{
390 struct ib_device *dev = ucontext->device;
391 size_t length = vma->vm_end - vma->vm_start;
392 struct rdma_user_mmap_entry *rdma_entry;
393 struct qedr_user_mmap_entry *entry;
394 int rc = 0;
395 u64 pfn;
396
397 ibdev_dbg(dev,
398 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
399 vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
400
401 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
402 if (!rdma_entry) {
403 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
404 vma->vm_pgoff);
ac1b36e5
RA		 405 return -EINVAL;
406 }
4c6bb02d
MK		 407 entry = get_qedr_mmap_entry(rdma_entry);
408 ibdev_dbg(dev,
409 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
410 entry->io_address, length, entry->mmap_flag);
411
412 switch (entry->mmap_flag) {
413 case QEDR_USER_MMAP_IO_WC:
414 pfn = entry->io_address >> PAGE_SHIFT;
415 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
416 pgprot_writecombine(vma->vm_page_prot),
417 rdma_entry);
418 break;
97f61250
MK		 419 case QEDR_USER_MMAP_PHYS_PAGE:
420 rc = vm_insert_page(vma, vma->vm_start,
421 virt_to_page(entry->address));
422 break;
4c6bb02d
MK		 423 default:
424 rc = -EINVAL;
30bf066c 425 }
ac1b36e5 426
4c6bb02d
MK		 427 if (rc)
428 ibdev_dbg(dev,
429 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
430 entry->io_address, length, entry->mmap_flag, rc);
30bf066c 431
4c6bb02d
MK		 432 rdma_user_mmap_entry_put(rdma_entry);
433 return rc;
ac1b36e5 434}
a7efd777 435
ff23dfa1 436int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777 437{
21a428a0 438 struct ib_device *ibdev = ibpd->device;
a7efd777 439 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 440 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 441 u16 pd_id;
442 int rc;
443
444 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
ff23dfa1 445 udata ? "User Lib" : "Kernel");
a7efd777
RA		 446
447 if (!dev->rdma_ctx) {
847cb1a3 448 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 449 return -EINVAL;
a7efd777
RA		 450 }
451
9c1e0228
RA		 452 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
453 if (rc)
21a428a0 454 return rc;
a7efd777 455
a7efd777
RA		 456 pd->pd_id = pd_id;
457
ff23dfa1 458 if (udata) {
57939021
JG		 459 struct qedr_alloc_pd_uresp uresp = {
460 .pd_id = pd_id,
461 };
ff23dfa1
SR		 462 struct qedr_ucontext *context = rdma_udata_to_drv_context(
463 udata, struct qedr_ucontext, ibucontext);
9c1e0228 464
c75d3ec8 465 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 466 if (rc) {
a7efd777 467 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 468 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 469 return rc;
9c1e0228
RA		 470 }
471
ff23dfa1 472 pd->uctx = context;
9c1e0228 473 pd->uctx->pd = pd;
a7efd777
RA		 474 }
475
21a428a0 476 return 0;
a7efd777
RA		 477}
478
91a7c58f 479int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 480{
481 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
482 struct qedr_pd *pd = get_qedr_pd(ibpd);
483
a7efd777
RA		 484 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
485 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
91a7c58f 486 return 0;
a7efd777
RA		 487}
488
06e8d1df
YB		 489
490int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
491{
492 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
493 struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
494
495 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
496}
497
498int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
499{
500 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
501 u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
502
503 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
504 return 0;
505}
a7efd777
RA		 506static void qedr_free_pbl(struct qedr_dev *dev,
507 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
508{
509 struct pci_dev *pdev = dev->pdev;
510 int i;
511
512 for (i = 0; i < pbl_info->num_pbls; i++) {
513 if (!pbl[i].va)
514 continue;
515 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
516 pbl[i].va, pbl[i].pa);
517 }
518
519 kfree(pbl);
520}
521
522#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
523#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
524
525#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
526#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
527#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
528
529static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
530 struct qedr_pbl_info *pbl_info,
531 gfp_t flags)
532{
533 struct pci_dev *pdev = dev->pdev;
534 struct qedr_pbl *pbl_table;
535 dma_addr_t *pbl_main_tbl;
536 dma_addr_t pa;
537 void *va;
538 int i;
539
540 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
541 if (!pbl_table)
542 return ERR_PTR(-ENOMEM);
543
544 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 545 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
546 flags);
a7efd777
RA		 547 if (!va)
548 goto err;
549
a7efd777
RA		 550 pbl_table[i].va = va;
551 pbl_table[i].pa = pa;
552 }
553
554 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
555 * the first one with physical pointers to all of the rest
556 */
557 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
558 for (i = 0; i < pbl_info->num_pbls - 1; i++)
559 pbl_main_tbl[i] = pbl_table[i + 1].pa;
560
561 return pbl_table;
562
563err:
564 for (i--; i >= 0; i--)
565 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
566 pbl_table[i].va, pbl_table[i].pa);
567
568 qedr_free_pbl(dev, pbl_info, pbl_table);
569
570 return ERR_PTR(-ENOMEM);
571}
572
573static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
574 struct qedr_pbl_info *pbl_info,
575 u32 num_pbes, int two_layer_capable)
576{
577 u32 pbl_capacity;
578 u32 pbl_size;
579 u32 num_pbls;
580
581 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
582 if (num_pbes > MAX_PBES_TWO_LAYER) {
583 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
584 num_pbes);
585 return -EINVAL;
586 }
587
588 /* calculate required pbl page size */
589 pbl_size = MIN_FW_PBL_PAGE_SIZE;
590 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
591 NUM_PBES_ON_PAGE(pbl_size);
592
593 while (pbl_capacity < num_pbes) {
594 pbl_size *= 2;
595 pbl_capacity = pbl_size / sizeof(u64);
596 pbl_capacity = pbl_capacity * pbl_capacity;
597 }
598
599 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
600 num_pbls++; /* One for the layer0 ( points to the pbls) */
601 pbl_info->two_layered = true;
602 } else {
603 /* One layered PBL */
604 num_pbls = 1;
605 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
606 roundup_pow_of_two((num_pbes * sizeof(u64))));
607 pbl_info->two_layered = false;
608 }
609
610 pbl_info->num_pbls = num_pbls;
611 pbl_info->pbl_size = pbl_size;
612 pbl_info->num_pbes = num_pbes;
613
614 DP_DEBUG(dev, QEDR_MSG_MR,
615 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
616 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
617
618 return 0;
619}
620
621static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
622 struct qedr_pbl *pbl,
e57bb6be 623 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 624{
95ad233f 625 int pbe_cnt, total_num_pbes = 0;
a7efd777 626 struct qedr_pbl *pbl_tbl;
68363052 627 struct ib_block_iter biter;
a7efd777 628 struct regpair *pbe;
a7efd777
RA		 629
630 if (!pbl_info->num_pbes)
631 return;
632
633 /* If we have a two layered pbl, the first pbl points to the rest
634 * of the pbls and the first entry lays on the second pbl in the table
635 */
636 if (pbl_info->two_layered)
637 pbl_tbl = &pbl[1];
638 else
639 pbl_tbl = pbl;
640
641 pbe = (struct regpair *)pbl_tbl->va;
642 if (!pbe) {
643 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
644 return;
645 }
646
647 pbe_cnt = 0;
648
68363052
JG		 649 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
650 u64 pg_addr = rdma_block_iter_dma_address(&biter);
95ad233f 651
68363052
JG		 652 pbe->lo = cpu_to_le32(pg_addr);
653 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
95ad233f 654
68363052
JG		 655 pbe_cnt++;
656 total_num_pbes++;
657 pbe++;
e57bb6be 658
68363052
JG		 659 if (total_num_pbes == pbl_info->num_pbes)
660 return;
95ad233f 661
68363052
JG		 662 /* If the given pbl is full storing the pbes, move to next pbl.
663 */
664 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
665 pbl_tbl++;
666 pbe = (struct regpair *)pbl_tbl->va;
667 pbe_cnt = 0;
a7efd777
RA		 668 }
669 }
670}
671
97f61250
MK		 672static int qedr_db_recovery_add(struct qedr_dev *dev,
673 void __iomem *db_addr,
674 void *db_data,
675 enum qed_db_rec_width db_width,
676 enum qed_db_rec_space db_space)
677{
678 if (!db_data) {
679 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
680 return 0;
681 }
682
683 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
684 db_width, db_space);
685}
686
687static void qedr_db_recovery_del(struct qedr_dev *dev,
688 void __iomem *db_addr,
689 void *db_data)
690{
691 if (!db_data) {
692 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
693 return;
694 }
695
696 /* Ignore return code as there is not much we can do about it. Error
697 * log will be printed inside.
698 */
699 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
700}
701
a7efd777 702static int qedr_copy_cq_uresp(struct qedr_dev *dev,
97f61250
MK		 703 struct qedr_cq *cq, struct ib_udata *udata,
704 u32 db_offset)
a7efd777
RA		 705{
706 struct qedr_create_cq_uresp uresp;
707 int rc;
708
709 memset(&uresp, 0, sizeof(uresp));
710
97f61250 711 uresp.db_offset = db_offset;
a7efd777 712 uresp.icid = cq->icid;
a25984f3
MK		 713 if (cq->q.db_mmap_entry)
714 uresp.db_rec_addr =
715 rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
a7efd777 716
c75d3ec8 717 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 718 if (rc)
719 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
720
721 return rc;
722}
723
724static void consume_cqe(struct qedr_cq *cq)
725{
726 if (cq->latest_cqe == cq->toggle_cqe)
727 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
728
729 cq->latest_cqe = qed_chain_consume(&cq->pbl);
730}
731
732static inline int qedr_align_cq_entries(int entries)
733{
734 u64 size, aligned_size;
735
736 /* We allocate an extra entry that we don't report to the FW. */
737 size = (entries + 1) * QEDR_CQE_SIZE;
738 aligned_size = ALIGN(size, PAGE_SIZE);
739
740 return aligned_size / QEDR_CQE_SIZE;
741}
742
97f61250
MK		 743static int qedr_init_user_db_rec(struct ib_udata *udata,
744 struct qedr_dev *dev, struct qedr_userq *q,
745 bool requires_db_rec)
746{
747 struct qedr_ucontext *uctx =
748 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
749 ibucontext);
750 struct qedr_user_mmap_entry *entry;
751 int rc;
752
753 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
754 if (requires_db_rec == 0 || !uctx->db_rec)
755 return 0;
756
757 /* Allocate a page for doorbell recovery, add to mmap */
758 q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
759 if (!q->db_rec_data) {
760 DP_ERR(dev, "get_zeroed_page failed\n");
761 return -ENOMEM;
762 }
763
764 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
765 if (!entry)
766 goto err_free_db_data;
767
768 entry->address = q->db_rec_data;
769 entry->length = PAGE_SIZE;
770 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
771 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
772 &entry->rdma_entry,
773 PAGE_SIZE);
774 if (rc)
775 goto err_free_entry;
776
777 q->db_mmap_entry = &entry->rdma_entry;
778
779 return 0;
780
781err_free_entry:
782 kfree(entry);
783
784err_free_db_data:
785 free_page((unsigned long)q->db_rec_data);
786 q->db_rec_data = NULL;
787 return -ENOMEM;
788}
789
b0ea0fa5 790static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 791 struct qedr_dev *dev,
b0ea0fa5 792 struct qedr_userq *q, u64 buf_addr,
97f61250 793 size_t buf_len, bool requires_db_rec,
72b894b0 794 int access,
69ad0e7f 795 int alloc_and_init)
a7efd777 796{
e57bb6be 797 u32 fw_pages;
a7efd777
RA		 798 int rc;
799
800 q->buf_addr = buf_addr;
801 q->buf_len = buf_len;
c320e527 802 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
a7efd777
RA		 803 if (IS_ERR(q->umem)) {
804 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
805 PTR_ERR(q->umem));
806 return PTR_ERR(q->umem);
807 }
808
901bca71 809 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
e57bb6be 810 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 811 if (rc)
812 goto err0;
813
69ad0e7f
KM		 814 if (alloc_and_init) {
815 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
816 if (IS_ERR(q->pbl_tbl)) {
817 rc = PTR_ERR(q->pbl_tbl);
818 goto err0;
819 }
e57bb6be
RA		 820 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
821 FW_PAGE_SHIFT);
69ad0e7f
KM		 822 } else {
823 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 824 if (!q->pbl_tbl) {
825 rc = -ENOMEM;
69ad0e7f 826 goto err0;
89fd2576 827 }
69ad0e7f 828 }
a7efd777 829
97f61250
MK		 830 /* mmap the user address used to store doorbell data for recovery */
831 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
a7efd777
RA		 832
833err0:
834 ib_umem_release(q->umem);
69ad0e7f 835 q->umem = NULL;
a7efd777
RA		 836
837 return rc;
838}
839
840static inline void qedr_init_cq_params(struct qedr_cq *cq,
841 struct qedr_ucontext *ctx,
842 struct qedr_dev *dev, int vector,
843 int chain_entries, int page_cnt,
844 u64 pbl_ptr,
845 struct qed_rdma_create_cq_in_params
846 *params)
847{
848 memset(params, 0, sizeof(*params));
849 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
850 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
851 params->cnq_id = vector;
852 params->cq_size = chain_entries - 1;
853 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
854 params->pbl_num_pages = page_cnt;
855 params->pbl_ptr = pbl_ptr;
856 params->pbl_two_level = 0;
857}
858
859static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
860{
a7efd777
RA		 861 cq->db.data.agg_flags = flags;
862 cq->db.data.value = cpu_to_le32(cons);
863 writeq(cq->db.raw, cq->db_addr);
a7efd777
RA		 864}
865
866int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
867{
868 struct qedr_cq *cq = get_qedr_cq(ibcq);
869 unsigned long sflags;
4dd72636
AR		 870 struct qedr_dev *dev;
871
872 dev = get_qedr_dev(ibcq->device);
873
874 if (cq->destroyed) {
875 DP_ERR(dev,
876 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
877 cq, cq->icid);
878 return -EINVAL;
879 }
880
a7efd777
RA		 881
882 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
883 return 0;
884
885 spin_lock_irqsave(&cq->cq_lock, sflags);
886
887 cq->arm_flags = 0;
888
889 if (flags & IB_CQ_SOLICITED)
890 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
891
892 if (flags & IB_CQ_NEXT_COMP)
893 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
894
895 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
896
897 spin_unlock_irqrestore(&cq->cq_lock, sflags);
898
899 return 0;
900}
901
e39afe3d
LR		 902int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
903 struct ib_udata *udata)
a7efd777 904{
e39afe3d 905 struct ib_device *ibdev = ibcq->device;
ff23dfa1
SR		 906 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
907 udata, struct qedr_ucontext, ibucontext);
a7efd777
RA		 908 struct qed_rdma_destroy_cq_out_params destroy_oparams;
909 struct qed_rdma_destroy_cq_in_params destroy_iparams;
b6db3f71
AL		 910 struct qed_chain_init_params chain_params = {
911 .mode = QED_CHAIN_MODE_PBL,
912 .intended_use = QED_CHAIN_USE_TO_CONSUME,
913 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
914 .elem_size = sizeof(union rdma_cqe),
915 };
a7efd777
RA		 916 struct qedr_dev *dev = get_qedr_dev(ibdev);
917 struct qed_rdma_create_cq_in_params params;
e39afe3d 918 struct qedr_create_cq_ureq ureq = {};
a7efd777
RA		 919 int vector = attr->comp_vector;
920 int entries = attr->cqe;
e39afe3d 921 struct qedr_cq *cq = get_qedr_cq(ibcq);
a7efd777 922 int chain_entries;
97f61250 923 u32 db_offset;
a7efd777
RA		 924 int page_cnt;
925 u64 pbl_ptr;
926 u16 icid;
927 int rc;
928
929 DP_DEBUG(dev, QEDR_MSG_INIT,
930 "create_cq: called from %s. entries=%d, vector=%d\n",
931 udata ? "User Lib" : "Kernel", entries, vector);
932
1c407cb5
JG		 933 if (attr->flags)
934 return -EOPNOTSUPP;
935
a7efd777
RA		 936 if (entries > QEDR_MAX_CQES) {
937 DP_ERR(dev,
938 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
939 entries, QEDR_MAX_CQES);
e39afe3d 940 return -EINVAL;
a7efd777
RA		 941 }
942
943 chain_entries = qedr_align_cq_entries(entries);
944 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
b6db3f71 945 chain_params.num_elems = chain_entries;
a7efd777 946
97f61250
MK		 947 /* calc db offset. user will add DPI base, kernel will add db addr */
948 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
949
a7efd777 950 if (udata) {
97f61250
MK		 951 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
952 udata->inlen))) {
a7efd777
RA		 953 DP_ERR(dev,
954 "create cq: problem copying data from user space\n");
955 goto err0;
956 }
957
958 if (!ureq.len) {
959 DP_ERR(dev,
960 "create cq: cannot create a cq with 0 entries\n");
961 goto err0;
962 }
963
964 cq->cq_type = QEDR_CQ_TYPE_USER;
965
b0ea0fa5 966 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
72b894b0
CH		 967 ureq.len, true, IB_ACCESS_LOCAL_WRITE,
968 1);
a7efd777
RA		 969 if (rc)
970 goto err0;
971
972 pbl_ptr = cq->q.pbl_tbl->pa;
973 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 974
975 cq->ibcq.cqe = chain_entries;
97f61250 976 cq->q.db_addr = ctx->dpi_addr + db_offset;
a7efd777
RA		 977 } else {
978 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
979
b6db3f71
AL		 980 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
981 &chain_params);
a7efd777 982 if (rc)
97f61250 983 goto err0;
a7efd777
RA		 984
985 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
986 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 987 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 988 }
989
990 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
991 pbl_ptr, &params);
992
993 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
994 if (rc)
97f61250 995 goto err1;
a7efd777
RA		 996
997 cq->icid = icid;
998 cq->sig = QEDR_CQ_MAGIC_NUMBER;
999 spin_lock_init(&cq->cq_lock);
1000
ff23dfa1 1001 if (udata) {
97f61250
MK		 1002 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003 if (rc)
1004 goto err2;
1005
1006 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1007 &cq->q.db_rec_data->db_data,
1008 DB_REC_WIDTH_64B,
1009 DB_REC_USER);
a7efd777 1010 if (rc)
97f61250
MK		 1011 goto err2;
1012
a7efd777
RA		 1013 } else {
1014 /* Generate doorbell address. */
a7efd777 1015 cq->db.data.icid = cq->icid;
97f61250 1016 cq->db_addr = dev->db_addr + db_offset;
0b1eddc1 1017 cq->db.data.params = DB_AGG_CMD_MAX <<
a7efd777
RA		 1018 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019
1020 /* point to the very last element, passing it we will toggle */
1021 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1022 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023 cq->latest_cqe = NULL;
1024 consume_cqe(cq);
1025 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
97f61250
MK		 1026
1027 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1028 DB_REC_WIDTH_64B, DB_REC_KERNEL);
1029 if (rc)
1030 goto err2;
a7efd777
RA		 1031 }
1032
1033 DP_DEBUG(dev, QEDR_MSG_CQ,
1034 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035 cq->icid, cq, params.cq_size);
1036
e39afe3d 1037 return 0;
a7efd777 1038
97f61250 1039err2:
a7efd777
RA		 1040 destroy_iparams.icid = cq->icid;
1041 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042 &destroy_oparams);
a7efd777 1043err1:
97f61250
MK		 1044 if (udata) {
1045 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
a7efd777 1046 ib_umem_release(cq->q.umem);
a25984f3 1047 if (cq->q.db_mmap_entry)
97f61250
MK		 1048 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1049 } else {
1050 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051 }
a7efd777 1052err0:
e39afe3d 1053 return -EINVAL;
a7efd777
RA		 1054}
1055
4dd72636
AR		 1056#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1057#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1058
43d781b9 1059int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 1060{
1061 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1062 struct qed_rdma_destroy_cq_out_params oparams;
1063 struct qed_rdma_destroy_cq_in_params iparams;
1064 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 1065 int iter;
a7efd777 1066
942b3b2c 1067 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 1068
4dd72636
AR		 1069 cq->destroyed = 1;
1070
a7efd777 1071 /* GSIs CQs are handled by driver, so they don't exist in the FW */
97f61250
MK		 1072 if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
43d781b9 1074 return 0;
97f61250 1075 }
a1211359 1076
942b3b2c 1077 iparams.icid = cq->icid;
a52c8e24 1078 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
942b3b2c 1079 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777 1080
bdeacabd 1081 if (udata) {
a7efd777
RA		 1082 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1083 ib_umem_release(cq->q.umem);
97f61250
MK		 1084
1085 if (cq->q.db_rec_data) {
1086 qedr_db_recovery_del(dev, cq->q.db_addr,
1087 &cq->q.db_rec_data->db_data);
1088 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1089 }
1090 } else {
1091 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
a7efd777
RA		 1092 }
1093
4dd72636
AR		 1094 /* We don't want the IRQ handler to handle a non-existing CQ so we
1095 * wait until all CNQ interrupts, if any, are received. This will always
1096 * happen and will always happen very fast. If not, then a serious error
1097 * has occured. That is why we can use a long delay.
1098 * We spin for a short time so we don’t lose time on context switching
1099 * in case all the completions are handled in that span. Otherwise
1100 * we sleep for a while and check again. Since the CNQ may be
1101 * associated with (only) the current CPU we use msleep to allow the
1102 * current CPU to be freed.
1103 * The CNQ notification is increased in qedr_irq_handler().
1104 */
1105 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108 iter--;
1109 }
1110
1111 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114 iter--;
1115 }
1116
4dd72636
AR		 1117 /* Note that we don't need to have explicit code to wait for the
1118 * completion of the event handler because it is invoked from the EQ.
1119 * Since the destroy CQ ramrod has also been received on the EQ we can
1120 * be certain that there's no event handler in process.
1121 */
43d781b9 1122 return 0;
a7efd777 1123}
cecbcddf
RA		 1124
1125static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126 struct ib_qp_attr *attr,
1127 int attr_mask,
1128 struct qed_rdma_modify_qp_in_params
1129 *qp_params)
1130{
47ec3866 1131 const struct ib_gid_attr *gid_attr;
cecbcddf 1132 enum rdma_network_type nw_type;
d8966fcd 1133 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1134 u32 ipv4_addr;
a70c0739 1135 int ret;
cecbcddf
RA		 1136 int i;
1137
47ec3866 1138 gid_attr = grh->sgid_attr;
a70c0739
PP		 1139 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1140 if (ret)
1141 return ret;
3e44e0ee 1142
47ec3866 1143 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1144 switch (nw_type) {
1145 case RDMA_NETWORK_IPV6:
47ec3866 1146 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1147 sizeof(qp_params->sgid));
1148 memcpy(&qp_params->dgid.bytes[0],
1149 &grh->dgid,
1150 sizeof(qp_params->dgid));
1151 qp_params->roce_mode = ROCE_V2_IPV6;
1152 SET_FIELD(qp_params->modify_flags,
1153 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154 break;
1c15b4f2 1155 case RDMA_NETWORK_ROCE_V1:
47ec3866 1156 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1157 sizeof(qp_params->sgid));
1158 memcpy(&qp_params->dgid.bytes[0],
1159 &grh->dgid,
1160 sizeof(qp_params->dgid));
1161 qp_params->roce_mode = ROCE_V1;
1162 break;
1163 case RDMA_NETWORK_IPV4:
1164 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1166 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1167 qp_params->sgid.ipv4_addr = ipv4_addr;
1168 ipv4_addr =
1169 qedr_get_ipv4_from_gid(grh->dgid.raw);
1170 qp_params->dgid.ipv4_addr = ipv4_addr;
1171 SET_FIELD(qp_params->modify_flags,
1172 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173 qp_params->roce_mode = ROCE_V2_IPV4;
1174 break;
1c15b4f2
AH		 1175 default:
1176 return -EINVAL;
cecbcddf
RA		 1177 }
1178
1179 for (i = 0; i < 4; i++) {
1180 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182 }
1183
1184 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185 qp_params->vlan_id = 0;
1186
1187 return 0;
1188}
1189
cecbcddf 1190static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1191 struct ib_qp_init_attr *attrs,
1192 struct ib_udata *udata)
cecbcddf
RA		 1193{
1194 struct qedr_device_attr *qattr = &dev->attr;
1195
1196 /* QP0... attrs->qp_type == IB_QPT_GSI */
06e8d1df
YB		 1197 if (attrs->qp_type != IB_QPT_RC &&
1198 attrs->qp_type != IB_QPT_GSI &&
1199 attrs->qp_type != IB_QPT_XRC_INI &&
1200 attrs->qp_type != IB_QPT_XRC_TGT) {
cecbcddf
RA		 1201 DP_DEBUG(dev, QEDR_MSG_QP,
1202 "create qp: unsupported qp type=0x%x requested\n",
1203 attrs->qp_type);
bb8865f4 1204 return -EOPNOTSUPP;
cecbcddf
RA		 1205 }
1206
1207 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208 DP_ERR(dev,
1209 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210 attrs->cap.max_send_wr, qattr->max_sqe);
1211 return -EINVAL;
1212 }
1213
1214 if (attrs->cap.max_inline_data > qattr->max_inline) {
1215 DP_ERR(dev,
1216 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217 attrs->cap.max_inline_data, qattr->max_inline);
1218 return -EINVAL;
1219 }
1220
1221 if (attrs->cap.max_send_sge > qattr->max_sge) {
1222 DP_ERR(dev,
1223 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224 attrs->cap.max_send_sge, qattr->max_sge);
1225 return -EINVAL;
1226 }
1227
1228 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229 DP_ERR(dev,
1230 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231 attrs->cap.max_recv_sge, qattr->max_sge);
1232 return -EINVAL;
1233 }
1234
06e8d1df
YB		 1235 /* verify consumer QPs are not trying to use GSI QP's CQ.
1236 * TGT QP isn't associated with RQ/SQ
1237 */
1238 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
e1ad897b
KH		 1239 (attrs->qp_type != IB_QPT_XRC_TGT) &&
1240 (attrs->qp_type != IB_QPT_XRC_INI)) {
06e8d1df
YB		 1241 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1242 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1243
1244 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245 (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246 DP_ERR(dev,
1247 "create qp: consumer QP cannot use GSI CQs.\n");
1248 return -EINVAL;
1249 }
1250 }
1251
cecbcddf
RA		 1252 return 0;
1253}
1254
40b173dd
YB		 1255static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256 struct qedr_srq *srq, struct ib_udata *udata)
1257{
1258 struct qedr_create_srq_uresp uresp = {};
1259 int rc;
1260
1261 uresp.srq_id = srq->srq_id;
1262
1263 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1264 if (rc)
1265 DP_ERR(dev, "create srq: problem copying data to user space\n");
1266
1267 return rc;
1268}
1269
69ad0e7f 1270static void qedr_copy_rq_uresp(struct qedr_dev *dev,
9e054b13
MK		 1271 struct qedr_create_qp_uresp *uresp,
1272 struct qedr_qp *qp)
cecbcddf 1273{
69ad0e7f
KM		 1274 /* iWARP requires two doorbells per RQ. */
1275 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1276 uresp->rq_db_offset =
1277 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279 } else {
1280 uresp->rq_db_offset =
1281 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282 }
1283
cecbcddf 1284 uresp->rq_icid = qp->icid;
a25984f3
MK		 1285 if (qp->urq.db_mmap_entry)
1286 uresp->rq_db_rec_addr =
1287 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
cecbcddf
RA		 1288}
1289
69ad0e7f
KM		 1290static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1292 struct qedr_qp *qp)
1293{
1294 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1295
1296 /* iWARP uses the same cid for rq and sq */
1297 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1298 uresp->sq_icid = qp->icid;
1299 else
1300 uresp->sq_icid = qp->icid + 1;
97f61250 1301
a25984f3
MK		 1302 if (qp->usq.db_mmap_entry)
1303 uresp->sq_db_rec_addr =
1304 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
cecbcddf
RA		 1305}
1306
1307static int qedr_copy_qp_uresp(struct qedr_dev *dev,
97f61250
MK		 1308 struct qedr_qp *qp, struct ib_udata *udata,
1309 struct qedr_create_qp_uresp *uresp)
cecbcddf 1310{
cecbcddf
RA		 1311 int rc;
1312
97f61250 1313 memset(uresp, 0, sizeof(*uresp));
06e8d1df
YB		 1314
1315 if (qedr_qp_has_sq(qp))
1316 qedr_copy_sq_uresp(dev, uresp, qp);
1317
1318 if (qedr_qp_has_rq(qp))
1319 qedr_copy_rq_uresp(dev, uresp, qp);
cecbcddf 1320
97f61250
MK		 1321 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322 uresp->qp_id = qp->qp_id;
cecbcddf 1323
97f61250 1324 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
cecbcddf
RA		 1325 if (rc)
1326 DP_ERR(dev,
1327 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328 qp->icid);
1329
1330 return rc;
1331}
1332
6ef793cb
PK		 1333static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334{
1335 qed_chain_reset(&qph->pbl);
1336 qph->prod = 0;
1337 qph->cons = 0;
1338 qph->wqe_cons = 0;
1339 qph->db_data.data.value = cpu_to_le16(0);
1340}
1341
df158561
AR		 1342static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343 struct qedr_qp *qp,
1344 struct qedr_pd *pd,
1345 struct ib_qp_init_attr *attrs)
cecbcddf 1346{
cecbcddf 1347 spin_lock_init(&qp->q_lock);
82af6d19
MK		 1348 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349 kref_init(&qp->refcnt);
1350 init_completion(&qp->iwarp_cm_comp);
60fab107 1351 init_completion(&qp->qp_rel_comp);
82af6d19 1352 }
06e8d1df 1353
df158561 1354 qp->pd = pd;
cecbcddf
RA		 1355 qp->qp_type = attrs->qp_type;
1356 qp->max_inline_data = attrs->cap.max_inline_data;
cecbcddf 1357 qp->state = QED_ROCE_QP_STATE_RESET;
6ef793cb
PK		 1358
1359 qp->prev_wqe_size = 0;
1360
cecbcddf 1361 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
cecbcddf 1362 qp->dev = dev;
06e8d1df 1363 if (qedr_qp_has_sq(qp)) {
6ef793cb 1364 qedr_reset_qp_hwq_info(&qp->sq);
06e8d1df
YB		 1365 qp->sq.max_sges = attrs->cap.max_send_sge;
1366 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1367 DP_DEBUG(dev, QEDR_MSG_QP,
1368 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369 qp->sq.max_sges, qp->sq_cq->icid);
1370 }
cecbcddf 1371
06e8d1df 1372 if (attrs->srq)
3491c9e7 1373 qp->srq = get_qedr_srq(attrs->srq);
06e8d1df
YB		 1374
1375 if (qedr_qp_has_rq(qp)) {
6ef793cb 1376 qedr_reset_qp_hwq_info(&qp->rq);
3491c9e7
YB		 1377 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378 qp->rq.max_sges = attrs->cap.max_recv_sge;
1379 DP_DEBUG(dev, QEDR_MSG_QP,
1380 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381 qp->rq.max_sges, qp->rq_cq->icid);
1382 }
1383
cecbcddf
RA		 1384 DP_DEBUG(dev, QEDR_MSG_QP,
1385 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386 pd->pd_id, qp->qp_type, qp->max_inline_data,
1387 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388 DP_DEBUG(dev, QEDR_MSG_QP,
1389 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1391}
1392
97f61250 1393static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1394{
06e8d1df 1395 int rc = 0;
97f61250 1396
06e8d1df
YB		 1397 if (qedr_qp_has_sq(qp)) {
1398 qp->sq.db = dev->db_addr +
1399 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400 qp->sq.db_data.data.icid = qp->icid + 1;
1401 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1402 DB_REC_WIDTH_32B, DB_REC_KERNEL);
1403 if (rc)
1404 return rc;
1405 }
97f61250 1406
06e8d1df 1407 if (qedr_qp_has_rq(qp)) {
3491c9e7
YB		 1408 qp->rq.db = dev->db_addr +
1409 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410 qp->rq.db_data.data.icid = qp->icid;
06e8d1df
YB		 1411 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1412 DB_REC_WIDTH_32B, DB_REC_KERNEL);
1413 if (rc && qedr_qp_has_sq(qp))
1414 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
3491c9e7 1415 }
97f61250
MK		 1416
1417 return rc;
3491c9e7
YB		 1418}
1419
68e326de 1420static int qedr_check_srq_params(struct qedr_dev *dev,
3491c9e7
YB		 1421 struct ib_srq_init_attr *attrs,
1422 struct ib_udata *udata)
1423{
1424 struct qedr_device_attr *qattr = &dev->attr;
1425
1426 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427 DP_ERR(dev,
1428 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429 attrs->attr.max_wr, qattr->max_srq_wr);
1430 return -EINVAL;
1431 }
1432
1433 if (attrs->attr.max_sge > qattr->max_sge) {
1434 DP_ERR(dev,
1435 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436 attrs->attr.max_sge, qattr->max_sge);
06e8d1df
YB		 1437 }
1438
1439 if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
3491c9e7
YB		 1441 return -EINVAL;
1442 }
1443
1444 return 0;
1445}
1446
40b173dd
YB		 1447static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448{
1449 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1450 ib_umem_release(srq->usrq.umem);
1451 ib_umem_release(srq->prod_umem);
1452}
1453
3491c9e7
YB		 1454static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455{
1456 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457 struct qedr_dev *dev = srq->dev;
1458
1459 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460
1461 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1462 hw_srq->virt_prod_pair_addr,
1463 hw_srq->phy_prod_pair_addr);
1464}
1465
b0ea0fa5 1466static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1467 struct qedr_srq *srq,
1468 struct qedr_create_srq_ureq *ureq,
72b894b0 1469 int access)
40b173dd
YB		 1470{
1471 struct scatterlist *sg;
1472 int rc;
1473
b0ea0fa5 1474 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
72b894b0 1475 ureq->srq_len, false, access, 1);
40b173dd
YB		 1476 if (rc)
1477 return rc;
1478
c320e527
MS		 1479 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1480 sizeof(struct rdma_srq_producers), access);
40b173dd
YB		 1481 if (IS_ERR(srq->prod_umem)) {
1482 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1483 ib_umem_release(srq->usrq.umem);
1484 DP_ERR(srq->dev,
1485 "create srq: failed ib_umem_get for producer, got %ld\n",
1486 PTR_ERR(srq->prod_umem));
1487 return PTR_ERR(srq->prod_umem);
1488 }
1489
79fbd3e1 1490 sg = srq->prod_umem->sgt_append.sgt.sgl;
40b173dd
YB		 1491 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492
1493 return 0;
1494}
1495
3491c9e7
YB		 1496static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497 struct qedr_dev *dev,
1498 struct ib_srq_init_attr *init_attr)
1499{
1500 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
b6db3f71
AL		 1501 struct qed_chain_init_params params = {
1502 .mode = QED_CHAIN_MODE_PBL,
1503 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1505 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE,
1506 };
3491c9e7
YB		 1507 dma_addr_t phy_prod_pair_addr;
1508 u32 num_elems;
1509 void *va;
1510 int rc;
1511
1512 va = dma_alloc_coherent(&dev->pdev->dev,
1513 sizeof(struct rdma_srq_producers),
1514 &phy_prod_pair_addr, GFP_KERNEL);
1515 if (!va) {
1516 DP_ERR(dev,
1517 "create srq: failed to allocate dma memory for producer\n");
1518 return -ENOMEM;
1519 }
1520
1521 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522 hw_srq->virt_prod_pair_addr = va;
1523
1524 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
b6db3f71
AL		 1525 params.num_elems = num_elems;
1526
1527 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
3491c9e7
YB		 1528 if (rc)
1529 goto err0;
1530
1531 hw_srq->num_elems = num_elems;
1532
1533 return 0;
1534
1535err0:
1536 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1537 va, phy_prod_pair_addr);
1538 return rc;
1539}
1540
68e326de
LR		 1541int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542 struct ib_udata *udata)
3491c9e7
YB		 1543{
1544 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545 struct qed_rdma_create_srq_in_params in_params = {};
68e326de 1546 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
3491c9e7 1547 struct qed_rdma_create_srq_out_params out_params;
68e326de 1548 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
40b173dd 1549 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1550 u64 pbl_base_addr, phy_prod_pair_addr;
1551 struct qedr_srq_hwq_info *hw_srq;
1552 u32 page_cnt, page_size;
68e326de 1553 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3491c9e7
YB		 1554 int rc = 0;
1555
1556 DP_DEBUG(dev, QEDR_MSG_QP,
1557 "create SRQ called from %s (pd %p)\n",
1558 (udata) ? "User lib" : "kernel", pd);
1559
652caba5
JG		 1560 if (init_attr->srq_type != IB_SRQT_BASIC &&
1561 init_attr->srq_type != IB_SRQT_XRC)
1562 return -EOPNOTSUPP;
1563
68e326de 1564 rc = qedr_check_srq_params(dev, init_attr, udata);
3491c9e7 1565 if (rc)
68e326de 1566 return -EINVAL;
3491c9e7
YB		 1567
1568 srq->dev = dev;
06e8d1df 1569 srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
3491c9e7
YB		 1570 hw_srq = &srq->hw_srq;
1571 spin_lock_init(&srq->lock);
1572
1573 hw_srq->max_wr = init_attr->attr.max_wr;
1574 hw_srq->max_sges = init_attr->attr.max_sge;
1575
89944450 1576 if (udata) {
97f61250
MK		 1577 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1578 udata->inlen))) {
40b173dd
YB		 1579 DP_ERR(dev,
1580 "create srq: problem copying data from user space\n");
1581 goto err0;
1582 }
1583
72b894b0 1584 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
40b173dd
YB		 1585 if (rc)
1586 goto err0;
1587
1588 page_cnt = srq->usrq.pbl_info.num_pbes;
1589 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1591 page_size = PAGE_SIZE;
40b173dd
YB		 1592 } else {
1593 struct qed_chain *pbl;
1594
1595 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596 if (rc)
1597 goto err0;
1598
1599 pbl = &hw_srq->pbl;
1600 page_cnt = qed_chain_get_page_cnt(pbl);
1601 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1602 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603 page_size = QED_CHAIN_PAGE_SIZE;
1604 }
3491c9e7 1605
3491c9e7
YB		 1606 in_params.pd_id = pd->pd_id;
1607 in_params.pbl_base_addr = pbl_base_addr;
1608 in_params.prod_pair_addr = phy_prod_pair_addr;
1609 in_params.num_pages = page_cnt;
1610 in_params.page_size = page_size;
06e8d1df
YB		 1611 if (srq->is_xrc) {
1612 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1613 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1614
1615 in_params.is_xrc = 1;
1616 in_params.xrcd_id = xrcd->xrcd_id;
1617 in_params.cq_cid = cq->icid;
1618 }
3491c9e7
YB		 1619
1620 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621 if (rc)
1622 goto err1;
1623
1624 srq->srq_id = out_params.srq_id;
1625
40b173dd
YB		 1626 if (udata) {
1627 rc = qedr_copy_srq_uresp(dev, srq, udata);
1628 if (rc)
1629 goto err2;
1630 }
1631
9fd15987 1632 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1633 if (rc)
1634 goto err2;
1635
1636 DP_DEBUG(dev, QEDR_MSG_SRQ,
1637 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
68e326de 1638 return 0;
3491c9e7
YB		 1639
1640err2:
1641 destroy_in_params.srq_id = srq->srq_id;
1642
1643 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644err1:
40b173dd
YB		 1645 if (udata)
1646 qedr_free_srq_user_params(srq);
1647 else
1648 qedr_free_srq_kernel_params(srq);
3491c9e7 1649err0:
68e326de 1650 return -EFAULT;
3491c9e7
YB		 1651}
1652
119181d1 1653int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1654{
1655 struct qed_rdma_destroy_srq_in_params in_params = {};
1656 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1657 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658
9fd15987 1659 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7 1660 in_params.srq_id = srq->srq_id;
06e8d1df 1661 in_params.is_xrc = srq->is_xrc;
3491c9e7
YB		 1662 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663
e00b64f7 1664 if (ibsrq->uobject)
40b173dd
YB		 1665 qedr_free_srq_user_params(srq);
1666 else
1667 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1668
1669 DP_DEBUG(dev, QEDR_MSG_SRQ,
1670 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1671 srq->srq_id);
119181d1 1672 return 0;
3491c9e7
YB		 1673}
1674
1675int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677{
1678 struct qed_rdma_modify_srq_in_params in_params = {};
1679 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1680 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681 int rc;
1682
1683 if (attr_mask & IB_SRQ_MAX_WR) {
1684 DP_ERR(dev,
1685 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1686 attr_mask, srq);
1687 return -EINVAL;
1688 }
1689
1690 if (attr_mask & IB_SRQ_LIMIT) {
1691 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692 DP_ERR(dev,
1693 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694 attr->srq_limit, srq->hw_srq.max_wr);
1695 return -EINVAL;
1696 }
1697
1698 in_params.srq_id = srq->srq_id;
1699 in_params.wqe_limit = attr->srq_limit;
1700 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701 if (rc)
1702 return rc;
1703 }
1704
1705 srq->srq_limit = attr->srq_limit;
1706
1707 DP_DEBUG(dev, QEDR_MSG_SRQ,
1708 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709
1710 return 0;
cecbcddf
RA		 1711}
1712
06e8d1df
YB		 1713static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714{
1715 switch (ib_qp_type) {
1716 case IB_QPT_RC:
1717 return QED_RDMA_QP_TYPE_RC;
1718 case IB_QPT_XRC_INI:
1719 return QED_RDMA_QP_TYPE_XRC_INI;
1720 case IB_QPT_XRC_TGT:
1721 return QED_RDMA_QP_TYPE_XRC_TGT;
1722 default:
1723 return QED_RDMA_QP_TYPE_INVAL;
1724 }
1725}
1726
cecbcddf 1727static inline void
df158561
AR		 1728qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729 struct qedr_pd *pd,
1730 struct qedr_qp *qp,
1731 struct ib_qp_init_attr *attrs,
1732 bool fmr_and_reserved_lkey,
1733 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1734{
df158561
AR		 1735 /* QP handle to be written in an async event */
1736 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738
1739 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
06e8d1df 1741 params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
df158561 1742 params->stats_queue = 0;
3491c9e7 1743
06e8d1df
YB		 1744 if (pd) {
1745 params->pd = pd->pd_id;
1746 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747 }
1748
1749 if (qedr_qp_has_sq(qp))
1750 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1751
1752 if (qedr_qp_has_rq(qp))
3491c9e7
YB		 1753 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1754
06e8d1df 1755 if (qedr_qp_has_srq(qp)) {
3491c9e7
YB		 1756 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1757 params->srq_id = qp->srq->srq_id;
1758 params->use_srq = true;
06e8d1df
YB		 1759 } else {
1760 params->srq_id = 0;
1761 params->use_srq = false;
3491c9e7 1762 }
cecbcddf
RA		 1763}
1764
df158561 1765static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1766{
df158561
AR		 1767 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768 "qp=%p. "
1769 "sq_addr=0x%llx, "
1770 "sq_len=%zd, "
1771 "rq_addr=0x%llx, "
1772 "rq_len=%zd"
1773 "\n",
1774 qp,
06e8d1df
YB		 1775 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
df158561 1779}
cecbcddf 1780
69ad0e7f
KM		 1781static inline void
1782qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783 struct qedr_qp *qp,
1784 struct qed_rdma_create_qp_out_params *out_params)
1785{
1786 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788
1789 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1791 if (!qp->srq) {
1792 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794 }
69ad0e7f
KM		 1795
1796 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798}
1799
97f61250
MK		 1800static void qedr_cleanup_user(struct qedr_dev *dev,
1801 struct qedr_ucontext *ctx,
1802 struct qedr_qp *qp)
df158561 1803{
06e8d1df
YB		 1804 if (qedr_qp_has_sq(qp)) {
1805 ib_umem_release(qp->usq.umem);
1806 qp->usq.umem = NULL;
1807 }
cecbcddf 1808
06e8d1df
YB		 1809 if (qedr_qp_has_rq(qp)) {
1810 ib_umem_release(qp->urq.umem);
1811 qp->urq.umem = NULL;
1812 }
24e412c1
MK		 1813
1814 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1817 } else {
1818 kfree(qp->usq.pbl_tbl);
1819 kfree(qp->urq.pbl_tbl);
1820 }
97f61250
MK		 1821
1822 if (qp->usq.db_rec_data) {
1823 qedr_db_recovery_del(dev, qp->usq.db_addr,
1824 &qp->usq.db_rec_data->db_data);
1825 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1826 }
1827
1828 if (qp->urq.db_rec_data) {
1829 qedr_db_recovery_del(dev, qp->urq.db_addr,
1830 &qp->urq.db_rec_data->db_data);
1831 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1832 }
b4bc7660
MK		 1833
1834 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836 &qp->urq.db_rec_db2_data);
cecbcddf
RA		 1837}
1838
df158561
AR		 1839static int qedr_create_user_qp(struct qedr_dev *dev,
1840 struct qedr_qp *qp,
1841 struct ib_pd *ibpd,
1842 struct ib_udata *udata,
1843 struct ib_qp_init_attr *attrs)
cecbcddf 1844{
df158561
AR		 1845 struct qed_rdma_create_qp_in_params in_params;
1846 struct qed_rdma_create_qp_out_params out_params;
06e8d1df
YB		 1847 struct qedr_create_qp_uresp uresp = {};
1848 struct qedr_create_qp_ureq ureq = {};
69ad0e7f 1849 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
06e8d1df
YB		 1850 struct qedr_ucontext *ctx = NULL;
1851 struct qedr_pd *pd = NULL;
1852 int rc = 0;
cecbcddf 1853
82af6d19 1854 qp->create_type = QEDR_QP_CREATE_USER;
06e8d1df
YB		 1855
1856 if (ibpd) {
1857 pd = get_qedr_pd(ibpd);
1858 ctx = pd->uctx;
df158561 1859 }
cecbcddf 1860
06e8d1df
YB		 1861 if (udata) {
1862 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1863 udata->inlen));
1864 if (rc) {
1865 DP_ERR(dev, "Problem copying data from user space\n");
1866 return rc;
1867 }
1868 }
cecbcddf 1869
06e8d1df
YB		 1870 if (qedr_qp_has_sq(qp)) {
1871 /* SQ - read access only (0) */
1872 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1873 ureq.sq_len, true, 0, alloc_and_init);
1874 if (rc)
1875 return rc;
1876 }
1877
1878 if (qedr_qp_has_rq(qp)) {
72b894b0 1879 /* RQ - read access only (0) */
b0ea0fa5 1880 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
72b894b0 1881 ureq.rq_len, true, 0, alloc_and_init);
40b173dd
YB		 1882 if (rc)
1883 return rc;
1884 }
df158561
AR		 1885
1886 memset(&in_params, 0, sizeof(in_params));
1887 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1888 in_params.qp_handle_lo = ureq.qp_handle_lo;
1889 in_params.qp_handle_hi = ureq.qp_handle_hi;
06e8d1df
YB		 1890
1891 if (qp->qp_type == IB_QPT_XRC_TGT) {
1892 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1893
1894 in_params.xrcd_id = xrcd->xrcd_id;
1895 in_params.qp_handle_lo = qp->qp_id;
1896 in_params.use_srq = 1;
1897 }
1898
1899 if (qedr_qp_has_sq(qp)) {
1900 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1901 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1902 }
1903
1904 if (qedr_qp_has_rq(qp)) {
40b173dd
YB		 1905 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1906 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1907 }
df158561 1908
bbe4f424
MK		 1909 if (ctx)
1910 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1911
df158561
AR		 1912 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1913 &in_params, &out_params);
1914
1915 if (!qp->qed_qp) {
1916 rc = -ENOMEM;
1917 goto err1;
1918 }
1919
69ad0e7f
KM		 1920 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1921 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1922
df158561
AR		 1923 qp->qp_id = out_params.qp_id;
1924 qp->icid = out_params.icid;
1925
06e8d1df
YB		 1926 if (udata) {
1927 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1928 if (rc)
1929 goto err;
1930 }
97f61250
MK		 1931
1932 /* db offset was calculated in copy_qp_uresp, now set in the user q */
06e8d1df
YB		 1933 if (qedr_qp_has_sq(qp)) {
1934 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
b1a4da64 1935 qp->sq.max_wr = attrs->cap.max_send_wr;
06e8d1df
YB		 1936 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1937 &qp->usq.db_rec_data->db_data,
1938 DB_REC_WIDTH_32B,
1939 DB_REC_USER);
1940 if (rc)
1941 goto err;
b4bc7660
MK		 1942 }
1943
06e8d1df
YB		 1944 if (qedr_qp_has_rq(qp)) {
1945 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
b1a4da64 1946 qp->rq.max_wr = attrs->cap.max_recv_wr;
06e8d1df
YB		 1947 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1948 &qp->urq.db_rec_data->db_data,
1949 DB_REC_WIDTH_32B,
1950 DB_REC_USER);
1951 if (rc)
1952 goto err;
1953 }
df158561 1954
b4bc7660 1955 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
0191c271
AP		 1956 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1957
1958 /* calculate the db_rec_db2 data since it is constant so no
1959 * need to reflect from user
1960 */
1961 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1962 qp->urq.db_rec_db2_data.data.value =
1963 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1964
b4bc7660
MK		 1965 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1966 &qp->urq.db_rec_db2_data,
1967 DB_REC_WIDTH_32B,
1968 DB_REC_USER);
1969 if (rc)
1970 goto err;
1971 }
df158561 1972 qedr_qp_user_print(dev, qp);
97f61250 1973 return rc;
df158561
AR		 1974err:
1975 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1976 if (rc)
1977 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1978
1979err1:
97f61250 1980 qedr_cleanup_user(dev, ctx, qp);
df158561 1981 return rc;
cecbcddf
RA		 1982}
1983
97f61250 1984static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
f5b1b177 1985{
97f61250
MK		 1986 int rc;
1987
f5b1b177
KM		 1988 qp->sq.db = dev->db_addr +
1989 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1990 qp->sq.db_data.data.icid = qp->icid;
1991
97f61250
MK		 1992 rc = qedr_db_recovery_add(dev, qp->sq.db,
1993 &qp->sq.db_data,
1994 DB_REC_WIDTH_32B,
1995 DB_REC_KERNEL);
1996 if (rc)
1997 return rc;
1998
f5b1b177
KM		 1999 qp->rq.db = dev->db_addr +
2000 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2001 qp->rq.db_data.data.icid = qp->icid;
2002 qp->rq.iwarp_db2 = dev->db_addr +
2003 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2004 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2005 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
97f61250
MK		 2006
2007 rc = qedr_db_recovery_add(dev, qp->rq.db,
2008 &qp->rq.db_data,
2009 DB_REC_WIDTH_32B,
2010 DB_REC_KERNEL);
b4bc7660
MK		 2011 if (rc)
2012 return rc;
2013
2014 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2015 &qp->rq.iwarp_db2_data,
2016 DB_REC_WIDTH_32B,
2017 DB_REC_KERNEL);
97f61250 2018 return rc;
f5b1b177
KM		 2019}
2020
df158561
AR		 2021static int
2022qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2023 struct qedr_qp *qp,
2024 struct qed_rdma_create_qp_in_params *in_params,
2025 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 2026{
df158561 2027 struct qed_rdma_create_qp_out_params out_params;
b6db3f71
AL		 2028 struct qed_chain_init_params params = {
2029 .mode = QED_CHAIN_MODE_PBL,
2030 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
2031 };
cecbcddf
RA		 2032 int rc;
2033
b6db3f71
AL		 2034 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2035 params.num_elems = n_sq_elems;
2036 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
cecbcddf 2037
b6db3f71 2038 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
df158561
AR		 2039 if (rc)
2040 return rc;
cecbcddf 2041
df158561
AR		 2042 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2043 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 2044
b6db3f71 2045 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
35406697 2046 params.num_elems = n_rq_elems;
b6db3f71
AL		 2047 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2048
2049 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
df158561
AR		 2050 if (rc)
2051 return rc;
cecbcddf 2052
df158561
AR		 2053 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2054 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 2055
df158561
AR		 2056 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2057 in_params, &out_params);
cecbcddf 2058
df158561
AR		 2059 if (!qp->qed_qp)
2060 return -EINVAL;
cecbcddf 2061
df158561
AR		 2062 qp->qp_id = out_params.qp_id;
2063 qp->icid = out_params.icid;
cecbcddf 2064
97f61250 2065 return qedr_set_roce_db_info(dev, qp);
f5b1b177 2066}
cecbcddf 2067
f5b1b177
KM		 2068static int
2069qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2070 struct qedr_qp *qp,
2071 struct qed_rdma_create_qp_in_params *in_params,
2072 u32 n_sq_elems, u32 n_rq_elems)
2073{
2074 struct qed_rdma_create_qp_out_params out_params;
b6db3f71
AL		 2075 struct qed_chain_init_params params = {
2076 .mode = QED_CHAIN_MODE_PBL,
2077 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
2078 };
f5b1b177
KM		 2079 int rc;
2080
2081 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2082 QEDR_SQE_ELEMENT_SIZE,
15506586 2083 QED_CHAIN_PAGE_SIZE,
f5b1b177
KM		 2084 QED_CHAIN_MODE_PBL);
2085 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2086 QEDR_RQE_ELEMENT_SIZE,
15506586 2087 QED_CHAIN_PAGE_SIZE,
f5b1b177
KM		 2088 QED_CHAIN_MODE_PBL);
2089
2090 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2091 in_params, &out_params);
2092
2093 if (!qp->qed_qp)
2094 return -EINVAL;
2095
2096 /* Now we allocate the chain */
f5b1b177 2097
b6db3f71
AL		 2098 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2099 params.num_elems = n_sq_elems;
2100 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2101 params.ext_pbl_virt = out_params.sq_pbl_virt;
2102 params.ext_pbl_phys = out_params.sq_pbl_phys;
f5b1b177 2103
b6db3f71 2104 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
f5b1b177
KM		 2105 if (rc)
2106 goto err;
2107
b6db3f71
AL		 2108 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2109 params.num_elems = n_rq_elems;
2110 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2111 params.ext_pbl_virt = out_params.rq_pbl_virt;
2112 params.ext_pbl_phys = out_params.rq_pbl_phys;
f5b1b177 2113
b6db3f71 2114 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
f5b1b177
KM		 2115 if (rc)
2116 goto err;
2117
2118 qp->qp_id = out_params.qp_id;
2119 qp->icid = out_params.icid;
2120
97f61250 2121 return qedr_set_iwarp_db_info(dev, qp);
f5b1b177
KM		 2122
2123err:
2124 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2125
2126 return rc;
cecbcddf
RA		 2127}
2128
df158561 2129static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 2130{
df158561
AR		 2131 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2132 kfree(qp->wqe_wr_id);
cecbcddf 2133
df158561
AR		 2134 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2135 kfree(qp->rqe_wr_id);
97f61250
MK		 2136
2137 /* GSI qp is not registered to db mechanism so no need to delete */
2138 if (qp->qp_type == IB_QPT_GSI)
2139 return;
2140
2141 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2142
b4bc7660 2143 if (!qp->srq) {
97f61250 2144 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
b4bc7660
MK		 2145
2146 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2147 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2148 &qp->rq.iwarp_db2_data);
2149 }
cecbcddf
RA		 2150}
2151
df158561
AR		 2152static int qedr_create_kernel_qp(struct qedr_dev *dev,
2153 struct qedr_qp *qp,
2154 struct ib_pd *ibpd,
2155 struct ib_qp_init_attr *attrs)
cecbcddf 2156{
df158561
AR		 2157 struct qed_rdma_create_qp_in_params in_params;
2158 struct qedr_pd *pd = get_qedr_pd(ibpd);
2159 int rc = -EINVAL;
2160 u32 n_rq_elems;
2161 u32 n_sq_elems;
2162 u32 n_sq_entries;
cecbcddf 2163
df158561 2164 memset(&in_params, 0, sizeof(in_params));
82af6d19 2165 qp->create_type = QEDR_QP_CREATE_KERNEL;
cecbcddf 2166
df158561
AR		 2167 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2168 * the ring. The ring should allow at least a single WR, even if the
2169 * user requested none, due to allocation issues.
2170 * We should add an extra WR since the prod and cons indices of
2171 * wqe_wr_id are managed in such a way that the WQ is considered full
2172 * when (prod+1)%max_wr==cons. We currently don't do that because we
2173 * double the number of entries due an iSER issue that pushes far more
2174 * WRs than indicated. If we decline its ib_post_send() then we get
2175 * error prints in the dmesg we'd like to avoid.
2176 */
2177 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2178 dev->attr.max_sqe);
cecbcddf 2179
6396bb22 2180 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 2181 GFP_KERNEL);
2182 if (!qp->wqe_wr_id) {
2183 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2184 return -ENOMEM;
2185 }
cecbcddf 2186
df158561
AR		 2187 /* QP handle to be written in CQE */
2188 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2189 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 2190
df158561
AR		 2191 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2192 * the ring. There ring should allow at least a single WR, even if the
2193 * user requested none, due to allocation issues.
2194 */
2195 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 2196
df158561 2197 /* Allocate driver internal RQ array */
6396bb22 2198 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 2199 GFP_KERNEL);
2200 if (!qp->rqe_wr_id) {
2201 DP_ERR(dev,
2202 "create qp: failed RQ shadow memory allocation\n");
2203 kfree(qp->wqe_wr_id);
2204 return -ENOMEM;
cecbcddf
RA		 2205 }
2206
df158561 2207 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 2208
df158561
AR		 2209 n_sq_entries = attrs->cap.max_send_wr;
2210 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2211 n_sq_entries = max_t(u32, n_sq_entries, 1);
2212 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 2213
df158561
AR		 2214 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2215
f5b1b177
KM		 2216 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2217 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2218 n_sq_elems, n_rq_elems);
2219 else
2220 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2221 n_sq_elems, n_rq_elems);
df158561
AR		 2222 if (rc)
2223 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 2224
2225 return rc;
2226}
2227
3e45410f
KS		 2228static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2229 struct ib_udata *udata)
2230{
2231 struct qedr_ucontext *ctx =
2232 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2233 ibucontext);
2234 int rc;
2235
2236 if (qp->qp_type != IB_QPT_GSI) {
2237 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2238 if (rc)
2239 return rc;
2240 }
2241
2242 if (qp->create_type == QEDR_QP_CREATE_USER)
2243 qedr_cleanup_user(dev, ctx, qp);
2244 else
2245 qedr_cleanup_kernel(dev, qp);
2246
2247 return 0;
2248}
2249
514aee66
LR		 2250int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2251 struct ib_udata *udata)
cecbcddf 2252{
06e8d1df 2253 struct qedr_xrcd *xrcd = NULL;
514aee66
LR		 2254 struct ib_pd *ibpd = ibqp->pd;
2255 struct qedr_pd *pd = get_qedr_pd(ibpd);
2256 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2257 struct qedr_qp *qp = get_qedr_qp(ibqp);
cecbcddf
RA		 2258 int rc = 0;
2259
1f11a761 2260 if (attrs->create_flags)
514aee66 2261 return -EOPNOTSUPP;
1f11a761 2262
514aee66 2263 if (attrs->qp_type == IB_QPT_XRC_TGT)
06e8d1df 2264 xrcd = get_qedr_xrcd(attrs->xrcd);
514aee66 2265 else
06e8d1df 2266 pd = get_qedr_pd(ibpd);
06e8d1df 2267
cecbcddf
RA		 2268 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2269 udata ? "user library" : "kernel", pd);
2270
e00b64f7 2271 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf 2272 if (rc)
514aee66 2273 return rc;
cecbcddf 2274
cecbcddf 2275 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 2276 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2277 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 2278 get_qedr_cq(attrs->send_cq),
2279 get_qedr_cq(attrs->send_cq)->icid,
2280 get_qedr_cq(attrs->recv_cq),
3491c9e7 2281 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 2282
df158561 2283 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 2284
514aee66
LR		 2285 if (attrs->qp_type == IB_QPT_GSI)
2286 return qedr_create_gsi_qp(dev, attrs, qp);
04886779 2287
06e8d1df 2288 if (udata || xrcd)
df158561
AR		 2289 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2290 else
2291 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 2292
df158561 2293 if (rc)
514aee66 2294 return rc;
cecbcddf 2295
cecbcddf
RA		 2296 qp->ibqp.qp_num = qp->qp_id;
2297
1212767e 2298 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 2299 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e 2300 if (rc)
3e45410f 2301 goto out_free_qp_resources;
1212767e 2302 }
de0089e6 2303
514aee66 2304 return 0;
cecbcddf 2305
3e45410f
KS		 2306out_free_qp_resources:
2307 qedr_free_qp_resources(dev, qp, udata);
514aee66 2308 return -EFAULT;
cecbcddf
RA		 2309}
2310
27a4b1a6 2311static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 2312{
2313 switch (qp_state) {
2314 case QED_ROCE_QP_STATE_RESET:
2315 return IB_QPS_RESET;
2316 case QED_ROCE_QP_STATE_INIT:
2317 return IB_QPS_INIT;
2318 case QED_ROCE_QP_STATE_RTR:
2319 return IB_QPS_RTR;
2320 case QED_ROCE_QP_STATE_RTS:
2321 return IB_QPS_RTS;
2322 case QED_ROCE_QP_STATE_SQD:
2323 return IB_QPS_SQD;
2324 case QED_ROCE_QP_STATE_ERR:
2325 return IB_QPS_ERR;
2326 case QED_ROCE_QP_STATE_SQE:
2327 return IB_QPS_SQE;
2328 }
2329 return IB_QPS_ERR;
2330}
2331
27a4b1a6
RA		 2332static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2333 enum ib_qp_state qp_state)
cecbcddf
RA		 2334{
2335 switch (qp_state) {
2336 case IB_QPS_RESET:
2337 return QED_ROCE_QP_STATE_RESET;
2338 case IB_QPS_INIT:
2339 return QED_ROCE_QP_STATE_INIT;
2340 case IB_QPS_RTR:
2341 return QED_ROCE_QP_STATE_RTR;
2342 case IB_QPS_RTS:
2343 return QED_ROCE_QP_STATE_RTS;
2344 case IB_QPS_SQD:
2345 return QED_ROCE_QP_STATE_SQD;
2346 case IB_QPS_ERR:
2347 return QED_ROCE_QP_STATE_ERR;
2348 default:
2349 return QED_ROCE_QP_STATE_ERR;
2350 }
2351}
2352
cecbcddf
RA		 2353static int qedr_update_qp_state(struct qedr_dev *dev,
2354 struct qedr_qp *qp,
caf61b1b 2355 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 2356 enum qed_roce_qp_state new_state)
2357{
2358 int status = 0;
2359
caf61b1b 2360 if (new_state == cur_state)
865cea40 2361 return 0;
cecbcddf 2362
caf61b1b 2363 switch (cur_state) {
cecbcddf
RA		 2364 case QED_ROCE_QP_STATE_RESET:
2365 switch (new_state) {
2366 case QED_ROCE_QP_STATE_INIT:
cecbcddf
RA		 2367 break;
2368 default:
2369 status = -EINVAL;
2370 break;
790b57f6 2371 }
cecbcddf
RA		 2372 break;
2373 case QED_ROCE_QP_STATE_INIT:
2374 switch (new_state) {
2375 case QED_ROCE_QP_STATE_RTR:
2376 /* Update doorbell (in case post_recv was
2377 * done before move to RTR)
2378 */
f5b1b177
KM		 2379
2380 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177 2381 writel(qp->rq.db_data.raw, qp->rq.db);
f5b1b177 2382 }
cecbcddf
RA		 2383 break;
2384 case QED_ROCE_QP_STATE_ERR:
2385 break;
2386 default:
2387 /* Invalid state change. */
2388 status = -EINVAL;
2389 break;
790b57f6 2390 }
cecbcddf
RA		 2391 break;
2392 case QED_ROCE_QP_STATE_RTR:
2393 /* RTR->XXX */
2394 switch (new_state) {
2395 case QED_ROCE_QP_STATE_RTS:
2396 break;
2397 case QED_ROCE_QP_STATE_ERR:
2398 break;
2399 default:
2400 /* Invalid state change. */
2401 status = -EINVAL;
2402 break;
790b57f6 2403 }
cecbcddf
RA		 2404 break;
2405 case QED_ROCE_QP_STATE_RTS:
2406 /* RTS->XXX */
2407 switch (new_state) {
2408 case QED_ROCE_QP_STATE_SQD:
2409 break;
2410 case QED_ROCE_QP_STATE_ERR:
2411 break;
2412 default:
2413 /* Invalid state change. */
2414 status = -EINVAL;
2415 break;
790b57f6 2416 }
cecbcddf
RA		 2417 break;
2418 case QED_ROCE_QP_STATE_SQD:
2419 /* SQD->XXX */
2420 switch (new_state) {
2421 case QED_ROCE_QP_STATE_RTS:
2422 case QED_ROCE_QP_STATE_ERR:
2423 break;
2424 default:
2425 /* Invalid state change. */
2426 status = -EINVAL;
2427 break;
790b57f6 2428 }
cecbcddf
RA		 2429 break;
2430 case QED_ROCE_QP_STATE_ERR:
2431 /* ERR->XXX */
2432 switch (new_state) {
2433 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2434 if ((qp->rq.prod != qp->rq.cons) ||
2435 (qp->sq.prod != qp->sq.cons)) {
2436 DP_NOTICE(dev,
2437 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2438 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2439 qp->sq.cons);
2440 status = -EINVAL;
2441 }
cecbcddf
RA		 2442 break;
2443 default:
2444 status = -EINVAL;
2445 break;
790b57f6 2446 }
cecbcddf
RA		 2447 break;
2448 default:
2449 status = -EINVAL;
2450 break;
790b57f6 2451 }
cecbcddf
RA		 2452
2453 return status;
2454}
2455
2456int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2457 int attr_mask, struct ib_udata *udata)
2458{
2459 struct qedr_qp *qp = get_qedr_qp(ibqp);
2460 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2461 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2462 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2463 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2464 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2465 int rc = 0;
2466
2467 DP_DEBUG(dev, QEDR_MSG_QP,
2468 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2469 attr->qp_state);
2470
26e990ba
JG		 2471 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2472 return -EOPNOTSUPP;
2473
cecbcddf
RA		 2474 old_qp_state = qedr_get_ibqp_state(qp->state);
2475 if (attr_mask & IB_QP_STATE)
2476 new_qp_state = attr->qp_state;
2477 else
2478 new_qp_state = old_qp_state;
2479
f5b1b177
KM		 2480 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2481 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2482 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2483 DP_ERR(dev,
2484 "modify qp: invalid attribute mask=0x%x specified for\n"
2485 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2486 attr_mask, qp->qp_id, ibqp->qp_type,
2487 old_qp_state, new_qp_state);
2488 rc = -EINVAL;
2489 goto err;
2490 }
cecbcddf
RA		 2491 }
2492
2493 /* Translate the masks... */
2494 if (attr_mask & IB_QP_STATE) {
2495 SET_FIELD(qp_params.modify_flags,
2496 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2497 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2498 }
2499
2500 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2501 qp_params.sqd_async = true;
2502
2503 if (attr_mask & IB_QP_PKEY_INDEX) {
2504 SET_FIELD(qp_params.modify_flags,
2505 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2506 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2507 rc = -EINVAL;
2508 goto err;
2509 }
2510
2511 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2512 }
2513
2514 if (attr_mask & IB_QP_QKEY)
2515 qp->qkey = attr->qkey;
2516
2517 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2518 SET_FIELD(qp_params.modify_flags,
2519 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2520 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2521 IB_ACCESS_REMOTE_READ;
2522 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2523 IB_ACCESS_REMOTE_WRITE;
2524 qp_params.incoming_atomic_en = attr->qp_access_flags &
2525 IB_ACCESS_REMOTE_ATOMIC;
2526 }
2527
2528 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2529 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2530 return -EINVAL;
2531
cecbcddf
RA		 2532 if (attr_mask & IB_QP_PATH_MTU) {
2533 if (attr->path_mtu < IB_MTU_256 ||
2534 attr->path_mtu > IB_MTU_4096) {
2535 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2536 rc = -EINVAL;
2537 goto err;
2538 }
2539 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2540 ib_mtu_enum_to_int(iboe_get_mtu
2541 (dev->ndev->mtu)));
2542 }
2543
2544 if (!qp->mtu) {
2545 qp->mtu =
2546 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2547 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2548 }
2549
2550 SET_FIELD(qp_params.modify_flags,
2551 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2552
d8966fcd
DC		 2553 qp_params.traffic_class_tos = grh->traffic_class;
2554 qp_params.flow_label = grh->flow_label;
2555 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2556
d8966fcd 2557 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2558
2559 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2560 if (rc) {
2561 DP_ERR(dev,
2562 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2563 grh->sgid_index, rc);
cecbcddf
RA		 2564 return rc;
2565 }
2566
2567 rc = qedr_get_dmac(dev, &attr->ah_attr,
2568 qp_params.remote_mac_addr);
2569 if (rc)
2570 return rc;
2571
2572 qp_params.use_local_mac = true;
2573 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2574
2575 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2576 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2577 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2578 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2579 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2580 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2581 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2582 qp_params.remote_mac_addr);
cecbcddf
RA		 2583
2584 qp_params.mtu = qp->mtu;
2585 qp_params.lb_indication = false;
2586 }
2587
2588 if (!qp_params.mtu) {
2589 /* Stay with current MTU */
2590 if (qp->mtu)
2591 qp_params.mtu = qp->mtu;
2592 else
2593 qp_params.mtu =
2594 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2595 }
2596
2597 if (attr_mask & IB_QP_TIMEOUT) {
2598 SET_FIELD(qp_params.modify_flags,
2599 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2600
c3594f22
KM		 2601 /* The received timeout value is an exponent used like this:
2602 * "12.7.34 LOCAL ACK TIMEOUT
2603 * Value representing the transport (ACK) timeout for use by
2604 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2605 * The FW expects timeout in msec so we need to divide the usec
2606 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2607 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2608 * The value of zero means infinite so we use a 'max_t' to make
2609 * sure that sub 1 msec values will be configured as 1 msec.
2610 */
2611 if (attr->timeout)
2612 qp_params.ack_timeout =
2613 1 << max_t(int, attr->timeout - 8, 0);
2614 else
cecbcddf 2615 qp_params.ack_timeout = 0;
118f7674
KH		 2616
2617 qp->timeout = attr->timeout;
cecbcddf 2618 }
c3594f22 2619
cecbcddf
RA		 2620 if (attr_mask & IB_QP_RETRY_CNT) {
2621 SET_FIELD(qp_params.modify_flags,
2622 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2623 qp_params.retry_cnt = attr->retry_cnt;
2624 }
2625
2626 if (attr_mask & IB_QP_RNR_RETRY) {
2627 SET_FIELD(qp_params.modify_flags,
2628 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2629 qp_params.rnr_retry_cnt = attr->rnr_retry;
2630 }
2631
2632 if (attr_mask & IB_QP_RQ_PSN) {
2633 SET_FIELD(qp_params.modify_flags,
2634 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2635 qp_params.rq_psn = attr->rq_psn;
2636 qp->rq_psn = attr->rq_psn;
2637 }
2638
2639 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2640 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2641 rc = -EINVAL;
2642 DP_ERR(dev,
2643 "unsupported max_rd_atomic=%d, supported=%d\n",
2644 attr->max_rd_atomic,
2645 dev->attr.max_qp_req_rd_atomic_resc);
2646 goto err;
2647 }
2648
2649 SET_FIELD(qp_params.modify_flags,
2650 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2651 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2652 }
2653
2654 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2655 SET_FIELD(qp_params.modify_flags,
2656 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2657 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2658 }
2659
2660 if (attr_mask & IB_QP_SQ_PSN) {
2661 SET_FIELD(qp_params.modify_flags,
2662 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2663 qp_params.sq_psn = attr->sq_psn;
2664 qp->sq_psn = attr->sq_psn;
2665 }
2666
2667 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2668 if (attr->max_dest_rd_atomic >
2669 dev->attr.max_qp_resp_rd_atomic_resc) {
2670 DP_ERR(dev,
2671 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2672 attr->max_dest_rd_atomic,
2673 dev->attr.max_qp_resp_rd_atomic_resc);
2674
2675 rc = -EINVAL;
2676 goto err;
2677 }
2678
2679 SET_FIELD(qp_params.modify_flags,
2680 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2681 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2682 }
2683
2684 if (attr_mask & IB_QP_DEST_QPN) {
2685 SET_FIELD(qp_params.modify_flags,
2686 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2687
2688 qp_params.dest_qp = attr->dest_qp_num;
2689 qp->dest_qp_num = attr->dest_qp_num;
2690 }
2691
caf61b1b
KM		 2692 cur_state = qp->state;
2693
2694 /* Update the QP state before the actual ramrod to prevent a race with
2695 * fast path. Modifying the QP state to error will cause the device to
2696 * flush the CQEs and while polling the flushed CQEs will considered as
2697 * a potential issue if the QP isn't in error state.
2698 */
2699 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2700 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2701 qp->state = QED_ROCE_QP_STATE_ERR;
2702
cecbcddf
RA		 2703 if (qp->qp_type != IB_QPT_GSI)
2704 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2705 qp->qed_qp, &qp_params);
2706
2707 if (attr_mask & IB_QP_STATE) {
2708 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2709 rc = qedr_update_qp_state(dev, qp, cur_state,
2710 qp_params.new_state);
cecbcddf
RA		 2711 qp->state = qp_params.new_state;
2712 }
2713
2714err:
2715 return rc;
2716}
2717
2718static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2719{
2720 int ib_qp_acc_flags = 0;
2721
2722 if (params->incoming_rdma_write_en)
2723 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2724 if (params->incoming_rdma_read_en)
2725 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2726 if (params->incoming_atomic_en)
2727 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2728 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2729 return ib_qp_acc_flags;
2730}
2731
2732int qedr_query_qp(struct ib_qp *ibqp,
2733 struct ib_qp_attr *qp_attr,
2734 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2735{
2736 struct qed_rdma_query_qp_out_params params;
2737 struct qedr_qp *qp = get_qedr_qp(ibqp);
2738 struct qedr_dev *dev = qp->dev;
2739 int rc = 0;
2740
2741 memset(&params, 0, sizeof(params));
cecbcddf
RA		 2742 memset(qp_attr, 0, sizeof(*qp_attr));
2743 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2744
4f960393
AP		 2745 if (qp->qp_type != IB_QPT_GSI) {
2746 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2747 if (rc)
2748 goto err;
2749 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2750 } else {
2751 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2752 }
2753
cecbcddf 2754 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2755 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2756 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2757 qp_attr->rq_psn = params.rq_psn;
2758 qp_attr->sq_psn = params.sq_psn;
2759 qp_attr->dest_qp_num = params.dest_qp;
2760
2761 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2762
2763 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2764 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2765 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2766 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
fbf58026 2767 qp_attr->cap.max_inline_data = dev->attr.max_inline;
cecbcddf
RA		 2768 qp_init_attr->cap = qp_attr->cap;
2769
44c58487 2770 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2771 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2772 params.flow_label, qp->sgid_idx,
2773 params.hop_limit_ttl, params.traffic_class_tos);
2774 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2775 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2776 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
118f7674 2777 qp_attr->timeout = qp->timeout;
cecbcddf
RA		 2778 qp_attr->rnr_retry = params.rnr_retry;
2779 qp_attr->retry_cnt = params.retry_cnt;
2780 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2781 qp_attr->pkey_index = params.pkey_index;
2782 qp_attr->port_num = 1;
d8966fcd
DC		 2783 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2784 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2785 qp_attr->alt_pkey_index = 0;
2786 qp_attr->alt_port_num = 0;
2787 qp_attr->alt_timeout = 0;
2788 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2789
2790 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2791 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2792 qp_attr->max_rd_atomic = params.max_rd_atomic;
2793 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2794
2795 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2796 qp_attr->cap.max_inline_data);
2797
2798err:
2799 return rc;
2800}
2801
c4367a26 2802int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2803{
2804 struct qedr_qp *qp = get_qedr_qp(ibqp);
2805 struct qedr_dev *dev = qp->dev;
2806 struct ib_qp_attr attr;
2807 int attr_mask = 0;
cecbcddf
RA		 2808
2809 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2810 qp, qp->qp_type);
2811
f5b1b177
KM		 2812 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2813 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2814 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2815 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2816
f5b1b177
KM		 2817 attr.qp_state = IB_QPS_ERR;
2818 attr_mask |= IB_QP_STATE;
cecbcddf 2819
f5b1b177
KM		 2820 /* Change the QP state to ERROR */
2821 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2822 }
e411e058 2823 } else {
82af6d19
MK		 2824 /* If connection establishment started the WAIT_FOR_CONNECT
2825 * bit will be on and we need to Wait for the establishment
2826 * to complete before destroying the qp.
2827 */
2828 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2829 &qp->iwarp_cm_flags))
2830 wait_for_completion(&qp->iwarp_cm_comp);
2831
2832 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2833 * bit will be on, and we need to wait for the disconnect to
2834 * complete before continuing. We can use the same completion,
2835 * iwarp_cm_comp, since this is the only place that waits for
2836 * this completion and it is sequential. In addition,
2837 * disconnect can't occur before the connection is fully
2838 * established, therefore if WAIT_FOR_DISCONNECT is on it
2839 * means WAIT_FOR_CONNECT is also on and the completion for
2840 * CONNECT already occurred.
2841 */
2842 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2843 &qp->iwarp_cm_flags))
2844 wait_for_completion(&qp->iwarp_cm_comp);
cecbcddf
RA		 2845 }
2846
df158561 2847 if (qp->qp_type == IB_QPT_GSI)
04886779 2848 qedr_destroy_gsi_qp(dev);
cecbcddf 2849
82af6d19
MK		 2850 /* We need to remove the entry from the xarray before we release the
2851 * qp_id to avoid a race of the qp_id being reallocated and failing
2852 * on xa_insert
2853 */
2854 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2855 xa_erase(&dev->qps, qp->qp_id);
2856
bdeacabd 2857 qedr_free_qp_resources(dev, qp, udata);
cecbcddf 2858
60fab107 2859 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
82af6d19 2860 qedr_iw_qp_rem_ref(&qp->ibqp);
60fab107
PK		 2861 wait_for_completion(&qp->qp_rel_comp);
2862 }
82af6d19 2863
cf167e5e 2864 return 0;
cecbcddf 2865}
e0290cce 2866
fa5d010c 2867int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
d3456914 2868 struct ib_udata *udata)
04886779 2869{
d3456914 2870 struct qedr_ah *ah = get_qedr_ah(ibah);
04886779 2871
fa5d010c 2872 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
04886779 2873
d3456914 2874 return 0;
04886779
RA		 2875}
2876
9a9ebf8c 2877int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
04886779
RA		 2878{
2879 struct qedr_ah *ah = get_qedr_ah(ibah);
2880
d97099fe 2881 rdma_destroy_ah_attr(&ah->attr);
9a9ebf8c 2882 return 0;
04886779
RA		 2883}
2884
e0290cce
RA		 2885static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2886{
2887 struct qedr_pbl *pbl, *tmp;
2888
2889 if (info->pbl_table)
2890 list_add_tail(&info->pbl_table->list_entry,
2891 &info->free_pbl_list);
2892
2893 if (!list_empty(&info->inuse_pbl_list))
2894 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2895
2896 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2897 list_del(&pbl->list_entry);
2898 qedr_free_pbl(dev, &info->pbl_info, pbl);
2899 }
2900}
2901
2902static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2903 size_t page_list_len, bool two_layered)
2904{
2905 struct qedr_pbl *tmp;
2906 int rc;
2907
2908 INIT_LIST_HEAD(&info->free_pbl_list);
2909 INIT_LIST_HEAD(&info->inuse_pbl_list);
2910
2911 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2912 page_list_len, two_layered);
2913 if (rc)
2914 goto done;
2915
2916 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2917 if (IS_ERR(info->pbl_table)) {
2918 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2919 goto done;
2920 }
2921
2922 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2923 &info->pbl_table->pa);
2924
2925 /* in usual case we use 2 PBLs, so we add one to free
2926 * list and allocating another one
2927 */
2928 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2929 if (IS_ERR(tmp)) {
e0290cce
RA		 2930 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2931 goto done;
2932 }
2933
2934 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2935
2936 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2937
2938done:
2939 if (rc)
2940 free_mr_info(dev, info);
2941
2942 return rc;
2943}
2944
2945struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2946 u64 usr_addr, int acc, struct ib_udata *udata)
2947{
2948 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2949 struct qedr_mr *mr;
2950 struct qedr_pd *pd;
2951 int rc = -ENOMEM;
2952
2953 pd = get_qedr_pd(ibpd);
2954 DP_DEBUG(dev, QEDR_MSG_MR,
2955 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2956 pd->pd_id, start, len, usr_addr, acc);
2957
2958 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2959 return ERR_PTR(-EINVAL);
2960
2961 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2962 if (!mr)
2963 return ERR_PTR(rc);
2964
2965 mr->type = QEDR_MR_USER;
2966
c320e527 2967 mr->umem = ib_umem_get(ibpd->device, start, len, acc);
e0290cce
RA		 2968 if (IS_ERR(mr->umem)) {
2969 rc = -EFAULT;
2970 goto err0;
2971 }
2972
901bca71
JG		 2973 rc = init_mr_info(dev, &mr->info,
2974 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
e0290cce
RA		 2975 if (rc)
2976 goto err1;
2977
2978 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2979 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2980
2981 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2982 if (rc) {
0050a576
PK		 2983 if (rc == -EINVAL)
2984 DP_ERR(dev, "Out of MR resources\n");
2985 else
2986 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2987
e0290cce
RA		 2988 goto err1;
2989 }
2990
2991 /* Index only, 18 bit long, lkey = itid << 8 | key */
2992 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2993 mr->hw_mr.key = 0;
2994 mr->hw_mr.pd = pd->pd_id;
2995 mr->hw_mr.local_read = 1;
2996 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2997 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2998 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2999 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3000 mr->hw_mr.mw_bind = false;
3001 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3002 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3003 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 3004 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 3005 mr->hw_mr.length = len;
3006 mr->hw_mr.vaddr = usr_addr;
e0290cce
RA		 3007 mr->hw_mr.phy_mr = false;
3008 mr->hw_mr.dma_mr = false;
3009
3010 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3011 if (rc) {
3012 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3013 goto err2;
3014 }
3015
3016 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3017 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3018 mr->hw_mr.remote_atomic)
3019 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3020
3021 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3022 mr->ibmr.lkey);
3023 return &mr->ibmr;
3024
3025err2:
3026 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3027err1:
3028 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3029err0:
3030 kfree(mr);
3031 return ERR_PTR(rc);
3032}
3033
c4367a26 3034int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 3035{
3036 struct qedr_mr *mr = get_qedr_mr(ib_mr);
3037 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3038 int rc = 0;
3039
3040 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3041 if (rc)
3042 return rc;
3043
3044 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3045
24e412c1
MK		 3046 if (mr->type != QEDR_MR_DMA)
3047 free_mr_info(dev, &mr->info);
e0290cce
RA		 3048
3049 /* it could be user registered memory. */
836a0fbb 3050 ib_umem_release(mr->umem);
e0290cce
RA		 3051
3052 kfree(mr);
3053
3054 return rc;
3055}
3056
27a4b1a6
RA		 3057static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3058 int max_page_list_len)
e0290cce
RA		 3059{
3060 struct qedr_pd *pd = get_qedr_pd(ibpd);
3061 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3062 struct qedr_mr *mr;
3063 int rc = -ENOMEM;
3064
3065 DP_DEBUG(dev, QEDR_MSG_MR,
3066 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3067 max_page_list_len);
3068
3069 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3070 if (!mr)
3071 return ERR_PTR(rc);
3072
3073 mr->dev = dev;
3074 mr->type = QEDR_MR_FRMR;
3075
3076 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3077 if (rc)
3078 goto err0;
3079
3080 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3081 if (rc) {
0050a576
PK		 3082 if (rc == -EINVAL)
3083 DP_ERR(dev, "Out of MR resources\n");
3084 else
3085 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3086
e0290cce
RA		 3087 goto err0;
3088 }
3089
3090 /* Index only, 18 bit long, lkey = itid << 8 | key */
3091 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3092 mr->hw_mr.key = 0;
3093 mr->hw_mr.pd = pd->pd_id;
3094 mr->hw_mr.local_read = 1;
3095 mr->hw_mr.local_write = 0;
3096 mr->hw_mr.remote_read = 0;
3097 mr->hw_mr.remote_write = 0;
3098 mr->hw_mr.remote_atomic = 0;
3099 mr->hw_mr.mw_bind = false;
3100 mr->hw_mr.pbl_ptr = 0;
3101 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3102 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
e0290cce
RA		 3103 mr->hw_mr.length = 0;
3104 mr->hw_mr.vaddr = 0;
e0290cce
RA		 3105 mr->hw_mr.phy_mr = true;
3106 mr->hw_mr.dma_mr = false;
3107
3108 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3109 if (rc) {
3110 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3111 goto err1;
3112 }
3113
3114 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3115 mr->ibmr.rkey = mr->ibmr.lkey;
3116
3117 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3118 return mr;
3119
3120err1:
3121 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3122err0:
3123 kfree(mr);
3124 return ERR_PTR(rc);
3125}
3126
c4367a26 3127struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
42a3b153 3128 u32 max_num_sg)
e0290cce 3129{
e0290cce
RA		 3130 struct qedr_mr *mr;
3131
3132 if (mr_type != IB_MR_TYPE_MEM_REG)
3133 return ERR_PTR(-EINVAL);
3134
3135 mr = __qedr_alloc_mr(ibpd, max_num_sg);
3136
3137 if (IS_ERR(mr))
3138 return ERR_PTR(-EINVAL);
3139
e0290cce
RA		 3140 return &mr->ibmr;
3141}
3142
3143static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3144{
3145 struct qedr_mr *mr = get_qedr_mr(ibmr);
3146 struct qedr_pbl *pbl_table;
3147 struct regpair *pbe;
3148 u32 pbes_in_page;
3149
3150 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 3151 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 3152 return -ENOMEM;
3153 }
3154
3155 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3156 mr->npages, addr);
3157
3158 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3159 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3160 pbe = (struct regpair *)pbl_table->va;
3161 pbe += mr->npages % pbes_in_page;
3162 pbe->lo = cpu_to_le32((u32)addr);
3163 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3164
3165 mr->npages++;
3166
3167 return 0;
3168}
3169
3170static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3171{
3172 int work = info->completed - info->completed_handled - 1;
3173
3174 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3175 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3176 struct qedr_pbl *pbl;
3177
3178 /* Free all the page list that are possible to be freed
3179 * (all the ones that were invalidated), under the assumption
3180 * that if an FMR was completed successfully that means that
3181 * if there was an invalidate operation before it also ended
3182 */
3183 pbl = list_first_entry(&info->inuse_pbl_list,
3184 struct qedr_pbl, list_entry);
aafec388 3185 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 3186 info->completed_handled++;
3187 }
3188}
3189
3190int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3191 int sg_nents, unsigned int *sg_offset)
3192{
3193 struct qedr_mr *mr = get_qedr_mr(ibmr);
3194
3195 mr->npages = 0;
3196
3197 handle_completed_mrs(mr->dev, &mr->info);
3198 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3199}
3200
3201struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3202{
3203 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3204 struct qedr_pd *pd = get_qedr_pd(ibpd);
3205 struct qedr_mr *mr;
3206 int rc;
3207
3208 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3209 if (!mr)
3210 return ERR_PTR(-ENOMEM);
3211
3212 mr->type = QEDR_MR_DMA;
3213
3214 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3215 if (rc) {
0050a576
PK		 3216 if (rc == -EINVAL)
3217 DP_ERR(dev, "Out of MR resources\n");
3218 else
3219 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3220
e0290cce
RA		 3221 goto err1;
3222 }
3223
3224 /* index only, 18 bit long, lkey = itid << 8 | key */
3225 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3226 mr->hw_mr.pd = pd->pd_id;
3227 mr->hw_mr.local_read = 1;
3228 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3229 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3230 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3231 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3232 mr->hw_mr.dma_mr = true;
3233
3234 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3235 if (rc) {
3236 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3237 goto err2;
3238 }
3239
3240 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3241 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3242 mr->hw_mr.remote_atomic)
3243 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3244
3245 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3246 return &mr->ibmr;
3247
3248err2:
3249 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3250err1:
3251 kfree(mr);
3252 return ERR_PTR(rc);
3253}
afa0e13b
RA		 3254
3255static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3256{
3257 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3258}
3259
3260static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3261{
3262 int i, len = 0;
3263
3264 for (i = 0; i < num_sge; i++)
3265 len += sg_list[i].length;
3266
3267 return len;
3268}
3269
3270static void swap_wqe_data64(u64 *p)
3271{
3272 int i;
3273
3274 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3275 *p = cpu_to_be64(cpu_to_le64(*p));
3276}
3277
3278static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3279 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 3280 const struct ib_send_wr *wr,
3281 const struct ib_send_wr **bad_wr,
3282 u8 *bits, u8 bit)
afa0e13b
RA		 3283{
3284 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3285 char *seg_prt, *wqe;
3286 int i, seg_siz;
3287
3288 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3289 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3290 *bad_wr = wr;
3291 return 0;
3292 }
3293
3294 if (!data_size)
3295 return data_size;
3296
3297 *bits |= bit;
3298
3299 seg_prt = NULL;
3300 wqe = NULL;
3301 seg_siz = 0;
3302
3303 /* Copy data inline */
3304 for (i = 0; i < wr->num_sge; i++) {
3305 u32 len = wr->sg_list[i].length;
3306 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3307
3308 while (len > 0) {
3309 u32 cur;
3310
3311 /* New segment required */
3312 if (!seg_siz) {
3313 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3314 seg_prt = wqe;
3315 seg_siz = sizeof(struct rdma_sq_common_wqe);
3316 (*wqe_size)++;
3317 }
3318
3319 /* Calculate currently allowed length */
3320 cur = min_t(u32, len, seg_siz);
3321 memcpy(seg_prt, src, cur);
3322
3323 /* Update segment variables */
3324 seg_prt += cur;
3325 seg_siz -= cur;
3326
3327 /* Update sge variables */
3328 src += cur;
3329 len -= cur;
3330
3331 /* Swap fully-completed segments */
3332 if (!seg_siz)
3333 swap_wqe_data64((u64 *)wqe);
3334 }
3335 }
3336
3337 /* swap last not completed segment */
3338 if (seg_siz)
3339 swap_wqe_data64((u64 *)wqe);
3340
3341 return data_size;
3342}
3343
3344#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3345 do { \
3346 DMA_REGPAIR_LE(sge->addr, vaddr); \
3347 (sge)->length = cpu_to_le32(vlength); \
3348 (sge)->flags = cpu_to_le32(vflags); \
3349 } while (0)
3350
3351#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3352 do { \
3353 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3354 (hdr)->num_sges = num_sge; \
3355 } while (0)
3356
3357#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3358 do { \
3359 DMA_REGPAIR_LE(sge->addr, vaddr); \
3360 (sge)->length = cpu_to_le32(vlength); \
3361 (sge)->l_key = cpu_to_le32(vlkey); \
3362 } while (0)
3363
3364static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 3365 const struct ib_send_wr *wr)
afa0e13b
RA		 3366{
3367 u32 data_size = 0;
3368 int i;
3369
3370 for (i = 0; i < wr->num_sge; i++) {
3371 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3372
3373 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3374 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3375 sge->length = cpu_to_le32(wr->sg_list[i].length);
3376 data_size += wr->sg_list[i].length;
3377 }
3378
3379 if (wqe_size)
3380 *wqe_size += wr->num_sge;
3381
3382 return data_size;
3383}
3384
3385static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3386 struct qedr_qp *qp,
3387 struct rdma_sq_rdma_wqe_1st *rwqe,
3388 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3389 const struct ib_send_wr *wr,
3390 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3391{
3392 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3393 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3394
8b0cabc6
AR		 3395 if (wr->send_flags & IB_SEND_INLINE &&
3396 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3397 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3398 u8 flags = 0;
3399
3400 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3401 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3402 bad_wr, &rwqe->flags, flags);
3403 }
3404
3405 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3406}
3407
3408static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3409 struct qedr_qp *qp,
3410 struct rdma_sq_send_wqe_1st *swqe,
3411 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3412 const struct ib_send_wr *wr,
3413 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3414{
3415 memset(swqe2, 0, sizeof(*swqe2));
3416 if (wr->send_flags & IB_SEND_INLINE) {
3417 u8 flags = 0;
3418
3419 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3420 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3421 bad_wr, &swqe->flags, flags);
3422 }
3423
3424 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3425}
3426
3427static int qedr_prepare_reg(struct qedr_qp *qp,
3428 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3429 const struct ib_reg_wr *wr)
afa0e13b
RA		 3430{
3431 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3432 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3433
3434 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3435 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3436 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3437 fwqe1->l_key = wr->key;
3438
08c4cf51
AR		 3439 fwqe2->access_ctrl = 0;
3440
afa0e13b
RA		 3441 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3442 !!(wr->access & IB_ACCESS_REMOTE_READ));
3443 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3444 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3445 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3446 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3447 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3448 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3449 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3450 fwqe2->fmr_ctrl = 0;
3451
3452 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3453 ilog2(mr->ibmr.page_size) - 12);
3454
3455 fwqe2->length_hi = 0;
3456 fwqe2->length_lo = mr->ibmr.length;
3457 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3458 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3459
3460 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3461
3462 return 0;
3463}
3464
27a4b1a6 3465static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3466{
3467 switch (opcode) {
3468 case IB_WR_RDMA_WRITE:
3469 case IB_WR_RDMA_WRITE_WITH_IMM:
3470 return IB_WC_RDMA_WRITE;
3471 case IB_WR_SEND_WITH_IMM:
3472 case IB_WR_SEND:
3473 case IB_WR_SEND_WITH_INV:
3474 return IB_WC_SEND;
3475 case IB_WR_RDMA_READ:
fb1a22be 3476 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3477 return IB_WC_RDMA_READ;
3478 case IB_WR_ATOMIC_CMP_AND_SWP:
3479 return IB_WC_COMP_SWAP;
3480 case IB_WR_ATOMIC_FETCH_AND_ADD:
3481 return IB_WC_FETCH_ADD;
3482 case IB_WR_REG_MR:
3483 return IB_WC_REG_MR;
3484 case IB_WR_LOCAL_INV:
3485 return IB_WC_LOCAL_INV;
3486 default:
3487 return IB_WC_SEND;
3488 }
3489}
3490
f696bf6d
BVA		 3491static inline bool qedr_can_post_send(struct qedr_qp *qp,
3492 const struct ib_send_wr *wr)
afa0e13b
RA		 3493{
3494 int wq_is_full, err_wr, pbl_is_full;
3495 struct qedr_dev *dev = qp->dev;
3496
3497 /* prevent SQ overflow and/or processing of a bad WR */
3498 err_wr = wr->num_sge > qp->sq.max_sges;
3499 wq_is_full = qedr_wq_is_full(&qp->sq);
3500 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3501 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3502 if (wq_is_full || err_wr || pbl_is_full) {
3503 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3504 DP_ERR(dev,
3505 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3506 qp);
3507 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3508 }
3509
3510 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3511 DP_ERR(dev,
3512 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3513 qp);
3514 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3515 }
3516
3517 if (pbl_is_full &&
3518 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3519 DP_ERR(dev,
3520 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3521 qp);
3522 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3523 }
3524 return false;
3525 }
3526 return true;
3527}
3528
d34ac5cd
BVA		 3529static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3530 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3531{
3532 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3533 struct qedr_qp *qp = get_qedr_qp(ibqp);
3534 struct rdma_sq_atomic_wqe_1st *awqe1;
3535 struct rdma_sq_atomic_wqe_2nd *awqe2;
3536 struct rdma_sq_atomic_wqe_3rd *awqe3;
3537 struct rdma_sq_send_wqe_2st *swqe2;
3538 struct rdma_sq_local_inv_wqe *iwqe;
3539 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3540 struct rdma_sq_send_wqe_1st *swqe;
3541 struct rdma_sq_rdma_wqe_1st *rwqe;
3542 struct rdma_sq_fmr_wqe_1st *fwqe1;
3543 struct rdma_sq_common_wqe *wqe;
3544 u32 length;
3545 int rc = 0;
3546 bool comp;
3547
3548 if (!qedr_can_post_send(qp, wr)) {
3549 *bad_wr = wr;
3550 return -ENOMEM;
3551 }
3552
3553 wqe = qed_chain_produce(&qp->sq.pbl);
3554 qp->wqe_wr_id[qp->sq.prod].signaled =
3555 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3556
3557 wqe->flags = 0;
3558 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3559 !!(wr->send_flags & IB_SEND_SOLICITED));
3560 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3561 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3562 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3563 !!(wr->send_flags & IB_SEND_FENCE));
3564 wqe->prev_wqe_size = qp->prev_wqe_size;
3565
3566 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3567
3568 switch (wr->opcode) {
3569 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3570 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3571 rc = -EINVAL;
3572 *bad_wr = wr;
3573 break;
3574 }
afa0e13b
RA		 3575 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3576 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3577 swqe->wqe_size = 2;
3578 swqe2 = qed_chain_produce(&qp->sq.pbl);
3579
7bed7ebc 3580 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3581 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3582 wr, bad_wr);
3583 swqe->length = cpu_to_le32(length);
3584 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3585 qp->prev_wqe_size = swqe->wqe_size;
3586 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3587 break;
3588 case IB_WR_SEND:
3589 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3590 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3591
3592 swqe->wqe_size = 2;
3593 swqe2 = qed_chain_produce(&qp->sq.pbl);
3594 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3595 wr, bad_wr);
3596 swqe->length = cpu_to_le32(length);
3597 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3598 qp->prev_wqe_size = swqe->wqe_size;
3599 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3600 break;
3601 case IB_WR_SEND_WITH_INV:
3602 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3603 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3604 swqe2 = qed_chain_produce(&qp->sq.pbl);
3605 swqe->wqe_size = 2;
3606 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3607 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3608 wr, bad_wr);
3609 swqe->length = cpu_to_le32(length);
3610 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3611 qp->prev_wqe_size = swqe->wqe_size;
3612 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3613 break;
3614
3615 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3616 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3617 rc = -EINVAL;
3618 *bad_wr = wr;
3619 break;
3620 }
afa0e13b
RA		 3621 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3622 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3623
3624 rwqe->wqe_size = 2;
3625 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3626 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3627 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3628 wr, bad_wr);
3629 rwqe->length = cpu_to_le32(length);
3630 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3631 qp->prev_wqe_size = rwqe->wqe_size;
3632 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3633 break;
3634 case IB_WR_RDMA_WRITE:
3635 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3636 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3637
3638 rwqe->wqe_size = 2;
3639 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3640 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3641 wr, bad_wr);
3642 rwqe->length = cpu_to_le32(length);
3643 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3644 qp->prev_wqe_size = rwqe->wqe_size;
3645 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3646 break;
3647 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3648 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
df561f66 3649 fallthrough; /* same is identical to RDMA READ */
afa0e13b
RA		 3650
3651 case IB_WR_RDMA_READ:
3652 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3653 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3654
3655 rwqe->wqe_size = 2;
3656 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3657 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3658 wr, bad_wr);
3659 rwqe->length = cpu_to_le32(length);
3660 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3661 qp->prev_wqe_size = rwqe->wqe_size;
3662 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3663 break;
3664
3665 case IB_WR_ATOMIC_CMP_AND_SWP:
3666 case IB_WR_ATOMIC_FETCH_AND_ADD:
3667 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3668 awqe1->wqe_size = 4;
3669
3670 awqe2 = qed_chain_produce(&qp->sq.pbl);
3671 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3672 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3673
3674 awqe3 = qed_chain_produce(&qp->sq.pbl);
3675
3676 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3677 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3678 DMA_REGPAIR_LE(awqe3->swap_data,
3679 atomic_wr(wr)->compare_add);
3680 } else {
3681 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3682 DMA_REGPAIR_LE(awqe3->swap_data,
3683 atomic_wr(wr)->swap);
3684 DMA_REGPAIR_LE(awqe3->cmp_data,
3685 atomic_wr(wr)->compare_add);
3686 }
3687
3688 qedr_prepare_sq_sges(qp, NULL, wr);
3689
3690 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3691 qp->prev_wqe_size = awqe1->wqe_size;
3692 break;
3693
3694 case IB_WR_LOCAL_INV:
3695 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3696 iwqe->wqe_size = 1;
3697
3698 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3699 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3700 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3701 qp->prev_wqe_size = iwqe->wqe_size;
3702 break;
3703 case IB_WR_REG_MR:
3704 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3705 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3706 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3707 fwqe1->wqe_size = 2;
3708
3709 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3710 if (rc) {
3711 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3712 *bad_wr = wr;
3713 break;
3714 }
3715
3716 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3717 qp->prev_wqe_size = fwqe1->wqe_size;
3718 break;
3719 default:
3720 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3721 rc = -EINVAL;
3722 *bad_wr = wr;
3723 break;
3724 }
3725
3726 if (*bad_wr) {
3727 u16 value;
3728
3729 /* Restore prod to its position before
3730 * this WR was processed
3731 */
3732 value = le16_to_cpu(qp->sq.db_data.data.value);
3733 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3734
3735 /* Restore prev_wqe_size */
3736 qp->prev_wqe_size = wqe->prev_wqe_size;
3737 rc = -EINVAL;
3738 DP_ERR(dev, "POST SEND FAILED\n");
3739 }
3740
3741 return rc;
3742}
3743
d34ac5cd
BVA		 3744int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3745 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3746{
3747 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3748 struct qedr_qp *qp = get_qedr_qp(ibqp);
3749 unsigned long flags;
3750 int rc = 0;
3751
3752 *bad_wr = NULL;
3753
04886779
RA		 3754 if (qp->qp_type == IB_QPT_GSI)
3755 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3756
afa0e13b
RA		 3757 spin_lock_irqsave(&qp->q_lock, flags);
3758
f5b1b177
KM		 3759 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3760 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3761 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3762 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3763 spin_unlock_irqrestore(&qp->q_lock, flags);
3764 *bad_wr = wr;
3765 DP_DEBUG(dev, QEDR_MSG_CQ,
3766 "QP in wrong state! QP icid=0x%x state %d\n",
3767 qp->icid, qp->state);
3768 return -EINVAL;
3769 }
afa0e13b
RA		 3770 }
3771
afa0e13b
RA		 3772 while (wr) {
3773 rc = __qedr_post_send(ibqp, wr, bad_wr);
3774 if (rc)
3775 break;
3776
3777 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3778
3779 qedr_inc_sw_prod(&qp->sq);
3780
3781 qp->sq.db_data.data.value++;
3782
3783 wr = wr->next;
3784 }
3785
3786 /* Trigger doorbell
3787 * If there was a failure in the first WR then it will be triggered in
3788 * vane. However this is not harmful (as long as the producer value is
3789 * unchanged). For performance reasons we avoid checking for this
3790 * redundant doorbell.
09c4854f
KM		 3791 *
3792 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3793 * soon as we give the doorbell, we could get a completion
3794 * for this wr, therefore we need to make sure that the
3795 * memory is updated before giving the doorbell.
3796 * During qedr_poll_cq, rmb is called before accessing the
3797 * cqe. This covers for the smp_rmb as well.
afa0e13b 3798 */
09c4854f 3799 smp_wmb();
afa0e13b
RA		 3800 writel(qp->sq.db_data.raw, qp->sq.db);
3801
afa0e13b
RA		 3802 spin_unlock_irqrestore(&qp->q_lock, flags);
3803
3804 return rc;
3805}
3806
3491c9e7
YB		 3807static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3808{
3809 u32 used;
3810
3811 /* Calculate number of elements used based on producer
3812 * count and consumer count and subtract it from max
3813 * work request supported so that we get elements left.
3814 */
acca72e2 3815 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3491c9e7
YB		 3816
3817 return hw_srq->max_wr - used;
3818}
3819
3820int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3821 const struct ib_recv_wr **bad_wr)
3822{
3823 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3824 struct qedr_srq_hwq_info *hw_srq;
3825 struct qedr_dev *dev = srq->dev;
3826 struct qed_chain *pbl;
3827 unsigned long flags;
3828 int status = 0;
3829 u32 num_sge;
3491c9e7
YB		 3830
3831 spin_lock_irqsave(&srq->lock, flags);
3832
3833 hw_srq = &srq->hw_srq;
3834 pbl = &srq->hw_srq.pbl;
3835 while (wr) {
3836 struct rdma_srq_wqe_header *hdr;
3837 int i;
3838
3839 if (!qedr_srq_elem_left(hw_srq) ||
3840 wr->num_sge > srq->hw_srq.max_sges) {
3841 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
acca72e2
YB		 3842 hw_srq->wr_prod_cnt,
3843 atomic_read(&hw_srq->wr_cons_cnt),
3491c9e7
YB		 3844 wr->num_sge, srq->hw_srq.max_sges);
3845 status = -ENOMEM;
3846 *bad_wr = wr;
3847 break;
3848 }
3849
3850 hdr = qed_chain_produce(pbl);
3851 num_sge = wr->num_sge;
3852 /* Set number of sge and work request id in header */
3853 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3854
3855 srq->hw_srq.wr_prod_cnt++;
3856 hw_srq->wqe_prod++;
3857 hw_srq->sge_prod++;
3858
3859 DP_DEBUG(dev, QEDR_MSG_SRQ,
3860 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3861 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3862
3863 for (i = 0; i < wr->num_sge; i++) {
3864 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3865
3866 /* Set SGE length, lkey and address */
3867 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3868 wr->sg_list[i].length, wr->sg_list[i].lkey);
3869
3870 DP_DEBUG(dev, QEDR_MSG_SRQ,
3871 "[%d]: len %d key %x addr %x:%x\n",
3872 i, srq_sge->length, srq_sge->l_key,
3873 srq_sge->addr.hi, srq_sge->addr.lo);
3874 hw_srq->sge_prod++;
3875 }
3876
acca72e2 3877 /* Update WQE and SGE information before
3491c9e7
YB		 3878 * updating producer.
3879 */
acca72e2 3880 dma_wmb();
3491c9e7
YB		 3881
3882 /* SRQ producer is 8 bytes. Need to update SGE producer index
3883 * in first 4 bytes and need to update WQE producer in
3884 * next 4 bytes.
3885 */
f45271ac 3886 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
acca72e2
YB		 3887 /* Make sure sge producer is updated first */
3888 dma_wmb();
f45271ac 3889 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3491c9e7 3890
3491c9e7
YB		 3891 wr = wr->next;
3892 }
3893
3894 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3895 qed_chain_get_elem_left(pbl));
3896 spin_unlock_irqrestore(&srq->lock, flags);
3897
3898 return status;
3899}
3900
d34ac5cd
BVA		 3901int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3902 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3903{
3904 struct qedr_qp *qp = get_qedr_qp(ibqp);
3905 struct qedr_dev *dev = qp->dev;
3906 unsigned long flags;
3907 int status = 0;
3908
04886779
RA		 3909 if (qp->qp_type == IB_QPT_GSI)
3910 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3911
afa0e13b
RA		 3912 spin_lock_irqsave(&qp->q_lock, flags);
3913
afa0e13b
RA		 3914 while (wr) {
3915 int i;
3916
3917 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3918 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3919 wr->num_sge > qp->rq.max_sges) {
3920 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3921 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3922 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3923 qp->rq.max_sges);
3924 status = -ENOMEM;
3925 *bad_wr = wr;
3926 break;
3927 }
3928 for (i = 0; i < wr->num_sge; i++) {
3929 u32 flags = 0;
3930 struct rdma_rq_sge *rqe =
3931 qed_chain_produce(&qp->rq.pbl);
3932
3933 /* First one must include the number
3934 * of SGE in the list
3935 */
3936 if (!i)
3937 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3938 wr->num_sge);
3939
d52c89f1 3940 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3941 wr->sg_list[i].lkey);
3942
3943 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3944 wr->sg_list[i].length, flags);
3945 }
3946
3947 /* Special case of no sges. FW requires between 1-4 sges...
3948 * in this case we need to post 1 sge with length zero. this is
3949 * because rdma write with immediate consumes an RQ.
3950 */
3951 if (!wr->num_sge) {
3952 u32 flags = 0;
3953 struct rdma_rq_sge *rqe =
3954 qed_chain_produce(&qp->rq.pbl);
3955
3956 /* First one must include the number
3957 * of SGE in the list
3958 */
d52c89f1 3959 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3960 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3961
3962 RQ_SGE_SET(rqe, 0, 0, flags);
3963 i = 1;
3964 }
3965
3966 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3967 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3968
3969 qedr_inc_sw_prod(&qp->rq);
3970
09c4854f
KM		 3971 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3972 * soon as we give the doorbell, we could get a completion
3973 * for this wr, therefore we need to make sure that the
3974 * memory is update before giving the doorbell.
3975 * During qedr_poll_cq, rmb is called before accessing the
3976 * cqe. This covers for the smp_rmb as well.
3977 */
3978 smp_wmb();
afa0e13b
RA		 3979
3980 qp->rq.db_data.data.value++;
3981
3982 writel(qp->rq.db_data.raw, qp->rq.db);
3983
f5b1b177
KM		 3984 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3985 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
f5b1b177
KM		 3986 }
3987
afa0e13b
RA		 3988 wr = wr->next;
3989 }
3990
3991 spin_unlock_irqrestore(&qp->q_lock, flags);
3992
3993 return status;
3994}
3995
3996static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3997{
3998 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3999
4000 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4001 cq->pbl_toggle;
4002}
4003
4004static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4005{
4006 struct rdma_cqe_requester *resp_cqe = &cqe->req;
4007 struct qedr_qp *qp;
4008
4009 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4010 resp_cqe->qp_handle.lo,
4011 u64);
4012 return qp;
4013}
4014
4015static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4016{
4017 struct rdma_cqe_requester *resp_cqe = &cqe->req;
4018
4019 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4020}
4021
4022/* Return latest CQE (needs processing) */
4023static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4024{
4025 return cq->latest_cqe;
4026}
4027
4028/* In fmr we need to increase the number of fmr completed counter for the fmr
4029 * algorithm determining whether we can free a pbl or not.
4030 * we need to perform this whether the work request was signaled or not. for
4031 * this purpose we call this function from the condition that checks if a wr
4032 * should be skipped, to make sure we don't miss it ( possibly this fmr
4033 * operation was not signalted)
4034 */
4035static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4036{
4037 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4038 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4039}
4040
4041static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4042 struct qedr_cq *cq, int num_entries,
4043 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4044 int force)
4045{
4046 u16 cnt = 0;
4047
4048 while (num_entries && qp->sq.wqe_cons != hw_cons) {
4049 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4050 qedr_chk_if_fmr(qp);
4051 /* skip WC */
4052 goto next_cqe;
4053 }
4054
4055 /* fill WC */
4056 wc->status = status;
27035a1b 4057 wc->vendor_err = 0;
afa0e13b
RA		 4058 wc->wc_flags = 0;
4059 wc->src_qp = qp->id;
4060 wc->qp = &qp->ibqp;
4061
4062 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4063 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4064
4065 switch (wc->opcode) {
4066 case IB_WC_RDMA_WRITE:
4067 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4068 break;
4069 case IB_WC_COMP_SWAP:
4070 case IB_WC_FETCH_ADD:
4071 wc->byte_len = 8;
4072 break;
4073 case IB_WC_REG_MR:
4074 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4075 break;
dac27386
MK		 4076 case IB_WC_RDMA_READ:
4077 case IB_WC_SEND:
4078 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4079 break;
afa0e13b
RA		 4080 default:
4081 break;
4082 }
4083
4084 num_entries--;
4085 wc++;
4086 cnt++;
4087next_cqe:
4088 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4089 qed_chain_consume(&qp->sq.pbl);
4090 qedr_inc_sw_cons(&qp->sq);
4091 }
4092
4093 return cnt;
4094}
4095
4096static int qedr_poll_cq_req(struct qedr_dev *dev,
4097 struct qedr_qp *qp, struct qedr_cq *cq,
4098 int num_entries, struct ib_wc *wc,
4099 struct rdma_cqe_requester *req)
4100{
4101 int cnt = 0;
4102
4103 switch (req->status) {
4104 case RDMA_CQE_REQ_STS_OK:
4105 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4106 IB_WC_SUCCESS, 0);
4107 break;
4108 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 4109 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 4110 DP_DEBUG(dev, QEDR_MSG_CQ,
4111 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4112 cq->icid, qp->icid);
afa0e13b 4113 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 4114 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 4115 break;
4116 default:
4117 /* process all WQE before the cosumer */
4118 qp->state = QED_ROCE_QP_STATE_ERR;
4119 cnt = process_req(dev, qp, cq, num_entries, wc,
4120 req->sq_cons - 1, IB_WC_SUCCESS, 0);
4121 wc += cnt;
4122 /* if we have extra WC fill it with actual error info */
4123 if (cnt < num_entries) {
4124 enum ib_wc_status wc_status;
4125
4126 switch (req->status) {
4127 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4128 DP_ERR(dev,
4129 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4130 cq->icid, qp->icid);
4131 wc_status = IB_WC_BAD_RESP_ERR;
4132 break;
4133 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4134 DP_ERR(dev,
4135 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4136 cq->icid, qp->icid);
4137 wc_status = IB_WC_LOC_LEN_ERR;
4138 break;
4139 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4140 DP_ERR(dev,
4141 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4142 cq->icid, qp->icid);
4143 wc_status = IB_WC_LOC_QP_OP_ERR;
4144 break;
4145 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4146 DP_ERR(dev,
4147 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4148 cq->icid, qp->icid);
4149 wc_status = IB_WC_LOC_PROT_ERR;
4150 break;
4151 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4152 DP_ERR(dev,
4153 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4154 cq->icid, qp->icid);
4155 wc_status = IB_WC_MW_BIND_ERR;
4156 break;
4157 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4158 DP_ERR(dev,
4159 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4160 cq->icid, qp->icid);
4161 wc_status = IB_WC_REM_INV_REQ_ERR;
4162 break;
4163 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4164 DP_ERR(dev,
4165 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4166 cq->icid, qp->icid);
4167 wc_status = IB_WC_REM_ACCESS_ERR;
4168 break;
4169 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4170 DP_ERR(dev,
4171 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4172 cq->icid, qp->icid);
4173 wc_status = IB_WC_REM_OP_ERR;
4174 break;
4175 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4176 DP_ERR(dev,
4177 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4178 cq->icid, qp->icid);
4179 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4180 break;
4181 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4182 DP_ERR(dev,
4183 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4184 cq->icid, qp->icid);
4185 wc_status = IB_WC_RETRY_EXC_ERR;
4186 break;
4187 default:
4188 DP_ERR(dev,
4189 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4190 cq->icid, qp->icid);
4191 wc_status = IB_WC_GENERAL_ERR;
4192 }
4193 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4194 wc_status, 1);
4195 }
4196 }
4197
4198 return cnt;
4199}
4200
b6acd71f 4201static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 4202{
b6acd71f 4203 switch (status) {
afa0e13b 4204 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 4205 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 4206 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 4207 return IB_WC_LOC_LEN_ERR;
afa0e13b 4208 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 4209 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 4210 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 4211 return IB_WC_LOC_PROT_ERR;
afa0e13b 4212 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 4213 return IB_WC_MW_BIND_ERR;
afa0e13b 4214 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 4215 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 4216 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 4217 return IB_WC_SUCCESS;
4218 default:
4219 return IB_WC_GENERAL_ERR;
4220 }
4221}
afa0e13b 4222
b6acd71f
AR		 4223static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4224 struct ib_wc *wc)
4225{
4226 wc->status = IB_WC_SUCCESS;
4227 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 4228
b6acd71f 4229 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 4230 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 4231 wc->wc_flags |= IB_WC_WITH_IMM;
4232
4233 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 4234 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4235
b6acd71f
AR		 4236 if (resp->flags & QEDR_RESP_INV)
4237 return -EINVAL;
4238
4239 } else if (resp->flags & QEDR_RESP_INV) {
4240 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4241 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4242
4243 if (resp->flags & QEDR_RESP_RDMA)
4244 return -EINVAL;
4245
4246 } else if (resp->flags & QEDR_RESP_RDMA) {
4247 return -EINVAL;
4248 }
4249
4250 return 0;
4251}
4252
4253static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4254 struct qedr_cq *cq, struct ib_wc *wc,
4255 struct rdma_cqe_responder *resp, u64 wr_id)
4256{
4257 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4258 wc->opcode = IB_WC_RECV;
4259 wc->wc_flags = 0;
4260
4261 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4262 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4263 DP_ERR(dev,
4264 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4265 cq, cq->icid, resp->flags);
4266
4267 } else {
4268 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4269 if (wc->status == IB_WC_GENERAL_ERR)
4270 DP_ERR(dev,
4271 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4272 cq, cq->icid, resp->status);
afa0e13b
RA		 4273 }
4274
b6acd71f 4275 /* Fill the rest of the WC */
27035a1b 4276 wc->vendor_err = 0;
afa0e13b
RA		 4277 wc->src_qp = qp->id;
4278 wc->qp = &qp->ibqp;
4279 wc->wr_id = wr_id;
4280}
4281
3491c9e7
YB		 4282static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4283 struct qedr_cq *cq, struct ib_wc *wc,
4284 struct rdma_cqe_responder *resp)
4285{
4286 struct qedr_srq *srq = qp->srq;
4287 u64 wr_id;
4288
4289 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4290 le32_to_cpu(resp->srq_wr_id.lo), u64);
4291
4292 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4293 wc->status = IB_WC_WR_FLUSH_ERR;
4294 wc->vendor_err = 0;
4295 wc->wr_id = wr_id;
4296 wc->byte_len = 0;
4297 wc->src_qp = qp->id;
4298 wc->qp = &qp->ibqp;
4299 wc->wr_id = wr_id;
4300 } else {
4301 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4302 }
acca72e2 4303 atomic_inc(&srq->hw_srq.wr_cons_cnt);
3491c9e7
YB		 4304
4305 return 1;
4306}
afa0e13b
RA		 4307static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4308 struct qedr_cq *cq, struct ib_wc *wc,
4309 struct rdma_cqe_responder *resp)
4310{
4311 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4312
4313 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4314
4315 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4316 qed_chain_consume(&qp->rq.pbl);
4317 qedr_inc_sw_cons(&qp->rq);
4318
4319 return 1;
4320}
4321
4322static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4323 int num_entries, struct ib_wc *wc, u16 hw_cons)
4324{
4325 u16 cnt = 0;
4326
4327 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4328 /* fill WC */
4329 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 4330 wc->vendor_err = 0;
afa0e13b
RA		 4331 wc->wc_flags = 0;
4332 wc->src_qp = qp->id;
4333 wc->byte_len = 0;
4334 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4335 wc->qp = &qp->ibqp;
4336 num_entries--;
4337 wc++;
4338 cnt++;
4339 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4340 qed_chain_consume(&qp->rq.pbl);
4341 qedr_inc_sw_cons(&qp->rq);
4342 }
4343
4344 return cnt;
4345}
4346
4347static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4348 struct rdma_cqe_responder *resp, int *update)
4349{
50bc60cb 4350 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 4351 consume_cqe(cq);
4352 *update |= 1;
4353 }
4354}
4355
3491c9e7
YB		 4356static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4357 struct qedr_cq *cq, int num_entries,
4358 struct ib_wc *wc,
4359 struct rdma_cqe_responder *resp)
4360{
4361 int cnt;
4362
4363 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4364 consume_cqe(cq);
4365
4366 return cnt;
4367}
4368
afa0e13b
RA		 4369static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4370 struct qedr_cq *cq, int num_entries,
4371 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4372 int *update)
4373{
4374 int cnt;
4375
4376 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4377 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4378 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4379 try_consume_resp_cqe(cq, qp, resp, update);
4380 } else {
4381 cnt = process_resp_one(dev, qp, cq, wc, resp);
4382 consume_cqe(cq);
4383 *update |= 1;
4384 }
4385
4386 return cnt;
4387}
4388
4389static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4390 struct rdma_cqe_requester *req, int *update)
4391{
4392 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4393 consume_cqe(cq);
4394 *update |= 1;
4395 }
4396}
4397
4398int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4399{
4400 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4401 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4402 union rdma_cqe *cqe;
afa0e13b
RA		 4403 u32 old_cons, new_cons;
4404 unsigned long flags;
4405 int update = 0;
4406 int done = 0;
4407
4dd72636
AR		 4408 if (cq->destroyed) {
4409 DP_ERR(dev,
4410 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4411 cq, cq->icid);
4412 return 0;
4413 }
4414
04886779
RA		 4415 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4416 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4417
afa0e13b 4418 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4419 cqe = cq->latest_cqe;
afa0e13b
RA		 4420 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4421 while (num_entries && is_valid_cqe(cq, cqe)) {
4422 struct qedr_qp *qp;
4423 int cnt = 0;
4424
4425 /* prevent speculative reads of any field of CQE */
4426 rmb();
4427
4428 qp = cqe_get_qp(cqe);
4429 if (!qp) {
4430 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4431 break;
4432 }
4433
4434 wc->qp = &qp->ibqp;
4435
4436 switch (cqe_get_type(cqe)) {
4437 case RDMA_CQE_TYPE_REQUESTER:
4438 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4439 &cqe->req);
4440 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4441 break;
4442 case RDMA_CQE_TYPE_RESPONDER_RQ:
4443 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4444 &cqe->resp, &update);
4445 break;
3491c9e7
YB		 4446 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4447 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4448 wc, &cqe->resp);
4449 update = 1;
4450 break;
afa0e13b
RA		 4451 case RDMA_CQE_TYPE_INVALID:
4452 default:
4453 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4454 cqe_get_type(cqe));
4455 }
4456 num_entries -= cnt;
4457 wc += cnt;
4458 done += cnt;
4459
4460 cqe = get_cqe(cq);
4461 }
4462 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4463
4464 cq->cq_cons += new_cons - old_cons;
4465
4466 if (update)
4467 /* doorbell notifies abount latest VALID entry,
4468 * but chain already point to the next INVALID one
4469 */
4470 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4471
4472 spin_unlock_irqrestore(&cq->cq_lock, flags);
4473 return done;
4474}
993d1b52
RA		 4475
4476int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
1fb7f897 4477 u32 port_num, const struct ib_wc *in_wc,
e26e7b88
LR		 4478 const struct ib_grh *in_grh, const struct ib_mad *in,
4479 struct ib_mad *out_mad, size_t *out_mad_size,
4480 u16 *out_mad_pkey_index)
993d1b52 4481{
993d1b52
RA		 4482 return IB_MAD_RESULT_SUCCESS;
4483}
Linux 6.x block layer and io_uring tree(s)