projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/qedr: Fix inline size returned for iWARP
[linux-2.6-block.git] / drivers / infiniband / hw / qedr / verbs.c
CommitLineData
ac1b36e5
RA		 1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
ff23dfa1 45#include <rdma/uverbs_ioctl.h>
ac1b36e5 46
be086e7c
MY		 47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
ac1b36e5
RA		 49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
99d195cc 53#include "qedr_roce_cm.h"
82af6d19 54#include "qedr_iw_cm.h"
ac1b36e5 55
3491c9e7
YB		 56#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57#define RDMA_MAX_SGE_PER_SRQ (4)
58#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
a7efd777
RA		 60#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
4c6bb02d
MK		 62enum {
63 QEDR_USER_MMAP_IO_WC = 0,
97f61250 64 QEDR_USER_MMAP_PHYS_PAGE,
4c6bb02d
MK		 65};
66
c75d3ec8
AR		 67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 size_t len)
69{
70 size_t min_len = min_t(size_t, len, udata->outlen);
71
72 return ib_copy_to_udata(udata, src, min_len);
73}
74
a7efd777
RA		 75int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
76{
dbe30dae 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
a7efd777
RA		 78 return -EINVAL;
79
80 *pkey = QEDR_ROCE_PKEY_DEFAULT;
81 return 0;
82}
83
e6a38c54
KM		 84int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
85 int index, union ib_gid *sgid)
86{
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89 memset(sgid->raw, 0, sizeof(sgid->raw));
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 sgid->global.interface_id, sgid->global.subnet_prefix);
94
95 return 0;
96}
97
3491c9e7
YB		 98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99{
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
102 struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104 srq_attr->srq_limit = srq->srq_limit;
105 srq_attr->max_wr = qattr->max_srq_wr;
106 srq_attr->max_sge = qattr->max_sge;
107
108 return 0;
109}
110
ac1b36e5
RA		 111int qedr_query_device(struct ib_device *ibdev,
112 struct ib_device_attr *attr, struct ib_udata *udata)
113{
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
116
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
119 "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 dev->rdma_ctx);
121 return -EINVAL;
122 }
123
124 memset(attr, 0, sizeof(*attr));
125
126 attr->fw_ver = qattr->fw_ver;
127 attr->sys_image_guid = qattr->sys_image_guid;
128 attr->max_mr_size = qattr->max_mr_size;
129 attr->page_size_cap = qattr->page_size_caps;
130 attr->vendor_id = qattr->vendor_id;
131 attr->vendor_part_id = qattr->vendor_part_id;
132 attr->hw_ver = qattr->hw_ver;
133 attr->max_qp = qattr->max_qp;
134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 IB_DEVICE_RC_RNR_NAK_GEN |
137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
138
33023fb8
SW		 139 attr->max_send_sge = qattr->max_sge;
140 attr->max_recv_sge = qattr->max_sge;
ac1b36e5
RA		 141 attr->max_sge_rd = qattr->max_sge;
142 attr->max_cq = qattr->max_cq;
143 attr->max_cqe = qattr->max_cqe;
144 attr->max_mr = qattr->max_mr;
145 attr->max_mw = qattr->max_mw;
146 attr->max_pd = qattr->max_pd;
147 attr->atomic_cap = dev->atomic_cap;
ac1b36e5
RA		 148 attr->max_qp_init_rd_atom =
149 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
150 attr->max_qp_rd_atom =
151 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
152 attr->max_qp_init_rd_atom);
153
154 attr->max_srq = qattr->max_srq;
155 attr->max_srq_sge = qattr->max_srq_sge;
156 attr->max_srq_wr = qattr->max_srq_wr;
157
158 attr->local_ca_ack_delay = qattr->dev_ack_delay;
159 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
7d11b478 160 attr->max_pkeys = qattr->max_pkey;
ac1b36e5
RA		 161 attr->max_ah = qattr->max_ah;
162
163 return 0;
164}
165
ac1b36e5
RA		 166static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
167 u8 *ib_width)
168{
169 switch (speed) {
170 case 1000:
69054666 171 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 172 *ib_width = IB_WIDTH_1X;
173 break;
174 case 10000:
69054666 175 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 176 *ib_width = IB_WIDTH_1X;
177 break;
178
179 case 20000:
69054666 180 *ib_speed = IB_SPEED_DDR;
ac1b36e5
RA		 181 *ib_width = IB_WIDTH_4X;
182 break;
183
184 case 25000:
69054666 185 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 186 *ib_width = IB_WIDTH_1X;
187 break;
188
189 case 40000:
69054666 190 *ib_speed = IB_SPEED_QDR;
ac1b36e5
RA		 191 *ib_width = IB_WIDTH_4X;
192 break;
193
194 case 50000:
69054666
SO		 195 *ib_speed = IB_SPEED_HDR;
196 *ib_width = IB_WIDTH_1X;
ac1b36e5
RA		 197 break;
198
199 case 100000:
69054666 200 *ib_speed = IB_SPEED_EDR;
ac1b36e5
RA		 201 *ib_width = IB_WIDTH_4X;
202 break;
203
204 default:
205 /* Unsupported */
69054666 206 *ib_speed = IB_SPEED_SDR;
ac1b36e5
RA		 207 *ib_width = IB_WIDTH_1X;
208 }
209}
210
211int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
212{
213 struct qedr_dev *dev;
214 struct qed_rdma_port *rdma_port;
215
216 dev = get_qedr_dev(ibdev);
ac1b36e5
RA		 217
218 if (!dev->rdma_ctx) {
219 DP_ERR(dev, "rdma_ctx is NULL\n");
220 return -EINVAL;
221 }
222
223 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
ac1b36e5 224
c4550c63 225 /* *attr being zeroed by the caller, avoid zeroing it here */
ac1b36e5
RA		 226 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
227 attr->state = IB_PORT_ACTIVE;
72a7720f 228 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
ac1b36e5
RA		 229 } else {
230 attr->state = IB_PORT_DOWN;
72a7720f 231 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
ac1b36e5
RA		 232 }
233 attr->max_mtu = IB_MTU_4096;
ac1b36e5
RA		 234 attr->lid = 0;
235 attr->lmc = 0;
236 attr->sm_lid = 0;
237 attr->sm_sl = 0;
2f944c0f 238 attr->ip_gids = true;
f5b1b177 239 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
cc293f54 240 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
f5b1b177 241 attr->gid_tbl_len = 1;
f5b1b177 242 } else {
cc293f54 243 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
f5b1b177
KM		 244 attr->gid_tbl_len = QEDR_MAX_SGID;
245 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
246 }
ac1b36e5
RA		 247 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
248 attr->qkey_viol_cntr = 0;
249 get_link_speed_and_width(rdma_port->link_speed,
250 &attr->active_speed, &attr->active_width);
251 attr->max_msg_sz = rdma_port->max_msg_size;
252 attr->max_vl_num = 4;
253
254 return 0;
255}
256
a2a074ef 257int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
ac1b36e5 258{
a2a074ef 259 struct ib_device *ibdev = uctx->device;
ac1b36e5 260 int rc;
a2a074ef
LR		 261 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
262 struct qedr_alloc_ucontext_resp uresp = {};
97f61250 263 struct qedr_alloc_ucontext_req ureq = {};
ac1b36e5
RA		 264 struct qedr_dev *dev = get_qedr_dev(ibdev);
265 struct qed_rdma_add_user_out_params oparams;
4c6bb02d 266 struct qedr_user_mmap_entry *entry;
ac1b36e5
RA		 267
268 if (!udata)
a2a074ef 269 return -EFAULT;
ac1b36e5 270
97f61250
MK		 271 if (udata->inlen) {
272 rc = ib_copy_from_udata(&ureq, udata,
273 min(sizeof(ureq), udata->inlen));
274 if (rc) {
275 DP_ERR(dev, "Problem copying data from user space\n");
276 return -EFAULT;
277 }
bbe4f424
MK		 278 ctx->edpm_mode = !!(ureq.context_flags &
279 QEDR_ALLOC_UCTX_EDPM_MODE);
97f61250
MK		 280 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
281 }
282
ac1b36e5
RA		 283 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
284 if (rc) {
285 DP_ERR(dev,
286 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
287 rc);
a2a074ef 288 return rc;
ac1b36e5
RA		 289 }
290
291 ctx->dpi = oparams.dpi;
292 ctx->dpi_addr = oparams.dpi_addr;
293 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
294 ctx->dpi_size = oparams.dpi_size;
4c6bb02d
MK		 295 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
296 if (!entry) {
297 rc = -ENOMEM;
298 goto err;
299 }
300
301 entry->io_address = ctx->dpi_phys_addr;
302 entry->length = ctx->dpi_size;
303 entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
304 entry->dpi = ctx->dpi;
305 entry->dev = dev;
306 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
307 ctx->dpi_size);
308 if (rc) {
309 kfree(entry);
310 goto err;
311 }
312 ctx->db_mmap_entry = &entry->rdma_entry;
ac1b36e5 313
93a3d05f
MK		 314 if (!dev->user_dpm_enabled)
315 uresp.dpm_flags = 0;
316 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
317 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
318 else
319 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
bbe4f424
MK		 320 QEDR_DPM_TYPE_ROCE_LEGACY |
321 QEDR_DPM_TYPE_ROCE_EDPM_MODE;
93a3d05f 322
eb7f84e3
MK		 323 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
324 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
325 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
326 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
327 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
328 }
93a3d05f 329
67cbe353
AR		 330 uresp.wids_enabled = 1;
331 uresp.wid_count = oparams.wid_count;
4c6bb02d 332 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
ac1b36e5
RA		 333 uresp.db_size = ctx->dpi_size;
334 uresp.max_send_wr = dev->attr.max_sqe;
335 uresp.max_recv_wr = dev->attr.max_rqe;
336 uresp.max_srq_wr = dev->attr.max_srq_wr;
337 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
338 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
339 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
340 uresp.max_cqes = QEDR_MAX_CQES;
341
c75d3ec8 342 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
ac1b36e5 343 if (rc)
4c6bb02d 344 goto err;
ac1b36e5
RA		 345
346 ctx->dev = dev;
347
ac1b36e5
RA		 348 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
349 &ctx->ibucontext);
a2a074ef 350 return 0;
4c6bb02d
MK		 351
352err:
353 if (!ctx->db_mmap_entry)
354 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
355 else
356 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
357
358 return rc;
ac1b36e5
RA		 359}
360
a2a074ef 361void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
ac1b36e5
RA		 362{
363 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
ac1b36e5
RA		 364
365 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
366 uctx);
4c6bb02d
MK		 367
368 rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
ac1b36e5
RA		 369}
370
4c6bb02d 371void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
ac1b36e5 372{
4c6bb02d
MK		 373 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
374 struct qedr_dev *dev = entry->dev;
ac1b36e5 375
97f61250
MK		 376 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
377 free_page((unsigned long)entry->address);
378 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
4c6bb02d 379 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
30bf066c 380
4c6bb02d
MK		 381 kfree(entry);
382}
ac1b36e5 383
4c6bb02d
MK		 384int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
385{
386 struct ib_device *dev = ucontext->device;
387 size_t length = vma->vm_end - vma->vm_start;
388 struct rdma_user_mmap_entry *rdma_entry;
389 struct qedr_user_mmap_entry *entry;
390 int rc = 0;
391 u64 pfn;
392
393 ibdev_dbg(dev,
394 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
395 vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
396
397 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
398 if (!rdma_entry) {
399 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
400 vma->vm_pgoff);
ac1b36e5
RA		 401 return -EINVAL;
402 }
4c6bb02d
MK		 403 entry = get_qedr_mmap_entry(rdma_entry);
404 ibdev_dbg(dev,
405 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
406 entry->io_address, length, entry->mmap_flag);
407
408 switch (entry->mmap_flag) {
409 case QEDR_USER_MMAP_IO_WC:
410 pfn = entry->io_address >> PAGE_SHIFT;
411 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
412 pgprot_writecombine(vma->vm_page_prot),
413 rdma_entry);
414 break;
97f61250
MK		 415 case QEDR_USER_MMAP_PHYS_PAGE:
416 rc = vm_insert_page(vma, vma->vm_start,
417 virt_to_page(entry->address));
418 break;
4c6bb02d
MK		 419 default:
420 rc = -EINVAL;
30bf066c 421 }
ac1b36e5 422
4c6bb02d
MK		 423 if (rc)
424 ibdev_dbg(dev,
425 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
426 entry->io_address, length, entry->mmap_flag, rc);
30bf066c 427
4c6bb02d
MK		 428 rdma_user_mmap_entry_put(rdma_entry);
429 return rc;
ac1b36e5 430}
a7efd777 431
ff23dfa1 432int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777 433{
21a428a0 434 struct ib_device *ibdev = ibpd->device;
a7efd777 435 struct qedr_dev *dev = get_qedr_dev(ibdev);
21a428a0 436 struct qedr_pd *pd = get_qedr_pd(ibpd);
a7efd777
RA		 437 u16 pd_id;
438 int rc;
439
440 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
ff23dfa1 441 udata ? "User Lib" : "Kernel");
a7efd777
RA		 442
443 if (!dev->rdma_ctx) {
847cb1a3 444 DP_ERR(dev, "invalid RDMA context\n");
21a428a0 445 return -EINVAL;
a7efd777
RA		 446 }
447
9c1e0228
RA		 448 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
449 if (rc)
21a428a0 450 return rc;
a7efd777 451
a7efd777
RA		 452 pd->pd_id = pd_id;
453
ff23dfa1 454 if (udata) {
57939021
JG		 455 struct qedr_alloc_pd_uresp uresp = {
456 .pd_id = pd_id,
457 };
ff23dfa1
SR		 458 struct qedr_ucontext *context = rdma_udata_to_drv_context(
459 udata, struct qedr_ucontext, ibucontext);
9c1e0228 460
c75d3ec8 461 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
9c1e0228 462 if (rc) {
a7efd777 463 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
9c1e0228 464 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
21a428a0 465 return rc;
9c1e0228
RA		 466 }
467
ff23dfa1 468 pd->uctx = context;
9c1e0228 469 pd->uctx->pd = pd;
a7efd777
RA		 470 }
471
21a428a0 472 return 0;
a7efd777
RA		 473}
474
91a7c58f 475int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
a7efd777
RA		 476{
477 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
478 struct qedr_pd *pd = get_qedr_pd(ibpd);
479
a7efd777
RA		 480 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
481 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
91a7c58f 482 return 0;
a7efd777
RA		 483}
484
485static void qedr_free_pbl(struct qedr_dev *dev,
486 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
487{
488 struct pci_dev *pdev = dev->pdev;
489 int i;
490
491 for (i = 0; i < pbl_info->num_pbls; i++) {
492 if (!pbl[i].va)
493 continue;
494 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
495 pbl[i].va, pbl[i].pa);
496 }
497
498 kfree(pbl);
499}
500
501#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
502#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
503
504#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
505#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
506#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
507
508static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
509 struct qedr_pbl_info *pbl_info,
510 gfp_t flags)
511{
512 struct pci_dev *pdev = dev->pdev;
513 struct qedr_pbl *pbl_table;
514 dma_addr_t *pbl_main_tbl;
515 dma_addr_t pa;
516 void *va;
517 int i;
518
519 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
520 if (!pbl_table)
521 return ERR_PTR(-ENOMEM);
522
523 for (i = 0; i < pbl_info->num_pbls; i++) {
750afb08
LC		 524 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
525 flags);
a7efd777
RA		 526 if (!va)
527 goto err;
528
a7efd777
RA		 529 pbl_table[i].va = va;
530 pbl_table[i].pa = pa;
531 }
532
533 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
534 * the first one with physical pointers to all of the rest
535 */
536 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
537 for (i = 0; i < pbl_info->num_pbls - 1; i++)
538 pbl_main_tbl[i] = pbl_table[i + 1].pa;
539
540 return pbl_table;
541
542err:
543 for (i--; i >= 0; i--)
544 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
545 pbl_table[i].va, pbl_table[i].pa);
546
547 qedr_free_pbl(dev, pbl_info, pbl_table);
548
549 return ERR_PTR(-ENOMEM);
550}
551
552static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
553 struct qedr_pbl_info *pbl_info,
554 u32 num_pbes, int two_layer_capable)
555{
556 u32 pbl_capacity;
557 u32 pbl_size;
558 u32 num_pbls;
559
560 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
561 if (num_pbes > MAX_PBES_TWO_LAYER) {
562 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
563 num_pbes);
564 return -EINVAL;
565 }
566
567 /* calculate required pbl page size */
568 pbl_size = MIN_FW_PBL_PAGE_SIZE;
569 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
570 NUM_PBES_ON_PAGE(pbl_size);
571
572 while (pbl_capacity < num_pbes) {
573 pbl_size *= 2;
574 pbl_capacity = pbl_size / sizeof(u64);
575 pbl_capacity = pbl_capacity * pbl_capacity;
576 }
577
578 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
579 num_pbls++; /* One for the layer0 ( points to the pbls) */
580 pbl_info->two_layered = true;
581 } else {
582 /* One layered PBL */
583 num_pbls = 1;
584 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
585 roundup_pow_of_two((num_pbes * sizeof(u64))));
586 pbl_info->two_layered = false;
587 }
588
589 pbl_info->num_pbls = num_pbls;
590 pbl_info->pbl_size = pbl_size;
591 pbl_info->num_pbes = num_pbes;
592
593 DP_DEBUG(dev, QEDR_MSG_MR,
594 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
595 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
596
597 return 0;
598}
599
600static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
601 struct qedr_pbl *pbl,
e57bb6be 602 struct qedr_pbl_info *pbl_info, u32 pg_shift)
a7efd777 603{
95ad233f 604 int pbe_cnt, total_num_pbes = 0;
a7efd777 605 struct qedr_pbl *pbl_tbl;
68363052 606 struct ib_block_iter biter;
a7efd777 607 struct regpair *pbe;
a7efd777
RA		 608
609 if (!pbl_info->num_pbes)
610 return;
611
612 /* If we have a two layered pbl, the first pbl points to the rest
613 * of the pbls and the first entry lays on the second pbl in the table
614 */
615 if (pbl_info->two_layered)
616 pbl_tbl = &pbl[1];
617 else
618 pbl_tbl = pbl;
619
620 pbe = (struct regpair *)pbl_tbl->va;
621 if (!pbe) {
622 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
623 return;
624 }
625
626 pbe_cnt = 0;
627
68363052
JG		 628 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
629 u64 pg_addr = rdma_block_iter_dma_address(&biter);
95ad233f 630
68363052
JG		 631 pbe->lo = cpu_to_le32(pg_addr);
632 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
95ad233f 633
68363052
JG		 634 pbe_cnt++;
635 total_num_pbes++;
636 pbe++;
e57bb6be 637
68363052
JG		 638 if (total_num_pbes == pbl_info->num_pbes)
639 return;
95ad233f 640
68363052
JG		 641 /* If the given pbl is full storing the pbes, move to next pbl.
642 */
643 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
644 pbl_tbl++;
645 pbe = (struct regpair *)pbl_tbl->va;
646 pbe_cnt = 0;
a7efd777
RA		 647 }
648 }
649}
650
97f61250
MK		 651static int qedr_db_recovery_add(struct qedr_dev *dev,
652 void __iomem *db_addr,
653 void *db_data,
654 enum qed_db_rec_width db_width,
655 enum qed_db_rec_space db_space)
656{
657 if (!db_data) {
658 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
659 return 0;
660 }
661
662 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
663 db_width, db_space);
664}
665
666static void qedr_db_recovery_del(struct qedr_dev *dev,
667 void __iomem *db_addr,
668 void *db_data)
669{
670 if (!db_data) {
671 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
672 return;
673 }
674
675 /* Ignore return code as there is not much we can do about it. Error
676 * log will be printed inside.
677 */
678 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
679}
680
a7efd777 681static int qedr_copy_cq_uresp(struct qedr_dev *dev,
97f61250
MK		 682 struct qedr_cq *cq, struct ib_udata *udata,
683 u32 db_offset)
a7efd777
RA		 684{
685 struct qedr_create_cq_uresp uresp;
686 int rc;
687
688 memset(&uresp, 0, sizeof(uresp));
689
97f61250 690 uresp.db_offset = db_offset;
a7efd777 691 uresp.icid = cq->icid;
a25984f3
MK		 692 if (cq->q.db_mmap_entry)
693 uresp.db_rec_addr =
694 rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
a7efd777 695
c75d3ec8 696 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
a7efd777
RA		 697 if (rc)
698 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
699
700 return rc;
701}
702
703static void consume_cqe(struct qedr_cq *cq)
704{
705 if (cq->latest_cqe == cq->toggle_cqe)
706 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
707
708 cq->latest_cqe = qed_chain_consume(&cq->pbl);
709}
710
711static inline int qedr_align_cq_entries(int entries)
712{
713 u64 size, aligned_size;
714
715 /* We allocate an extra entry that we don't report to the FW. */
716 size = (entries + 1) * QEDR_CQE_SIZE;
717 aligned_size = ALIGN(size, PAGE_SIZE);
718
719 return aligned_size / QEDR_CQE_SIZE;
720}
721
97f61250
MK		 722static int qedr_init_user_db_rec(struct ib_udata *udata,
723 struct qedr_dev *dev, struct qedr_userq *q,
724 bool requires_db_rec)
725{
726 struct qedr_ucontext *uctx =
727 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
728 ibucontext);
729 struct qedr_user_mmap_entry *entry;
730 int rc;
731
732 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
733 if (requires_db_rec == 0 || !uctx->db_rec)
734 return 0;
735
736 /* Allocate a page for doorbell recovery, add to mmap */
737 q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
738 if (!q->db_rec_data) {
739 DP_ERR(dev, "get_zeroed_page failed\n");
740 return -ENOMEM;
741 }
742
743 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
744 if (!entry)
745 goto err_free_db_data;
746
747 entry->address = q->db_rec_data;
748 entry->length = PAGE_SIZE;
749 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
750 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
751 &entry->rdma_entry,
752 PAGE_SIZE);
753 if (rc)
754 goto err_free_entry;
755
756 q->db_mmap_entry = &entry->rdma_entry;
757
758 return 0;
759
760err_free_entry:
761 kfree(entry);
762
763err_free_db_data:
764 free_page((unsigned long)q->db_rec_data);
765 q->db_rec_data = NULL;
766 return -ENOMEM;
767}
768
b0ea0fa5 769static inline int qedr_init_user_queue(struct ib_udata *udata,
a7efd777 770 struct qedr_dev *dev,
b0ea0fa5 771 struct qedr_userq *q, u64 buf_addr,
97f61250 772 size_t buf_len, bool requires_db_rec,
72b894b0 773 int access,
69ad0e7f 774 int alloc_and_init)
a7efd777 775{
e57bb6be 776 u32 fw_pages;
a7efd777
RA		 777 int rc;
778
779 q->buf_addr = buf_addr;
780 q->buf_len = buf_len;
c320e527 781 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
a7efd777
RA		 782 if (IS_ERR(q->umem)) {
783 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
784 PTR_ERR(q->umem));
785 return PTR_ERR(q->umem);
786 }
787
901bca71 788 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
e57bb6be 789 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
a7efd777
RA		 790 if (rc)
791 goto err0;
792
69ad0e7f
KM		 793 if (alloc_and_init) {
794 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
795 if (IS_ERR(q->pbl_tbl)) {
796 rc = PTR_ERR(q->pbl_tbl);
797 goto err0;
798 }
e57bb6be
RA		 799 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
800 FW_PAGE_SHIFT);
69ad0e7f
KM		 801 } else {
802 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
89fd2576
DC		 803 if (!q->pbl_tbl) {
804 rc = -ENOMEM;
69ad0e7f 805 goto err0;
89fd2576 806 }
69ad0e7f 807 }
a7efd777 808
97f61250
MK		 809 /* mmap the user address used to store doorbell data for recovery */
810 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
a7efd777
RA		 811
812err0:
813 ib_umem_release(q->umem);
69ad0e7f 814 q->umem = NULL;
a7efd777
RA		 815
816 return rc;
817}
818
819static inline void qedr_init_cq_params(struct qedr_cq *cq,
820 struct qedr_ucontext *ctx,
821 struct qedr_dev *dev, int vector,
822 int chain_entries, int page_cnt,
823 u64 pbl_ptr,
824 struct qed_rdma_create_cq_in_params
825 *params)
826{
827 memset(params, 0, sizeof(*params));
828 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
829 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
830 params->cnq_id = vector;
831 params->cq_size = chain_entries - 1;
832 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
833 params->pbl_num_pages = page_cnt;
834 params->pbl_ptr = pbl_ptr;
835 params->pbl_two_level = 0;
836}
837
838static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
839{
a7efd777
RA		 840 cq->db.data.agg_flags = flags;
841 cq->db.data.value = cpu_to_le32(cons);
842 writeq(cq->db.raw, cq->db_addr);
a7efd777
RA		 843}
844
845int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
846{
847 struct qedr_cq *cq = get_qedr_cq(ibcq);
848 unsigned long sflags;
4dd72636
AR		 849 struct qedr_dev *dev;
850
851 dev = get_qedr_dev(ibcq->device);
852
853 if (cq->destroyed) {
854 DP_ERR(dev,
855 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
856 cq, cq->icid);
857 return -EINVAL;
858 }
859
a7efd777
RA		 860
861 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
862 return 0;
863
864 spin_lock_irqsave(&cq->cq_lock, sflags);
865
866 cq->arm_flags = 0;
867
868 if (flags & IB_CQ_SOLICITED)
869 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
870
871 if (flags & IB_CQ_NEXT_COMP)
872 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
873
874 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
875
876 spin_unlock_irqrestore(&cq->cq_lock, sflags);
877
878 return 0;
879}
880
e39afe3d
LR		 881int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
882 struct ib_udata *udata)
a7efd777 883{
e39afe3d 884 struct ib_device *ibdev = ibcq->device;
ff23dfa1
SR		 885 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
886 udata, struct qedr_ucontext, ibucontext);
a7efd777
RA		 887 struct qed_rdma_destroy_cq_out_params destroy_oparams;
888 struct qed_rdma_destroy_cq_in_params destroy_iparams;
b6db3f71
AL		 889 struct qed_chain_init_params chain_params = {
890 .mode = QED_CHAIN_MODE_PBL,
891 .intended_use = QED_CHAIN_USE_TO_CONSUME,
892 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
893 .elem_size = sizeof(union rdma_cqe),
894 };
a7efd777
RA		 895 struct qedr_dev *dev = get_qedr_dev(ibdev);
896 struct qed_rdma_create_cq_in_params params;
e39afe3d 897 struct qedr_create_cq_ureq ureq = {};
a7efd777
RA		 898 int vector = attr->comp_vector;
899 int entries = attr->cqe;
e39afe3d 900 struct qedr_cq *cq = get_qedr_cq(ibcq);
a7efd777 901 int chain_entries;
97f61250 902 u32 db_offset;
a7efd777
RA		 903 int page_cnt;
904 u64 pbl_ptr;
905 u16 icid;
906 int rc;
907
908 DP_DEBUG(dev, QEDR_MSG_INIT,
909 "create_cq: called from %s. entries=%d, vector=%d\n",
910 udata ? "User Lib" : "Kernel", entries, vector);
911
912 if (entries > QEDR_MAX_CQES) {
913 DP_ERR(dev,
914 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
915 entries, QEDR_MAX_CQES);
e39afe3d 916 return -EINVAL;
a7efd777
RA		 917 }
918
919 chain_entries = qedr_align_cq_entries(entries);
920 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
b6db3f71 921 chain_params.num_elems = chain_entries;
a7efd777 922
97f61250
MK		 923 /* calc db offset. user will add DPI base, kernel will add db addr */
924 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
925
a7efd777 926 if (udata) {
97f61250
MK		 927 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
928 udata->inlen))) {
a7efd777
RA		 929 DP_ERR(dev,
930 "create cq: problem copying data from user space\n");
931 goto err0;
932 }
933
934 if (!ureq.len) {
935 DP_ERR(dev,
936 "create cq: cannot create a cq with 0 entries\n");
937 goto err0;
938 }
939
940 cq->cq_type = QEDR_CQ_TYPE_USER;
941
b0ea0fa5 942 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
72b894b0
CH		 943 ureq.len, true, IB_ACCESS_LOCAL_WRITE,
944 1);
a7efd777
RA		 945 if (rc)
946 goto err0;
947
948 pbl_ptr = cq->q.pbl_tbl->pa;
949 page_cnt = cq->q.pbl_info.num_pbes;
c7eb3bce
AR		 950
951 cq->ibcq.cqe = chain_entries;
97f61250 952 cq->q.db_addr = ctx->dpi_addr + db_offset;
a7efd777
RA		 953 } else {
954 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
955
b6db3f71
AL		 956 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
957 &chain_params);
a7efd777 958 if (rc)
97f61250 959 goto err0;
a7efd777
RA		 960
961 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
962 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
c7eb3bce 963 cq->ibcq.cqe = cq->pbl.capacity;
a7efd777
RA		 964 }
965
966 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
967 pbl_ptr, &params);
968
969 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
970 if (rc)
97f61250 971 goto err1;
a7efd777
RA		 972
973 cq->icid = icid;
974 cq->sig = QEDR_CQ_MAGIC_NUMBER;
975 spin_lock_init(&cq->cq_lock);
976
ff23dfa1 977 if (udata) {
97f61250
MK		 978 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
979 if (rc)
980 goto err2;
981
982 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
983 &cq->q.db_rec_data->db_data,
984 DB_REC_WIDTH_64B,
985 DB_REC_USER);
a7efd777 986 if (rc)
97f61250
MK		 987 goto err2;
988
a7efd777
RA		 989 } else {
990 /* Generate doorbell address. */
a7efd777 991 cq->db.data.icid = cq->icid;
97f61250 992 cq->db_addr = dev->db_addr + db_offset;
0b1eddc1 993 cq->db.data.params = DB_AGG_CMD_MAX <<
a7efd777
RA		 994 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
995
996 /* point to the very last element, passing it we will toggle */
997 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
998 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
999 cq->latest_cqe = NULL;
1000 consume_cqe(cq);
1001 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
97f61250
MK		 1002
1003 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1004 DB_REC_WIDTH_64B, DB_REC_KERNEL);
1005 if (rc)
1006 goto err2;
a7efd777
RA		 1007 }
1008
1009 DP_DEBUG(dev, QEDR_MSG_CQ,
1010 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1011 cq->icid, cq, params.cq_size);
1012
e39afe3d 1013 return 0;
a7efd777 1014
97f61250 1015err2:
a7efd777
RA		 1016 destroy_iparams.icid = cq->icid;
1017 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1018 &destroy_oparams);
a7efd777 1019err1:
97f61250
MK		 1020 if (udata) {
1021 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
a7efd777 1022 ib_umem_release(cq->q.umem);
a25984f3 1023 if (cq->q.db_mmap_entry)
97f61250
MK		 1024 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1025 } else {
1026 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1027 }
a7efd777 1028err0:
e39afe3d 1029 return -EINVAL;
a7efd777
RA		 1030}
1031
1032int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1033{
1034 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1035 struct qedr_cq *cq = get_qedr_cq(ibcq);
1036
1037 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1038
1039 return 0;
1040}
1041
4dd72636
AR		 1042#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1043#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1044
43d781b9 1045int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
a7efd777
RA		 1046{
1047 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1048 struct qed_rdma_destroy_cq_out_params oparams;
1049 struct qed_rdma_destroy_cq_in_params iparams;
1050 struct qedr_cq *cq = get_qedr_cq(ibcq);
4dd72636 1051 int iter;
a7efd777 1052
942b3b2c 1053 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
a7efd777 1054
4dd72636
AR		 1055 cq->destroyed = 1;
1056
a7efd777 1057 /* GSIs CQs are handled by driver, so they don't exist in the FW */
97f61250
MK		 1058 if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1059 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
43d781b9 1060 return 0;
97f61250 1061 }
a1211359 1062
942b3b2c 1063 iparams.icid = cq->icid;
a52c8e24 1064 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
942b3b2c 1065 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
a7efd777 1066
bdeacabd 1067 if (udata) {
a7efd777
RA		 1068 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1069 ib_umem_release(cq->q.umem);
97f61250
MK		 1070
1071 if (cq->q.db_rec_data) {
1072 qedr_db_recovery_del(dev, cq->q.db_addr,
1073 &cq->q.db_rec_data->db_data);
1074 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1075 }
1076 } else {
1077 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
a7efd777
RA		 1078 }
1079
4dd72636
AR		 1080 /* We don't want the IRQ handler to handle a non-existing CQ so we
1081 * wait until all CNQ interrupts, if any, are received. This will always
1082 * happen and will always happen very fast. If not, then a serious error
1083 * has occured. That is why we can use a long delay.
1084 * We spin for a short time so we don’t lose time on context switching
1085 * in case all the completions are handled in that span. Otherwise
1086 * we sleep for a while and check again. Since the CNQ may be
1087 * associated with (only) the current CPU we use msleep to allow the
1088 * current CPU to be freed.
1089 * The CNQ notification is increased in qedr_irq_handler().
1090 */
1091 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1092 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1093 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1094 iter--;
1095 }
1096
1097 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1098 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1099 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1100 iter--;
1101 }
1102
4dd72636
AR		 1103 /* Note that we don't need to have explicit code to wait for the
1104 * completion of the event handler because it is invoked from the EQ.
1105 * Since the destroy CQ ramrod has also been received on the EQ we can
1106 * be certain that there's no event handler in process.
1107 */
43d781b9 1108 return 0;
a7efd777 1109}
cecbcddf
RA		 1110
1111static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1112 struct ib_qp_attr *attr,
1113 int attr_mask,
1114 struct qed_rdma_modify_qp_in_params
1115 *qp_params)
1116{
47ec3866 1117 const struct ib_gid_attr *gid_attr;
cecbcddf 1118 enum rdma_network_type nw_type;
d8966fcd 1119 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 1120 u32 ipv4_addr;
a70c0739 1121 int ret;
cecbcddf
RA		 1122 int i;
1123
47ec3866 1124 gid_attr = grh->sgid_attr;
a70c0739
PP		 1125 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1126 if (ret)
1127 return ret;
3e44e0ee 1128
47ec3866 1129 nw_type = rdma_gid_attr_network_type(gid_attr);
3e44e0ee
PP		 1130 switch (nw_type) {
1131 case RDMA_NETWORK_IPV6:
47ec3866 1132 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1133 sizeof(qp_params->sgid));
1134 memcpy(&qp_params->dgid.bytes[0],
1135 &grh->dgid,
1136 sizeof(qp_params->dgid));
1137 qp_params->roce_mode = ROCE_V2_IPV6;
1138 SET_FIELD(qp_params->modify_flags,
1139 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1140 break;
1141 case RDMA_NETWORK_IB:
47ec3866 1142 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
3e44e0ee
PP		 1143 sizeof(qp_params->sgid));
1144 memcpy(&qp_params->dgid.bytes[0],
1145 &grh->dgid,
1146 sizeof(qp_params->dgid));
1147 qp_params->roce_mode = ROCE_V1;
1148 break;
1149 case RDMA_NETWORK_IPV4:
1150 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1151 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
47ec3866 1152 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
3e44e0ee
PP		 1153 qp_params->sgid.ipv4_addr = ipv4_addr;
1154 ipv4_addr =
1155 qedr_get_ipv4_from_gid(grh->dgid.raw);
1156 qp_params->dgid.ipv4_addr = ipv4_addr;
1157 SET_FIELD(qp_params->modify_flags,
1158 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1159 qp_params->roce_mode = ROCE_V2_IPV4;
1160 break;
cecbcddf
RA		 1161 }
1162
1163 for (i = 0; i < 4; i++) {
1164 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1165 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1166 }
1167
1168 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1169 qp_params->vlan_id = 0;
1170
1171 return 0;
1172}
1173
cecbcddf 1174static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
e00b64f7
SR		 1175 struct ib_qp_init_attr *attrs,
1176 struct ib_udata *udata)
cecbcddf
RA		 1177{
1178 struct qedr_device_attr *qattr = &dev->attr;
1179
1180 /* QP0... attrs->qp_type == IB_QPT_GSI */
1181 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1182 DP_DEBUG(dev, QEDR_MSG_QP,
1183 "create qp: unsupported qp type=0x%x requested\n",
1184 attrs->qp_type);
bb8865f4 1185 return -EOPNOTSUPP;
cecbcddf
RA		 1186 }
1187
1188 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1189 DP_ERR(dev,
1190 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1191 attrs->cap.max_send_wr, qattr->max_sqe);
1192 return -EINVAL;
1193 }
1194
1195 if (attrs->cap.max_inline_data > qattr->max_inline) {
1196 DP_ERR(dev,
1197 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1198 attrs->cap.max_inline_data, qattr->max_inline);
1199 return -EINVAL;
1200 }
1201
1202 if (attrs->cap.max_send_sge > qattr->max_sge) {
1203 DP_ERR(dev,
1204 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1205 attrs->cap.max_send_sge, qattr->max_sge);
1206 return -EINVAL;
1207 }
1208
1209 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1210 DP_ERR(dev,
1211 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1212 attrs->cap.max_recv_sge, qattr->max_sge);
1213 return -EINVAL;
1214 }
1215
1216 /* Unprivileged user space cannot create special QP */
e00b64f7 1217 if (udata && attrs->qp_type == IB_QPT_GSI) {
cecbcddf
RA		 1218 DP_ERR(dev,
1219 "create qp: userspace can't create special QPs of type=0x%x\n",
1220 attrs->qp_type);
1221 return -EINVAL;
1222 }
1223
1224 return 0;
1225}
1226
40b173dd
YB		 1227static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1228 struct qedr_srq *srq, struct ib_udata *udata)
1229{
1230 struct qedr_create_srq_uresp uresp = {};
1231 int rc;
1232
1233 uresp.srq_id = srq->srq_id;
1234
1235 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1236 if (rc)
1237 DP_ERR(dev, "create srq: problem copying data to user space\n");
1238
1239 return rc;
1240}
1241
69ad0e7f 1242static void qedr_copy_rq_uresp(struct qedr_dev *dev,
97f61250
MK		 1243 struct qedr_create_qp_uresp *uresp,
1244 struct qedr_qp *qp)
cecbcddf 1245{
69ad0e7f
KM		 1246 /* iWARP requires two doorbells per RQ. */
1247 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1248 uresp->rq_db_offset =
1249 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1250 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1251 } else {
1252 uresp->rq_db_offset =
1253 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1254 }
1255
cecbcddf 1256 uresp->rq_icid = qp->icid;
a25984f3
MK		 1257 if (qp->urq.db_mmap_entry)
1258 uresp->rq_db_rec_addr =
1259 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
cecbcddf
RA		 1260}
1261
69ad0e7f
KM		 1262static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1263 struct qedr_create_qp_uresp *uresp,
cecbcddf
RA		 1264 struct qedr_qp *qp)
1265{
1266 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
69ad0e7f
KM		 1267
1268 /* iWARP uses the same cid for rq and sq */
1269 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1270 uresp->sq_icid = qp->icid;
1271 else
1272 uresp->sq_icid = qp->icid + 1;
97f61250 1273
a25984f3
MK		 1274 if (qp->usq.db_mmap_entry)
1275 uresp->sq_db_rec_addr =
1276 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
cecbcddf
RA		 1277}
1278
1279static int qedr_copy_qp_uresp(struct qedr_dev *dev,
97f61250
MK		 1280 struct qedr_qp *qp, struct ib_udata *udata,
1281 struct qedr_create_qp_uresp *uresp)
cecbcddf 1282{
cecbcddf
RA		 1283 int rc;
1284
97f61250
MK		 1285 memset(uresp, 0, sizeof(*uresp));
1286 qedr_copy_sq_uresp(dev, uresp, qp);
1287 qedr_copy_rq_uresp(dev, uresp, qp);
cecbcddf 1288
97f61250
MK		 1289 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1290 uresp->qp_id = qp->qp_id;
cecbcddf 1291
97f61250 1292 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
cecbcddf
RA		 1293 if (rc)
1294 DP_ERR(dev,
1295 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1296 qp->icid);
1297
1298 return rc;
1299}
1300
df158561
AR		 1301static void qedr_set_common_qp_params(struct qedr_dev *dev,
1302 struct qedr_qp *qp,
1303 struct qedr_pd *pd,
1304 struct ib_qp_init_attr *attrs)
cecbcddf 1305{
cecbcddf 1306 spin_lock_init(&qp->q_lock);
82af6d19
MK		 1307 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1308 kref_init(&qp->refcnt);
1309 init_completion(&qp->iwarp_cm_comp);
1310 }
df158561 1311 qp->pd = pd;
cecbcddf
RA		 1312 qp->qp_type = attrs->qp_type;
1313 qp->max_inline_data = attrs->cap.max_inline_data;
1314 qp->sq.max_sges = attrs->cap.max_send_sge;
1315 qp->state = QED_ROCE_QP_STATE_RESET;
1316 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1317 qp->sq_cq = get_qedr_cq(attrs->send_cq);
cecbcddf
RA		 1318 qp->dev = dev;
1319
3491c9e7
YB		 1320 if (attrs->srq) {
1321 qp->srq = get_qedr_srq(attrs->srq);
1322 } else {
1323 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1324 qp->rq.max_sges = attrs->cap.max_recv_sge;
1325 DP_DEBUG(dev, QEDR_MSG_QP,
1326 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1327 qp->rq.max_sges, qp->rq_cq->icid);
1328 }
1329
cecbcddf
RA		 1330 DP_DEBUG(dev, QEDR_MSG_QP,
1331 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1332 pd->pd_id, qp->qp_type, qp->max_inline_data,
1333 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1334 DP_DEBUG(dev, QEDR_MSG_QP,
1335 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1336 qp->sq.max_sges, qp->sq_cq->icid);
cecbcddf
RA		 1337}
1338
97f61250 1339static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1340{
97f61250
MK		 1341 int rc;
1342
cecbcddf
RA		 1343 qp->sq.db = dev->db_addr +
1344 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1345 qp->sq.db_data.data.icid = qp->icid + 1;
97f61250
MK		 1346 rc = qedr_db_recovery_add(dev, qp->sq.db,
1347 &qp->sq.db_data,
1348 DB_REC_WIDTH_32B,
1349 DB_REC_KERNEL);
1350 if (rc)
1351 return rc;
1352
3491c9e7
YB		 1353 if (!qp->srq) {
1354 qp->rq.db = dev->db_addr +
1355 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1356 qp->rq.db_data.data.icid = qp->icid;
97f61250
MK		 1357
1358 rc = qedr_db_recovery_add(dev, qp->rq.db,
1359 &qp->rq.db_data,
1360 DB_REC_WIDTH_32B,
1361 DB_REC_KERNEL);
1362 if (rc)
1363 qedr_db_recovery_del(dev, qp->sq.db,
1364 &qp->sq.db_data);
3491c9e7 1365 }
97f61250
MK		 1366
1367 return rc;
3491c9e7
YB		 1368}
1369
68e326de 1370static int qedr_check_srq_params(struct qedr_dev *dev,
3491c9e7
YB		 1371 struct ib_srq_init_attr *attrs,
1372 struct ib_udata *udata)
1373{
1374 struct qedr_device_attr *qattr = &dev->attr;
1375
1376 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1377 DP_ERR(dev,
1378 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1379 attrs->attr.max_wr, qattr->max_srq_wr);
1380 return -EINVAL;
1381 }
1382
1383 if (attrs->attr.max_sge > qattr->max_sge) {
1384 DP_ERR(dev,
1385 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1386 attrs->attr.max_sge, qattr->max_sge);
1387 return -EINVAL;
1388 }
1389
1390 return 0;
1391}
1392
40b173dd
YB		 1393static void qedr_free_srq_user_params(struct qedr_srq *srq)
1394{
1395 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1396 ib_umem_release(srq->usrq.umem);
1397 ib_umem_release(srq->prod_umem);
1398}
1399
3491c9e7
YB		 1400static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1401{
1402 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1403 struct qedr_dev *dev = srq->dev;
1404
1405 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1406
1407 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1408 hw_srq->virt_prod_pair_addr,
1409 hw_srq->phy_prod_pair_addr);
1410}
1411
b0ea0fa5 1412static int qedr_init_srq_user_params(struct ib_udata *udata,
40b173dd
YB		 1413 struct qedr_srq *srq,
1414 struct qedr_create_srq_ureq *ureq,
72b894b0 1415 int access)
40b173dd
YB		 1416{
1417 struct scatterlist *sg;
1418 int rc;
1419
b0ea0fa5 1420 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
72b894b0 1421 ureq->srq_len, false, access, 1);
40b173dd
YB		 1422 if (rc)
1423 return rc;
1424
c320e527
MS		 1425 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1426 sizeof(struct rdma_srq_producers), access);
40b173dd
YB		 1427 if (IS_ERR(srq->prod_umem)) {
1428 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1429 ib_umem_release(srq->usrq.umem);
1430 DP_ERR(srq->dev,
1431 "create srq: failed ib_umem_get for producer, got %ld\n",
1432 PTR_ERR(srq->prod_umem));
1433 return PTR_ERR(srq->prod_umem);
1434 }
1435
1436 sg = srq->prod_umem->sg_head.sgl;
1437 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1438
1439 return 0;
1440}
1441
3491c9e7
YB		 1442static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1443 struct qedr_dev *dev,
1444 struct ib_srq_init_attr *init_attr)
1445{
1446 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
b6db3f71
AL		 1447 struct qed_chain_init_params params = {
1448 .mode = QED_CHAIN_MODE_PBL,
1449 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1450 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1451 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE,
1452 };
3491c9e7
YB		 1453 dma_addr_t phy_prod_pair_addr;
1454 u32 num_elems;
1455 void *va;
1456 int rc;
1457
1458 va = dma_alloc_coherent(&dev->pdev->dev,
1459 sizeof(struct rdma_srq_producers),
1460 &phy_prod_pair_addr, GFP_KERNEL);
1461 if (!va) {
1462 DP_ERR(dev,
1463 "create srq: failed to allocate dma memory for producer\n");
1464 return -ENOMEM;
1465 }
1466
1467 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1468 hw_srq->virt_prod_pair_addr = va;
1469
1470 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
b6db3f71
AL		 1471 params.num_elems = num_elems;
1472
1473 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
3491c9e7
YB		 1474 if (rc)
1475 goto err0;
1476
1477 hw_srq->num_elems = num_elems;
1478
1479 return 0;
1480
1481err0:
1482 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1483 va, phy_prod_pair_addr);
1484 return rc;
1485}
1486
68e326de
LR		 1487int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1488 struct ib_udata *udata)
3491c9e7
YB		 1489{
1490 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1491 struct qed_rdma_create_srq_in_params in_params = {};
68e326de 1492 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
3491c9e7 1493 struct qed_rdma_create_srq_out_params out_params;
68e326de 1494 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
40b173dd 1495 struct qedr_create_srq_ureq ureq = {};
3491c9e7
YB		 1496 u64 pbl_base_addr, phy_prod_pair_addr;
1497 struct qedr_srq_hwq_info *hw_srq;
1498 u32 page_cnt, page_size;
68e326de 1499 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3491c9e7
YB		 1500 int rc = 0;
1501
1502 DP_DEBUG(dev, QEDR_MSG_QP,
1503 "create SRQ called from %s (pd %p)\n",
1504 (udata) ? "User lib" : "kernel", pd);
1505
68e326de 1506 rc = qedr_check_srq_params(dev, init_attr, udata);
3491c9e7 1507 if (rc)
68e326de 1508 return -EINVAL;
3491c9e7
YB		 1509
1510 srq->dev = dev;
1511 hw_srq = &srq->hw_srq;
1512 spin_lock_init(&srq->lock);
1513
1514 hw_srq->max_wr = init_attr->attr.max_wr;
1515 hw_srq->max_sges = init_attr->attr.max_sge;
1516
89944450 1517 if (udata) {
97f61250
MK		 1518 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1519 udata->inlen))) {
40b173dd
YB		 1520 DP_ERR(dev,
1521 "create srq: problem copying data from user space\n");
1522 goto err0;
1523 }
1524
72b894b0 1525 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
40b173dd
YB		 1526 if (rc)
1527 goto err0;
1528
1529 page_cnt = srq->usrq.pbl_info.num_pbes;
1530 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1531 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
95ad233f 1532 page_size = PAGE_SIZE;
40b173dd
YB		 1533 } else {
1534 struct qed_chain *pbl;
1535
1536 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1537 if (rc)
1538 goto err0;
1539
1540 pbl = &hw_srq->pbl;
1541 page_cnt = qed_chain_get_page_cnt(pbl);
1542 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1543 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1544 page_size = QED_CHAIN_PAGE_SIZE;
1545 }
3491c9e7 1546
3491c9e7
YB		 1547 in_params.pd_id = pd->pd_id;
1548 in_params.pbl_base_addr = pbl_base_addr;
1549 in_params.prod_pair_addr = phy_prod_pair_addr;
1550 in_params.num_pages = page_cnt;
1551 in_params.page_size = page_size;
1552
1553 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1554 if (rc)
1555 goto err1;
1556
1557 srq->srq_id = out_params.srq_id;
1558
40b173dd
YB		 1559 if (udata) {
1560 rc = qedr_copy_srq_uresp(dev, srq, udata);
1561 if (rc)
1562 goto err2;
1563 }
1564
9fd15987 1565 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
3491c9e7
YB		 1566 if (rc)
1567 goto err2;
1568
1569 DP_DEBUG(dev, QEDR_MSG_SRQ,
1570 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
68e326de 1571 return 0;
3491c9e7
YB		 1572
1573err2:
1574 destroy_in_params.srq_id = srq->srq_id;
1575
1576 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1577err1:
40b173dd
YB		 1578 if (udata)
1579 qedr_free_srq_user_params(srq);
1580 else
1581 qedr_free_srq_kernel_params(srq);
3491c9e7 1582err0:
68e326de 1583 return -EFAULT;
3491c9e7
YB		 1584}
1585
119181d1 1586int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3491c9e7
YB		 1587{
1588 struct qed_rdma_destroy_srq_in_params in_params = {};
1589 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1590 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1591
9fd15987 1592 xa_erase_irq(&dev->srqs, srq->srq_id);
3491c9e7
YB		 1593 in_params.srq_id = srq->srq_id;
1594 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1595
e00b64f7 1596 if (ibsrq->uobject)
40b173dd
YB		 1597 qedr_free_srq_user_params(srq);
1598 else
1599 qedr_free_srq_kernel_params(srq);
3491c9e7
YB		 1600
1601 DP_DEBUG(dev, QEDR_MSG_SRQ,
1602 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1603 srq->srq_id);
119181d1 1604 return 0;
3491c9e7
YB		 1605}
1606
1607int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1608 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1609{
1610 struct qed_rdma_modify_srq_in_params in_params = {};
1611 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1612 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1613 int rc;
1614
1615 if (attr_mask & IB_SRQ_MAX_WR) {
1616 DP_ERR(dev,
1617 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1618 attr_mask, srq);
1619 return -EINVAL;
1620 }
1621
1622 if (attr_mask & IB_SRQ_LIMIT) {
1623 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1624 DP_ERR(dev,
1625 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1626 attr->srq_limit, srq->hw_srq.max_wr);
1627 return -EINVAL;
1628 }
1629
1630 in_params.srq_id = srq->srq_id;
1631 in_params.wqe_limit = attr->srq_limit;
1632 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1633 if (rc)
1634 return rc;
1635 }
1636
1637 srq->srq_limit = attr->srq_limit;
1638
1639 DP_DEBUG(dev, QEDR_MSG_SRQ,
1640 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1641
1642 return 0;
cecbcddf
RA		 1643}
1644
1645static inline void
df158561
AR		 1646qedr_init_common_qp_in_params(struct qedr_dev *dev,
1647 struct qedr_pd *pd,
1648 struct qedr_qp *qp,
1649 struct ib_qp_init_attr *attrs,
1650 bool fmr_and_reserved_lkey,
1651 struct qed_rdma_create_qp_in_params *params)
cecbcddf 1652{
df158561
AR		 1653 /* QP handle to be written in an async event */
1654 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1655 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1656
1657 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1658 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1659 params->pd = pd->pd_id;
1660 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1661 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1662 params->stats_queue = 0;
df158561
AR		 1663 params->srq_id = 0;
1664 params->use_srq = false;
3491c9e7
YB		 1665
1666 if (!qp->srq) {
1667 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1668
1669 } else {
1670 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1671 params->srq_id = qp->srq->srq_id;
1672 params->use_srq = true;
1673 }
cecbcddf
RA		 1674}
1675
df158561 1676static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 1677{
df158561
AR		 1678 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1679 "qp=%p. "
1680 "sq_addr=0x%llx, "
1681 "sq_len=%zd, "
1682 "rq_addr=0x%llx, "
1683 "rq_len=%zd"
1684 "\n",
1685 qp,
1686 qp->usq.buf_addr,
1687 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1688}
cecbcddf 1689
69ad0e7f
KM		 1690static inline void
1691qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1692 struct qedr_qp *qp,
1693 struct qed_rdma_create_qp_out_params *out_params)
1694{
1695 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1696 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1697
1698 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1699 &qp->usq.pbl_info, FW_PAGE_SHIFT);
40b173dd
YB		 1700 if (!qp->srq) {
1701 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1702 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1703 }
69ad0e7f
KM		 1704
1705 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1706 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1707}
1708
97f61250
MK		 1709static void qedr_cleanup_user(struct qedr_dev *dev,
1710 struct qedr_ucontext *ctx,
1711 struct qedr_qp *qp)
df158561 1712{
836a0fbb 1713 ib_umem_release(qp->usq.umem);
df158561 1714 qp->usq.umem = NULL;
cecbcddf 1715
836a0fbb 1716 ib_umem_release(qp->urq.umem);
df158561 1717 qp->urq.umem = NULL;
24e412c1
MK		 1718
1719 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1720 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1721 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1722 } else {
1723 kfree(qp->usq.pbl_tbl);
1724 kfree(qp->urq.pbl_tbl);
1725 }
97f61250
MK		 1726
1727 if (qp->usq.db_rec_data) {
1728 qedr_db_recovery_del(dev, qp->usq.db_addr,
1729 &qp->usq.db_rec_data->db_data);
1730 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1731 }
1732
1733 if (qp->urq.db_rec_data) {
1734 qedr_db_recovery_del(dev, qp->urq.db_addr,
1735 &qp->urq.db_rec_data->db_data);
1736 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1737 }
b4bc7660
MK		 1738
1739 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1740 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1741 &qp->urq.db_rec_db2_data);
cecbcddf
RA		 1742}
1743
df158561
AR		 1744static int qedr_create_user_qp(struct qedr_dev *dev,
1745 struct qedr_qp *qp,
1746 struct ib_pd *ibpd,
1747 struct ib_udata *udata,
1748 struct ib_qp_init_attr *attrs)
cecbcddf 1749{
df158561
AR		 1750 struct qed_rdma_create_qp_in_params in_params;
1751 struct qed_rdma_create_qp_out_params out_params;
1752 struct qedr_pd *pd = get_qedr_pd(ibpd);
97f61250 1753 struct qedr_create_qp_uresp uresp;
bbe4f424 1754 struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
df158561 1755 struct qedr_create_qp_ureq ureq;
69ad0e7f 1756 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
df158561 1757 int rc = -EINVAL;
cecbcddf 1758
82af6d19 1759 qp->create_type = QEDR_QP_CREATE_USER;
df158561 1760 memset(&ureq, 0, sizeof(ureq));
97f61250 1761 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
df158561
AR		 1762 if (rc) {
1763 DP_ERR(dev, "Problem copying data from user space\n");
1764 return rc;
1765 }
cecbcddf 1766
72b894b0 1767 /* SQ - read access only (0) */
b0ea0fa5 1768 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
72b894b0 1769 ureq.sq_len, true, 0, alloc_and_init);
df158561
AR		 1770 if (rc)
1771 return rc;
cecbcddf 1772
40b173dd 1773 if (!qp->srq) {
72b894b0 1774 /* RQ - read access only (0) */
b0ea0fa5 1775 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
72b894b0 1776 ureq.rq_len, true, 0, alloc_and_init);
40b173dd
YB		 1777 if (rc)
1778 return rc;
1779 }
df158561
AR		 1780
1781 memset(&in_params, 0, sizeof(in_params));
1782 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1783 in_params.qp_handle_lo = ureq.qp_handle_lo;
1784 in_params.qp_handle_hi = ureq.qp_handle_hi;
1785 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1786 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
40b173dd
YB		 1787 if (!qp->srq) {
1788 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1789 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1790 }
df158561 1791
bbe4f424
MK		 1792 if (ctx)
1793 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1794
df158561
AR		 1795 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1796 &in_params, &out_params);
1797
1798 if (!qp->qed_qp) {
1799 rc = -ENOMEM;
1800 goto err1;
1801 }
1802
69ad0e7f
KM		 1803 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1804 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1805
df158561
AR		 1806 qp->qp_id = out_params.qp_id;
1807 qp->icid = out_params.icid;
1808
97f61250
MK		 1809 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1810 if (rc)
1811 goto err;
1812
1813 /* db offset was calculated in copy_qp_uresp, now set in the user q */
1814 ctx = pd->uctx;
1815 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1816 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1817
b4bc7660
MK		 1818 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1819 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1820
1821 /* calculate the db_rec_db2 data since it is constant so no
1822 * need to reflect from user
1823 */
1824 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1825 qp->urq.db_rec_db2_data.data.value =
1826 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1827 }
1828
97f61250
MK		 1829 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1830 &qp->usq.db_rec_data->db_data,
1831 DB_REC_WIDTH_32B,
1832 DB_REC_USER);
1833 if (rc)
1834 goto err;
1835
1836 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1837 &qp->urq.db_rec_data->db_data,
1838 DB_REC_WIDTH_32B,
1839 DB_REC_USER);
df158561
AR		 1840 if (rc)
1841 goto err;
1842
b4bc7660
MK		 1843 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1844 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1845 &qp->urq.db_rec_db2_data,
1846 DB_REC_WIDTH_32B,
1847 DB_REC_USER);
1848 if (rc)
1849 goto err;
1850 }
df158561 1851 qedr_qp_user_print(dev, qp);
cecbcddf 1852
97f61250 1853 return rc;
df158561
AR		 1854err:
1855 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1856 if (rc)
1857 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1858
1859err1:
97f61250 1860 qedr_cleanup_user(dev, ctx, qp);
df158561 1861 return rc;
cecbcddf
RA		 1862}
1863
97f61250 1864static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
f5b1b177 1865{
97f61250
MK		 1866 int rc;
1867
f5b1b177
KM		 1868 qp->sq.db = dev->db_addr +
1869 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1870 qp->sq.db_data.data.icid = qp->icid;
1871
97f61250
MK		 1872 rc = qedr_db_recovery_add(dev, qp->sq.db,
1873 &qp->sq.db_data,
1874 DB_REC_WIDTH_32B,
1875 DB_REC_KERNEL);
1876 if (rc)
1877 return rc;
1878
f5b1b177
KM		 1879 qp->rq.db = dev->db_addr +
1880 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1881 qp->rq.db_data.data.icid = qp->icid;
1882 qp->rq.iwarp_db2 = dev->db_addr +
1883 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1884 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1885 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
97f61250
MK		 1886
1887 rc = qedr_db_recovery_add(dev, qp->rq.db,
1888 &qp->rq.db_data,
1889 DB_REC_WIDTH_32B,
1890 DB_REC_KERNEL);
b4bc7660
MK		 1891 if (rc)
1892 return rc;
1893
1894 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1895 &qp->rq.iwarp_db2_data,
1896 DB_REC_WIDTH_32B,
1897 DB_REC_KERNEL);
97f61250 1898 return rc;
f5b1b177
KM		 1899}
1900
df158561
AR		 1901static int
1902qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1903 struct qedr_qp *qp,
1904 struct qed_rdma_create_qp_in_params *in_params,
1905 u32 n_sq_elems, u32 n_rq_elems)
cecbcddf 1906{
df158561 1907 struct qed_rdma_create_qp_out_params out_params;
b6db3f71
AL		 1908 struct qed_chain_init_params params = {
1909 .mode = QED_CHAIN_MODE_PBL,
1910 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1911 };
cecbcddf
RA		 1912 int rc;
1913
b6db3f71
AL		 1914 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
1915 params.num_elems = n_sq_elems;
1916 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
cecbcddf 1917
b6db3f71 1918 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
df158561
AR		 1919 if (rc)
1920 return rc;
cecbcddf 1921
df158561
AR		 1922 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1923 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
cecbcddf 1924
b6db3f71 1925 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
35406697 1926 params.num_elems = n_rq_elems;
b6db3f71
AL		 1927 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
1928
1929 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
df158561
AR		 1930 if (rc)
1931 return rc;
cecbcddf 1932
df158561
AR		 1933 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1934 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
cecbcddf 1935
df158561
AR		 1936 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1937 in_params, &out_params);
cecbcddf 1938
df158561
AR		 1939 if (!qp->qed_qp)
1940 return -EINVAL;
cecbcddf 1941
df158561
AR		 1942 qp->qp_id = out_params.qp_id;
1943 qp->icid = out_params.icid;
cecbcddf 1944
97f61250 1945 return qedr_set_roce_db_info(dev, qp);
f5b1b177 1946}
cecbcddf 1947
f5b1b177
KM		 1948static int
1949qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1950 struct qedr_qp *qp,
1951 struct qed_rdma_create_qp_in_params *in_params,
1952 u32 n_sq_elems, u32 n_rq_elems)
1953{
1954 struct qed_rdma_create_qp_out_params out_params;
b6db3f71
AL		 1955 struct qed_chain_init_params params = {
1956 .mode = QED_CHAIN_MODE_PBL,
1957 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1958 };
f5b1b177
KM		 1959 int rc;
1960
1961 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1962 QEDR_SQE_ELEMENT_SIZE,
15506586 1963 QED_CHAIN_PAGE_SIZE,
f5b1b177
KM		 1964 QED_CHAIN_MODE_PBL);
1965 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1966 QEDR_RQE_ELEMENT_SIZE,
15506586 1967 QED_CHAIN_PAGE_SIZE,
f5b1b177
KM		 1968 QED_CHAIN_MODE_PBL);
1969
1970 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1971 in_params, &out_params);
1972
1973 if (!qp->qed_qp)
1974 return -EINVAL;
1975
1976 /* Now we allocate the chain */
f5b1b177 1977
b6db3f71
AL		 1978 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
1979 params.num_elems = n_sq_elems;
1980 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
1981 params.ext_pbl_virt = out_params.sq_pbl_virt;
1982 params.ext_pbl_phys = out_params.sq_pbl_phys;
f5b1b177 1983
b6db3f71 1984 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
f5b1b177
KM		 1985 if (rc)
1986 goto err;
1987
b6db3f71
AL		 1988 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1989 params.num_elems = n_rq_elems;
1990 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
1991 params.ext_pbl_virt = out_params.rq_pbl_virt;
1992 params.ext_pbl_phys = out_params.rq_pbl_phys;
f5b1b177 1993
b6db3f71 1994 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
f5b1b177
KM		 1995 if (rc)
1996 goto err;
1997
1998 qp->qp_id = out_params.qp_id;
1999 qp->icid = out_params.icid;
2000
97f61250 2001 return qedr_set_iwarp_db_info(dev, qp);
f5b1b177
KM		 2002
2003err:
2004 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2005
2006 return rc;
cecbcddf
RA		 2007}
2008
df158561 2009static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
cecbcddf 2010{
df158561
AR		 2011 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2012 kfree(qp->wqe_wr_id);
cecbcddf 2013
df158561
AR		 2014 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2015 kfree(qp->rqe_wr_id);
97f61250
MK		 2016
2017 /* GSI qp is not registered to db mechanism so no need to delete */
2018 if (qp->qp_type == IB_QPT_GSI)
2019 return;
2020
2021 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2022
b4bc7660 2023 if (!qp->srq) {
97f61250 2024 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
b4bc7660
MK		 2025
2026 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2027 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2028 &qp->rq.iwarp_db2_data);
2029 }
cecbcddf
RA		 2030}
2031
df158561
AR		 2032static int qedr_create_kernel_qp(struct qedr_dev *dev,
2033 struct qedr_qp *qp,
2034 struct ib_pd *ibpd,
2035 struct ib_qp_init_attr *attrs)
cecbcddf 2036{
df158561
AR		 2037 struct qed_rdma_create_qp_in_params in_params;
2038 struct qedr_pd *pd = get_qedr_pd(ibpd);
2039 int rc = -EINVAL;
2040 u32 n_rq_elems;
2041 u32 n_sq_elems;
2042 u32 n_sq_entries;
cecbcddf 2043
df158561 2044 memset(&in_params, 0, sizeof(in_params));
82af6d19 2045 qp->create_type = QEDR_QP_CREATE_KERNEL;
cecbcddf 2046
df158561
AR		 2047 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2048 * the ring. The ring should allow at least a single WR, even if the
2049 * user requested none, due to allocation issues.
2050 * We should add an extra WR since the prod and cons indices of
2051 * wqe_wr_id are managed in such a way that the WQ is considered full
2052 * when (prod+1)%max_wr==cons. We currently don't do that because we
2053 * double the number of entries due an iSER issue that pushes far more
2054 * WRs than indicated. If we decline its ib_post_send() then we get
2055 * error prints in the dmesg we'd like to avoid.
2056 */
2057 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2058 dev->attr.max_sqe);
cecbcddf 2059
6396bb22 2060 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
df158561
AR		 2061 GFP_KERNEL);
2062 if (!qp->wqe_wr_id) {
2063 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2064 return -ENOMEM;
2065 }
cecbcddf 2066
df158561
AR		 2067 /* QP handle to be written in CQE */
2068 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2069 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
cecbcddf 2070
df158561
AR		 2071 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2072 * the ring. There ring should allow at least a single WR, even if the
2073 * user requested none, due to allocation issues.
2074 */
2075 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
cecbcddf 2076
df158561 2077 /* Allocate driver internal RQ array */
6396bb22 2078 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
df158561
AR		 2079 GFP_KERNEL);
2080 if (!qp->rqe_wr_id) {
2081 DP_ERR(dev,
2082 "create qp: failed RQ shadow memory allocation\n");
2083 kfree(qp->wqe_wr_id);
2084 return -ENOMEM;
cecbcddf
RA		 2085 }
2086
df158561 2087 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
cecbcddf 2088
df158561
AR		 2089 n_sq_entries = attrs->cap.max_send_wr;
2090 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2091 n_sq_entries = max_t(u32, n_sq_entries, 1);
2092 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
cecbcddf 2093
df158561
AR		 2094 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2095
f5b1b177
KM		 2096 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2097 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2098 n_sq_elems, n_rq_elems);
2099 else
2100 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2101 n_sq_elems, n_rq_elems);
df158561
AR		 2102 if (rc)
2103 qedr_cleanup_kernel(dev, qp);
cecbcddf
RA		 2104
2105 return rc;
2106}
2107
2108struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2109 struct ib_qp_init_attr *attrs,
2110 struct ib_udata *udata)
2111{
2112 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
cecbcddf 2113 struct qedr_pd *pd = get_qedr_pd(ibpd);
cecbcddf 2114 struct qedr_qp *qp;
181d8015 2115 struct ib_qp *ibqp;
cecbcddf
RA		 2116 int rc = 0;
2117
2118 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2119 udata ? "user library" : "kernel", pd);
2120
e00b64f7 2121 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
cecbcddf
RA		 2122 if (rc)
2123 return ERR_PTR(rc);
2124
cecbcddf 2125 DP_DEBUG(dev, QEDR_MSG_QP,
df158561
AR		 2126 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2127 udata ? "user library" : "kernel", attrs->event_handler, pd,
cecbcddf
RA		 2128 get_qedr_cq(attrs->send_cq),
2129 get_qedr_cq(attrs->send_cq)->icid,
2130 get_qedr_cq(attrs->recv_cq),
3491c9e7 2131 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
cecbcddf 2132
df158561
AR		 2133 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2134 if (!qp) {
2135 DP_ERR(dev, "create qp: failed allocating memory\n");
2136 return ERR_PTR(-ENOMEM);
2137 }
2138
2139 qedr_set_common_qp_params(dev, qp, pd, attrs);
cecbcddf 2140
04886779 2141 if (attrs->qp_type == IB_QPT_GSI) {
181d8015
WY		 2142 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2143 if (IS_ERR(ibqp))
2144 kfree(qp);
2145 return ibqp;
04886779
RA		 2146 }
2147
df158561
AR		 2148 if (udata)
2149 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2150 else
2151 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
cecbcddf 2152
df158561
AR		 2153 if (rc)
2154 goto err;
cecbcddf 2155
cecbcddf
RA		 2156 qp->ibqp.qp_num = qp->qp_id;
2157
1212767e 2158 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
5fdff18b 2159 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1212767e
YB		 2160 if (rc)
2161 goto err;
2162 }
de0089e6 2163
cecbcddf
RA		 2164 return &qp->ibqp;
2165
df158561 2166err:
cecbcddf
RA		 2167 kfree(qp);
2168
2169 return ERR_PTR(-EFAULT);
2170}
2171
27a4b1a6 2172static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
cecbcddf
RA		 2173{
2174 switch (qp_state) {
2175 case QED_ROCE_QP_STATE_RESET:
2176 return IB_QPS_RESET;
2177 case QED_ROCE_QP_STATE_INIT:
2178 return IB_QPS_INIT;
2179 case QED_ROCE_QP_STATE_RTR:
2180 return IB_QPS_RTR;
2181 case QED_ROCE_QP_STATE_RTS:
2182 return IB_QPS_RTS;
2183 case QED_ROCE_QP_STATE_SQD:
2184 return IB_QPS_SQD;
2185 case QED_ROCE_QP_STATE_ERR:
2186 return IB_QPS_ERR;
2187 case QED_ROCE_QP_STATE_SQE:
2188 return IB_QPS_SQE;
2189 }
2190 return IB_QPS_ERR;
2191}
2192
27a4b1a6
RA		 2193static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2194 enum ib_qp_state qp_state)
cecbcddf
RA		 2195{
2196 switch (qp_state) {
2197 case IB_QPS_RESET:
2198 return QED_ROCE_QP_STATE_RESET;
2199 case IB_QPS_INIT:
2200 return QED_ROCE_QP_STATE_INIT;
2201 case IB_QPS_RTR:
2202 return QED_ROCE_QP_STATE_RTR;
2203 case IB_QPS_RTS:
2204 return QED_ROCE_QP_STATE_RTS;
2205 case IB_QPS_SQD:
2206 return QED_ROCE_QP_STATE_SQD;
2207 case IB_QPS_ERR:
2208 return QED_ROCE_QP_STATE_ERR;
2209 default:
2210 return QED_ROCE_QP_STATE_ERR;
2211 }
2212}
2213
2214static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2215{
2216 qed_chain_reset(&qph->pbl);
2217 qph->prod = 0;
2218 qph->cons = 0;
2219 qph->wqe_cons = 0;
2220 qph->db_data.data.value = cpu_to_le16(0);
2221}
2222
2223static int qedr_update_qp_state(struct qedr_dev *dev,
2224 struct qedr_qp *qp,
caf61b1b 2225 enum qed_roce_qp_state cur_state,
cecbcddf
RA		 2226 enum qed_roce_qp_state new_state)
2227{
2228 int status = 0;
2229
caf61b1b 2230 if (new_state == cur_state)
865cea40 2231 return 0;
cecbcddf 2232
caf61b1b 2233 switch (cur_state) {
cecbcddf
RA		 2234 case QED_ROCE_QP_STATE_RESET:
2235 switch (new_state) {
2236 case QED_ROCE_QP_STATE_INIT:
2237 qp->prev_wqe_size = 0;
2238 qedr_reset_qp_hwq_info(&qp->sq);
2239 qedr_reset_qp_hwq_info(&qp->rq);
2240 break;
2241 default:
2242 status = -EINVAL;
2243 break;
790b57f6 2244 }
cecbcddf
RA		 2245 break;
2246 case QED_ROCE_QP_STATE_INIT:
2247 switch (new_state) {
2248 case QED_ROCE_QP_STATE_RTR:
2249 /* Update doorbell (in case post_recv was
2250 * done before move to RTR)
2251 */
f5b1b177
KM		 2252
2253 if (rdma_protocol_roce(&dev->ibdev, 1)) {
f5b1b177 2254 writel(qp->rq.db_data.raw, qp->rq.db);
f5b1b177 2255 }
cecbcddf
RA		 2256 break;
2257 case QED_ROCE_QP_STATE_ERR:
2258 break;
2259 default:
2260 /* Invalid state change. */
2261 status = -EINVAL;
2262 break;
790b57f6 2263 }
cecbcddf
RA		 2264 break;
2265 case QED_ROCE_QP_STATE_RTR:
2266 /* RTR->XXX */
2267 switch (new_state) {
2268 case QED_ROCE_QP_STATE_RTS:
2269 break;
2270 case QED_ROCE_QP_STATE_ERR:
2271 break;
2272 default:
2273 /* Invalid state change. */
2274 status = -EINVAL;
2275 break;
790b57f6 2276 }
cecbcddf
RA		 2277 break;
2278 case QED_ROCE_QP_STATE_RTS:
2279 /* RTS->XXX */
2280 switch (new_state) {
2281 case QED_ROCE_QP_STATE_SQD:
2282 break;
2283 case QED_ROCE_QP_STATE_ERR:
2284 break;
2285 default:
2286 /* Invalid state change. */
2287 status = -EINVAL;
2288 break;
790b57f6 2289 }
cecbcddf
RA		 2290 break;
2291 case QED_ROCE_QP_STATE_SQD:
2292 /* SQD->XXX */
2293 switch (new_state) {
2294 case QED_ROCE_QP_STATE_RTS:
2295 case QED_ROCE_QP_STATE_ERR:
2296 break;
2297 default:
2298 /* Invalid state change. */
2299 status = -EINVAL;
2300 break;
790b57f6 2301 }
cecbcddf
RA		 2302 break;
2303 case QED_ROCE_QP_STATE_ERR:
2304 /* ERR->XXX */
2305 switch (new_state) {
2306 case QED_ROCE_QP_STATE_RESET:
933e6dca
RA		 2307 if ((qp->rq.prod != qp->rq.cons) ||
2308 (qp->sq.prod != qp->sq.cons)) {
2309 DP_NOTICE(dev,
2310 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2311 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2312 qp->sq.cons);
2313 status = -EINVAL;
2314 }
cecbcddf
RA		 2315 break;
2316 default:
2317 status = -EINVAL;
2318 break;
790b57f6 2319 }
cecbcddf
RA		 2320 break;
2321 default:
2322 status = -EINVAL;
2323 break;
790b57f6 2324 }
cecbcddf
RA		 2325
2326 return status;
2327}
2328
2329int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2330 int attr_mask, struct ib_udata *udata)
2331{
2332 struct qedr_qp *qp = get_qedr_qp(ibqp);
2333 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2334 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
d8966fcd 2335 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
cecbcddf 2336 enum ib_qp_state old_qp_state, new_qp_state;
caf61b1b 2337 enum qed_roce_qp_state cur_state;
cecbcddf
RA		 2338 int rc = 0;
2339
2340 DP_DEBUG(dev, QEDR_MSG_QP,
2341 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2342 attr->qp_state);
2343
2344 old_qp_state = qedr_get_ibqp_state(qp->state);
2345 if (attr_mask & IB_QP_STATE)
2346 new_qp_state = attr->qp_state;
2347 else
2348 new_qp_state = old_qp_state;
2349
f5b1b177
KM		 2350 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2351 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
d31131bb 2352 ibqp->qp_type, attr_mask)) {
f5b1b177
KM		 2353 DP_ERR(dev,
2354 "modify qp: invalid attribute mask=0x%x specified for\n"
2355 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2356 attr_mask, qp->qp_id, ibqp->qp_type,
2357 old_qp_state, new_qp_state);
2358 rc = -EINVAL;
2359 goto err;
2360 }
cecbcddf
RA		 2361 }
2362
2363 /* Translate the masks... */
2364 if (attr_mask & IB_QP_STATE) {
2365 SET_FIELD(qp_params.modify_flags,
2366 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2367 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2368 }
2369
2370 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2371 qp_params.sqd_async = true;
2372
2373 if (attr_mask & IB_QP_PKEY_INDEX) {
2374 SET_FIELD(qp_params.modify_flags,
2375 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2376 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2377 rc = -EINVAL;
2378 goto err;
2379 }
2380
2381 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2382 }
2383
2384 if (attr_mask & IB_QP_QKEY)
2385 qp->qkey = attr->qkey;
2386
2387 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2388 SET_FIELD(qp_params.modify_flags,
2389 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2390 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2391 IB_ACCESS_REMOTE_READ;
2392 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2393 IB_ACCESS_REMOTE_WRITE;
2394 qp_params.incoming_atomic_en = attr->qp_access_flags &
2395 IB_ACCESS_REMOTE_ATOMIC;
2396 }
2397
2398 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
425cf5c1
KM		 2399 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2400 return -EINVAL;
2401
cecbcddf
RA		 2402 if (attr_mask & IB_QP_PATH_MTU) {
2403 if (attr->path_mtu < IB_MTU_256 ||
2404 attr->path_mtu > IB_MTU_4096) {
2405 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2406 rc = -EINVAL;
2407 goto err;
2408 }
2409 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2410 ib_mtu_enum_to_int(iboe_get_mtu
2411 (dev->ndev->mtu)));
2412 }
2413
2414 if (!qp->mtu) {
2415 qp->mtu =
2416 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2417 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2418 }
2419
2420 SET_FIELD(qp_params.modify_flags,
2421 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2422
d8966fcd
DC		 2423 qp_params.traffic_class_tos = grh->traffic_class;
2424 qp_params.flow_label = grh->flow_label;
2425 qp_params.hop_limit_ttl = grh->hop_limit;
cecbcddf 2426
d8966fcd 2427 qp->sgid_idx = grh->sgid_index;
cecbcddf
RA		 2428
2429 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2430 if (rc) {
2431 DP_ERR(dev,
2432 "modify qp: problems with GID index %d (rc=%d)\n",
d8966fcd 2433 grh->sgid_index, rc);
cecbcddf
RA		 2434 return rc;
2435 }
2436
2437 rc = qedr_get_dmac(dev, &attr->ah_attr,
2438 qp_params.remote_mac_addr);
2439 if (rc)
2440 return rc;
2441
2442 qp_params.use_local_mac = true;
2443 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2444
2445 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2446 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2447 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2448 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2449 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2450 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2451 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2452 qp_params.remote_mac_addr);
cecbcddf
RA		 2453
2454 qp_params.mtu = qp->mtu;
2455 qp_params.lb_indication = false;
2456 }
2457
2458 if (!qp_params.mtu) {
2459 /* Stay with current MTU */
2460 if (qp->mtu)
2461 qp_params.mtu = qp->mtu;
2462 else
2463 qp_params.mtu =
2464 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2465 }
2466
2467 if (attr_mask & IB_QP_TIMEOUT) {
2468 SET_FIELD(qp_params.modify_flags,
2469 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2470
c3594f22
KM		 2471 /* The received timeout value is an exponent used like this:
2472 * "12.7.34 LOCAL ACK TIMEOUT
2473 * Value representing the transport (ACK) timeout for use by
2474 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2475 * The FW expects timeout in msec so we need to divide the usec
2476 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2477 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2478 * The value of zero means infinite so we use a 'max_t' to make
2479 * sure that sub 1 msec values will be configured as 1 msec.
2480 */
2481 if (attr->timeout)
2482 qp_params.ack_timeout =
2483 1 << max_t(int, attr->timeout - 8, 0);
2484 else
cecbcddf 2485 qp_params.ack_timeout = 0;
cecbcddf 2486 }
c3594f22 2487
cecbcddf
RA		 2488 if (attr_mask & IB_QP_RETRY_CNT) {
2489 SET_FIELD(qp_params.modify_flags,
2490 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2491 qp_params.retry_cnt = attr->retry_cnt;
2492 }
2493
2494 if (attr_mask & IB_QP_RNR_RETRY) {
2495 SET_FIELD(qp_params.modify_flags,
2496 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2497 qp_params.rnr_retry_cnt = attr->rnr_retry;
2498 }
2499
2500 if (attr_mask & IB_QP_RQ_PSN) {
2501 SET_FIELD(qp_params.modify_flags,
2502 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2503 qp_params.rq_psn = attr->rq_psn;
2504 qp->rq_psn = attr->rq_psn;
2505 }
2506
2507 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2508 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2509 rc = -EINVAL;
2510 DP_ERR(dev,
2511 "unsupported max_rd_atomic=%d, supported=%d\n",
2512 attr->max_rd_atomic,
2513 dev->attr.max_qp_req_rd_atomic_resc);
2514 goto err;
2515 }
2516
2517 SET_FIELD(qp_params.modify_flags,
2518 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2519 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2520 }
2521
2522 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2523 SET_FIELD(qp_params.modify_flags,
2524 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2525 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2526 }
2527
2528 if (attr_mask & IB_QP_SQ_PSN) {
2529 SET_FIELD(qp_params.modify_flags,
2530 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2531 qp_params.sq_psn = attr->sq_psn;
2532 qp->sq_psn = attr->sq_psn;
2533 }
2534
2535 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2536 if (attr->max_dest_rd_atomic >
2537 dev->attr.max_qp_resp_rd_atomic_resc) {
2538 DP_ERR(dev,
2539 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2540 attr->max_dest_rd_atomic,
2541 dev->attr.max_qp_resp_rd_atomic_resc);
2542
2543 rc = -EINVAL;
2544 goto err;
2545 }
2546
2547 SET_FIELD(qp_params.modify_flags,
2548 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2549 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2550 }
2551
2552 if (attr_mask & IB_QP_DEST_QPN) {
2553 SET_FIELD(qp_params.modify_flags,
2554 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2555
2556 qp_params.dest_qp = attr->dest_qp_num;
2557 qp->dest_qp_num = attr->dest_qp_num;
2558 }
2559
caf61b1b
KM		 2560 cur_state = qp->state;
2561
2562 /* Update the QP state before the actual ramrod to prevent a race with
2563 * fast path. Modifying the QP state to error will cause the device to
2564 * flush the CQEs and while polling the flushed CQEs will considered as
2565 * a potential issue if the QP isn't in error state.
2566 */
2567 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2568 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2569 qp->state = QED_ROCE_QP_STATE_ERR;
2570
cecbcddf
RA		 2571 if (qp->qp_type != IB_QPT_GSI)
2572 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2573 qp->qed_qp, &qp_params);
2574
2575 if (attr_mask & IB_QP_STATE) {
2576 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
caf61b1b
KM		 2577 rc = qedr_update_qp_state(dev, qp, cur_state,
2578 qp_params.new_state);
cecbcddf
RA		 2579 qp->state = qp_params.new_state;
2580 }
2581
2582err:
2583 return rc;
2584}
2585
2586static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2587{
2588 int ib_qp_acc_flags = 0;
2589
2590 if (params->incoming_rdma_write_en)
2591 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2592 if (params->incoming_rdma_read_en)
2593 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2594 if (params->incoming_atomic_en)
2595 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2596 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2597 return ib_qp_acc_flags;
2598}
2599
2600int qedr_query_qp(struct ib_qp *ibqp,
2601 struct ib_qp_attr *qp_attr,
2602 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2603{
2604 struct qed_rdma_query_qp_out_params params;
2605 struct qedr_qp *qp = get_qedr_qp(ibqp);
2606 struct qedr_dev *dev = qp->dev;
2607 int rc = 0;
2608
2609 memset(&params, 0, sizeof(params));
2610
2611 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2612 if (rc)
2613 goto err;
2614
2615 memset(qp_attr, 0, sizeof(*qp_attr));
2616 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2617
2618 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2619 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
097b6159 2620 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
cecbcddf
RA		 2621 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2622 qp_attr->rq_psn = params.rq_psn;
2623 qp_attr->sq_psn = params.sq_psn;
2624 qp_attr->dest_qp_num = params.dest_qp;
2625
2626 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2627
2628 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2629 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2630 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2631 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
fbf58026 2632 qp_attr->cap.max_inline_data = dev->attr.max_inline;
cecbcddf
RA		 2633 qp_init_attr->cap = qp_attr->cap;
2634
44c58487 2635 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
d8966fcd
DC		 2636 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2637 params.flow_label, qp->sgid_idx,
2638 params.hop_limit_ttl, params.traffic_class_tos);
2639 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2640 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2641 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2642 qp_attr->timeout = params.timeout;
2643 qp_attr->rnr_retry = params.rnr_retry;
2644 qp_attr->retry_cnt = params.retry_cnt;
2645 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2646 qp_attr->pkey_index = params.pkey_index;
2647 qp_attr->port_num = 1;
d8966fcd
DC		 2648 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2649 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
cecbcddf
RA		 2650 qp_attr->alt_pkey_index = 0;
2651 qp_attr->alt_port_num = 0;
2652 qp_attr->alt_timeout = 0;
2653 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2654
2655 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2656 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2657 qp_attr->max_rd_atomic = params.max_rd_atomic;
2658 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2659
2660 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2661 qp_attr->cap.max_inline_data);
2662
2663err:
2664 return rc;
2665}
2666
bdeacabd
SR		 2667static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2668 struct ib_udata *udata)
df158561 2669{
97f61250
MK		 2670 struct qedr_ucontext *ctx =
2671 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2672 ibucontext);
2673 int rc;
df158561
AR		 2674
2675 if (qp->qp_type != IB_QPT_GSI) {
2676 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2677 if (rc)
2678 return rc;
2679 }
2680
82af6d19 2681 if (qp->create_type == QEDR_QP_CREATE_USER)
97f61250 2682 qedr_cleanup_user(dev, ctx, qp);
df158561
AR		 2683 else
2684 qedr_cleanup_kernel(dev, qp);
2685
2686 return 0;
2687}
2688
c4367a26 2689int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
cecbcddf
RA		 2690{
2691 struct qedr_qp *qp = get_qedr_qp(ibqp);
2692 struct qedr_dev *dev = qp->dev;
2693 struct ib_qp_attr attr;
2694 int attr_mask = 0;
cecbcddf
RA		 2695
2696 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2697 qp, qp->qp_type);
2698
f5b1b177
KM		 2699 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2700 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2701 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2702 (qp->state != QED_ROCE_QP_STATE_INIT)) {
b4c2cc48 2703
f5b1b177
KM		 2704 attr.qp_state = IB_QPS_ERR;
2705 attr_mask |= IB_QP_STATE;
cecbcddf 2706
f5b1b177
KM		 2707 /* Change the QP state to ERROR */
2708 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2709 }
e411e058 2710 } else {
82af6d19
MK		 2711 /* If connection establishment started the WAIT_FOR_CONNECT
2712 * bit will be on and we need to Wait for the establishment
2713 * to complete before destroying the qp.
2714 */
2715 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2716 &qp->iwarp_cm_flags))
2717 wait_for_completion(&qp->iwarp_cm_comp);
2718
2719 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2720 * bit will be on, and we need to wait for the disconnect to
2721 * complete before continuing. We can use the same completion,
2722 * iwarp_cm_comp, since this is the only place that waits for
2723 * this completion and it is sequential. In addition,
2724 * disconnect can't occur before the connection is fully
2725 * established, therefore if WAIT_FOR_DISCONNECT is on it
2726 * means WAIT_FOR_CONNECT is also on and the completion for
2727 * CONNECT already occurred.
2728 */
2729 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2730 &qp->iwarp_cm_flags))
2731 wait_for_completion(&qp->iwarp_cm_comp);
cecbcddf
RA		 2732 }
2733
df158561 2734 if (qp->qp_type == IB_QPT_GSI)
04886779 2735 qedr_destroy_gsi_qp(dev);
cecbcddf 2736
82af6d19
MK		 2737 /* We need to remove the entry from the xarray before we release the
2738 * qp_id to avoid a race of the qp_id being reallocated and failing
2739 * on xa_insert
2740 */
2741 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2742 xa_erase(&dev->qps, qp->qp_id);
2743
bdeacabd 2744 qedr_free_qp_resources(dev, qp, udata);
cecbcddf 2745
82af6d19
MK		 2746 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2747 qedr_iw_qp_rem_ref(&qp->ibqp);
098e345a
MK		 2748 else
2749 kfree(qp);
82af6d19 2750
cf167e5e 2751 return 0;
cecbcddf 2752}
e0290cce 2753
fa5d010c 2754int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
d3456914 2755 struct ib_udata *udata)
04886779 2756{
d3456914 2757 struct qedr_ah *ah = get_qedr_ah(ibah);
04886779 2758
fa5d010c 2759 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
04886779 2760
d3456914 2761 return 0;
04886779
RA		 2762}
2763
9a9ebf8c 2764int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
04886779
RA		 2765{
2766 struct qedr_ah *ah = get_qedr_ah(ibah);
2767
d97099fe 2768 rdma_destroy_ah_attr(&ah->attr);
9a9ebf8c 2769 return 0;
04886779
RA		 2770}
2771
e0290cce
RA		 2772static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2773{
2774 struct qedr_pbl *pbl, *tmp;
2775
2776 if (info->pbl_table)
2777 list_add_tail(&info->pbl_table->list_entry,
2778 &info->free_pbl_list);
2779
2780 if (!list_empty(&info->inuse_pbl_list))
2781 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2782
2783 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2784 list_del(&pbl->list_entry);
2785 qedr_free_pbl(dev, &info->pbl_info, pbl);
2786 }
2787}
2788
2789static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2790 size_t page_list_len, bool two_layered)
2791{
2792 struct qedr_pbl *tmp;
2793 int rc;
2794
2795 INIT_LIST_HEAD(&info->free_pbl_list);
2796 INIT_LIST_HEAD(&info->inuse_pbl_list);
2797
2798 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2799 page_list_len, two_layered);
2800 if (rc)
2801 goto done;
2802
2803 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf
CJ		 2804 if (IS_ERR(info->pbl_table)) {
2805 rc = PTR_ERR(info->pbl_table);
e0290cce
RA		 2806 goto done;
2807 }
2808
2809 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2810 &info->pbl_table->pa);
2811
2812 /* in usual case we use 2 PBLs, so we add one to free
2813 * list and allocating another one
2814 */
2815 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
4cd33aaf 2816 if (IS_ERR(tmp)) {
e0290cce
RA		 2817 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2818 goto done;
2819 }
2820
2821 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2822
2823 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2824
2825done:
2826 if (rc)
2827 free_mr_info(dev, info);
2828
2829 return rc;
2830}
2831
2832struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2833 u64 usr_addr, int acc, struct ib_udata *udata)
2834{
2835 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2836 struct qedr_mr *mr;
2837 struct qedr_pd *pd;
2838 int rc = -ENOMEM;
2839
2840 pd = get_qedr_pd(ibpd);
2841 DP_DEBUG(dev, QEDR_MSG_MR,
2842 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2843 pd->pd_id, start, len, usr_addr, acc);
2844
2845 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2846 return ERR_PTR(-EINVAL);
2847
2848 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2849 if (!mr)
2850 return ERR_PTR(rc);
2851
2852 mr->type = QEDR_MR_USER;
2853
c320e527 2854 mr->umem = ib_umem_get(ibpd->device, start, len, acc);
e0290cce
RA		 2855 if (IS_ERR(mr->umem)) {
2856 rc = -EFAULT;
2857 goto err0;
2858 }
2859
901bca71
JG		 2860 rc = init_mr_info(dev, &mr->info,
2861 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
e0290cce
RA		 2862 if (rc)
2863 goto err1;
2864
2865 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
95ad233f 2866 &mr->info.pbl_info, PAGE_SHIFT);
e0290cce
RA		 2867
2868 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2869 if (rc) {
2870 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2871 goto err1;
2872 }
2873
2874 /* Index only, 18 bit long, lkey = itid << 8 | key */
2875 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2876 mr->hw_mr.key = 0;
2877 mr->hw_mr.pd = pd->pd_id;
2878 mr->hw_mr.local_read = 1;
2879 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2880 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2881 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2882 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2883 mr->hw_mr.mw_bind = false;
2884 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2885 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2886 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
95ad233f 2887 mr->hw_mr.page_size_log = PAGE_SHIFT;
e0290cce
RA		 2888 mr->hw_mr.length = len;
2889 mr->hw_mr.vaddr = usr_addr;
e0290cce
RA		 2890 mr->hw_mr.phy_mr = false;
2891 mr->hw_mr.dma_mr = false;
2892
2893 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2894 if (rc) {
2895 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2896 goto err2;
2897 }
2898
2899 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2900 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2901 mr->hw_mr.remote_atomic)
2902 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2903
2904 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2905 mr->ibmr.lkey);
2906 return &mr->ibmr;
2907
2908err2:
2909 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2910err1:
2911 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2912err0:
2913 kfree(mr);
2914 return ERR_PTR(rc);
2915}
2916
c4367a26 2917int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
e0290cce
RA		 2918{
2919 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2920 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2921 int rc = 0;
2922
2923 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2924 if (rc)
2925 return rc;
2926
2927 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2928
24e412c1
MK		 2929 if (mr->type != QEDR_MR_DMA)
2930 free_mr_info(dev, &mr->info);
e0290cce
RA		 2931
2932 /* it could be user registered memory. */
836a0fbb 2933 ib_umem_release(mr->umem);
e0290cce
RA		 2934
2935 kfree(mr);
2936
2937 return rc;
2938}
2939
27a4b1a6
RA		 2940static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2941 int max_page_list_len)
e0290cce
RA		 2942{
2943 struct qedr_pd *pd = get_qedr_pd(ibpd);
2944 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2945 struct qedr_mr *mr;
2946 int rc = -ENOMEM;
2947
2948 DP_DEBUG(dev, QEDR_MSG_MR,
2949 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2950 max_page_list_len);
2951
2952 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2953 if (!mr)
2954 return ERR_PTR(rc);
2955
2956 mr->dev = dev;
2957 mr->type = QEDR_MR_FRMR;
2958
2959 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2960 if (rc)
2961 goto err0;
2962
2963 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2964 if (rc) {
2965 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2966 goto err0;
2967 }
2968
2969 /* Index only, 18 bit long, lkey = itid << 8 | key */
2970 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2971 mr->hw_mr.key = 0;
2972 mr->hw_mr.pd = pd->pd_id;
2973 mr->hw_mr.local_read = 1;
2974 mr->hw_mr.local_write = 0;
2975 mr->hw_mr.remote_read = 0;
2976 mr->hw_mr.remote_write = 0;
2977 mr->hw_mr.remote_atomic = 0;
2978 mr->hw_mr.mw_bind = false;
2979 mr->hw_mr.pbl_ptr = 0;
2980 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2981 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
e0290cce
RA		 2982 mr->hw_mr.length = 0;
2983 mr->hw_mr.vaddr = 0;
e0290cce
RA		 2984 mr->hw_mr.phy_mr = true;
2985 mr->hw_mr.dma_mr = false;
2986
2987 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2988 if (rc) {
2989 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2990 goto err1;
2991 }
2992
2993 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2994 mr->ibmr.rkey = mr->ibmr.lkey;
2995
2996 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2997 return mr;
2998
2999err1:
3000 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3001err0:
3002 kfree(mr);
3003 return ERR_PTR(rc);
3004}
3005
c4367a26 3006struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
42a3b153 3007 u32 max_num_sg)
e0290cce 3008{
e0290cce
RA		 3009 struct qedr_mr *mr;
3010
3011 if (mr_type != IB_MR_TYPE_MEM_REG)
3012 return ERR_PTR(-EINVAL);
3013
3014 mr = __qedr_alloc_mr(ibpd, max_num_sg);
3015
3016 if (IS_ERR(mr))
3017 return ERR_PTR(-EINVAL);
3018
e0290cce
RA		 3019 return &mr->ibmr;
3020}
3021
3022static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3023{
3024 struct qedr_mr *mr = get_qedr_mr(ibmr);
3025 struct qedr_pbl *pbl_table;
3026 struct regpair *pbe;
3027 u32 pbes_in_page;
3028
3029 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
ffab8c89 3030 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
e0290cce
RA		 3031 return -ENOMEM;
3032 }
3033
3034 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3035 mr->npages, addr);
3036
3037 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3038 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3039 pbe = (struct regpair *)pbl_table->va;
3040 pbe += mr->npages % pbes_in_page;
3041 pbe->lo = cpu_to_le32((u32)addr);
3042 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3043
3044 mr->npages++;
3045
3046 return 0;
3047}
3048
3049static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3050{
3051 int work = info->completed - info->completed_handled - 1;
3052
3053 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3054 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3055 struct qedr_pbl *pbl;
3056
3057 /* Free all the page list that are possible to be freed
3058 * (all the ones that were invalidated), under the assumption
3059 * that if an FMR was completed successfully that means that
3060 * if there was an invalidate operation before it also ended
3061 */
3062 pbl = list_first_entry(&info->inuse_pbl_list,
3063 struct qedr_pbl, list_entry);
aafec388 3064 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
e0290cce
RA		 3065 info->completed_handled++;
3066 }
3067}
3068
3069int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3070 int sg_nents, unsigned int *sg_offset)
3071{
3072 struct qedr_mr *mr = get_qedr_mr(ibmr);
3073
3074 mr->npages = 0;
3075
3076 handle_completed_mrs(mr->dev, &mr->info);
3077 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3078}
3079
3080struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3081{
3082 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3083 struct qedr_pd *pd = get_qedr_pd(ibpd);
3084 struct qedr_mr *mr;
3085 int rc;
3086
3087 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3088 if (!mr)
3089 return ERR_PTR(-ENOMEM);
3090
3091 mr->type = QEDR_MR_DMA;
3092
3093 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3094 if (rc) {
3095 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3096 goto err1;
3097 }
3098
3099 /* index only, 18 bit long, lkey = itid << 8 | key */
3100 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3101 mr->hw_mr.pd = pd->pd_id;
3102 mr->hw_mr.local_read = 1;
3103 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3104 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3105 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3106 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3107 mr->hw_mr.dma_mr = true;
3108
3109 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3110 if (rc) {
3111 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3112 goto err2;
3113 }
3114
3115 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3116 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3117 mr->hw_mr.remote_atomic)
3118 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3119
3120 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3121 return &mr->ibmr;
3122
3123err2:
3124 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3125err1:
3126 kfree(mr);
3127 return ERR_PTR(rc);
3128}
afa0e13b
RA		 3129
3130static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3131{
3132 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3133}
3134
3135static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3136{
3137 int i, len = 0;
3138
3139 for (i = 0; i < num_sge; i++)
3140 len += sg_list[i].length;
3141
3142 return len;
3143}
3144
3145static void swap_wqe_data64(u64 *p)
3146{
3147 int i;
3148
3149 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3150 *p = cpu_to_be64(cpu_to_le64(*p));
3151}
3152
3153static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3154 struct qedr_qp *qp, u8 *wqe_size,
d34ac5cd
BVA		 3155 const struct ib_send_wr *wr,
3156 const struct ib_send_wr **bad_wr,
3157 u8 *bits, u8 bit)
afa0e13b
RA		 3158{
3159 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3160 char *seg_prt, *wqe;
3161 int i, seg_siz;
3162
3163 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3164 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3165 *bad_wr = wr;
3166 return 0;
3167 }
3168
3169 if (!data_size)
3170 return data_size;
3171
3172 *bits |= bit;
3173
3174 seg_prt = NULL;
3175 wqe = NULL;
3176 seg_siz = 0;
3177
3178 /* Copy data inline */
3179 for (i = 0; i < wr->num_sge; i++) {
3180 u32 len = wr->sg_list[i].length;
3181 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3182
3183 while (len > 0) {
3184 u32 cur;
3185
3186 /* New segment required */
3187 if (!seg_siz) {
3188 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3189 seg_prt = wqe;
3190 seg_siz = sizeof(struct rdma_sq_common_wqe);
3191 (*wqe_size)++;
3192 }
3193
3194 /* Calculate currently allowed length */
3195 cur = min_t(u32, len, seg_siz);
3196 memcpy(seg_prt, src, cur);
3197
3198 /* Update segment variables */
3199 seg_prt += cur;
3200 seg_siz -= cur;
3201
3202 /* Update sge variables */
3203 src += cur;
3204 len -= cur;
3205
3206 /* Swap fully-completed segments */
3207 if (!seg_siz)
3208 swap_wqe_data64((u64 *)wqe);
3209 }
3210 }
3211
3212 /* swap last not completed segment */
3213 if (seg_siz)
3214 swap_wqe_data64((u64 *)wqe);
3215
3216 return data_size;
3217}
3218
3219#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3220 do { \
3221 DMA_REGPAIR_LE(sge->addr, vaddr); \
3222 (sge)->length = cpu_to_le32(vlength); \
3223 (sge)->flags = cpu_to_le32(vflags); \
3224 } while (0)
3225
3226#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3227 do { \
3228 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3229 (hdr)->num_sges = num_sge; \
3230 } while (0)
3231
3232#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3233 do { \
3234 DMA_REGPAIR_LE(sge->addr, vaddr); \
3235 (sge)->length = cpu_to_le32(vlength); \
3236 (sge)->l_key = cpu_to_le32(vlkey); \
3237 } while (0)
3238
3239static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
f696bf6d 3240 const struct ib_send_wr *wr)
afa0e13b
RA		 3241{
3242 u32 data_size = 0;
3243 int i;
3244
3245 for (i = 0; i < wr->num_sge; i++) {
3246 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3247
3248 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3249 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3250 sge->length = cpu_to_le32(wr->sg_list[i].length);
3251 data_size += wr->sg_list[i].length;
3252 }
3253
3254 if (wqe_size)
3255 *wqe_size += wr->num_sge;
3256
3257 return data_size;
3258}
3259
3260static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3261 struct qedr_qp *qp,
3262 struct rdma_sq_rdma_wqe_1st *rwqe,
3263 struct rdma_sq_rdma_wqe_2nd *rwqe2,
d34ac5cd
BVA		 3264 const struct ib_send_wr *wr,
3265 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3266{
3267 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3268 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3269
8b0cabc6
AR		 3270 if (wr->send_flags & IB_SEND_INLINE &&
3271 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3272 wr->opcode == IB_WR_RDMA_WRITE)) {
afa0e13b
RA		 3273 u8 flags = 0;
3274
3275 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3276 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3277 bad_wr, &rwqe->flags, flags);
3278 }
3279
3280 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3281}
3282
3283static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3284 struct qedr_qp *qp,
3285 struct rdma_sq_send_wqe_1st *swqe,
3286 struct rdma_sq_send_wqe_2st *swqe2,
d34ac5cd
BVA		 3287 const struct ib_send_wr *wr,
3288 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3289{
3290 memset(swqe2, 0, sizeof(*swqe2));
3291 if (wr->send_flags & IB_SEND_INLINE) {
3292 u8 flags = 0;
3293
3294 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3295 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3296 bad_wr, &swqe->flags, flags);
3297 }
3298
3299 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3300}
3301
3302static int qedr_prepare_reg(struct qedr_qp *qp,
3303 struct rdma_sq_fmr_wqe_1st *fwqe1,
f696bf6d 3304 const struct ib_reg_wr *wr)
afa0e13b
RA		 3305{
3306 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3307 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3308
3309 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3310 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3311 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3312 fwqe1->l_key = wr->key;
3313
08c4cf51
AR		 3314 fwqe2->access_ctrl = 0;
3315
afa0e13b
RA		 3316 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3317 !!(wr->access & IB_ACCESS_REMOTE_READ));
3318 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3319 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3320 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3321 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3322 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3323 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3324 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3325 fwqe2->fmr_ctrl = 0;
3326
3327 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3328 ilog2(mr->ibmr.page_size) - 12);
3329
3330 fwqe2->length_hi = 0;
3331 fwqe2->length_lo = mr->ibmr.length;
3332 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3333 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3334
3335 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3336
3337 return 0;
3338}
3339
27a4b1a6 3340static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
afa0e13b
RA		 3341{
3342 switch (opcode) {
3343 case IB_WR_RDMA_WRITE:
3344 case IB_WR_RDMA_WRITE_WITH_IMM:
3345 return IB_WC_RDMA_WRITE;
3346 case IB_WR_SEND_WITH_IMM:
3347 case IB_WR_SEND:
3348 case IB_WR_SEND_WITH_INV:
3349 return IB_WC_SEND;
3350 case IB_WR_RDMA_READ:
fb1a22be 3351 case IB_WR_RDMA_READ_WITH_INV:
afa0e13b
RA		 3352 return IB_WC_RDMA_READ;
3353 case IB_WR_ATOMIC_CMP_AND_SWP:
3354 return IB_WC_COMP_SWAP;
3355 case IB_WR_ATOMIC_FETCH_AND_ADD:
3356 return IB_WC_FETCH_ADD;
3357 case IB_WR_REG_MR:
3358 return IB_WC_REG_MR;
3359 case IB_WR_LOCAL_INV:
3360 return IB_WC_LOCAL_INV;
3361 default:
3362 return IB_WC_SEND;
3363 }
3364}
3365
f696bf6d
BVA		 3366static inline bool qedr_can_post_send(struct qedr_qp *qp,
3367 const struct ib_send_wr *wr)
afa0e13b
RA		 3368{
3369 int wq_is_full, err_wr, pbl_is_full;
3370 struct qedr_dev *dev = qp->dev;
3371
3372 /* prevent SQ overflow and/or processing of a bad WR */
3373 err_wr = wr->num_sge > qp->sq.max_sges;
3374 wq_is_full = qedr_wq_is_full(&qp->sq);
3375 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3376 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3377 if (wq_is_full || err_wr || pbl_is_full) {
3378 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3379 DP_ERR(dev,
3380 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3381 qp);
3382 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3383 }
3384
3385 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3386 DP_ERR(dev,
3387 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3388 qp);
3389 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3390 }
3391
3392 if (pbl_is_full &&
3393 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3394 DP_ERR(dev,
3395 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3396 qp);
3397 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3398 }
3399 return false;
3400 }
3401 return true;
3402}
3403
d34ac5cd
BVA		 3404static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3405 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3406{
3407 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3408 struct qedr_qp *qp = get_qedr_qp(ibqp);
3409 struct rdma_sq_atomic_wqe_1st *awqe1;
3410 struct rdma_sq_atomic_wqe_2nd *awqe2;
3411 struct rdma_sq_atomic_wqe_3rd *awqe3;
3412 struct rdma_sq_send_wqe_2st *swqe2;
3413 struct rdma_sq_local_inv_wqe *iwqe;
3414 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3415 struct rdma_sq_send_wqe_1st *swqe;
3416 struct rdma_sq_rdma_wqe_1st *rwqe;
3417 struct rdma_sq_fmr_wqe_1st *fwqe1;
3418 struct rdma_sq_common_wqe *wqe;
3419 u32 length;
3420 int rc = 0;
3421 bool comp;
3422
3423 if (!qedr_can_post_send(qp, wr)) {
3424 *bad_wr = wr;
3425 return -ENOMEM;
3426 }
3427
3428 wqe = qed_chain_produce(&qp->sq.pbl);
3429 qp->wqe_wr_id[qp->sq.prod].signaled =
3430 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3431
3432 wqe->flags = 0;
3433 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3434 !!(wr->send_flags & IB_SEND_SOLICITED));
3435 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3436 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3437 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3438 !!(wr->send_flags & IB_SEND_FENCE));
3439 wqe->prev_wqe_size = qp->prev_wqe_size;
3440
3441 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3442
3443 switch (wr->opcode) {
3444 case IB_WR_SEND_WITH_IMM:
551e1c67
KM		 3445 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3446 rc = -EINVAL;
3447 *bad_wr = wr;
3448 break;
3449 }
afa0e13b
RA		 3450 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3451 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3452 swqe->wqe_size = 2;
3453 swqe2 = qed_chain_produce(&qp->sq.pbl);
3454
7bed7ebc 3455 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
afa0e13b
RA		 3456 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3457 wr, bad_wr);
3458 swqe->length = cpu_to_le32(length);
3459 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3460 qp->prev_wqe_size = swqe->wqe_size;
3461 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3462 break;
3463 case IB_WR_SEND:
3464 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3465 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3466
3467 swqe->wqe_size = 2;
3468 swqe2 = qed_chain_produce(&qp->sq.pbl);
3469 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3470 wr, bad_wr);
3471 swqe->length = cpu_to_le32(length);
3472 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3473 qp->prev_wqe_size = swqe->wqe_size;
3474 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3475 break;
3476 case IB_WR_SEND_WITH_INV:
3477 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3478 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3479 swqe2 = qed_chain_produce(&qp->sq.pbl);
3480 swqe->wqe_size = 2;
3481 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3482 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3483 wr, bad_wr);
3484 swqe->length = cpu_to_le32(length);
3485 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3486 qp->prev_wqe_size = swqe->wqe_size;
3487 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3488 break;
3489
3490 case IB_WR_RDMA_WRITE_WITH_IMM:
551e1c67
KM		 3491 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3492 rc = -EINVAL;
3493 *bad_wr = wr;
3494 break;
3495 }
afa0e13b
RA		 3496 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3497 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3498
3499 rwqe->wqe_size = 2;
3500 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3501 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3502 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3503 wr, bad_wr);
3504 rwqe->length = cpu_to_le32(length);
3505 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3506 qp->prev_wqe_size = rwqe->wqe_size;
3507 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3508 break;
3509 case IB_WR_RDMA_WRITE:
3510 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3511 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3512
3513 rwqe->wqe_size = 2;
3514 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3515 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3516 wr, bad_wr);
3517 rwqe->length = cpu_to_le32(length);
3518 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3519 qp->prev_wqe_size = rwqe->wqe_size;
3520 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3521 break;
3522 case IB_WR_RDMA_READ_WITH_INV:
fb1a22be 3523 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
df561f66 3524 fallthrough; /* same is identical to RDMA READ */
afa0e13b
RA		 3525
3526 case IB_WR_RDMA_READ:
3527 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3528 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3529
3530 rwqe->wqe_size = 2;
3531 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3532 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3533 wr, bad_wr);
3534 rwqe->length = cpu_to_le32(length);
3535 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3536 qp->prev_wqe_size = rwqe->wqe_size;
3537 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3538 break;
3539
3540 case IB_WR_ATOMIC_CMP_AND_SWP:
3541 case IB_WR_ATOMIC_FETCH_AND_ADD:
3542 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3543 awqe1->wqe_size = 4;
3544
3545 awqe2 = qed_chain_produce(&qp->sq.pbl);
3546 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3547 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3548
3549 awqe3 = qed_chain_produce(&qp->sq.pbl);
3550
3551 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3552 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3553 DMA_REGPAIR_LE(awqe3->swap_data,
3554 atomic_wr(wr)->compare_add);
3555 } else {
3556 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3557 DMA_REGPAIR_LE(awqe3->swap_data,
3558 atomic_wr(wr)->swap);
3559 DMA_REGPAIR_LE(awqe3->cmp_data,
3560 atomic_wr(wr)->compare_add);
3561 }
3562
3563 qedr_prepare_sq_sges(qp, NULL, wr);
3564
3565 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3566 qp->prev_wqe_size = awqe1->wqe_size;
3567 break;
3568
3569 case IB_WR_LOCAL_INV:
3570 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3571 iwqe->wqe_size = 1;
3572
3573 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3574 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3575 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3576 qp->prev_wqe_size = iwqe->wqe_size;
3577 break;
3578 case IB_WR_REG_MR:
3579 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3580 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3581 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3582 fwqe1->wqe_size = 2;
3583
3584 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3585 if (rc) {
3586 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3587 *bad_wr = wr;
3588 break;
3589 }
3590
3591 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3592 qp->prev_wqe_size = fwqe1->wqe_size;
3593 break;
3594 default:
3595 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3596 rc = -EINVAL;
3597 *bad_wr = wr;
3598 break;
3599 }
3600
3601 if (*bad_wr) {
3602 u16 value;
3603
3604 /* Restore prod to its position before
3605 * this WR was processed
3606 */
3607 value = le16_to_cpu(qp->sq.db_data.data.value);
3608 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3609
3610 /* Restore prev_wqe_size */
3611 qp->prev_wqe_size = wqe->prev_wqe_size;
3612 rc = -EINVAL;
3613 DP_ERR(dev, "POST SEND FAILED\n");
3614 }
3615
3616 return rc;
3617}
3618
d34ac5cd
BVA		 3619int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3620 const struct ib_send_wr **bad_wr)
afa0e13b
RA		 3621{
3622 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3623 struct qedr_qp *qp = get_qedr_qp(ibqp);
3624 unsigned long flags;
3625 int rc = 0;
3626
3627 *bad_wr = NULL;
3628
04886779
RA		 3629 if (qp->qp_type == IB_QPT_GSI)
3630 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3631
afa0e13b
RA		 3632 spin_lock_irqsave(&qp->q_lock, flags);
3633
f5b1b177
KM		 3634 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3635 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3636 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3637 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3638 spin_unlock_irqrestore(&qp->q_lock, flags);
3639 *bad_wr = wr;
3640 DP_DEBUG(dev, QEDR_MSG_CQ,
3641 "QP in wrong state! QP icid=0x%x state %d\n",
3642 qp->icid, qp->state);
3643 return -EINVAL;
3644 }
afa0e13b
RA		 3645 }
3646
afa0e13b
RA		 3647 while (wr) {
3648 rc = __qedr_post_send(ibqp, wr, bad_wr);
3649 if (rc)
3650 break;
3651
3652 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3653
3654 qedr_inc_sw_prod(&qp->sq);
3655
3656 qp->sq.db_data.data.value++;
3657
3658 wr = wr->next;
3659 }
3660
3661 /* Trigger doorbell
3662 * If there was a failure in the first WR then it will be triggered in
3663 * vane. However this is not harmful (as long as the producer value is
3664 * unchanged). For performance reasons we avoid checking for this
3665 * redundant doorbell.
09c4854f
KM		 3666 *
3667 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3668 * soon as we give the doorbell, we could get a completion
3669 * for this wr, therefore we need to make sure that the
3670 * memory is updated before giving the doorbell.
3671 * During qedr_poll_cq, rmb is called before accessing the
3672 * cqe. This covers for the smp_rmb as well.
afa0e13b 3673 */
09c4854f 3674 smp_wmb();
afa0e13b
RA		 3675 writel(qp->sq.db_data.raw, qp->sq.db);
3676
afa0e13b
RA		 3677 spin_unlock_irqrestore(&qp->q_lock, flags);
3678
3679 return rc;
3680}
3681
3491c9e7
YB		 3682static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3683{
3684 u32 used;
3685
3686 /* Calculate number of elements used based on producer
3687 * count and consumer count and subtract it from max
3688 * work request supported so that we get elements left.
3689 */
acca72e2 3690 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3491c9e7
YB		 3691
3692 return hw_srq->max_wr - used;
3693}
3694
3695int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3696 const struct ib_recv_wr **bad_wr)
3697{
3698 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3699 struct qedr_srq_hwq_info *hw_srq;
3700 struct qedr_dev *dev = srq->dev;
3701 struct qed_chain *pbl;
3702 unsigned long flags;
3703 int status = 0;
3704 u32 num_sge;
3491c9e7
YB		 3705
3706 spin_lock_irqsave(&srq->lock, flags);
3707
3708 hw_srq = &srq->hw_srq;
3709 pbl = &srq->hw_srq.pbl;
3710 while (wr) {
3711 struct rdma_srq_wqe_header *hdr;
3712 int i;
3713
3714 if (!qedr_srq_elem_left(hw_srq) ||
3715 wr->num_sge > srq->hw_srq.max_sges) {
3716 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
acca72e2
YB		 3717 hw_srq->wr_prod_cnt,
3718 atomic_read(&hw_srq->wr_cons_cnt),
3491c9e7
YB		 3719 wr->num_sge, srq->hw_srq.max_sges);
3720 status = -ENOMEM;
3721 *bad_wr = wr;
3722 break;
3723 }
3724
3725 hdr = qed_chain_produce(pbl);
3726 num_sge = wr->num_sge;
3727 /* Set number of sge and work request id in header */
3728 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3729
3730 srq->hw_srq.wr_prod_cnt++;
3731 hw_srq->wqe_prod++;
3732 hw_srq->sge_prod++;
3733
3734 DP_DEBUG(dev, QEDR_MSG_SRQ,
3735 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3736 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3737
3738 for (i = 0; i < wr->num_sge; i++) {
3739 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3740
3741 /* Set SGE length, lkey and address */
3742 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3743 wr->sg_list[i].length, wr->sg_list[i].lkey);
3744
3745 DP_DEBUG(dev, QEDR_MSG_SRQ,
3746 "[%d]: len %d key %x addr %x:%x\n",
3747 i, srq_sge->length, srq_sge->l_key,
3748 srq_sge->addr.hi, srq_sge->addr.lo);
3749 hw_srq->sge_prod++;
3750 }
3751
acca72e2 3752 /* Update WQE and SGE information before
3491c9e7
YB		 3753 * updating producer.
3754 */
acca72e2 3755 dma_wmb();
3491c9e7
YB		 3756
3757 /* SRQ producer is 8 bytes. Need to update SGE producer index
3758 * in first 4 bytes and need to update WQE producer in
3759 * next 4 bytes.
3760 */
acca72e2
YB		 3761 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3762 /* Make sure sge producer is updated first */
3763 dma_wmb();
3764 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3491c9e7 3765
3491c9e7
YB		 3766 wr = wr->next;
3767 }
3768
3769 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3770 qed_chain_get_elem_left(pbl));
3771 spin_unlock_irqrestore(&srq->lock, flags);
3772
3773 return status;
3774}
3775
d34ac5cd
BVA		 3776int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3777 const struct ib_recv_wr **bad_wr)
afa0e13b
RA		 3778{
3779 struct qedr_qp *qp = get_qedr_qp(ibqp);
3780 struct qedr_dev *dev = qp->dev;
3781 unsigned long flags;
3782 int status = 0;
3783
04886779
RA		 3784 if (qp->qp_type == IB_QPT_GSI)
3785 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3786
afa0e13b
RA		 3787 spin_lock_irqsave(&qp->q_lock, flags);
3788
922d9a40 3789 if (qp->state == QED_ROCE_QP_STATE_RESET) {
afa0e13b
RA		 3790 spin_unlock_irqrestore(&qp->q_lock, flags);
3791 *bad_wr = wr;
3792 return -EINVAL;
3793 }
3794
3795 while (wr) {
3796 int i;
3797
3798 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3799 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3800 wr->num_sge > qp->rq.max_sges) {
3801 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3802 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3803 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3804 qp->rq.max_sges);
3805 status = -ENOMEM;
3806 *bad_wr = wr;
3807 break;
3808 }
3809 for (i = 0; i < wr->num_sge; i++) {
3810 u32 flags = 0;
3811 struct rdma_rq_sge *rqe =
3812 qed_chain_produce(&qp->rq.pbl);
3813
3814 /* First one must include the number
3815 * of SGE in the list
3816 */
3817 if (!i)
3818 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3819 wr->num_sge);
3820
d52c89f1 3821 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
afa0e13b
RA		 3822 wr->sg_list[i].lkey);
3823
3824 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3825 wr->sg_list[i].length, flags);
3826 }
3827
3828 /* Special case of no sges. FW requires between 1-4 sges...
3829 * in this case we need to post 1 sge with length zero. this is
3830 * because rdma write with immediate consumes an RQ.
3831 */
3832 if (!wr->num_sge) {
3833 u32 flags = 0;
3834 struct rdma_rq_sge *rqe =
3835 qed_chain_produce(&qp->rq.pbl);
3836
3837 /* First one must include the number
3838 * of SGE in the list
3839 */
d52c89f1 3840 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
afa0e13b
RA		 3841 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3842
3843 RQ_SGE_SET(rqe, 0, 0, flags);
3844 i = 1;
3845 }
3846
3847 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3848 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3849
3850 qedr_inc_sw_prod(&qp->rq);
3851
09c4854f
KM		 3852 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3853 * soon as we give the doorbell, we could get a completion
3854 * for this wr, therefore we need to make sure that the
3855 * memory is update before giving the doorbell.
3856 * During qedr_poll_cq, rmb is called before accessing the
3857 * cqe. This covers for the smp_rmb as well.
3858 */
3859 smp_wmb();
afa0e13b
RA		 3860
3861 qp->rq.db_data.data.value++;
3862
3863 writel(qp->rq.db_data.raw, qp->rq.db);
3864
f5b1b177
KM		 3865 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3866 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
f5b1b177
KM		 3867 }
3868
afa0e13b
RA		 3869 wr = wr->next;
3870 }
3871
3872 spin_unlock_irqrestore(&qp->q_lock, flags);
3873
3874 return status;
3875}
3876
3877static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3878{
3879 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3880
3881 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3882 cq->pbl_toggle;
3883}
3884
3885static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3886{
3887 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3888 struct qedr_qp *qp;
3889
3890 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3891 resp_cqe->qp_handle.lo,
3892 u64);
3893 return qp;
3894}
3895
3896static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3897{
3898 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3899
3900 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3901}
3902
3903/* Return latest CQE (needs processing) */
3904static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3905{
3906 return cq->latest_cqe;
3907}
3908
3909/* In fmr we need to increase the number of fmr completed counter for the fmr
3910 * algorithm determining whether we can free a pbl or not.
3911 * we need to perform this whether the work request was signaled or not. for
3912 * this purpose we call this function from the condition that checks if a wr
3913 * should be skipped, to make sure we don't miss it ( possibly this fmr
3914 * operation was not signalted)
3915 */
3916static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3917{
3918 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3919 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3920}
3921
3922static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3923 struct qedr_cq *cq, int num_entries,
3924 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3925 int force)
3926{
3927 u16 cnt = 0;
3928
3929 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3930 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3931 qedr_chk_if_fmr(qp);
3932 /* skip WC */
3933 goto next_cqe;
3934 }
3935
3936 /* fill WC */
3937 wc->status = status;
27035a1b 3938 wc->vendor_err = 0;
afa0e13b
RA		 3939 wc->wc_flags = 0;
3940 wc->src_qp = qp->id;
3941 wc->qp = &qp->ibqp;
3942
3943 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3944 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3945
3946 switch (wc->opcode) {
3947 case IB_WC_RDMA_WRITE:
3948 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3949 break;
3950 case IB_WC_COMP_SWAP:
3951 case IB_WC_FETCH_ADD:
3952 wc->byte_len = 8;
3953 break;
3954 case IB_WC_REG_MR:
3955 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3956 break;
dac27386
MK		 3957 case IB_WC_RDMA_READ:
3958 case IB_WC_SEND:
3959 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3960 break;
afa0e13b
RA		 3961 default:
3962 break;
3963 }
3964
3965 num_entries--;
3966 wc++;
3967 cnt++;
3968next_cqe:
3969 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3970 qed_chain_consume(&qp->sq.pbl);
3971 qedr_inc_sw_cons(&qp->sq);
3972 }
3973
3974 return cnt;
3975}
3976
3977static int qedr_poll_cq_req(struct qedr_dev *dev,
3978 struct qedr_qp *qp, struct qedr_cq *cq,
3979 int num_entries, struct ib_wc *wc,
3980 struct rdma_cqe_requester *req)
3981{
3982 int cnt = 0;
3983
3984 switch (req->status) {
3985 case RDMA_CQE_REQ_STS_OK:
3986 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3987 IB_WC_SUCCESS, 0);
3988 break;
3989 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
c78c3149 3990 if (qp->state != QED_ROCE_QP_STATE_ERR)
dc728f77
KM		 3991 DP_DEBUG(dev, QEDR_MSG_CQ,
3992 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3993 cq->icid, qp->icid);
afa0e13b 3994 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
74c3875c 3995 IB_WC_WR_FLUSH_ERR, 1);
afa0e13b
RA		 3996 break;
3997 default:
3998 /* process all WQE before the cosumer */
3999 qp->state = QED_ROCE_QP_STATE_ERR;
4000 cnt = process_req(dev, qp, cq, num_entries, wc,
4001 req->sq_cons - 1, IB_WC_SUCCESS, 0);
4002 wc += cnt;
4003 /* if we have extra WC fill it with actual error info */
4004 if (cnt < num_entries) {
4005 enum ib_wc_status wc_status;
4006
4007 switch (req->status) {
4008 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4009 DP_ERR(dev,
4010 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4011 cq->icid, qp->icid);
4012 wc_status = IB_WC_BAD_RESP_ERR;
4013 break;
4014 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4015 DP_ERR(dev,
4016 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4017 cq->icid, qp->icid);
4018 wc_status = IB_WC_LOC_LEN_ERR;
4019 break;
4020 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4021 DP_ERR(dev,
4022 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4023 cq->icid, qp->icid);
4024 wc_status = IB_WC_LOC_QP_OP_ERR;
4025 break;
4026 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4027 DP_ERR(dev,
4028 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4029 cq->icid, qp->icid);
4030 wc_status = IB_WC_LOC_PROT_ERR;
4031 break;
4032 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4033 DP_ERR(dev,
4034 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4035 cq->icid, qp->icid);
4036 wc_status = IB_WC_MW_BIND_ERR;
4037 break;
4038 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4039 DP_ERR(dev,
4040 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4041 cq->icid, qp->icid);
4042 wc_status = IB_WC_REM_INV_REQ_ERR;
4043 break;
4044 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4045 DP_ERR(dev,
4046 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4047 cq->icid, qp->icid);
4048 wc_status = IB_WC_REM_ACCESS_ERR;
4049 break;
4050 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4051 DP_ERR(dev,
4052 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4053 cq->icid, qp->icid);
4054 wc_status = IB_WC_REM_OP_ERR;
4055 break;
4056 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4057 DP_ERR(dev,
4058 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4059 cq->icid, qp->icid);
4060 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4061 break;
4062 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4063 DP_ERR(dev,
4064 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4065 cq->icid, qp->icid);
4066 wc_status = IB_WC_RETRY_EXC_ERR;
4067 break;
4068 default:
4069 DP_ERR(dev,
4070 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4071 cq->icid, qp->icid);
4072 wc_status = IB_WC_GENERAL_ERR;
4073 }
4074 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4075 wc_status, 1);
4076 }
4077 }
4078
4079 return cnt;
4080}
4081
b6acd71f 4082static inline int qedr_cqe_resp_status_to_ib(u8 status)
afa0e13b 4083{
b6acd71f 4084 switch (status) {
afa0e13b 4085 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
b6acd71f 4086 return IB_WC_LOC_ACCESS_ERR;
afa0e13b 4087 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
b6acd71f 4088 return IB_WC_LOC_LEN_ERR;
afa0e13b 4089 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
b6acd71f 4090 return IB_WC_LOC_QP_OP_ERR;
afa0e13b 4091 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
b6acd71f 4092 return IB_WC_LOC_PROT_ERR;
afa0e13b 4093 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
b6acd71f 4094 return IB_WC_MW_BIND_ERR;
afa0e13b 4095 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
b6acd71f 4096 return IB_WC_REM_INV_RD_REQ_ERR;
afa0e13b 4097 case RDMA_CQE_RESP_STS_OK:
b6acd71f
AR		 4098 return IB_WC_SUCCESS;
4099 default:
4100 return IB_WC_GENERAL_ERR;
4101 }
4102}
afa0e13b 4103
b6acd71f
AR		 4104static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4105 struct ib_wc *wc)
4106{
4107 wc->status = IB_WC_SUCCESS;
4108 wc->byte_len = le32_to_cpu(resp->length);
afa0e13b 4109
b6acd71f 4110 if (resp->flags & QEDR_RESP_IMM) {
7bed7ebc 4111 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
b6acd71f
AR		 4112 wc->wc_flags |= IB_WC_WITH_IMM;
4113
4114 if (resp->flags & QEDR_RESP_RDMA)
afa0e13b
RA		 4115 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4116
b6acd71f
AR		 4117 if (resp->flags & QEDR_RESP_INV)
4118 return -EINVAL;
4119
4120 } else if (resp->flags & QEDR_RESP_INV) {
4121 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4122 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4123
4124 if (resp->flags & QEDR_RESP_RDMA)
4125 return -EINVAL;
4126
4127 } else if (resp->flags & QEDR_RESP_RDMA) {
4128 return -EINVAL;
4129 }
4130
4131 return 0;
4132}
4133
4134static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4135 struct qedr_cq *cq, struct ib_wc *wc,
4136 struct rdma_cqe_responder *resp, u64 wr_id)
4137{
4138 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4139 wc->opcode = IB_WC_RECV;
4140 wc->wc_flags = 0;
4141
4142 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4143 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4144 DP_ERR(dev,
4145 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4146 cq, cq->icid, resp->flags);
4147
4148 } else {
4149 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4150 if (wc->status == IB_WC_GENERAL_ERR)
4151 DP_ERR(dev,
4152 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4153 cq, cq->icid, resp->status);
afa0e13b
RA		 4154 }
4155
b6acd71f 4156 /* Fill the rest of the WC */
27035a1b 4157 wc->vendor_err = 0;
afa0e13b
RA		 4158 wc->src_qp = qp->id;
4159 wc->qp = &qp->ibqp;
4160 wc->wr_id = wr_id;
4161}
4162
3491c9e7
YB		 4163static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4164 struct qedr_cq *cq, struct ib_wc *wc,
4165 struct rdma_cqe_responder *resp)
4166{
4167 struct qedr_srq *srq = qp->srq;
4168 u64 wr_id;
4169
4170 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4171 le32_to_cpu(resp->srq_wr_id.lo), u64);
4172
4173 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4174 wc->status = IB_WC_WR_FLUSH_ERR;
4175 wc->vendor_err = 0;
4176 wc->wr_id = wr_id;
4177 wc->byte_len = 0;
4178 wc->src_qp = qp->id;
4179 wc->qp = &qp->ibqp;
4180 wc->wr_id = wr_id;
4181 } else {
4182 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4183 }
acca72e2 4184 atomic_inc(&srq->hw_srq.wr_cons_cnt);
3491c9e7
YB		 4185
4186 return 1;
4187}
afa0e13b
RA		 4188static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4189 struct qedr_cq *cq, struct ib_wc *wc,
4190 struct rdma_cqe_responder *resp)
4191{
4192 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4193
4194 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4195
4196 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4197 qed_chain_consume(&qp->rq.pbl);
4198 qedr_inc_sw_cons(&qp->rq);
4199
4200 return 1;
4201}
4202
4203static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4204 int num_entries, struct ib_wc *wc, u16 hw_cons)
4205{
4206 u16 cnt = 0;
4207
4208 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4209 /* fill WC */
4210 wc->status = IB_WC_WR_FLUSH_ERR;
27035a1b 4211 wc->vendor_err = 0;
afa0e13b
RA		 4212 wc->wc_flags = 0;
4213 wc->src_qp = qp->id;
4214 wc->byte_len = 0;
4215 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4216 wc->qp = &qp->ibqp;
4217 num_entries--;
4218 wc++;
4219 cnt++;
4220 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4221 qed_chain_consume(&qp->rq.pbl);
4222 qedr_inc_sw_cons(&qp->rq);
4223 }
4224
4225 return cnt;
4226}
4227
4228static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4229 struct rdma_cqe_responder *resp, int *update)
4230{
50bc60cb 4231 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
afa0e13b
RA		 4232 consume_cqe(cq);
4233 *update |= 1;
4234 }
4235}
4236
3491c9e7
YB		 4237static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4238 struct qedr_cq *cq, int num_entries,
4239 struct ib_wc *wc,
4240 struct rdma_cqe_responder *resp)
4241{
4242 int cnt;
4243
4244 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4245 consume_cqe(cq);
4246
4247 return cnt;
4248}
4249
afa0e13b
RA		 4250static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4251 struct qedr_cq *cq, int num_entries,
4252 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4253 int *update)
4254{
4255 int cnt;
4256
4257 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4258 cnt = process_resp_flush(qp, cq, num_entries, wc,
50bc60cb 4259 resp->rq_cons_or_srq_id);
afa0e13b
RA		 4260 try_consume_resp_cqe(cq, qp, resp, update);
4261 } else {
4262 cnt = process_resp_one(dev, qp, cq, wc, resp);
4263 consume_cqe(cq);
4264 *update |= 1;
4265 }
4266
4267 return cnt;
4268}
4269
4270static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4271 struct rdma_cqe_requester *req, int *update)
4272{
4273 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4274 consume_cqe(cq);
4275 *update |= 1;
4276 }
4277}
4278
4279int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4280{
4281 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4282 struct qedr_cq *cq = get_qedr_cq(ibcq);
e3fd112c 4283 union rdma_cqe *cqe;
afa0e13b
RA		 4284 u32 old_cons, new_cons;
4285 unsigned long flags;
4286 int update = 0;
4287 int done = 0;
4288
4dd72636
AR		 4289 if (cq->destroyed) {
4290 DP_ERR(dev,
4291 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4292 cq, cq->icid);
4293 return 0;
4294 }
4295
04886779
RA		 4296 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4297 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4298
afa0e13b 4299 spin_lock_irqsave(&cq->cq_lock, flags);
e3fd112c 4300 cqe = cq->latest_cqe;
afa0e13b
RA		 4301 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4302 while (num_entries && is_valid_cqe(cq, cqe)) {
4303 struct qedr_qp *qp;
4304 int cnt = 0;
4305
4306 /* prevent speculative reads of any field of CQE */
4307 rmb();
4308
4309 qp = cqe_get_qp(cqe);
4310 if (!qp) {
4311 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4312 break;
4313 }
4314
4315 wc->qp = &qp->ibqp;
4316
4317 switch (cqe_get_type(cqe)) {
4318 case RDMA_CQE_TYPE_REQUESTER:
4319 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4320 &cqe->req);
4321 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4322 break;
4323 case RDMA_CQE_TYPE_RESPONDER_RQ:
4324 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4325 &cqe->resp, &update);
4326 break;
3491c9e7
YB		 4327 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4328 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4329 wc, &cqe->resp);
4330 update = 1;
4331 break;
afa0e13b
RA		 4332 case RDMA_CQE_TYPE_INVALID:
4333 default:
4334 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4335 cqe_get_type(cqe));
4336 }
4337 num_entries -= cnt;
4338 wc += cnt;
4339 done += cnt;
4340
4341 cqe = get_cqe(cq);
4342 }
4343 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4344
4345 cq->cq_cons += new_cons - old_cons;
4346
4347 if (update)
4348 /* doorbell notifies abount latest VALID entry,
4349 * but chain already point to the next INVALID one
4350 */
4351 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4352
4353 spin_unlock_irqrestore(&cq->cq_lock, flags);
4354 return done;
4355}
993d1b52
RA		 4356
4357int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
e26e7b88
LR		 4358 u8 port_num, const struct ib_wc *in_wc,
4359 const struct ib_grh *in_grh, const struct ib_mad *in,
4360 struct ib_mad *out_mad, size_t *out_mad_size,
4361 u16 *out_mad_pkey_index)
993d1b52 4362{
993d1b52
RA		 4363 return IB_MAD_RESULT_SUCCESS;
4364}
Linux 6.x block layer and io_uring tree(s)