projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'asoc/topic/pcm5102a' into asoc-next
[linux-2.6-block.git] / drivers / infiniband / hw / bnxt_re / qplib_fp.c
CommitLineData
1ac5a404
SX		 1/*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: Fast Path Operators
37 */
38
39#include <linux/interrupt.h>
40#include <linux/spinlock.h>
41#include <linux/sched.h>
42#include <linux/slab.h>
43#include <linux/pci.h>
44#include <linux/prefetch.h>
45
46#include "roce_hsi.h"
47
48#include "qplib_res.h"
49#include "qplib_rcfw.h"
50#include "qplib_sp.h"
51#include "qplib_fp.h"
52
53static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
f218d67e 54static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
37cb11ac 55static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
f218d67e
SX		 56
57static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
58{
59 qp->sq.condition = false;
60 qp->sq.send_phantom = false;
61 qp->sq.single = false;
62}
63
64/* Flush list */
65static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
66{
67 struct bnxt_qplib_cq *scq, *rcq;
68
69 scq = qp->scq;
70 rcq = qp->rcq;
71
72 if (!qp->sq.flushed) {
73 dev_dbg(&scq->hwq.pdev->dev,
74 "QPLIB: FP: Adding to SQ Flush list = %p",
75 qp);
76 bnxt_qplib_cancel_phantom_processing(qp);
77 list_add_tail(&qp->sq_flush, &scq->sqf_head);
78 qp->sq.flushed = true;
79 }
80 if (!qp->srq) {
81 if (!qp->rq.flushed) {
82 dev_dbg(&rcq->hwq.pdev->dev,
83 "QPLIB: FP: Adding to RQ Flush list = %p",
84 qp);
85 list_add_tail(&qp->rq_flush, &rcq->rqf_head);
86 qp->rq.flushed = true;
87 }
88 }
89}
90
942c9b6c
SX		 91static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp,
92 unsigned long *flags)
93 __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
f218d67e 94{
942c9b6c 95 spin_lock_irqsave(&qp->scq->flush_lock, *flags);
f218d67e 96 if (qp->scq == qp->rcq)
942c9b6c 97 __acquire(&qp->rcq->flush_lock);
f218d67e 98 else
942c9b6c 99 spin_lock(&qp->rcq->flush_lock);
f218d67e
SX		 100}
101
942c9b6c
SX		 102static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp,
103 unsigned long *flags)
104 __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
f218d67e
SX		 105{
106 if (qp->scq == qp->rcq)
942c9b6c 107 __release(&qp->rcq->flush_lock);
f218d67e 108 else
942c9b6c
SX		 109 spin_unlock(&qp->rcq->flush_lock);
110 spin_unlock_irqrestore(&qp->scq->flush_lock, *flags);
f218d67e
SX		 111}
112
113void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
114{
115 unsigned long flags;
116
942c9b6c 117 bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
f218d67e 118 __bnxt_qplib_add_flush_qp(qp);
942c9b6c 119 bnxt_qplib_release_cq_flush_locks(qp, &flags);
f218d67e
SX		 120}
121
122static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
123{
f218d67e
SX		 124 if (qp->sq.flushed) {
125 qp->sq.flushed = false;
126 list_del(&qp->sq_flush);
127 }
128 if (!qp->srq) {
129 if (qp->rq.flushed) {
130 qp->rq.flushed = false;
131 list_del(&qp->rq_flush);
132 }
133 }
134}
135
3b921e3b 136void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
f218d67e
SX		 137{
138 unsigned long flags;
139
942c9b6c 140 bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
f218d67e
SX		 141 __clean_cq(qp->scq, (u64)(unsigned long)qp);
142 qp->sq.hwq.prod = 0;
143 qp->sq.hwq.cons = 0;
144 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
145 qp->rq.hwq.prod = 0;
146 qp->rq.hwq.cons = 0;
147
148 __bnxt_qplib_del_flush_qp(qp);
942c9b6c 149 bnxt_qplib_release_cq_flush_locks(qp, &flags);
f218d67e
SX		 150}
151
152static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
153{
154 struct bnxt_qplib_nq_work *nq_work =
155 container_of(work, struct bnxt_qplib_nq_work, work);
156
157 struct bnxt_qplib_cq *cq = nq_work->cq;
158 struct bnxt_qplib_nq *nq = nq_work->nq;
159
160 if (cq && nq) {
161 spin_lock_bh(&cq->compl_lock);
162 if (atomic_read(&cq->arm_state) && nq->cqn_handler) {
163 dev_dbg(&nq->pdev->dev,
164 "%s:Trigger cq = %p event nq = %p\n",
165 __func__, cq, nq);
166 nq->cqn_handler(nq, cq);
167 }
168 spin_unlock_bh(&cq->compl_lock);
169 }
170 kfree(nq_work);
171}
1ac5a404
SX		 172
173static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
174 struct bnxt_qplib_qp *qp)
175{
176 struct bnxt_qplib_q *rq = &qp->rq;
177 struct bnxt_qplib_q *sq = &qp->sq;
178
179 if (qp->rq_hdr_buf)
180 dma_free_coherent(&res->pdev->dev,
181 rq->hwq.max_elements * qp->rq_hdr_buf_size,
182 qp->rq_hdr_buf, qp->rq_hdr_buf_map);
183 if (qp->sq_hdr_buf)
184 dma_free_coherent(&res->pdev->dev,
185 sq->hwq.max_elements * qp->sq_hdr_buf_size,
186 qp->sq_hdr_buf, qp->sq_hdr_buf_map);
187 qp->rq_hdr_buf = NULL;
188 qp->sq_hdr_buf = NULL;
189 qp->rq_hdr_buf_map = 0;
190 qp->sq_hdr_buf_map = 0;
191 qp->sq_hdr_buf_size = 0;
192 qp->rq_hdr_buf_size = 0;
193}
194
195static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
196 struct bnxt_qplib_qp *qp)
197{
198 struct bnxt_qplib_q *rq = &qp->rq;
199 struct bnxt_qplib_q *sq = &qp->rq;
200 int rc = 0;
201
202 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
203 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
204 sq->hwq.max_elements *
205 qp->sq_hdr_buf_size,
206 &qp->sq_hdr_buf_map, GFP_KERNEL);
207 if (!qp->sq_hdr_buf) {
208 rc = -ENOMEM;
209 dev_err(&res->pdev->dev,
210 "QPLIB: Failed to create sq_hdr_buf");
211 goto fail;
212 }
213 }
214
215 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
216 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
217 rq->hwq.max_elements *
218 qp->rq_hdr_buf_size,
219 &qp->rq_hdr_buf_map,
220 GFP_KERNEL);
221 if (!qp->rq_hdr_buf) {
222 rc = -ENOMEM;
223 dev_err(&res->pdev->dev,
224 "QPLIB: Failed to create rq_hdr_buf");
225 goto fail;
226 }
227 }
228 return 0;
229
230fail:
231 bnxt_qplib_free_qp_hdr_buf(res, qp);
232 return rc;
233}
234
235static void bnxt_qplib_service_nq(unsigned long data)
236{
237 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
238 struct bnxt_qplib_hwq *hwq = &nq->hwq;
239 struct nq_base *nqe, **nq_ptr;
f218d67e 240 struct bnxt_qplib_cq *cq;
1ac5a404 241 int num_cqne_processed = 0;
37cb11ac 242 int num_srqne_processed = 0;
1ac5a404
SX		 243 u32 sw_cons, raw_cons;
244 u16 type;
245 int budget = nq->budget;
5388a508 246 uintptr_t q_handle;
1ac5a404
SX		 247
248 /* Service the NQ until empty */
249 raw_cons = hwq->cons;
250 while (budget--) {
251 sw_cons = HWQ_CMP(raw_cons, hwq);
252 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
253 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
254 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
255 break;
256
9b40183c
SK		 257 /*
258 * The valid test of the entry must be done first before
259 * reading any further.
260 */
261 dma_rmb();
262
1ac5a404
SX		 263 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
264 switch (type) {
265 case NQ_BASE_TYPE_CQ_NOTIFICATION:
266 {
267 struct nq_cn *nqcne = (struct nq_cn *)nqe;
268
269 q_handle = le32_to_cpu(nqcne->cq_handle_low);
270 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
271 << 32;
f218d67e
SX		 272 cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
273 bnxt_qplib_arm_cq_enable(cq);
274 spin_lock_bh(&cq->compl_lock);
275 atomic_set(&cq->arm_state, 0);
276 if (!nq->cqn_handler(nq, (cq)))
1ac5a404
SX		 277 num_cqne_processed++;
278 else
279 dev_warn(&nq->pdev->dev,
280 "QPLIB: cqn - type 0x%x not handled",
281 type);
f218d67e 282 spin_unlock_bh(&cq->compl_lock);
1ac5a404
SX		 283 break;
284 }
37cb11ac
DS		 285 case NQ_BASE_TYPE_SRQ_EVENT:
286 {
287 struct nq_srq_event *nqsrqe =
288 (struct nq_srq_event *)nqe;
289
290 q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
291 q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
292 << 32;
293 bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
294 DBR_DBR_TYPE_SRQ_ARMENA);
295 if (!nq->srqn_handler(nq,
296 (struct bnxt_qplib_srq *)q_handle,
297 nqsrqe->event))
298 num_srqne_processed++;
299 else
300 dev_warn(&nq->pdev->dev,
301 "QPLIB: SRQ event 0x%x not handled",
302 nqsrqe->event);
303 break;
304 }
1ac5a404
SX		 305 case NQ_BASE_TYPE_DBQ_EVENT:
306 break;
307 default:
308 dev_warn(&nq->pdev->dev,
309 "QPLIB: nqe with type = 0x%x not handled",
310 type);
311 break;
312 }
313 raw_cons++;
314 }
315 if (hwq->cons != raw_cons) {
316 hwq->cons = raw_cons;
317 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
318 }
319}
320
321static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
322{
323 struct bnxt_qplib_nq *nq = dev_instance;
324 struct bnxt_qplib_hwq *hwq = &nq->hwq;
325 struct nq_base **nq_ptr;
326 u32 sw_cons;
327
328 /* Prefetch the NQ element */
329 sw_cons = HWQ_CMP(hwq->cons, hwq);
330 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
331 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
332
333 /* Fan out to CPU affinitized kthreads? */
334 tasklet_schedule(&nq->worker);
335
336 return IRQ_HANDLED;
337}
338
339void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
340{
f218d67e
SX		 341 if (nq->cqn_wq) {
342 destroy_workqueue(nq->cqn_wq);
343 nq->cqn_wq = NULL;
344 }
1ac5a404
SX		 345 /* Make sure the HW is stopped! */
346 synchronize_irq(nq->vector);
347 tasklet_disable(&nq->worker);
348 tasklet_kill(&nq->worker);
349
350 if (nq->requested) {
6a5df91b 351 irq_set_affinity_hint(nq->vector, NULL);
1ac5a404
SX		 352 free_irq(nq->vector, nq);
353 nq->requested = false;
354 }
355 if (nq->bar_reg_iomem)
356 iounmap(nq->bar_reg_iomem);
357 nq->bar_reg_iomem = NULL;
358
359 nq->cqn_handler = NULL;
360 nq->srqn_handler = NULL;
361 nq->vector = 0;
362}
363
364int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
6a5df91b 365 int nq_idx, int msix_vector, int bar_reg_offset,
1ac5a404
SX		 366 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
367 struct bnxt_qplib_cq *),
368 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
37cb11ac
DS		 369 struct bnxt_qplib_srq *,
370 u8 event))
1ac5a404
SX		 371{
372 resource_size_t nq_base;
f218d67e 373 int rc = -1;
1ac5a404
SX		 374
375 nq->pdev = pdev;
376 nq->vector = msix_vector;
37cb11ac
DS		 377 if (cqn_handler)
378 nq->cqn_handler = cqn_handler;
1ac5a404 379
37cb11ac
DS		 380 if (srqn_handler)
381 nq->srqn_handler = srqn_handler;
1ac5a404
SX		 382
383 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
384
f218d67e
SX		 385 /* Have a task to schedule CQ notifiers in post send case */
386 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq");
387 if (!nq->cqn_wq)
388 goto fail;
389
1ac5a404 390 nq->requested = false;
6a5df91b
SX		 391 memset(nq->name, 0, 32);
392 sprintf(nq->name, "bnxt_qplib_nq-%d", nq_idx);
393 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
1ac5a404
SX		 394 if (rc) {
395 dev_err(&nq->pdev->dev,
396 "Failed to request IRQ for NQ: %#x", rc);
1ac5a404
SX		 397 goto fail;
398 }
6a5df91b
SX		 399
400 cpumask_clear(&nq->mask);
401 cpumask_set_cpu(nq_idx, &nq->mask);
402 rc = irq_set_affinity_hint(nq->vector, &nq->mask);
403 if (rc) {
404 dev_warn(&nq->pdev->dev,
405 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
406 nq->vector, nq_idx);
407 }
408
1ac5a404
SX		 409 nq->requested = true;
410 nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
411 nq->bar_reg_off = bar_reg_offset;
412 nq_base = pci_resource_start(pdev, nq->bar_reg);
413 if (!nq_base) {
414 rc = -ENOMEM;
415 goto fail;
416 }
417 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
418 if (!nq->bar_reg_iomem) {
419 rc = -ENOMEM;
420 goto fail;
421 }
422 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
423
424 return 0;
425fail:
426 bnxt_qplib_disable_nq(nq);
427 return rc;
428}
429
430void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
431{
6a5df91b 432 if (nq->hwq.max_elements) {
1ac5a404 433 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
6a5df91b
SX		 434 nq->hwq.max_elements = 0;
435 }
1ac5a404
SX		 436}
437
438int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
439{
440 nq->pdev = pdev;
441 if (!nq->hwq.max_elements ||
442 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
443 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
444
445 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
446 &nq->hwq.max_elements,
447 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
448 PAGE_SIZE, HWQ_TYPE_L2_CMPL))
449 return -ENOMEM;
450
451 nq->budget = 8;
452 return 0;
453}
454
37cb11ac
DS		 455/* SRQ */
456static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
457{
458 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
459 struct dbr_dbr db_msg = { 0 };
460 void __iomem *db;
461 u32 sw_prod = 0;
462
463 /* Ring DB */
464 sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold :
465 HWQ_CMP(srq_hwq->prod, srq_hwq);
466 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
467 DBR_DBR_INDEX_MASK);
468 db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) &
469 DBR_DBR_XID_MASK) | arm_type);
470 db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ?
471 srq->dbr_base : srq->dpi->dbr;
472 wmb(); /* barrier before db ring */
473 __iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64));
474}
475
476int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
477 struct bnxt_qplib_srq *srq)
478{
479 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
480 struct cmdq_destroy_srq req;
481 struct creq_destroy_srq_resp resp;
482 u16 cmd_flags = 0;
483 int rc;
484
485 RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);
486
487 /* Configure the request */
488 req.srq_cid = cpu_to_le32(srq->id);
489
490 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
491 (void *)&resp, NULL, 0);
492 if (rc)
493 return rc;
494
495 bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
496 kfree(srq->swq);
497 return 0;
498}
499
500int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
501 struct bnxt_qplib_srq *srq)
502{
503 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
504 struct cmdq_create_srq req;
505 struct creq_create_srq_resp resp;
506 struct bnxt_qplib_pbl *pbl;
507 u16 cmd_flags = 0;
508 int rc, idx;
509
510 srq->hwq.max_elements = srq->max_wqe;
511 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
512 srq->nmap, &srq->hwq.max_elements,
513 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
514 PAGE_SIZE, HWQ_TYPE_QUEUE);
515 if (rc)
516 goto exit;
517
518 srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),
519 GFP_KERNEL);
8de9399a
DC		 520 if (!srq->swq) {
521 rc = -ENOMEM;
37cb11ac 522 goto fail;
8de9399a 523 }
37cb11ac
DS		 524
525 RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);
526
527 /* Configure the request */
528 req.dpi = cpu_to_le32(srq->dpi->dpi);
5388a508 529 req.srq_handle = cpu_to_le64((uintptr_t)srq);
37cb11ac
DS		 530
531 req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
532 pbl = &srq->hwq.pbl[PBL_LVL_0];
533 req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
534 CMDQ_CREATE_SRQ_LVL_MASK) <<
535 CMDQ_CREATE_SRQ_LVL_SFT) |
536 (pbl->pg_size == ROCE_PG_SIZE_4K ?
537 CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
538 pbl->pg_size == ROCE_PG_SIZE_8K ?
539 CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
540 pbl->pg_size == ROCE_PG_SIZE_64K ?
541 CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
542 pbl->pg_size == ROCE_PG_SIZE_2M ?
543 CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
544 pbl->pg_size == ROCE_PG_SIZE_8M ?
545 CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
546 pbl->pg_size == ROCE_PG_SIZE_1G ?
547 CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
548 CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
549 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
550 req.pd_id = cpu_to_le32(srq->pd->id);
551 req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
552
553 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
554 (void *)&resp, NULL, 0);
555 if (rc)
556 goto fail;
557
558 spin_lock_init(&srq->lock);
559 srq->start_idx = 0;
560 srq->last_idx = srq->hwq.max_elements - 1;
561 for (idx = 0; idx < srq->hwq.max_elements; idx++)
562 srq->swq[idx].next_idx = idx + 1;
563 srq->swq[srq->last_idx].next_idx = -1;
564
565 srq->id = le32_to_cpu(resp.xid);
566 srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
567 if (srq->threshold)
568 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA);
569 srq->arm_req = false;
570
571 return 0;
572fail:
573 bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
574 kfree(srq->swq);
575exit:
576 return rc;
577}
578
579int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
580 struct bnxt_qplib_srq *srq)
581{
582 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
583 u32 sw_prod, sw_cons, count = 0;
584
585 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
586 sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
587
588 count = sw_prod > sw_cons ? sw_prod - sw_cons :
589 srq_hwq->max_elements - sw_cons + sw_prod;
590 if (count > srq->threshold) {
591 srq->arm_req = false;
592 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
593 } else {
594 /* Deferred arming */
595 srq->arm_req = true;
596 }
597
598 return 0;
599}
600
601int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
602 struct bnxt_qplib_srq *srq)
603{
604 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
605 struct cmdq_query_srq req;
606 struct creq_query_srq_resp resp;
607 struct bnxt_qplib_rcfw_sbuf *sbuf;
608 struct creq_query_srq_resp_sb *sb;
609 u16 cmd_flags = 0;
610 int rc = 0;
611
612 RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);
613 req.srq_cid = cpu_to_le32(srq->id);
614
615 /* Configure the request */
616 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
617 if (!sbuf)
618 return -ENOMEM;
619 sb = sbuf->sb;
620 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
621 (void *)sbuf, 0);
622 srq->threshold = le16_to_cpu(sb->srq_limit);
623 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
624
625 return rc;
626}
627
628int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
629 struct bnxt_qplib_swqe *wqe)
630{
631 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
632 struct rq_wqe *srqe, **srqe_ptr;
633 struct sq_sge *hw_sge;
634 u32 sw_prod, sw_cons, count = 0;
635 int i, rc = 0, next;
636
637 spin_lock(&srq_hwq->lock);
638 if (srq->start_idx == srq->last_idx) {
639 dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!",
640 srq->id);
641 rc = -EINVAL;
642 spin_unlock(&srq_hwq->lock);
643 goto done;
644 }
645 next = srq->start_idx;
646 srq->start_idx = srq->swq[next].next_idx;
647 spin_unlock(&srq_hwq->lock);
648
649 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
650 srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr;
651 srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
652 memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
653 /* Calculate wqe_size16 and data_len */
654 for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
655 i < wqe->num_sge; i++, hw_sge++) {
656 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
657 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
658 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
659 }
660 srqe->wqe_type = wqe->type;
661 srqe->flags = wqe->flags;
662 srqe->wqe_size = wqe->num_sge +
663 ((offsetof(typeof(*srqe), data) + 15) >> 4);
664 srqe->wr_id[0] = cpu_to_le32((u32)next);
665 srq->swq[next].wr_id = wqe->wr_id;
666
667 srq_hwq->prod++;
668
669 spin_lock(&srq_hwq->lock);
670 sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
671 /* retaining srq_hwq->cons for this logic
672 * actually the lock is only required to
673 * read srq_hwq->cons.
674 */
675 sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
676 count = sw_prod > sw_cons ? sw_prod - sw_cons :
677 srq_hwq->max_elements - sw_cons + sw_prod;
678 spin_unlock(&srq_hwq->lock);
679 /* Ring DB */
680 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ);
681 if (srq->arm_req == true && count > srq->threshold) {
682 srq->arm_req = false;
683 bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
684 }
685done:
686 return rc;
687}
688
1ac5a404
SX		 689/* QP */
690int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
691{
692 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
693 struct cmdq_create_qp1 req;
cc1ec769 694 struct creq_create_qp1_resp resp;
1ac5a404
SX		 695 struct bnxt_qplib_pbl *pbl;
696 struct bnxt_qplib_q *sq = &qp->sq;
697 struct bnxt_qplib_q *rq = &qp->rq;
698 int rc;
699 u16 cmd_flags = 0;
700 u32 qp_flags = 0;
701
702 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
703
704 /* General */
705 req.type = qp->type;
706 req.dpi = cpu_to_le32(qp->dpi->dpi);
707 req.qp_handle = cpu_to_le64(qp->qp_handle);
708
709 /* SQ */
710 sq->hwq.max_elements = sq->max_wqe;
711 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
712 &sq->hwq.max_elements,
713 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
714 PAGE_SIZE, HWQ_TYPE_QUEUE);
715 if (rc)
716 goto exit;
717
718 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
719 if (!sq->swq) {
720 rc = -ENOMEM;
721 goto fail_sq;
722 }
723 pbl = &sq->hwq.pbl[PBL_LVL_0];
724 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
725 req.sq_pg_size_sq_lvl =
726 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
727 << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
728 (pbl->pg_size == ROCE_PG_SIZE_4K ?
729 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
730 pbl->pg_size == ROCE_PG_SIZE_8K ?
731 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
732 pbl->pg_size == ROCE_PG_SIZE_64K ?
733 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
734 pbl->pg_size == ROCE_PG_SIZE_2M ?
735 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
736 pbl->pg_size == ROCE_PG_SIZE_8M ?
737 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
738 pbl->pg_size == ROCE_PG_SIZE_1G ?
739 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
740 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
741
742 if (qp->scq)
743 req.scq_cid = cpu_to_le32(qp->scq->id);
744
745 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
746
747 /* RQ */
748 if (rq->max_wqe) {
749 rq->hwq.max_elements = qp->rq.max_wqe;
750 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
751 &rq->hwq.max_elements,
752 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
753 PAGE_SIZE, HWQ_TYPE_QUEUE);
754 if (rc)
755 goto fail_sq;
756
757 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
758 GFP_KERNEL);
759 if (!rq->swq) {
760 rc = -ENOMEM;
761 goto fail_rq;
762 }
763 pbl = &rq->hwq.pbl[PBL_LVL_0];
764 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
765 req.rq_pg_size_rq_lvl =
766 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
767 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
768 (pbl->pg_size == ROCE_PG_SIZE_4K ?
769 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
770 pbl->pg_size == ROCE_PG_SIZE_8K ?
771 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
772 pbl->pg_size == ROCE_PG_SIZE_64K ?
773 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
774 pbl->pg_size == ROCE_PG_SIZE_2M ?
775 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
776 pbl->pg_size == ROCE_PG_SIZE_8M ?
777 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
778 pbl->pg_size == ROCE_PG_SIZE_1G ?
779 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
780 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
781 if (qp->rcq)
782 req.rcq_cid = cpu_to_le32(qp->rcq->id);
783 }
784
785 /* Header buffer - allow hdr_buf pass in */
786 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
787 if (rc) {
788 rc = -ENOMEM;
789 goto fail;
790 }
791 req.qp_flags = cpu_to_le32(qp_flags);
792 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
793 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
794
795 req.sq_fwo_sq_sge =
796 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
797 CMDQ_CREATE_QP1_SQ_SGE_SFT);
798 req.rq_fwo_rq_sge =
799 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
800 CMDQ_CREATE_QP1_RQ_SGE_SFT);
801
802 req.pd_id = cpu_to_le32(qp->pd->id);
803
cc1ec769
DS		 804 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
805 (void *)&resp, NULL, 0);
806 if (rc)
1ac5a404 807 goto fail;
cc1ec769
DS		 808
809 qp->id = le32_to_cpu(resp.xid);
1ac5a404 810 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
f218d67e
SX		 811 rcfw->qp_tbl[qp->id].qp_id = qp->id;
812 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
1ac5a404
SX		 813
814 return 0;
815
816fail:
817 bnxt_qplib_free_qp_hdr_buf(res, qp);
818fail_rq:
819 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
820 kfree(rq->swq);
821fail_sq:
822 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
823 kfree(sq->swq);
824exit:
825 return rc;
826}
827
828int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
829{
830 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
831 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
832 struct cmdq_create_qp req;
cc1ec769 833 struct creq_create_qp_resp resp;
1ac5a404
SX		 834 struct bnxt_qplib_pbl *pbl;
835 struct sq_psn_search **psn_search_ptr;
836 unsigned long int psn_search, poff = 0;
837 struct bnxt_qplib_q *sq = &qp->sq;
838 struct bnxt_qplib_q *rq = &qp->rq;
839 struct bnxt_qplib_hwq *xrrq;
840 int i, rc, req_size, psn_sz;
841 u16 cmd_flags = 0, max_ssge;
842 u32 sw_prod, qp_flags = 0;
843
844 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
845
846 /* General */
847 req.type = qp->type;
848 req.dpi = cpu_to_le32(qp->dpi->dpi);
849 req.qp_handle = cpu_to_le64(qp->qp_handle);
850
851 /* SQ */
852 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
853 sizeof(struct sq_psn_search) : 0;
854 sq->hwq.max_elements = sq->max_wqe;
855 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
856 sq->nmap, &sq->hwq.max_elements,
857 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
858 psn_sz,
859 PAGE_SIZE, HWQ_TYPE_QUEUE);
860 if (rc)
861 goto exit;
862
863 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
864 if (!sq->swq) {
865 rc = -ENOMEM;
866 goto fail_sq;
867 }
868 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
869 if (psn_sz) {
870 psn_search_ptr = (struct sq_psn_search **)
871 &hw_sq_send_ptr[get_sqe_pg
872 (sq->hwq.max_elements)];
873 psn_search = (unsigned long int)
874 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
875 [get_sqe_idx(sq->hwq.max_elements)];
876 if (psn_search & ~PAGE_MASK) {
877 /* If the psn_search does not start on a page boundary,
878 * then calculate the offset
879 */
880 poff = (psn_search & ~PAGE_MASK) /
881 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
882 }
883 for (i = 0; i < sq->hwq.max_elements; i++)
884 sq->swq[i].psn_search =
885 &psn_search_ptr[get_psne_pg(i + poff)]
886 [get_psne_idx(i + poff)];
887 }
888 pbl = &sq->hwq.pbl[PBL_LVL_0];
889 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
890 req.sq_pg_size_sq_lvl =
891 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
892 << CMDQ_CREATE_QP_SQ_LVL_SFT) |
893 (pbl->pg_size == ROCE_PG_SIZE_4K ?
894 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
895 pbl->pg_size == ROCE_PG_SIZE_8K ?
896 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
897 pbl->pg_size == ROCE_PG_SIZE_64K ?
898 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
899 pbl->pg_size == ROCE_PG_SIZE_2M ?
900 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
901 pbl->pg_size == ROCE_PG_SIZE_8M ?
902 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
903 pbl->pg_size == ROCE_PG_SIZE_1G ?
904 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
905 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
906
907 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
908 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
909 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
910 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
911 [get_sqe_idx(sw_prod)];
912 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
913 }
914
915 if (qp->scq)
916 req.scq_cid = cpu_to_le32(qp->scq->id);
917
918 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
919 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
920 if (qp->sig_type)
921 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
922
923 /* RQ */
924 if (rq->max_wqe) {
925 rq->hwq.max_elements = rq->max_wqe;
926 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
927 rq->nmap, &rq->hwq.max_elements,
928 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
929 PAGE_SIZE, HWQ_TYPE_QUEUE);
930 if (rc)
931 goto fail_sq;
932
933 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
934 GFP_KERNEL);
935 if (!rq->swq) {
936 rc = -ENOMEM;
937 goto fail_rq;
938 }
939 pbl = &rq->hwq.pbl[PBL_LVL_0];
940 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
941 req.rq_pg_size_rq_lvl =
942 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
943 CMDQ_CREATE_QP_RQ_LVL_SFT) |
944 (pbl->pg_size == ROCE_PG_SIZE_4K ?
945 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
946 pbl->pg_size == ROCE_PG_SIZE_8K ?
947 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
948 pbl->pg_size == ROCE_PG_SIZE_64K ?
949 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
950 pbl->pg_size == ROCE_PG_SIZE_2M ?
951 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
952 pbl->pg_size == ROCE_PG_SIZE_8M ?
953 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
954 pbl->pg_size == ROCE_PG_SIZE_1G ?
955 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
956 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
37cb11ac
DS		 957 } else {
958 /* SRQ */
959 if (qp->srq) {
960 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
961 req.srq_cid = cpu_to_le32(qp->srq->id);
962 }
1ac5a404
SX		 963 }
964
965 if (qp->rcq)
966 req.rcq_cid = cpu_to_le32(qp->rcq->id);
967 req.qp_flags = cpu_to_le32(qp_flags);
968 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
969 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
970 qp->sq_hdr_buf = NULL;
971 qp->rq_hdr_buf = NULL;
972
973 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
974 if (rc)
975 goto fail_rq;
976
977 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
978 * always create the QP with max send sges possible if the requested
979 * inline size is greater than 0.
980 */
981 max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
982 req.sq_fwo_sq_sge = cpu_to_le16(
983 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
984 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
985 req.rq_fwo_rq_sge = cpu_to_le16(
986 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
987 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
988 /* ORRQ and IRRQ */
989 if (psn_sz) {
990 xrrq = &qp->orrq;
991 xrrq->max_elements =
992 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
993 req_size = xrrq->max_elements *
994 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
995 req_size &= ~(PAGE_SIZE - 1);
996 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
997 &xrrq->max_elements,
998 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
999 0, req_size, HWQ_TYPE_CTX);
1000 if (rc)
1001 goto fail_buf_free;
1002 pbl = &xrrq->pbl[PBL_LVL_0];
1003 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1004
1005 xrrq = &qp->irrq;
1006 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
1007 qp->max_dest_rd_atomic);
1008 req_size = xrrq->max_elements *
1009 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1010 req_size &= ~(PAGE_SIZE - 1);
1011
1012 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
1013 &xrrq->max_elements,
1014 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
1015 0, req_size, HWQ_TYPE_CTX);
1016 if (rc)
1017 goto fail_orrq;
1018
1019 pbl = &xrrq->pbl[PBL_LVL_0];
1020 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
1021 }
1022 req.pd_id = cpu_to_le32(qp->pd->id);
1023
cc1ec769
DS		 1024 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1025 (void *)&resp, NULL, 0);
1026 if (rc)
1ac5a404 1027 goto fail;
cc1ec769
DS		 1028
1029 qp->id = le32_to_cpu(resp.xid);
1ac5a404 1030 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
f218d67e
SX		 1031 INIT_LIST_HEAD(&qp->sq_flush);
1032 INIT_LIST_HEAD(&qp->rq_flush);
1033 rcfw->qp_tbl[qp->id].qp_id = qp->id;
1034 rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
1ac5a404
SX		 1035
1036 return 0;
1037
1038fail:
1039 if (qp->irrq.max_elements)
1040 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1041fail_orrq:
1042 if (qp->orrq.max_elements)
1043 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1044fail_buf_free:
1045 bnxt_qplib_free_qp_hdr_buf(res, qp);
1046fail_rq:
1047 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
1048 kfree(rq->swq);
1049fail_sq:
1050 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
1051 kfree(sq->swq);
1052exit:
1053 return rc;
1054}
1055
1056static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
1057{
1058 switch (qp->state) {
1059 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1060 /* INIT->RTR, configure the path_mtu to the default
1061 * 2048 if not being requested
1062 */
1063 if (!(qp->modify_flags &
1064 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
1065 qp->modify_flags |=
1066 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1067 qp->path_mtu =
1068 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1069 }
1070 qp->modify_flags &=
1071 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1072 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
1073 if (qp->max_dest_rd_atomic < 1)
1074 qp->max_dest_rd_atomic = 1;
1075 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
1076 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
1077 if (!(qp->modify_flags &
1078 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
1079 qp->modify_flags |=
1080 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
1081 qp->ah.sgid_index = 0;
1082 }
1083 break;
1084 default:
1085 break;
1086 }
1087}
1088
1089static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
1090{
1091 switch (qp->state) {
1092 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1093 /* Bono FW requires the max_rd_atomic to be >= 1 */
1094 if (qp->max_rd_atomic < 1)
1095 qp->max_rd_atomic = 1;
1096 /* Bono FW does not allow PKEY_INDEX,
1097 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
1098 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
1099 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
1100 * modification
1101 */
1102 qp->modify_flags &=
1103 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
1104 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1105 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1106 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1107 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1108 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1109 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1110 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
1111 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
1112 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
1113 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
1114 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
1115 break;
1116 default:
1117 break;
1118 }
1119}
1120
1121static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
1122{
1123 switch (qp->cur_qp_state) {
1124 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1125 break;
1126 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1127 __modify_flags_from_init_state(qp);
1128 break;
1129 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1130 __modify_flags_from_rtr_state(qp);
1131 break;
1132 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1133 break;
1134 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1135 break;
1136 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1137 break;
1138 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1139 break;
1140 default:
1141 break;
1142 }
1143}
1144
1145int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1146{
1147 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1148 struct cmdq_modify_qp req;
cc1ec769 1149 struct creq_modify_qp_resp resp;
1ac5a404
SX		 1150 u16 cmd_flags = 0, pkey;
1151 u32 temp32[4];
1152 u32 bmask;
cc1ec769 1153 int rc;
1ac5a404
SX		 1154
1155 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
1156
1157 /* Filter out the qp_attr_mask based on the state->new transition */
1158 __filter_modify_flags(qp);
1159 bmask = qp->modify_flags;
1160 req.modify_mask = cpu_to_le32(qp->modify_flags);
1161 req.qp_cid = cpu_to_le32(qp->id);
1162 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
1163 req.network_type_en_sqd_async_notify_new_state =
1164 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
1165 (qp->en_sqd_async_notify ?
1166 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
1167 }
1168 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
1169
1170 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
1171 req.access = qp->access;
1172
1173 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
1174 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
1175 qp->pkey_index, &pkey))
1176 req.pkey = cpu_to_le16(pkey);
1177 }
1178 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
1179 req.qkey = cpu_to_le32(qp->qkey);
1180
1181 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
1182 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
1183 req.dgid[0] = cpu_to_le32(temp32[0]);
1184 req.dgid[1] = cpu_to_le32(temp32[1]);
1185 req.dgid[2] = cpu_to_le32(temp32[2]);
1186 req.dgid[3] = cpu_to_le32(temp32[3]);
1187 }
1188 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
1189 req.flow_label = cpu_to_le32(qp->ah.flow_label);
1190
1191 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
1192 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
1193 [qp->ah.sgid_index]);
1194
1195 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
1196 req.hop_limit = qp->ah.hop_limit;
1197
1198 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
1199 req.traffic_class = qp->ah.traffic_class;
1200
1201 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
1202 memcpy(req.dest_mac, qp->ah.dmac, 6);
1203
1204 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
1205 req.path_mtu = qp->path_mtu;
1206
1207 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
1208 req.timeout = qp->timeout;
1209
1210 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
1211 req.retry_cnt = qp->retry_cnt;
1212
1213 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
1214 req.rnr_retry = qp->rnr_retry;
1215
1216 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
1217 req.min_rnr_timer = qp->min_rnr_timer;
1218
1219 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
1220 req.rq_psn = cpu_to_le32(qp->rq.psn);
1221
1222 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
1223 req.sq_psn = cpu_to_le32(qp->sq.psn);
1224
1225 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
1226 req.max_rd_atomic =
1227 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
1228
1229 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
1230 req.max_dest_rd_atomic =
1231 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
1232
1233 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
1234 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
1235 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
1236 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
1237 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
1238 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
1239 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
1240
1241 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
1242
cc1ec769
DS		 1243 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1244 (void *)&resp, NULL, 0);
1245 if (rc)
1246 return rc;
1ac5a404
SX		 1247 qp->cur_qp_state = qp->state;
1248 return 0;
1249}
1250
1251int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
1252{
1253 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1254 struct cmdq_query_qp req;
cc1ec769
DS		 1255 struct creq_query_qp_resp resp;
1256 struct bnxt_qplib_rcfw_sbuf *sbuf;
1ac5a404
SX		 1257 struct creq_query_qp_resp_sb *sb;
1258 u16 cmd_flags = 0;
1259 u32 temp32[4];
cc1ec769 1260 int i, rc = 0;
1ac5a404
SX		 1261
1262 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
1263
cc1ec769
DS		 1264 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
1265 if (!sbuf)
1266 return -ENOMEM;
1267 sb = sbuf->sb;
1268
1ac5a404
SX		 1269 req.qp_cid = cpu_to_le32(qp->id);
1270 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
cc1ec769
DS		 1271 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
1272 (void *)sbuf, 0);
1273 if (rc)
1274 goto bail;
1ac5a404
SX		 1275 /* Extract the context from the side buffer */
1276 qp->state = sb->en_sqd_async_notify_state &
1277 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
1278 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
1279 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
1280 true : false;
1281 qp->access = sb->access;
1282 qp->pkey_index = le16_to_cpu(sb->pkey);
1283 qp->qkey = le32_to_cpu(sb->qkey);
1284
1285 temp32[0] = le32_to_cpu(sb->dgid[0]);
1286 temp32[1] = le32_to_cpu(sb->dgid[1]);
1287 temp32[2] = le32_to_cpu(sb->dgid[2]);
1288 temp32[3] = le32_to_cpu(sb->dgid[3]);
1289 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
1290
1291 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
1292
1293 qp->ah.sgid_index = 0;
1294 for (i = 0; i < res->sgid_tbl.max; i++) {
1295 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
1296 qp->ah.sgid_index = i;
1297 break;
1298 }
1299 }
1300 if (i == res->sgid_tbl.max)
1301 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
1302
1303 qp->ah.hop_limit = sb->hop_limit;
1304 qp->ah.traffic_class = sb->traffic_class;
1305 memcpy(qp->ah.dmac, sb->dest_mac, 6);
1306 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1307 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
1308 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
1309 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
1310 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
1311 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
1312 qp->timeout = sb->timeout;
1313 qp->retry_cnt = sb->retry_cnt;
1314 qp->rnr_retry = sb->rnr_retry;
1315 qp->min_rnr_timer = sb->min_rnr_timer;
1316 qp->rq.psn = le32_to_cpu(sb->rq_psn);
1317 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
1318 qp->sq.psn = le32_to_cpu(sb->sq_psn);
1319 qp->max_dest_rd_atomic =
1320 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
1321 qp->sq.max_wqe = qp->sq.hwq.max_elements;
1322 qp->rq.max_wqe = qp->rq.hwq.max_elements;
1323 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
1324 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
1325 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
1326 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
1327 memcpy(qp->smac, sb->src_mac, 6);
1328 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
cc1ec769
DS		 1329bail:
1330 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
1331 return rc;
1ac5a404
SX		 1332}
1333
1334static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
1335{
1336 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1337 struct cq_base *hw_cqe, **hw_cqe_ptr;
1338 int i;
1339
1340 for (i = 0; i < cq_hwq->max_elements; i++) {
1341 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
1342 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
1343 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
1344 continue;
9b40183c
SK		 1345 /*
1346 * The valid test of the entry must be done first before
1347 * reading any further.
1348 */
1349 dma_rmb();
1ac5a404
SX		 1350 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
1351 case CQ_BASE_CQE_TYPE_REQ:
1352 case CQ_BASE_CQE_TYPE_TERMINAL:
1353 {
1354 struct cq_req *cqe = (struct cq_req *)hw_cqe;
1355
1356 if (qp == le64_to_cpu(cqe->qp_handle))
1357 cqe->qp_handle = 0;
1358 break;
1359 }
1360 case CQ_BASE_CQE_TYPE_RES_RC:
1361 case CQ_BASE_CQE_TYPE_RES_UD:
1362 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
1363 {
1364 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
1365
1366 if (qp == le64_to_cpu(cqe->qp_handle))
1367 cqe->qp_handle = 0;
1368 break;
1369 }
1370 default:
1371 break;
1372 }
1373 }
1374}
1375
1376int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
1377 struct bnxt_qplib_qp *qp)
1378{
1379 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1380 struct cmdq_destroy_qp req;
cc1ec769 1381 struct creq_destroy_qp_resp resp;
1ac5a404 1382 u16 cmd_flags = 0;
cc1ec769 1383 int rc;
1ac5a404 1384
f218d67e
SX		 1385 rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
1386 rcfw->qp_tbl[qp->id].qp_handle = NULL;
1387
1ac5a404
SX		 1388 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
1389
1390 req.qp_cid = cpu_to_le32(qp->id);
cc1ec769
DS		 1391 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1392 (void *)&resp, NULL, 0);
f218d67e
SX		 1393 if (rc) {
1394 rcfw->qp_tbl[qp->id].qp_id = qp->id;
1395 rcfw->qp_tbl[qp->id].qp_handle = qp;
cc1ec769 1396 return rc;
f218d67e 1397 }
1ac5a404 1398
3b921e3b
SX		 1399 return 0;
1400}
1ac5a404 1401
3b921e3b
SX		 1402void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
1403 struct bnxt_qplib_qp *qp)
1404{
1ac5a404
SX		 1405 bnxt_qplib_free_qp_hdr_buf(res, qp);
1406 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
1407 kfree(qp->sq.swq);
1408
1409 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
1410 kfree(qp->rq.swq);
1411
1412 if (qp->irrq.max_elements)
1413 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
1414 if (qp->orrq.max_elements)
1415 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1416
1ac5a404
SX		 1417}
1418
1419void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1420 struct bnxt_qplib_sge *sge)
1421{
1422 struct bnxt_qplib_q *sq = &qp->sq;
1423 u32 sw_prod;
1424
1425 memset(sge, 0, sizeof(*sge));
1426
1427 if (qp->sq_hdr_buf) {
1428 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1429 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1430 sw_prod * qp->sq_hdr_buf_size);
1431 sge->lkey = 0xFFFFFFFF;
1432 sge->size = qp->sq_hdr_buf_size;
1433 return qp->sq_hdr_buf + sw_prod * sge->size;
1434 }
1435 return NULL;
1436}
1437
1438u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1439{
1440 struct bnxt_qplib_q *rq = &qp->rq;
1441
1442 return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1443}
1444
1445dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1446{
1447 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1448}
1449
1450void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1451 struct bnxt_qplib_sge *sge)
1452{
1453 struct bnxt_qplib_q *rq = &qp->rq;
1454 u32 sw_prod;
1455
1456 memset(sge, 0, sizeof(*sge));
1457
1458 if (qp->rq_hdr_buf) {
1459 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1460 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1461 sw_prod * qp->rq_hdr_buf_size);
1462 sge->lkey = 0xFFFFFFFF;
1463 sge->size = qp->rq_hdr_buf_size;
1464 return qp->rq_hdr_buf + sw_prod * sge->size;
1465 }
1466 return NULL;
1467}
1468
1469void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1470{
1471 struct bnxt_qplib_q *sq = &qp->sq;
1472 struct dbr_dbr db_msg = { 0 };
1473 u32 sw_prod;
1474
1475 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1476
1477 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1478 DBR_DBR_INDEX_MASK);
1479 db_msg.type_xid =
1480 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1481 DBR_DBR_TYPE_SQ);
1482 /* Flush all the WQE writes to HW */
1483 wmb();
1484 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1485}
1486
1487int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1488 struct bnxt_qplib_swqe *wqe)
1489{
1490 struct bnxt_qplib_q *sq = &qp->sq;
1491 struct bnxt_qplib_swq *swq;
1492 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1493 struct sq_sge *hw_sge;
f218d67e
SX		 1494 struct bnxt_qplib_nq_work *nq_work = NULL;
1495 bool sch_handler = false;
1ac5a404
SX		 1496 u32 sw_prod;
1497 u8 wqe_size16;
1498 int i, rc = 0, data_len = 0, pkt_num = 0;
1499 __le32 temp32;
1500
1501 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
f218d67e
SX		 1502 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1503 sch_handler = true;
1504 dev_dbg(&sq->hwq.pdev->dev,
1505 "%s Error QP. Scheduling for poll_cq\n",
1506 __func__);
1507 goto queue_err;
1508 }
1ac5a404 1509 }
9152e0b7
EW		 1510
1511 if (bnxt_qplib_queue_full(sq)) {
1512 dev_err(&sq->hwq.pdev->dev,
1513 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1514 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1515 sq->q_full_delta);
1ac5a404
SX		 1516 rc = -ENOMEM;
1517 goto done;
1518 }
1519 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1520 swq = &sq->swq[sw_prod];
1521 swq->wr_id = wqe->wr_id;
1522 swq->type = wqe->type;
1523 swq->flags = wqe->flags;
1524 if (qp->sig_type)
1525 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1526 swq->start_psn = sq->psn & BTH_PSN_MASK;
1527
1528 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1529 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1530 [get_sqe_idx(sw_prod)];
1531
1532 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1533
1534 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1535 /* Copy the inline data */
1536 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1537 dev_warn(&sq->hwq.pdev->dev,
1538 "QPLIB: Inline data length > 96 detected");
1539 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1540 } else {
1541 data_len = wqe->inline_len;
1542 }
1543 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1544 wqe_size16 = (data_len + 15) >> 4;
1545 } else {
1546 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1547 i < wqe->num_sge; i++, hw_sge++) {
1548 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1549 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1550 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1551 data_len += wqe->sg_list[i].size;
1552 }
1553 /* Each SGE entry = 1 WQE size16 */
1554 wqe_size16 = wqe->num_sge;
ab69d4c8
SK		 1555 /* HW requires wqe size has room for atleast one SGE even if
1556 * none was supplied by ULP
1557 */
1558 if (!wqe->num_sge)
1559 wqe_size16++;
1ac5a404
SX		 1560 }
1561
1562 /* Specifics */
1563 switch (wqe->type) {
1564 case BNXT_QPLIB_SWQE_TYPE_SEND:
1565 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1566 /* Assemble info for Raw Ethertype QPs */
1567 struct sq_send_raweth_qp1 *sqe =
1568 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1569
1570 sqe->wqe_type = wqe->type;
1571 sqe->flags = wqe->flags;
1572 sqe->wqe_size = wqe_size16 +
1573 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1574 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1575 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1576 sqe->length = cpu_to_le32(data_len);
1577 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1578 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1579 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1580
1581 break;
1582 }
38c8a710 1583 /* fall thru */
1ac5a404
SX		 1584 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1585 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1586 {
1587 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1588
1589 sqe->wqe_type = wqe->type;
1590 sqe->flags = wqe->flags;
1591 sqe->wqe_size = wqe_size16 +
1592 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1593 sqe->inv_key_or_imm_data = cpu_to_le32(
1594 wqe->send.inv_key);
1595 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1596 sqe->q_key = cpu_to_le32(wqe->send.q_key);
1597 sqe->dst_qp = cpu_to_le32(
1598 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1599 sqe->length = cpu_to_le32(data_len);
1600 sqe->avid = cpu_to_le32(wqe->send.avid &
1601 SQ_SEND_AVID_MASK);
1602 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1603 } else {
1604 sqe->length = cpu_to_le32(data_len);
1605 sqe->dst_qp = 0;
1606 sqe->avid = 0;
1607 if (qp->mtu)
1608 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1609 if (!pkt_num)
1610 pkt_num = 1;
1611 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1612 }
1613 break;
1614 }
1615 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1616 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1617 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1618 {
1619 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1620
1621 sqe->wqe_type = wqe->type;
1622 sqe->flags = wqe->flags;
1623 sqe->wqe_size = wqe_size16 +
1624 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1625 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1626 sqe->length = cpu_to_le32((u32)data_len);
1627 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1628 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1629 if (qp->mtu)
1630 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1631 if (!pkt_num)
1632 pkt_num = 1;
1633 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1634 break;
1635 }
1636 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1637 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1638 {
1639 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1640
1641 sqe->wqe_type = wqe->type;
1642 sqe->flags = wqe->flags;
1643 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1644 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1645 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1646 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1647 if (qp->mtu)
1648 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1649 if (!pkt_num)
1650 pkt_num = 1;
1651 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1652 break;
1653 }
1654 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1655 {
1656 struct sq_localinvalidate *sqe =
1657 (struct sq_localinvalidate *)hw_sq_send_hdr;
1658
1659 sqe->wqe_type = wqe->type;
1660 sqe->flags = wqe->flags;
1661 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1662
1663 break;
1664 }
1665 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1666 {
1667 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1668
1669 sqe->wqe_type = wqe->type;
1670 sqe->flags = wqe->flags;
1671 sqe->access_cntl = wqe->frmr.access_cntl |
1672 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1673 sqe->zero_based_page_size_log =
1674 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1675 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1676 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1677 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1678 temp32 = cpu_to_le32(wqe->frmr.length);
1679 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1680 sqe->numlevels_pbl_page_size_log =
1681 ((wqe->frmr.pbl_pg_sz_log <<
1682 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1683 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1684 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1685 SQ_FR_PMR_NUMLEVELS_MASK);
1686
1687 for (i = 0; i < wqe->frmr.page_list_len; i++)
1688 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1689 wqe->frmr.page_list[i] |
1690 PTU_PTE_VALID);
1691 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1692 sqe->va = cpu_to_le64(wqe->frmr.va);
1693
1694 break;
1695 }
1696 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1697 {
1698 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1699
1700 sqe->wqe_type = wqe->type;
1701 sqe->flags = wqe->flags;
1702 sqe->access_cntl = wqe->bind.access_cntl;
1703 sqe->mw_type_zero_based = wqe->bind.mw_type |
1704 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1705 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1706 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1707 sqe->va = cpu_to_le64(wqe->bind.va);
1708 temp32 = cpu_to_le32(wqe->bind.length);
1709 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1710 break;
1711 }
1712 default:
1713 /* Bad wqe, return error */
1714 rc = -EINVAL;
1715 goto done;
1716 }
1717 swq->next_psn = sq->psn & BTH_PSN_MASK;
1718 if (swq->psn_search) {
1719 swq->psn_search->opcode_start_psn = cpu_to_le32(
1720 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1721 SQ_PSN_SEARCH_START_PSN_MASK) |
1722 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1723 SQ_PSN_SEARCH_OPCODE_MASK));
1724 swq->psn_search->flags_next_psn = cpu_to_le32(
1725 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1726 SQ_PSN_SEARCH_NEXT_PSN_MASK));
1727 }
f218d67e
SX		 1728queue_err:
1729 if (sch_handler) {
1730 /* Store the ULP info in the software structures */
1731 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1732 swq = &sq->swq[sw_prod];
1733 swq->wr_id = wqe->wr_id;
1734 swq->type = wqe->type;
1735 swq->flags = wqe->flags;
1736 if (qp->sig_type)
1737 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1738 swq->start_psn = sq->psn & BTH_PSN_MASK;
1739 }
1ac5a404 1740 sq->hwq.prod++;
3fb755b3
SK		 1741 qp->wqe_cnt++;
1742
1ac5a404 1743done:
f218d67e
SX		 1744 if (sch_handler) {
1745 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1746 if (nq_work) {
1747 nq_work->cq = qp->scq;
1748 nq_work->nq = qp->scq->nq;
1749 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1750 queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
1751 } else {
1752 dev_err(&sq->hwq.pdev->dev,
1753 "QPLIB: FP: Failed to allocate SQ nq_work!");
1754 rc = -ENOMEM;
1755 }
1756 }
1ac5a404
SX		 1757 return rc;
1758}
1759
1760void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1761{
1762 struct bnxt_qplib_q *rq = &qp->rq;
1763 struct dbr_dbr db_msg = { 0 };
1764 u32 sw_prod;
1765
1766 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1767 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1768 DBR_DBR_INDEX_MASK);
1769 db_msg.type_xid =
1770 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1771 DBR_DBR_TYPE_RQ);
1772
1773 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1774 wmb();
1775 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1776}
1777
1778int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1779 struct bnxt_qplib_swqe *wqe)
1780{
1781 struct bnxt_qplib_q *rq = &qp->rq;
1782 struct rq_wqe *rqe, **rqe_ptr;
1783 struct sq_sge *hw_sge;
f218d67e
SX		 1784 struct bnxt_qplib_nq_work *nq_work = NULL;
1785 bool sch_handler = false;
1ac5a404
SX		 1786 u32 sw_prod;
1787 int i, rc = 0;
1788
1789 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
f218d67e
SX		 1790 sch_handler = true;
1791 dev_dbg(&rq->hwq.pdev->dev,
1792 "%s Error QP. Scheduling for poll_cq\n",
1793 __func__);
1794 goto queue_err;
1ac5a404 1795 }
9152e0b7 1796 if (bnxt_qplib_queue_full(rq)) {
1ac5a404
SX		 1797 dev_err(&rq->hwq.pdev->dev,
1798 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1799 rc = -EINVAL;
1800 goto done;
1801 }
1802 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1803 rq->swq[sw_prod].wr_id = wqe->wr_id;
1804
1805 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1806 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1807
1808 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1809
1810 /* Calculate wqe_size16 and data_len */
1811 for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1812 i < wqe->num_sge; i++, hw_sge++) {
1813 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1814 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1815 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1816 }
1817 rqe->wqe_type = wqe->type;
1818 rqe->flags = wqe->flags;
1819 rqe->wqe_size = wqe->num_sge +
1820 ((offsetof(typeof(*rqe), data) + 15) >> 4);
ab69d4c8
SK		 1821 /* HW requires wqe size has room for atleast one SGE even if none
1822 * was supplied by ULP
1823 */
1824 if (!wqe->num_sge)
1825 rqe->wqe_size++;
1ac5a404
SX		 1826
1827 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1828 rqe->wr_id[0] = cpu_to_le32(sw_prod);
1829
f218d67e
SX		 1830queue_err:
1831 if (sch_handler) {
1832 /* Store the ULP info in the software structures */
1833 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1834 rq->swq[sw_prod].wr_id = wqe->wr_id;
1835 }
1836
1ac5a404 1837 rq->hwq.prod++;
f218d67e
SX		 1838 if (sch_handler) {
1839 nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
1840 if (nq_work) {
1841 nq_work->cq = qp->rcq;
1842 nq_work->nq = qp->rcq->nq;
1843 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
1844 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
1845 } else {
1846 dev_err(&rq->hwq.pdev->dev,
1847 "QPLIB: FP: Failed to allocate RQ nq_work!");
1848 rc = -ENOMEM;
1849 }
1850 }
1ac5a404
SX		 1851done:
1852 return rc;
1853}
1854
1855/* CQ */
1856
1857/* Spinlock must be held */
1858static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1859{
1860 struct dbr_dbr db_msg = { 0 };
1861
1862 db_msg.type_xid =
1863 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1864 DBR_DBR_TYPE_CQ_ARMENA);
1865 /* Flush memory writes before enabling the CQ */
1866 wmb();
1867 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1868}
1869
1870static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1871{
1872 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1873 struct dbr_dbr db_msg = { 0 };
1874 u32 sw_cons;
1875
1876 /* Ring DB */
1877 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1878 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1879 DBR_DBR_INDEX_MASK);
1880 db_msg.type_xid =
1881 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1882 arm_type);
1883 /* flush memory writes before arming the CQ */
1884 wmb();
1885 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1886}
1887
1888int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1889{
1890 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1891 struct cmdq_create_cq req;
cc1ec769 1892 struct creq_create_cq_resp resp;
1ac5a404
SX		 1893 struct bnxt_qplib_pbl *pbl;
1894 u16 cmd_flags = 0;
1895 int rc;
1896
1897 cq->hwq.max_elements = cq->max_wqe;
1898 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1899 cq->nmap, &cq->hwq.max_elements,
1900 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1901 PAGE_SIZE, HWQ_TYPE_QUEUE);
1902 if (rc)
1903 goto exit;
1904
1905 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1906
1907 if (!cq->dpi) {
1908 dev_err(&rcfw->pdev->dev,
1909 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1910 return -EINVAL;
1911 }
1912 req.dpi = cpu_to_le32(cq->dpi->dpi);
1913 req.cq_handle = cpu_to_le64(cq->cq_handle);
1914
1915 req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1916 pbl = &cq->hwq.pbl[PBL_LVL_0];
1917 req.pg_size_lvl = cpu_to_le32(
1918 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1919 CMDQ_CREATE_CQ_LVL_SFT) |
1920 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1921 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1922 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1923 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1924 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1925 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1926 CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1927
1928 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1929
1930 req.cq_fco_cnq_id = cpu_to_le32(
1931 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1932 CMDQ_CREATE_CQ_CNQ_ID_SFT);
1933
cc1ec769
DS		 1934 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1935 (void *)&resp, NULL, 0);
1936 if (rc)
1ac5a404 1937 goto fail;
cc1ec769
DS		 1938
1939 cq->id = le32_to_cpu(resp.xid);
1ac5a404
SX		 1940 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1941 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1942 init_waitqueue_head(&cq->waitq);
f218d67e
SX		 1943 INIT_LIST_HEAD(&cq->sqf_head);
1944 INIT_LIST_HEAD(&cq->rqf_head);
1945 spin_lock_init(&cq->compl_lock);
1ac5a404
SX		 1946
1947 bnxt_qplib_arm_cq_enable(cq);
1948 return 0;
1949
1950fail:
1951 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1952exit:
1953 return rc;
1954}
1955
1956int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1957{
1958 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1959 struct cmdq_destroy_cq req;
cc1ec769 1960 struct creq_destroy_cq_resp resp;
1ac5a404 1961 u16 cmd_flags = 0;
cc1ec769 1962 int rc;
1ac5a404
SX		 1963
1964 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1965
1966 req.cq_cid = cpu_to_le32(cq->id);
cc1ec769
DS		 1967 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1968 (void *)&resp, NULL, 0);
1969 if (rc)
1970 return rc;
1ac5a404
SX		 1971 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1972 return 0;
1973}
1974
1975static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1976 struct bnxt_qplib_cqe **pcqe, int *budget)
1977{
1978 u32 sw_prod, sw_cons;
1979 struct bnxt_qplib_cqe *cqe;
1980 int rc = 0;
1981
1982 /* Now complete all outstanding SQEs with FLUSHED_ERR */
1983 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1984 cqe = *pcqe;
1985 while (*budget) {
1986 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1987 if (sw_cons == sw_prod) {
1ac5a404
SX		 1988 break;
1989 }
f218d67e
SX		 1990 /* Skip the FENCE WQE completions */
1991 if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
1992 bnxt_qplib_cancel_phantom_processing(qp);
1993 goto skip_compl;
1994 }
1ac5a404
SX		 1995 memset(cqe, 0, sizeof(*cqe));
1996 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1997 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1998 cqe->qp_handle = (u64)(unsigned long)qp;
1999 cqe->wr_id = sq->swq[sw_cons].wr_id;
2000 cqe->src_qp = qp->id;
2001 cqe->type = sq->swq[sw_cons].type;
2002 cqe++;
2003 (*budget)--;
f218d67e 2004skip_compl:
1ac5a404
SX		 2005 sq->hwq.cons++;
2006 }
2007 *pcqe = cqe;
2008 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
2009 /* Out of budget */
2010 rc = -EAGAIN;
2011
2012 return rc;
2013}
2014
2015static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
f218d67e 2016 struct bnxt_qplib_cqe **pcqe, int *budget)
1ac5a404
SX		 2017{
2018 struct bnxt_qplib_cqe *cqe;
2019 u32 sw_prod, sw_cons;
2020 int rc = 0;
f218d67e
SX		 2021 int opcode = 0;
2022
2023 switch (qp->type) {
2024 case CMDQ_CREATE_QP1_TYPE_GSI:
2025 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2026 break;
2027 case CMDQ_CREATE_QP_TYPE_RC:
2028 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2029 break;
2030 case CMDQ_CREATE_QP_TYPE_UD:
2031 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2032 break;
2033 }
1ac5a404
SX		 2034
2035 /* Flush the rest of the RQ */
2036 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
2037 cqe = *pcqe;
2038 while (*budget) {
2039 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
2040 if (sw_cons == sw_prod)
2041 break;
2042 memset(cqe, 0, sizeof(*cqe));
2043 cqe->status =
2044 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
2045 cqe->opcode = opcode;
2046 cqe->qp_handle = (unsigned long)qp;
2047 cqe->wr_id = rq->swq[sw_cons].wr_id;
2048 cqe++;
2049 (*budget)--;
2050 rq->hwq.cons++;
2051 }
2052 *pcqe = cqe;
2053 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
2054 /* Out of budget */
2055 rc = -EAGAIN;
2056
2057 return rc;
2058}
2059
f218d67e
SX		 2060void bnxt_qplib_mark_qp_error(void *qp_handle)
2061{
2062 struct bnxt_qplib_qp *qp = qp_handle;
2063
2064 if (!qp)
2065 return;
2066
2067 /* Must block new posting of SQ and RQ */
2068 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2069 bnxt_qplib_cancel_phantom_processing(qp);
f218d67e
SX		 2070}
2071
9152e0b7
EW		 2072/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
2073 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
2074 */
2075static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
2076 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
2077{
2078 struct bnxt_qplib_q *sq = &qp->sq;
2079 struct bnxt_qplib_swq *swq;
2080 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
2081 struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
2082 struct cq_req *peek_req_hwcqe;
2083 struct bnxt_qplib_qp *peek_qp;
2084 struct bnxt_qplib_q *peek_sq;
2085 int i, rc = 0;
2086
2087 /* Normal mode */
2088 /* Check for the psn_search marking before completing */
2089 swq = &sq->swq[sw_sq_cons];
2090 if (swq->psn_search &&
2091 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
2092 /* Unmark */
2093 swq->psn_search->flags_next_psn = cpu_to_le32
2094 (le32_to_cpu(swq->psn_search->flags_next_psn)
2095 & ~0x80000000);
2096 dev_dbg(&cq->hwq.pdev->dev,
2097 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
2098 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
2099 sq->condition = true;
2100 sq->send_phantom = true;
2101
2102 /* TODO: Only ARM if the previous SQE is ARMALL */
2103 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
2104
2105 rc = -EAGAIN;
2106 goto out;
2107 }
2108 if (sq->condition) {
2109 /* Peek at the completions */
2110 peek_raw_cq_cons = cq->hwq.cons;
2111 peek_sw_cq_cons = cq_cons;
2112 i = cq->hwq.max_elements;
2113 while (i--) {
2114 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
2115 peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2116 peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
2117 [CQE_IDX(peek_sw_cq_cons)];
2118 /* If the next hwcqe is VALID */
2119 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
2120 cq->hwq.max_elements)) {
9b40183c
SK		 2121 /*
2122 * The valid test of the entry must be done first before
2123 * reading any further.
2124 */
2125 dma_rmb();
9152e0b7
EW		 2126 /* If the next hwcqe is a REQ */
2127 if ((peek_hwcqe->cqe_type_toggle &
2128 CQ_BASE_CQE_TYPE_MASK) ==
2129 CQ_BASE_CQE_TYPE_REQ) {
2130 peek_req_hwcqe = (struct cq_req *)
2131 peek_hwcqe;
2132 peek_qp = (struct bnxt_qplib_qp *)
2133 ((unsigned long)
2134 le64_to_cpu
2135 (peek_req_hwcqe->qp_handle));
2136 peek_sq = &peek_qp->sq;
2137 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
2138 peek_req_hwcqe->sq_cons_idx) - 1
2139 , &sq->hwq);
2140 /* If the hwcqe's sq's wr_id matches */
2141 if (peek_sq == sq &&
2142 sq->swq[peek_sq_cons_idx].wr_id ==
2143 BNXT_QPLIB_FENCE_WRID) {
2144 /*
2145 * Unbreak only if the phantom
2146 * comes back
2147 */
2148 dev_dbg(&cq->hwq.pdev->dev,
2149 "FP:Got Phantom CQE");
2150 sq->condition = false;
2151 sq->single = true;
2152 rc = 0;
2153 goto out;
2154 }
2155 }
2156 /* Valid but not the phantom, so keep looping */
2157 } else {
2158 /* Not valid yet, just exit and wait */
2159 rc = -EINVAL;
2160 goto out;
2161 }
2162 peek_sw_cq_cons++;
2163 peek_raw_cq_cons++;
2164 }
2165 dev_err(&cq->hwq.pdev->dev,
2166 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
2167 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
2168 rc = -EINVAL;
2169 }
2170out:
2171 return rc;
2172}
2173
1ac5a404
SX		 2174static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
2175 struct cq_req *hwcqe,
9152e0b7
EW		 2176 struct bnxt_qplib_cqe **pcqe, int *budget,
2177 u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
1ac5a404
SX		 2178{
2179 struct bnxt_qplib_qp *qp;
2180 struct bnxt_qplib_q *sq;
2181 struct bnxt_qplib_cqe *cqe;
9152e0b7
EW		 2182 u32 sw_sq_cons, cqe_sq_cons;
2183 struct bnxt_qplib_swq *swq;
1ac5a404
SX		 2184 int rc = 0;
2185
2186 qp = (struct bnxt_qplib_qp *)((unsigned long)
2187 le64_to_cpu(hwcqe->qp_handle));
2188 if (!qp) {
2189 dev_err(&cq->hwq.pdev->dev,
2190 "QPLIB: FP: Process Req qp is NULL");
2191 return -EINVAL;
2192 }
2193 sq = &qp->sq;
2194
9152e0b7
EW		 2195 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
2196 if (cqe_sq_cons > sq->hwq.max_elements) {
1ac5a404
SX		 2197 dev_err(&cq->hwq.pdev->dev,
2198 "QPLIB: FP: CQ Process req reported ");
2199 dev_err(&cq->hwq.pdev->dev,
2200 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
9152e0b7 2201 cqe_sq_cons, sq->hwq.max_elements);
1ac5a404
SX		 2202 return -EINVAL;
2203 }
1ac5a404 2204
f218d67e
SX		 2205 if (qp->sq.flushed) {
2206 dev_dbg(&cq->hwq.pdev->dev,
2207 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2208 goto done;
2209 }
1ac5a404
SX		 2210 /* Require to walk the sq's swq to fabricate CQEs for all previously
2211 * signaled SWQEs due to CQE aggregation from the current sq cons
9152e0b7 2212 * to the cqe_sq_cons
1ac5a404
SX		 2213 */
2214 cqe = *pcqe;
2215 while (*budget) {
9152e0b7
EW		 2216 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2217 if (sw_sq_cons == cqe_sq_cons)
2218 /* Done */
1ac5a404 2219 break;
9152e0b7
EW		 2220
2221 swq = &sq->swq[sw_sq_cons];
1ac5a404
SX		 2222 memset(cqe, 0, sizeof(*cqe));
2223 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2224 cqe->qp_handle = (u64)(unsigned long)qp;
2225 cqe->src_qp = qp->id;
9152e0b7
EW		 2226 cqe->wr_id = swq->wr_id;
2227 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
2228 goto skip;
2229 cqe->type = swq->type;
1ac5a404
SX		 2230
2231 /* For the last CQE, check for status. For errors, regardless
2232 * of the request being signaled or not, it must complete with
2233 * the hwcqe error status
2234 */
9152e0b7 2235 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
1ac5a404
SX		 2236 hwcqe->status != CQ_REQ_STATUS_OK) {
2237 cqe->status = hwcqe->status;
2238 dev_err(&cq->hwq.pdev->dev,
2239 "QPLIB: FP: CQ Processed Req ");
2240 dev_err(&cq->hwq.pdev->dev,
2241 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
9152e0b7 2242 sw_sq_cons, cqe->wr_id, cqe->status);
1ac5a404
SX		 2243 cqe++;
2244 (*budget)--;
f218d67e 2245 bnxt_qplib_mark_qp_error(qp);
942c9b6c
SX		 2246 /* Add qp to flush list of the CQ */
2247 bnxt_qplib_add_flush_qp(qp);
1ac5a404 2248 } else {
9152e0b7
EW		 2249 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2250 /* Before we complete, do WA 9060 */
2251 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
2252 cqe_sq_cons)) {
2253 *lib_qp = qp;
2254 goto out;
2255 }
1ac5a404
SX		 2256 cqe->status = CQ_REQ_STATUS_OK;
2257 cqe++;
2258 (*budget)--;
2259 }
2260 }
9152e0b7 2261skip:
1ac5a404 2262 sq->hwq.cons++;
9152e0b7
EW		 2263 if (sq->single)
2264 break;
1ac5a404 2265 }
9152e0b7 2266out:
1ac5a404 2267 *pcqe = cqe;
9152e0b7 2268 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
1ac5a404
SX		 2269 /* Out of budget */
2270 rc = -EAGAIN;
2271 goto done;
2272 }
9152e0b7
EW		 2273 /*
2274 * Back to normal completion mode only after it has completed all of
2275 * the WC for this CQE
2276 */
2277 sq->single = false;
1ac5a404
SX		 2278done:
2279 return rc;
2280}
2281
37cb11ac
DS		 2282static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
2283{
2284 spin_lock(&srq->hwq.lock);
2285 srq->swq[srq->last_idx].next_idx = (int)tag;
2286 srq->last_idx = (int)tag;
2287 srq->swq[srq->last_idx].next_idx = -1;
2288 srq->hwq.cons++; /* Support for SRQE counter */
2289 spin_unlock(&srq->hwq.lock);
2290}
2291
1ac5a404
SX		 2292static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
2293 struct cq_res_rc *hwcqe,
2294 struct bnxt_qplib_cqe **pcqe,
2295 int *budget)
2296{
2297 struct bnxt_qplib_qp *qp;
2298 struct bnxt_qplib_q *rq;
37cb11ac 2299 struct bnxt_qplib_srq *srq;
1ac5a404
SX		 2300 struct bnxt_qplib_cqe *cqe;
2301 u32 wr_id_idx;
2302 int rc = 0;
2303
2304 qp = (struct bnxt_qplib_qp *)((unsigned long)
2305 le64_to_cpu(hwcqe->qp_handle));
2306 if (!qp) {
2307 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
2308 return -EINVAL;
2309 }
f218d67e
SX		 2310 if (qp->rq.flushed) {
2311 dev_dbg(&cq->hwq.pdev->dev,
2312 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2313 goto done;
2314 }
2315
1ac5a404
SX		 2316 cqe = *pcqe;
2317 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2318 cqe->length = le32_to_cpu(hwcqe->length);
2319 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
2320 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
2321 cqe->flags = le16_to_cpu(hwcqe->flags);
2322 cqe->status = hwcqe->status;
2323 cqe->qp_handle = (u64)(unsigned long)qp;
2324
2325 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
2326 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
37cb11ac
DS		 2327 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
2328 srq = qp->srq;
2329 if (!srq)
2330 return -EINVAL;
2331 if (wr_id_idx > srq->hwq.max_elements) {
2332 dev_err(&cq->hwq.pdev->dev,
2333 "QPLIB: FP: CQ Process RC ");
2334 dev_err(&cq->hwq.pdev->dev,
2335 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
2336 wr_id_idx, srq->hwq.max_elements);
2337 return -EINVAL;
2338 }
2339 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2340 bnxt_qplib_release_srqe(srq, wr_id_idx);
2341 cqe++;
2342 (*budget)--;
2343 *pcqe = cqe;
2344 } else {
2345 rq = &qp->rq;
2346 if (wr_id_idx > rq->hwq.max_elements) {
2347 dev_err(&cq->hwq.pdev->dev,
2348 "QPLIB: FP: CQ Process RC ");
2349 dev_err(&cq->hwq.pdev->dev,
2350 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2351 wr_id_idx, rq->hwq.max_elements);
2352 return -EINVAL;
2353 }
2354 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2355 cqe++;
2356 (*budget)--;
2357 rq->hwq.cons++;
2358 *pcqe = cqe;
1ac5a404 2359
37cb11ac
DS		 2360 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2361 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2362 /* Add qp to flush list of the CQ */
942c9b6c 2363 bnxt_qplib_add_flush_qp(qp);
37cb11ac 2364 }
1ac5a404 2365 }
f218d67e
SX		 2366
2367done:
1ac5a404
SX		 2368 return rc;
2369}
2370
2371static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
2372 struct cq_res_ud *hwcqe,
2373 struct bnxt_qplib_cqe **pcqe,
2374 int *budget)
2375{
2376 struct bnxt_qplib_qp *qp;
2377 struct bnxt_qplib_q *rq;
37cb11ac 2378 struct bnxt_qplib_srq *srq;
1ac5a404
SX		 2379 struct bnxt_qplib_cqe *cqe;
2380 u32 wr_id_idx;
2381 int rc = 0;
2382
2383 qp = (struct bnxt_qplib_qp *)((unsigned long)
2384 le64_to_cpu(hwcqe->qp_handle));
2385 if (!qp) {
2386 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
2387 return -EINVAL;
2388 }
f218d67e
SX		 2389 if (qp->rq.flushed) {
2390 dev_dbg(&cq->hwq.pdev->dev,
2391 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2392 goto done;
2393 }
1ac5a404
SX		 2394 cqe = *pcqe;
2395 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2396 cqe->length = le32_to_cpu(hwcqe->length);
2397 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
2398 cqe->flags = le16_to_cpu(hwcqe->flags);
2399 cqe->status = hwcqe->status;
2400 cqe->qp_handle = (u64)(unsigned long)qp;
2401 memcpy(cqe->smac, hwcqe->src_mac, 6);
2402 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
2403 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
2404 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
2405 ((le32_to_cpu(
2406 hwcqe->src_qp_high_srq_or_rq_wr_id) &
2407 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
2408
37cb11ac
DS		 2409 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
2410 srq = qp->srq;
2411 if (!srq)
2412 return -EINVAL;
1ac5a404 2413
37cb11ac
DS		 2414 if (wr_id_idx > srq->hwq.max_elements) {
2415 dev_err(&cq->hwq.pdev->dev,
2416 "QPLIB: FP: CQ Process UD ");
2417 dev_err(&cq->hwq.pdev->dev,
2418 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
2419 wr_id_idx, srq->hwq.max_elements);
2420 return -EINVAL;
2421 }
2422 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2423 bnxt_qplib_release_srqe(srq, wr_id_idx);
2424 cqe++;
2425 (*budget)--;
2426 *pcqe = cqe;
2427 } else {
2428 rq = &qp->rq;
2429 if (wr_id_idx > rq->hwq.max_elements) {
2430 dev_err(&cq->hwq.pdev->dev,
2431 "QPLIB: FP: CQ Process UD ");
2432 dev_err(&cq->hwq.pdev->dev,
2433 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
2434 wr_id_idx, rq->hwq.max_elements);
2435 return -EINVAL;
2436 }
2437
2438 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2439 cqe++;
2440 (*budget)--;
2441 rq->hwq.cons++;
2442 *pcqe = cqe;
1ac5a404 2443
37cb11ac
DS		 2444 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2445 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2446 /* Add qp to flush list of the CQ */
942c9b6c 2447 bnxt_qplib_add_flush_qp(qp);
37cb11ac 2448 }
1ac5a404 2449 }
f218d67e 2450done:
1ac5a404
SX		 2451 return rc;
2452}
2453
499e4569
SX		 2454bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
2455{
2456 struct cq_base *hw_cqe, **hw_cqe_ptr;
499e4569
SX		 2457 u32 sw_cons, raw_cons;
2458 bool rc = true;
2459
499e4569
SX		 2460 raw_cons = cq->hwq.cons;
2461 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2462 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2463 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2464
2465 /* Check for Valid bit. If the CQE is valid, return false */
2466 rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
499e4569
SX		 2467 return rc;
2468}
2469
1ac5a404
SX		 2470static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
2471 struct cq_res_raweth_qp1 *hwcqe,
2472 struct bnxt_qplib_cqe **pcqe,
2473 int *budget)
2474{
2475 struct bnxt_qplib_qp *qp;
2476 struct bnxt_qplib_q *rq;
37cb11ac 2477 struct bnxt_qplib_srq *srq;
1ac5a404
SX		 2478 struct bnxt_qplib_cqe *cqe;
2479 u32 wr_id_idx;
2480 int rc = 0;
2481
2482 qp = (struct bnxt_qplib_qp *)((unsigned long)
2483 le64_to_cpu(hwcqe->qp_handle));
2484 if (!qp) {
2485 dev_err(&cq->hwq.pdev->dev,
2486 "QPLIB: process_cq Raw/QP1 qp is NULL");
2487 return -EINVAL;
2488 }
f218d67e
SX		 2489 if (qp->rq.flushed) {
2490 dev_dbg(&cq->hwq.pdev->dev,
2491 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2492 goto done;
2493 }
1ac5a404
SX		 2494 cqe = *pcqe;
2495 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
2496 cqe->flags = le16_to_cpu(hwcqe->flags);
2497 cqe->qp_handle = (u64)(unsigned long)qp;
2498
2499 wr_id_idx =
2500 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
2501 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
2502 cqe->src_qp = qp->id;
2503 if (qp->id == 1 && !cqe->length) {
2504 /* Add workaround for the length misdetection */
2505 cqe->length = 296;
2506 } else {
2507 cqe->length = le16_to_cpu(hwcqe->length);
2508 }
2509 cqe->pkey_index = qp->pkey_index;
2510 memcpy(cqe->smac, qp->smac, 6);
2511
2512 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
2513 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
84511455 2514 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
1ac5a404 2515
37cb11ac
DS		 2516 if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
2517 srq = qp->srq;
2518 if (!srq) {
2519 dev_err(&cq->hwq.pdev->dev,
2520 "QPLIB: FP: SRQ used but not defined??");
2521 return -EINVAL;
2522 }
2523 if (wr_id_idx > srq->hwq.max_elements) {
2524 dev_err(&cq->hwq.pdev->dev,
2525 "QPLIB: FP: CQ Process Raw/QP1 ");
2526 dev_err(&cq->hwq.pdev->dev,
2527 "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
2528 wr_id_idx, srq->hwq.max_elements);
2529 return -EINVAL;
2530 }
2531 cqe->wr_id = srq->swq[wr_id_idx].wr_id;
2532 bnxt_qplib_release_srqe(srq, wr_id_idx);
2533 cqe++;
2534 (*budget)--;
2535 *pcqe = cqe;
2536 } else {
2537 rq = &qp->rq;
2538 if (wr_id_idx > rq->hwq.max_elements) {
2539 dev_err(&cq->hwq.pdev->dev,
2540 "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
2541 dev_err(&cq->hwq.pdev->dev,
2542 "QPLIB: ix 0x%x exceeded RQ max 0x%x",
2543 wr_id_idx, rq->hwq.max_elements);
2544 return -EINVAL;
2545 }
2546 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
2547 cqe++;
2548 (*budget)--;
2549 rq->hwq.cons++;
2550 *pcqe = cqe;
1ac5a404 2551
37cb11ac
DS		 2552 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
2553 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2554 /* Add qp to flush list of the CQ */
942c9b6c 2555 bnxt_qplib_add_flush_qp(qp);
37cb11ac 2556 }
1ac5a404 2557 }
f218d67e
SX		 2558
2559done:
1ac5a404
SX		 2560 return rc;
2561}
2562
2563static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
2564 struct cq_terminal *hwcqe,
2565 struct bnxt_qplib_cqe **pcqe,
2566 int *budget)
2567{
2568 struct bnxt_qplib_qp *qp;
2569 struct bnxt_qplib_q *sq, *rq;
2570 struct bnxt_qplib_cqe *cqe;
2571 u32 sw_cons = 0, cqe_cons;
2572 int rc = 0;
1ac5a404
SX		 2573
2574 /* Check the Status */
2575 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
2576 dev_warn(&cq->hwq.pdev->dev,
2577 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
2578 hwcqe->status);
2579
2580 qp = (struct bnxt_qplib_qp *)((unsigned long)
2581 le64_to_cpu(hwcqe->qp_handle));
2582 if (!qp) {
2583 dev_err(&cq->hwq.pdev->dev,
2584 "QPLIB: FP: CQ Process terminal qp is NULL");
2585 return -EINVAL;
2586 }
f218d67e 2587
1ac5a404
SX		 2588 /* Must block new posting of SQ and RQ */
2589 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
2590
2591 sq = &qp->sq;
2592 rq = &qp->rq;
2593
2594 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
2595 if (cqe_cons == 0xFFFF)
2596 goto do_rq;
2597
2598 if (cqe_cons > sq->hwq.max_elements) {
2599 dev_err(&cq->hwq.pdev->dev,
2600 "QPLIB: FP: CQ Process terminal reported ");
2601 dev_err(&cq->hwq.pdev->dev,
2602 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
2603 cqe_cons, sq->hwq.max_elements);
2604 goto do_rq;
2605 }
f218d67e
SX		 2606
2607 if (qp->sq.flushed) {
2608 dev_dbg(&cq->hwq.pdev->dev,
2609 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2610 goto sq_done;
2611 }
1ac5a404
SX		 2612
2613 /* Terminal CQE can also include aggregated successful CQEs prior.
2614 * So we must complete all CQEs from the current sq's cons to the
2615 * cq_cons with status OK
2616 */
2617 cqe = *pcqe;
2618 while (*budget) {
2619 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2620 if (sw_cons == cqe_cons)
2621 break;
2622 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2623 memset(cqe, 0, sizeof(*cqe));
2624 cqe->status = CQ_REQ_STATUS_OK;
2625 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2626 cqe->qp_handle = (u64)(unsigned long)qp;
2627 cqe->src_qp = qp->id;
2628 cqe->wr_id = sq->swq[sw_cons].wr_id;
2629 cqe->type = sq->swq[sw_cons].type;
2630 cqe++;
2631 (*budget)--;
2632 }
2633 sq->hwq.cons++;
2634 }
2635 *pcqe = cqe;
2636 if (!(*budget) && sw_cons != cqe_cons) {
2637 /* Out of budget */
2638 rc = -EAGAIN;
2639 goto sq_done;
2640 }
1ac5a404
SX		 2641sq_done:
2642 if (rc)
2643 return rc;
2644do_rq:
2645 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2646 if (cqe_cons == 0xFFFF) {
2647 goto done;
2648 } else if (cqe_cons > rq->hwq.max_elements) {
2649 dev_err(&cq->hwq.pdev->dev,
2650 "QPLIB: FP: CQ Processed terminal ");
2651 dev_err(&cq->hwq.pdev->dev,
2652 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2653 cqe_cons, rq->hwq.max_elements);
2654 goto done;
2655 }
f218d67e
SX		 2656
2657 if (qp->rq.flushed) {
2658 dev_dbg(&cq->hwq.pdev->dev,
2659 "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
2660 rc = 0;
2661 goto done;
2662 }
2663
1ac5a404
SX		 2664 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2665 * from the current rq->cons to the rq->prod regardless what the
2666 * rq->cons the terminal CQE indicates
2667 */
1ac5a404 2668
f218d67e 2669 /* Add qp to flush list of the CQ */
942c9b6c 2670 bnxt_qplib_add_flush_qp(qp);
1ac5a404
SX		 2671done:
2672 return rc;
2673}
2674
2675static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2676 struct cq_cutoff *hwcqe)
2677{
2678 /* Check the Status */
2679 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2680 dev_err(&cq->hwq.pdev->dev,
2681 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2682 hwcqe->status);
2683 return -EINVAL;
2684 }
2685 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2686 wake_up_interruptible(&cq->waitq);
2687
2688 return 0;
2689}
2690
f218d67e
SX		 2691int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
2692 struct bnxt_qplib_cqe *cqe,
2693 int num_cqes)
2694{
2695 struct bnxt_qplib_qp *qp = NULL;
2696 u32 budget = num_cqes;
2697 unsigned long flags;
2698
942c9b6c 2699 spin_lock_irqsave(&cq->flush_lock, flags);
f218d67e
SX		 2700 list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
2701 dev_dbg(&cq->hwq.pdev->dev,
2702 "QPLIB: FP: Flushing SQ QP= %p",
2703 qp);
2704 __flush_sq(&qp->sq, qp, &cqe, &budget);
2705 }
2706
2707 list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
2708 dev_dbg(&cq->hwq.pdev->dev,
2709 "QPLIB: FP: Flushing RQ QP= %p",
2710 qp);
2711 __flush_rq(&qp->rq, qp, &cqe, &budget);
2712 }
942c9b6c 2713 spin_unlock_irqrestore(&cq->flush_lock, flags);
f218d67e
SX		 2714
2715 return num_cqes - budget;
2716}
2717
1ac5a404 2718int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
9152e0b7 2719 int num_cqes, struct bnxt_qplib_qp **lib_qp)
1ac5a404
SX		 2720{
2721 struct cq_base *hw_cqe, **hw_cqe_ptr;
1ac5a404
SX		 2722 u32 sw_cons, raw_cons;
2723 int budget, rc = 0;
2724
1ac5a404
SX		 2725 raw_cons = cq->hwq.cons;
2726 budget = num_cqes;
2727
2728 while (budget) {
2729 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2730 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2731 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2732
2733 /* Check for Valid bit */
2734 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2735 break;
2736
9b40183c
SK		 2737 /*
2738 * The valid test of the entry must be done first before
2739 * reading any further.
2740 */
2741 dma_rmb();
1ac5a404
SX		 2742 /* From the device's respective CQE format to qplib_wc*/
2743 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2744 case CQ_BASE_CQE_TYPE_REQ:
2745 rc = bnxt_qplib_cq_process_req(cq,
2746 (struct cq_req *)hw_cqe,
9152e0b7
EW		 2747 &cqe, &budget,
2748 sw_cons, lib_qp);
1ac5a404
SX		 2749 break;
2750 case CQ_BASE_CQE_TYPE_RES_RC:
2751 rc = bnxt_qplib_cq_process_res_rc(cq,
2752 (struct cq_res_rc *)
2753 hw_cqe, &cqe,
2754 &budget);
2755 break;
2756 case CQ_BASE_CQE_TYPE_RES_UD:
2757 rc = bnxt_qplib_cq_process_res_ud
2758 (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2759 &budget);
2760 break;
2761 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2762 rc = bnxt_qplib_cq_process_res_raweth_qp1
2763 (cq, (struct cq_res_raweth_qp1 *)
2764 hw_cqe, &cqe, &budget);
2765 break;
2766 case CQ_BASE_CQE_TYPE_TERMINAL:
2767 rc = bnxt_qplib_cq_process_terminal
2768 (cq, (struct cq_terminal *)hw_cqe,
2769 &cqe, &budget);
2770 break;
2771 case CQ_BASE_CQE_TYPE_CUT_OFF:
2772 bnxt_qplib_cq_process_cutoff
2773 (cq, (struct cq_cutoff *)hw_cqe);
2774 /* Done processing this CQ */
2775 goto exit;
2776 default:
2777 dev_err(&cq->hwq.pdev->dev,
2778 "QPLIB: process_cq unknown type 0x%lx",
2779 hw_cqe->cqe_type_toggle &
2780 CQ_BASE_CQE_TYPE_MASK);
2781 rc = -EINVAL;
2782 break;
2783 }
2784 if (rc < 0) {
2785 if (rc == -EAGAIN)
2786 break;
2787 /* Error while processing the CQE, just skip to the
2788 * next one
2789 */
2790 dev_err(&cq->hwq.pdev->dev,
2791 "QPLIB: process_cqe error rc = 0x%x", rc);
2792 }
2793 raw_cons++;
2794 }
2795 if (cq->hwq.cons != raw_cons) {
2796 cq->hwq.cons = raw_cons;
2797 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2798 }
2799exit:
1ac5a404
SX		 2800 return num_cqes - budget;
2801}
2802
2803void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2804{
1ac5a404
SX		 2805 if (arm_type)
2806 bnxt_qplib_arm_cq(cq, arm_type);
f218d67e
SX		 2807 /* Using cq->arm_state variable to track whether to issue cq handler */
2808 atomic_set(&cq->arm_state, 1);
1ac5a404 2809}
c88a7858
SX		 2810
2811void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
2812{
2813 flush_workqueue(qp->scq->nq->cqn_wq);
2814 if (qp->scq != qp->rcq)
2815 flush_workqueue(qp->rcq->nq->cqn_wq);
2816}
Linux 6.x block layer and io_uring tree(s)