Commit | Line | Data |
---|---|---|
1ac5a404 SX |
1 | /* |
2 | * Broadcom NetXtreme-E RoCE driver. | |
3 | * | |
4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
6 | * | |
7 | * This software is available to you under a choice of one of two | |
8 | * licenses. You may choose to be licensed under the terms of the GNU | |
9 | * General Public License (GPL) Version 2, available from the file | |
10 | * COPYING in the main directory of this source tree, or the | |
11 | * BSD license below: | |
12 | * | |
13 | * Redistribution and use in source and binary forms, with or without | |
14 | * modification, are permitted provided that the following conditions | |
15 | * are met: | |
16 | * | |
17 | * 1. Redistributions of source code must retain the above copyright | |
18 | * notice, this list of conditions and the following disclaimer. | |
19 | * 2. Redistributions in binary form must reproduce the above copyright | |
20 | * notice, this list of conditions and the following disclaimer in | |
21 | * the documentation and/or other materials provided with the | |
22 | * distribution. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
35 | * | |
36 | * Description: Fast Path Operators (header) | |
37 | */ | |
38 | ||
39 | #ifndef __BNXT_QPLIB_FP_H__ | |
40 | #define __BNXT_QPLIB_FP_H__ | |
41 | ||
87974051 DS |
42 | #include <rdma/bnxt_re-abi.h> |
43 | ||
54ace984 DS |
44 | /* Few helper structures temporarily defined here |
45 | * should get rid of these when roce_hsi.h is updated | |
46 | * in original code base | |
47 | */ | |
48 | struct sq_ud_ext_hdr { | |
49 | __le32 dst_qp; | |
50 | __le32 avid; | |
51 | __le64 rsvd; | |
52 | }; | |
53 | ||
54 | struct sq_raw_ext_hdr { | |
55 | __le32 cfa_meta; | |
56 | __le32 rsvd0; | |
57 | __le64 rsvd1; | |
58 | }; | |
59 | ||
60 | struct sq_rdma_ext_hdr { | |
61 | __le64 remote_va; | |
62 | __le32 remote_key; | |
63 | __le32 rsvd; | |
64 | }; | |
65 | ||
66 | struct sq_atomic_ext_hdr { | |
67 | __le64 swap_data; | |
68 | __le64 cmp_data; | |
69 | }; | |
70 | ||
71 | struct sq_fr_pmr_ext_hdr { | |
72 | __le64 pblptr; | |
73 | __le64 va; | |
74 | }; | |
75 | ||
76 | struct sq_bind_ext_hdr { | |
77 | __le64 va; | |
78 | __le32 length_lo; | |
79 | __le32 length_hi; | |
80 | }; | |
81 | ||
82 | struct rq_ext_hdr { | |
83 | __le64 rsvd1; | |
84 | __le64 rsvd2; | |
85 | }; | |
86 | ||
87 | /* Helper structures end */ | |
88 | ||
37cb11ac DS |
89 | struct bnxt_qplib_srq { |
90 | struct bnxt_qplib_pd *pd; | |
91 | struct bnxt_qplib_dpi *dpi; | |
6f53196b | 92 | struct bnxt_qplib_db_info dbinfo; |
37cb11ac DS |
93 | u64 srq_handle; |
94 | u32 id; | |
fddcbbb0 | 95 | u16 wqe_size; |
37cb11ac DS |
96 | u32 max_wqe; |
97 | u32 max_sge; | |
98 | u32 threshold; | |
99 | bool arm_req; | |
100 | struct bnxt_qplib_cq *cq; | |
101 | struct bnxt_qplib_hwq hwq; | |
102 | struct bnxt_qplib_swq *swq; | |
37cb11ac DS |
103 | int start_idx; |
104 | int last_idx; | |
5aa84840 | 105 | struct bnxt_qplib_sg_info sg_info; |
37cb11ac DS |
106 | u16 eventq_hw_ring_id; |
107 | spinlock_t lock; /* protect SRQE link list */ | |
108 | }; | |
109 | ||
1ac5a404 SX |
110 | struct bnxt_qplib_sge { |
111 | u64 addr; | |
112 | u32 lkey; | |
113 | u32 size; | |
114 | }; | |
115 | ||
1ac5a404 | 116 | #define BNXT_QPLIB_QP_MAX_SGL 6 |
1ac5a404 SX |
117 | struct bnxt_qplib_swq { |
118 | u64 wr_id; | |
37cb11ac | 119 | int next_idx; |
1ac5a404 SX |
120 | u8 type; |
121 | u8 flags; | |
122 | u32 start_psn; | |
123 | u32 next_psn; | |
2bb3c32c | 124 | u32 slot_idx; |
159fb4ce | 125 | u8 slots; |
1ac5a404 | 126 | struct sq_psn_search *psn_search; |
37f91cff | 127 | struct sq_psn_search_ext *psn_ext; |
1ac5a404 SX |
128 | }; |
129 | ||
130 | struct bnxt_qplib_swqe { | |
131 | /* General */ | |
9152e0b7 | 132 | #define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ |
1ac5a404 SX |
133 | u64 wr_id; |
134 | u8 reqs_type; | |
135 | u8 type; | |
136 | #define BNXT_QPLIB_SWQE_TYPE_SEND 0 | |
137 | #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 | |
138 | #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 | |
139 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 | |
140 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 | |
141 | #define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 | |
142 | #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 | |
143 | #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 | |
144 | #define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 | |
145 | #define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 | |
146 | #define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 | |
147 | #define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 | |
148 | #define BNXT_QPLIB_SWQE_TYPE_RECV 128 | |
149 | #define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 | |
150 | u8 flags; | |
151 | #define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP BIT(0) | |
152 | #define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE BIT(1) | |
153 | #define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE BIT(2) | |
154 | #define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT BIT(3) | |
155 | #define BNXT_QPLIB_SWQE_FLAGS_INLINE BIT(4) | |
156 | struct bnxt_qplib_sge sg_list[BNXT_QPLIB_QP_MAX_SGL]; | |
157 | int num_sge; | |
158 | /* Max inline data is 96 bytes */ | |
159 | u32 inline_len; | |
160 | #define BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH 96 | |
161 | u8 inline_data[BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH]; | |
162 | ||
163 | union { | |
164 | /* Send, with imm, inval key */ | |
165 | struct { | |
166 | union { | |
167 | __be32 imm_data; | |
168 | u32 inv_key; | |
169 | }; | |
170 | u32 q_key; | |
171 | u32 dst_qp; | |
172 | u16 avid; | |
173 | } send; | |
174 | ||
175 | /* Send Raw Ethernet and QP1 */ | |
176 | struct { | |
177 | u16 lflags; | |
178 | u16 cfa_action; | |
179 | u32 cfa_meta; | |
180 | } rawqp1; | |
181 | ||
182 | /* RDMA write, with imm, read */ | |
183 | struct { | |
184 | union { | |
185 | __be32 imm_data; | |
186 | u32 inv_key; | |
187 | }; | |
188 | u64 remote_va; | |
189 | u32 r_key; | |
190 | } rdma; | |
191 | ||
192 | /* Atomic cmp/swap, fetch/add */ | |
193 | struct { | |
194 | u64 remote_va; | |
195 | u32 r_key; | |
196 | u64 swap_data; | |
197 | u64 cmp_data; | |
198 | } atomic; | |
199 | ||
200 | /* Local Invalidate */ | |
201 | struct { | |
202 | u32 inv_l_key; | |
203 | } local_inv; | |
204 | ||
205 | /* FR-PMR */ | |
206 | struct { | |
207 | u8 access_cntl; | |
208 | u8 pg_sz_log; | |
209 | bool zero_based; | |
210 | u32 l_key; | |
211 | u32 length; | |
212 | u8 pbl_pg_sz_log; | |
213 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 | |
214 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 | |
215 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 | |
216 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 | |
217 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 | |
218 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 | |
219 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 | |
220 | #define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 | |
221 | u8 levels; | |
222 | #define PAGE_SHIFT_4K 12 | |
223 | __le64 *pbl_ptr; | |
224 | dma_addr_t pbl_dma_ptr; | |
225 | u64 *page_list; | |
226 | u16 page_list_len; | |
227 | u64 va; | |
228 | } frmr; | |
229 | ||
230 | /* Bind */ | |
231 | struct { | |
232 | u8 access_cntl; | |
233 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE BIT(0) | |
234 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ BIT(1) | |
235 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE BIT(2) | |
236 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC BIT(3) | |
237 | #define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND BIT(4) | |
238 | bool zero_based; | |
239 | u8 mw_type; | |
240 | u32 parent_l_key; | |
241 | u32 r_key; | |
242 | u64 va; | |
243 | u32 length; | |
244 | } bind; | |
245 | }; | |
246 | }; | |
247 | ||
1ac5a404 SX |
248 | struct bnxt_qplib_q { |
249 | struct bnxt_qplib_hwq hwq; | |
250 | struct bnxt_qplib_swq *swq; | |
6f53196b | 251 | struct bnxt_qplib_db_info dbinfo; |
5aa84840 | 252 | struct bnxt_qplib_sg_info sg_info; |
1ac5a404 | 253 | u32 max_wqe; |
fddcbbb0 | 254 | u16 wqe_size; |
9152e0b7 | 255 | u16 q_full_delta; |
1ac5a404 SX |
256 | u16 max_sge; |
257 | u32 psn; | |
9152e0b7 EW |
258 | bool condition; |
259 | bool single; | |
260 | bool send_phantom; | |
261 | u32 phantom_wqe_cnt; | |
262 | u32 phantom_cqe_cnt; | |
263 | u32 next_cq_cons; | |
f218d67e | 264 | bool flushed; |
159fb4ce DS |
265 | u32 swq_start; |
266 | u32 swq_last; | |
1ac5a404 SX |
267 | }; |
268 | ||
269 | struct bnxt_qplib_qp { | |
270 | struct bnxt_qplib_pd *pd; | |
271 | struct bnxt_qplib_dpi *dpi; | |
37f91cff | 272 | struct bnxt_qplib_chip_ctx *cctx; |
1ac5a404 | 273 | u64 qp_handle; |
fddcbbb0 | 274 | #define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF |
1ac5a404 SX |
275 | u32 id; |
276 | u8 type; | |
277 | u8 sig_type; | |
1da968e0 | 278 | u8 wqe_mode; |
1ac5a404 SX |
279 | u8 state; |
280 | u8 cur_qp_state; | |
1da968e0 | 281 | u64 modify_flags; |
1ac5a404 SX |
282 | u32 max_inline_data; |
283 | u32 mtu; | |
284 | u8 path_mtu; | |
285 | bool en_sqd_async_notify; | |
286 | u16 pkey_index; | |
287 | u32 qkey; | |
288 | u32 dest_qp_id; | |
289 | u8 access; | |
290 | u8 timeout; | |
291 | u8 retry_cnt; | |
292 | u8 rnr_retry; | |
3fb755b3 | 293 | u64 wqe_cnt; |
1ac5a404 SX |
294 | u32 min_rnr_timer; |
295 | u32 max_rd_atomic; | |
296 | u32 max_dest_rd_atomic; | |
297 | u32 dest_qpn; | |
298 | u8 smac[6]; | |
299 | u16 vlan_id; | |
300 | u8 nw_type; | |
301 | struct bnxt_qplib_ah ah; | |
302 | ||
303 | #define BTH_PSN_MASK ((1 << 24) - 1) | |
304 | /* SQ */ | |
305 | struct bnxt_qplib_q sq; | |
306 | /* RQ */ | |
307 | struct bnxt_qplib_q rq; | |
308 | /* SRQ */ | |
309 | struct bnxt_qplib_srq *srq; | |
310 | /* CQ */ | |
311 | struct bnxt_qplib_cq *scq; | |
312 | struct bnxt_qplib_cq *rcq; | |
313 | /* IRRQ and ORRQ */ | |
314 | struct bnxt_qplib_hwq irrq; | |
315 | struct bnxt_qplib_hwq orrq; | |
316 | /* Header buffer for QP1 */ | |
317 | int sq_hdr_buf_size; | |
318 | int rq_hdr_buf_size; | |
319 | /* | |
320 | * Buffer space for ETH(14), IP or GRH(40), UDP header(8) | |
321 | * and ib_bth + ib_deth (20). | |
322 | * Max required is 82 when RoCE V2 is enabled | |
323 | */ | |
324 | #define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 | |
325 | /* Ethernet header = 14 */ | |
326 | /* ib_grh = 40 (provided by MAD) */ | |
327 | /* ib_bth + ib_deth = 20 */ | |
328 | /* MAD = 256 (provided by MAD) */ | |
329 | /* iCRC = 4 */ | |
330 | #define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 | |
331 | #define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 | |
332 | #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 | |
333 | #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 | |
334 | #define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 | |
335 | void *sq_hdr_buf; | |
336 | dma_addr_t sq_hdr_buf_map; | |
337 | void *rq_hdr_buf; | |
338 | dma_addr_t rq_hdr_buf_map; | |
f218d67e SX |
339 | struct list_head sq_flush; |
340 | struct list_head rq_flush; | |
1ac5a404 SX |
341 | }; |
342 | ||
343 | #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) | |
344 | ||
345 | #define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE) | |
346 | #define CQE_MAX_IDX_PER_PG (CQE_CNT_PER_PG - 1) | |
347 | #define CQE_PG(x) (((x) & ~CQE_MAX_IDX_PER_PG) / CQE_CNT_PER_PG) | |
348 | #define CQE_IDX(x) ((x) & CQE_MAX_IDX_PER_PG) | |
349 | ||
350 | #define ROCE_CQE_CMP_V 0 | |
351 | #define CQE_CMP_VALID(hdr, raw_cons, cp_bit) \ | |
352 | (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ | |
353 | !((raw_cons) & (cp_bit))) | |
354 | ||
2bb3c32c DS |
355 | static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que, |
356 | u8 slots) | |
9152e0b7 | 357 | { |
2bb3c32c DS |
358 | struct bnxt_qplib_hwq *hwq; |
359 | int avail; | |
360 | ||
361 | hwq = &que->hwq; | |
362 | /* False full is possible, retrying post-send makes sense */ | |
363 | avail = hwq->cons - hwq->prod; | |
364 | if (hwq->cons <= hwq->prod) | |
365 | avail += hwq->depth; | |
366 | return avail <= slots; | |
9152e0b7 EW |
367 | } |
368 | ||
1ac5a404 SX |
369 | struct bnxt_qplib_cqe { |
370 | u8 status; | |
371 | u8 type; | |
372 | u8 opcode; | |
373 | u32 length; | |
374c5285 | 374 | u16 cfa_meta; |
1ac5a404 SX |
375 | u64 wr_id; |
376 | union { | |
377 | __be32 immdata; | |
378 | u32 invrkey; | |
379 | }; | |
380 | u64 qp_handle; | |
381 | u64 mr_handle; | |
382 | u16 flags; | |
383 | u8 smac[6]; | |
384 | u32 src_qp; | |
385 | u16 raweth_qp1_flags; | |
386 | u16 raweth_qp1_errors; | |
387 | u16 raweth_qp1_cfa_code; | |
388 | u32 raweth_qp1_flags2; | |
389 | u32 raweth_qp1_metadata; | |
390 | u8 raweth_qp1_payload_offset; | |
391 | u16 pkey_index; | |
392 | }; | |
393 | ||
394 | #define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 | |
395 | struct bnxt_qplib_cq { | |
396 | struct bnxt_qplib_dpi *dpi; | |
6f53196b | 397 | struct bnxt_qplib_db_info dbinfo; |
1ac5a404 SX |
398 | u32 max_wqe; |
399 | u32 id; | |
400 | u16 count; | |
401 | u16 period; | |
402 | struct bnxt_qplib_hwq hwq; | |
d54bd5ab | 403 | struct bnxt_qplib_hwq resize_hwq; |
1ac5a404 | 404 | u32 cnq_hw_ring_id; |
f218d67e | 405 | struct bnxt_qplib_nq *nq; |
1ac5a404 | 406 | bool resize_in_progress; |
5aa84840 | 407 | struct bnxt_qplib_sg_info sg_info; |
1ac5a404 SX |
408 | u64 cq_handle; |
409 | ||
410 | #define CQ_RESIZE_WAIT_TIME_MS 500 | |
411 | unsigned long flags; | |
412 | #define CQ_FLAGS_RESIZE_IN_PROG 1 | |
413 | wait_queue_head_t waitq; | |
f218d67e SX |
414 | struct list_head sqf_head, rqf_head; |
415 | atomic_t arm_state; | |
416 | spinlock_t compl_lock; /* synch CQ handlers */ | |
942c9b6c SX |
417 | /* Locking Notes: |
418 | * QP can move to error state from modify_qp, async error event or error | |
419 | * CQE as part of poll_cq. When QP is moved to error state, it gets added | |
420 | * to two flush lists, one each for SQ and RQ. | |
421 | * Each flush list is protected by qplib_cq->flush_lock. Both scq and rcq | |
422 | * flush_locks should be acquired when QP is moved to error. The control path | |
423 | * operations(modify_qp and async error events) are synchronized with poll_cq | |
424 | * using upper level CQ locks (bnxt_re_cq->cq_lock) of both SCQ and RCQ. | |
425 | * The qplib_cq->flush_lock is required to synchronize two instances of poll_cq | |
426 | * of the same QP while manipulating the flush list. | |
427 | */ | |
428 | spinlock_t flush_lock; /* QP flush management */ | |
b1d56fdc | 429 | u16 cnq_events; |
1ac5a404 SX |
430 | }; |
431 | ||
432 | #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) | |
433 | #define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) | |
434 | #define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * (x) + 2) | |
435 | #define IRRQ_SLOTS_TO_IRD_LIMIT(s) (((s) >> 1) - 1) | |
436 | #define ORD_LIMIT_TO_ORRQ_SLOTS(x) ((x) + 1) | |
437 | #define ORRQ_SLOTS_TO_ORD_LIMIT(s) ((s) - 1) | |
438 | ||
439 | #define BNXT_QPLIB_MAX_NQE_ENTRY_SIZE sizeof(struct nq_base) | |
440 | ||
441 | #define NQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_NQE_ENTRY_SIZE) | |
442 | #define NQE_MAX_IDX_PER_PG (NQE_CNT_PER_PG - 1) | |
443 | #define NQE_PG(x) (((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG) | |
444 | #define NQE_IDX(x) ((x) & NQE_MAX_IDX_PER_PG) | |
445 | ||
446 | #define NQE_CMP_VALID(hdr, raw_cons, cp_bit) \ | |
447 | (!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) == \ | |
448 | !((raw_cons) & (cp_bit))) | |
449 | ||
450 | #define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) | |
451 | ||
452 | #define NQ_CONS_PCI_BAR_REGION 2 | |
453 | #define NQ_DB_KEY_CP (0x2 << CMPL_DOORBELL_KEY_SFT) | |
454 | #define NQ_DB_IDX_VALID CMPL_DOORBELL_IDX_VALID | |
455 | #define NQ_DB_IRQ_DIS CMPL_DOORBELL_MASK | |
456 | #define NQ_DB_CP_FLAGS_REARM (NQ_DB_KEY_CP | \ | |
457 | NQ_DB_IDX_VALID) | |
458 | #define NQ_DB_CP_FLAGS (NQ_DB_KEY_CP | \ | |
459 | NQ_DB_IDX_VALID | \ | |
460 | NQ_DB_IRQ_DIS) | |
b353ce55 | 461 | |
9555352b DS |
462 | struct bnxt_qplib_nq_db { |
463 | struct bnxt_qplib_reg_desc reg; | |
6f53196b | 464 | struct bnxt_qplib_db_info dbinfo; |
9555352b DS |
465 | }; |
466 | ||
467 | typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq, | |
468 | struct bnxt_qplib_cq *cq); | |
469 | typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq, | |
470 | struct bnxt_qplib_srq *srq, u8 event); | |
471 | ||
1ac5a404 | 472 | struct bnxt_qplib_nq { |
9555352b DS |
473 | struct pci_dev *pdev; |
474 | struct bnxt_qplib_res *res; | |
ff2e4bfd | 475 | char *name; |
9555352b DS |
476 | struct bnxt_qplib_hwq hwq; |
477 | struct bnxt_qplib_nq_db nq_db; | |
478 | u16 ring_id; | |
479 | int msix_vec; | |
480 | cpumask_t mask; | |
481 | struct tasklet_struct nq_tasklet; | |
482 | bool requested; | |
483 | int budget; | |
484 | ||
485 | cqn_handler_t cqn_handler; | |
486 | srqn_handler_t srqn_handler; | |
487 | struct workqueue_struct *cqn_wq; | |
f218d67e SX |
488 | }; |
489 | ||
490 | struct bnxt_qplib_nq_work { | |
491 | struct work_struct work; | |
492 | struct bnxt_qplib_nq *nq; | |
493 | struct bnxt_qplib_cq *cq; | |
1ac5a404 SX |
494 | }; |
495 | ||
6e04b103 | 496 | void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill); |
1ac5a404 | 497 | void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); |
6e04b103 DS |
498 | int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, |
499 | int msix_vector, bool need_init); | |
1ac5a404 | 500 | int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq, |
6a5df91b | 501 | int nq_idx, int msix_vector, int bar_reg_offset, |
9555352b DS |
502 | cqn_handler_t cqn_handler, |
503 | srqn_handler_t srq_handler); | |
37cb11ac DS |
504 | int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, |
505 | struct bnxt_qplib_srq *srq); | |
506 | int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, | |
507 | struct bnxt_qplib_srq *srq); | |
508 | int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, | |
509 | struct bnxt_qplib_srq *srq); | |
68e326de LR |
510 | void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, |
511 | struct bnxt_qplib_srq *srq); | |
37cb11ac DS |
512 | int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, |
513 | struct bnxt_qplib_swqe *wqe); | |
1ac5a404 SX |
514 | int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); |
515 | int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
516 | int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
517 | int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
518 | int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); | |
3b921e3b SX |
519 | void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); |
520 | void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, | |
521 | struct bnxt_qplib_qp *qp); | |
1ac5a404 SX |
522 | void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, |
523 | struct bnxt_qplib_sge *sge); | |
524 | void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, | |
525 | struct bnxt_qplib_sge *sge); | |
526 | u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); | |
527 | dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, | |
528 | u32 index); | |
529 | void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); | |
530 | int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, | |
531 | struct bnxt_qplib_swqe *wqe); | |
532 | void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); | |
533 | int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, | |
534 | struct bnxt_qplib_swqe *wqe); | |
535 | int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); | |
d54bd5ab SX |
536 | int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, |
537 | int new_cqes); | |
538 | void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, | |
539 | struct bnxt_qplib_cq *cq); | |
1ac5a404 SX |
540 | int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); |
541 | int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, | |
9152e0b7 | 542 | int num, struct bnxt_qplib_qp **qp); |
499e4569 | 543 | bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); |
1ac5a404 SX |
544 | void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); |
545 | void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq); | |
0c4dcd60 | 546 | int bnxt_qplib_alloc_nq(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq); |
f218d67e | 547 | void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); |
f218d67e SX |
548 | void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp, |
549 | unsigned long *flags); | |
550 | void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp, | |
551 | unsigned long *flags); | |
552 | int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, | |
553 | struct bnxt_qplib_cqe *cqe, | |
554 | int num_cqes); | |
c88a7858 | 555 | void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); |
b5bbc655 | 556 | void bnxt_re_synchronize_nq(struct bnxt_qplib_nq *nq); |
159fb4ce DS |
557 | |
558 | static inline void *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx) | |
559 | { | |
560 | u32 idx; | |
561 | ||
562 | idx = que->swq_start; | |
563 | if (swq_idx) | |
564 | *swq_idx = idx; | |
565 | return &que->swq[idx]; | |
566 | } | |
567 | ||
568 | static inline void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) | |
569 | { | |
570 | que->swq_start = que->swq[idx].next_idx; | |
571 | } | |
572 | ||
2bb3c32c DS |
573 | static inline u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) |
574 | { | |
575 | return (que->wqe_size * que->max_wqe) / sizeof(struct sq_sge); | |
576 | } | |
577 | ||
578 | static inline u32 bnxt_qplib_set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) | |
579 | { | |
580 | return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? | |
581 | que->max_wqe : bnxt_qplib_get_depth(que); | |
582 | } | |
583 | ||
584 | static inline u32 bnxt_qplib_set_sq_max_slot(u8 wqe_mode) | |
585 | { | |
586 | return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? | |
587 | sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; | |
588 | } | |
589 | ||
590 | static inline u32 bnxt_qplib_set_rq_max_slot(u32 wqe_size) | |
591 | { | |
592 | return (wqe_size / sizeof(struct sq_sge)); | |
593 | } | |
594 | ||
595 | static inline u16 __xlate_qfd(u16 delta, u16 wqe_bytes) | |
596 | { | |
597 | /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 | |
598 | * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. | |
599 | * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. | |
600 | * when 8916 is disabled. | |
601 | */ | |
602 | return (delta * wqe_bytes) / sizeof(struct sq_sge); | |
603 | } | |
604 | ||
605 | static inline u16 bnxt_qplib_calc_ilsize(struct bnxt_qplib_swqe *wqe, u16 max) | |
606 | { | |
607 | u16 size = 0; | |
608 | int indx; | |
609 | ||
610 | for (indx = 0; indx < wqe->num_sge; indx++) | |
611 | size += wqe->sg_list[indx].size; | |
612 | if (size > max) | |
613 | size = max; | |
614 | ||
615 | return size; | |
616 | } | |
1ac5a404 | 617 | #endif /* __BNXT_QPLIB_FP_H__ */ |