Commit | Line | Data |
---|---|---|
3ec648c6 | 1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
551c46ed MI |
2 | /* Copyright (c) 2015 - 2021 Intel Corporation */ |
3 | #include "osdep.h" | |
551c46ed MI |
4 | #include "defs.h" |
5 | #include "user.h" | |
6 | #include "irdma.h" | |
7 | ||
8 | /** | |
9 | * irdma_set_fragment - set fragment in wqe | |
10 | * @wqe: wqe for setting fragment | |
11 | * @offset: offset value | |
12 | * @sge: sge length and stag | |
13 | * @valid: The wqe valid | |
14 | */ | |
9ed8110c | 15 | static void irdma_set_fragment(__le64 *wqe, u32 offset, struct ib_sge *sge, |
551c46ed MI |
16 | u8 valid) |
17 | { | |
18 | if (sge) { | |
19 | set_64bit_val(wqe, offset, | |
9ed8110c | 20 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); |
551c46ed MI |
21 | set_64bit_val(wqe, offset + 8, |
22 | FIELD_PREP(IRDMAQPSQ_VALID, valid) | | |
9ed8110c ZY |
23 | FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) | |
24 | FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey)); | |
551c46ed MI |
25 | } else { |
26 | set_64bit_val(wqe, offset, 0); | |
27 | set_64bit_val(wqe, offset + 8, | |
28 | FIELD_PREP(IRDMAQPSQ_VALID, valid)); | |
29 | } | |
30 | } | |
31 | ||
32 | /** | |
33 | * irdma_set_fragment_gen_1 - set fragment in wqe | |
34 | * @wqe: wqe for setting fragment | |
35 | * @offset: offset value | |
36 | * @sge: sge length and stag | |
37 | * @valid: wqe valid flag | |
38 | */ | |
39 | static void irdma_set_fragment_gen_1(__le64 *wqe, u32 offset, | |
9ed8110c | 40 | struct ib_sge *sge, u8 valid) |
551c46ed MI |
41 | { |
42 | if (sge) { | |
43 | set_64bit_val(wqe, offset, | |
9ed8110c | 44 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); |
551c46ed | 45 | set_64bit_val(wqe, offset + 8, |
9ed8110c ZY |
46 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) | |
47 | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey)); | |
551c46ed MI |
48 | } else { |
49 | set_64bit_val(wqe, offset, 0); | |
50 | set_64bit_val(wqe, offset + 8, 0); | |
51 | } | |
52 | } | |
53 | ||
54 | /** | |
55 | * irdma_nop_1 - insert a NOP wqe | |
56 | * @qp: hw qp ptr | |
57 | */ | |
2c4b14ea | 58 | static int irdma_nop_1(struct irdma_qp_uk *qp) |
551c46ed MI |
59 | { |
60 | u64 hdr; | |
61 | __le64 *wqe; | |
62 | u32 wqe_idx; | |
63 | bool signaled = false; | |
64 | ||
65 | if (!qp->sq_ring.head) | |
2c4b14ea | 66 | return -EINVAL; |
551c46ed MI |
67 | |
68 | wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); | |
69 | wqe = qp->sq_base[wqe_idx].elem; | |
70 | ||
71 | qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA; | |
72 | ||
73 | set_64bit_val(wqe, 0, 0); | |
74 | set_64bit_val(wqe, 8, 0); | |
75 | set_64bit_val(wqe, 16, 0); | |
76 | ||
77 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | | |
78 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | | |
79 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
80 | ||
81 | /* make sure WQE is written before valid bit is set */ | |
82 | dma_wmb(); | |
83 | ||
84 | set_64bit_val(wqe, 24, hdr); | |
85 | ||
86 | return 0; | |
87 | } | |
88 | ||
89 | /** | |
90 | * irdma_clr_wqes - clear next 128 sq entries | |
91 | * @qp: hw qp ptr | |
92 | * @qp_wqe_idx: wqe_idx | |
93 | */ | |
94 | void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx) | |
95 | { | |
b002760f | 96 | struct irdma_qp_quanta *sq; |
551c46ed MI |
97 | u32 wqe_idx; |
98 | ||
99 | if (!(qp_wqe_idx & 0x7F)) { | |
100 | wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size; | |
b002760f | 101 | sq = qp->sq_base + wqe_idx; |
551c46ed | 102 | if (wqe_idx) |
b002760f AB |
103 | memset(sq, qp->swqe_polarity ? 0 : 0xFF, |
104 | 128 * sizeof(*sq)); | |
551c46ed | 105 | else |
b002760f AB |
106 | memset(sq, qp->swqe_polarity ? 0xFF : 0, |
107 | 128 * sizeof(*sq)); | |
551c46ed MI |
108 | } |
109 | } | |
110 | ||
111 | /** | |
112 | * irdma_uk_qp_post_wr - ring doorbell | |
113 | * @qp: hw qp ptr | |
114 | */ | |
115 | void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp) | |
116 | { | |
117 | u64 temp; | |
118 | u32 hw_sq_tail; | |
119 | u32 sw_sq_head; | |
120 | ||
121 | /* valid bit is written and loads completed before reading shadow */ | |
122 | mb(); | |
123 | ||
124 | /* read the doorbell shadow area */ | |
125 | get_64bit_val(qp->shadow_area, 0, &temp); | |
126 | ||
127 | hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp); | |
128 | sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); | |
129 | if (sw_sq_head != qp->initial_ring.head) { | |
295c95aa | 130 | if (sw_sq_head != hw_sq_tail) { |
551c46ed MI |
131 | if (sw_sq_head > qp->initial_ring.head) { |
132 | if (hw_sq_tail >= qp->initial_ring.head && | |
133 | hw_sq_tail < sw_sq_head) | |
134 | writel(qp->qp_id, qp->wqe_alloc_db); | |
135 | } else { | |
136 | if (hw_sq_tail >= qp->initial_ring.head || | |
137 | hw_sq_tail < sw_sq_head) | |
138 | writel(qp->qp_id, qp->wqe_alloc_db); | |
139 | } | |
140 | } | |
141 | } | |
142 | ||
143 | qp->initial_ring.head = qp->sq_ring.head; | |
144 | } | |
145 | ||
551c46ed MI |
146 | /** |
147 | * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go | |
148 | * @qp: hw qp ptr | |
149 | * @wqe_idx: return wqe index | |
150 | * @quanta: size of WR in quanta | |
151 | * @total_size: size of WR in bytes | |
152 | * @info: info on WR | |
153 | */ | |
154 | __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, | |
155 | u16 quanta, u32 total_size, | |
156 | struct irdma_post_sq_info *info) | |
157 | { | |
158 | __le64 *wqe; | |
159 | __le64 *wqe_0 = NULL; | |
551c46ed MI |
160 | u16 avail_quanta; |
161 | u16 i; | |
162 | ||
163 | avail_quanta = qp->uk_attrs->max_hw_sq_chunk - | |
164 | (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % | |
165 | qp->uk_attrs->max_hw_sq_chunk); | |
166 | if (quanta <= avail_quanta) { | |
167 | /* WR fits in current chunk */ | |
168 | if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) | |
169 | return NULL; | |
170 | } else { | |
171 | /* Need to pad with NOP */ | |
172 | if (quanta + avail_quanta > | |
173 | IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) | |
174 | return NULL; | |
175 | ||
551c46ed MI |
176 | for (i = 0; i < avail_quanta; i++) { |
177 | irdma_nop_1(qp); | |
178 | IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); | |
179 | } | |
551c46ed MI |
180 | } |
181 | ||
182 | *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); | |
183 | if (!*wqe_idx) | |
184 | qp->swqe_polarity = !qp->swqe_polarity; | |
185 | ||
186 | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); | |
187 | ||
188 | wqe = qp->sq_base[*wqe_idx].elem; | |
189 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 && | |
190 | (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { | |
191 | wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; | |
192 | wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity ? 0 : 1)); | |
193 | } | |
194 | qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; | |
195 | qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; | |
196 | qp->sq_wrtrk_array[*wqe_idx].quanta = quanta; | |
197 | ||
198 | return wqe; | |
199 | } | |
200 | ||
201 | /** | |
202 | * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe | |
203 | * @qp: hw qp ptr | |
204 | * @wqe_idx: return wqe index | |
205 | */ | |
206 | __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx) | |
207 | { | |
208 | __le64 *wqe; | |
2c4b14ea | 209 | int ret_code; |
551c46ed MI |
210 | |
211 | if (IRDMA_RING_FULL_ERR(qp->rq_ring)) | |
212 | return NULL; | |
213 | ||
214 | IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); | |
215 | if (ret_code) | |
216 | return NULL; | |
217 | ||
218 | if (!*wqe_idx) | |
219 | qp->rwqe_polarity = !qp->rwqe_polarity; | |
220 | /* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */ | |
221 | wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem; | |
222 | ||
223 | return wqe; | |
224 | } | |
225 | ||
226 | /** | |
227 | * irdma_uk_rdma_write - rdma write operation | |
228 | * @qp: hw qp ptr | |
229 | * @info: post sq information | |
230 | * @post_sq: flag to post sq | |
231 | */ | |
2c4b14ea SS |
232 | int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, |
233 | bool post_sq) | |
551c46ed MI |
234 | { |
235 | u64 hdr; | |
236 | __le64 *wqe; | |
237 | struct irdma_rdma_write *op_info; | |
238 | u32 i, wqe_idx; | |
239 | u32 total_size = 0, byte_off; | |
2c4b14ea | 240 | int ret_code; |
551c46ed MI |
241 | u32 frag_cnt, addl_frag_cnt; |
242 | bool read_fence = false; | |
243 | u16 quanta; | |
244 | ||
551c46ed MI |
245 | op_info = &info->op.rdma_write; |
246 | if (op_info->num_lo_sges > qp->max_sq_frag_cnt) | |
2c4b14ea | 247 | return -EINVAL; |
551c46ed MI |
248 | |
249 | for (i = 0; i < op_info->num_lo_sges; i++) | |
9ed8110c | 250 | total_size += op_info->lo_sg_list[i].length; |
551c46ed MI |
251 | |
252 | read_fence |= info->read_fence; | |
253 | ||
254 | if (info->imm_data_valid) | |
255 | frag_cnt = op_info->num_lo_sges + 1; | |
256 | else | |
257 | frag_cnt = op_info->num_lo_sges; | |
258 | addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; | |
259 | ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta); | |
260 | if (ret_code) | |
261 | return ret_code; | |
262 | ||
263 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | |
264 | info); | |
265 | if (!wqe) | |
2c4b14ea | 266 | return -ENOMEM; |
551c46ed MI |
267 | |
268 | irdma_clr_wqes(qp, wqe_idx); | |
269 | ||
270 | set_64bit_val(wqe, 16, | |
9ed8110c | 271 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); |
551c46ed MI |
272 | |
273 | if (info->imm_data_valid) { | |
274 | set_64bit_val(wqe, 0, | |
275 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | |
276 | i = 0; | |
277 | } else { | |
278 | qp->wqe_ops.iw_set_fragment(wqe, 0, | |
279 | op_info->lo_sg_list, | |
280 | qp->swqe_polarity); | |
281 | i = 1; | |
282 | } | |
283 | ||
284 | for (byte_off = 32; i < op_info->num_lo_sges; i++) { | |
285 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, | |
286 | &op_info->lo_sg_list[i], | |
287 | qp->swqe_polarity); | |
288 | byte_off += 16; | |
289 | } | |
290 | ||
291 | /* if not an odd number set valid bit in next fragment */ | |
292 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && | |
293 | frag_cnt) { | |
294 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, | |
295 | qp->swqe_polarity); | |
296 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) | |
297 | ++addl_frag_cnt; | |
298 | } | |
299 | ||
9ed8110c | 300 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | |
551c46ed MI |
301 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | |
302 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) | | |
303 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) | | |
304 | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | |
551c46ed MI |
305 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
306 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | |
307 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | |
308 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
309 | ||
310 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
311 | ||
312 | set_64bit_val(wqe, 24, hdr); | |
295c95aa SS |
313 | |
314 | if (post_sq) | |
315 | irdma_uk_qp_post_wr(qp); | |
551c46ed MI |
316 | |
317 | return 0; | |
318 | } | |
319 | ||
320 | /** | |
321 | * irdma_uk_rdma_read - rdma read command | |
322 | * @qp: hw qp ptr | |
323 | * @info: post sq information | |
324 | * @inv_stag: flag for inv_stag | |
325 | * @post_sq: flag to post sq | |
326 | */ | |
2c4b14ea SS |
327 | int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, |
328 | bool inv_stag, bool post_sq) | |
551c46ed MI |
329 | { |
330 | struct irdma_rdma_read *op_info; | |
2c4b14ea | 331 | int ret_code; |
551c46ed MI |
332 | u32 i, byte_off, total_size = 0; |
333 | bool local_fence = false; | |
334 | u32 addl_frag_cnt; | |
335 | __le64 *wqe; | |
336 | u32 wqe_idx; | |
337 | u16 quanta; | |
338 | u64 hdr; | |
339 | ||
551c46ed MI |
340 | op_info = &info->op.rdma_read; |
341 | if (qp->max_sq_frag_cnt < op_info->num_lo_sges) | |
2c4b14ea | 342 | return -EINVAL; |
551c46ed MI |
343 | |
344 | for (i = 0; i < op_info->num_lo_sges; i++) | |
9ed8110c | 345 | total_size += op_info->lo_sg_list[i].length; |
551c46ed MI |
346 | |
347 | ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta); | |
348 | if (ret_code) | |
349 | return ret_code; | |
350 | ||
351 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | |
352 | info); | |
353 | if (!wqe) | |
2c4b14ea | 354 | return -ENOMEM; |
551c46ed MI |
355 | |
356 | irdma_clr_wqes(qp, wqe_idx); | |
357 | ||
358 | addl_frag_cnt = op_info->num_lo_sges > 1 ? | |
359 | (op_info->num_lo_sges - 1) : 0; | |
360 | local_fence |= info->local_fence; | |
361 | ||
362 | qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list, | |
363 | qp->swqe_polarity); | |
364 | for (i = 1, byte_off = 32; i < op_info->num_lo_sges; ++i) { | |
365 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, | |
366 | &op_info->lo_sg_list[i], | |
367 | qp->swqe_polarity); | |
368 | byte_off += 16; | |
369 | } | |
370 | ||
371 | /* if not an odd number set valid bit in next fragment */ | |
372 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && | |
373 | !(op_info->num_lo_sges & 0x01) && op_info->num_lo_sges) { | |
374 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, | |
375 | qp->swqe_polarity); | |
376 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) | |
377 | ++addl_frag_cnt; | |
378 | } | |
379 | set_64bit_val(wqe, 16, | |
9ed8110c ZY |
380 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); |
381 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | | |
551c46ed MI |
382 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | |
383 | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | |
384 | FIELD_PREP(IRDMAQPSQ_OPCODE, | |
385 | (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) | | |
551c46ed MI |
386 | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | |
387 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | | |
388 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | |
389 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
390 | ||
391 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
392 | ||
393 | set_64bit_val(wqe, 24, hdr); | |
295c95aa SS |
394 | |
395 | if (post_sq) | |
396 | irdma_uk_qp_post_wr(qp); | |
551c46ed MI |
397 | |
398 | return 0; | |
399 | } | |
400 | ||
401 | /** | |
402 | * irdma_uk_send - rdma send command | |
403 | * @qp: hw qp ptr | |
404 | * @info: post sq information | |
405 | * @post_sq: flag to post sq | |
406 | */ | |
2c4b14ea SS |
407 | int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, |
408 | bool post_sq) | |
551c46ed MI |
409 | { |
410 | __le64 *wqe; | |
411 | struct irdma_post_send *op_info; | |
412 | u64 hdr; | |
413 | u32 i, wqe_idx, total_size = 0, byte_off; | |
2c4b14ea | 414 | int ret_code; |
551c46ed MI |
415 | u32 frag_cnt, addl_frag_cnt; |
416 | bool read_fence = false; | |
417 | u16 quanta; | |
418 | ||
551c46ed MI |
419 | op_info = &info->op.send; |
420 | if (qp->max_sq_frag_cnt < op_info->num_sges) | |
2c4b14ea | 421 | return -EINVAL; |
551c46ed MI |
422 | |
423 | for (i = 0; i < op_info->num_sges; i++) | |
9ed8110c | 424 | total_size += op_info->sg_list[i].length; |
551c46ed MI |
425 | |
426 | if (info->imm_data_valid) | |
427 | frag_cnt = op_info->num_sges + 1; | |
428 | else | |
429 | frag_cnt = op_info->num_sges; | |
430 | ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta); | |
431 | if (ret_code) | |
432 | return ret_code; | |
433 | ||
434 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | |
435 | info); | |
436 | if (!wqe) | |
2c4b14ea | 437 | return -ENOMEM; |
551c46ed MI |
438 | |
439 | irdma_clr_wqes(qp, wqe_idx); | |
440 | ||
441 | read_fence |= info->read_fence; | |
442 | addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; | |
443 | if (info->imm_data_valid) { | |
444 | set_64bit_val(wqe, 0, | |
445 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | |
446 | i = 0; | |
447 | } else { | |
2c884443 SD |
448 | qp->wqe_ops.iw_set_fragment(wqe, 0, |
449 | frag_cnt ? op_info->sg_list : NULL, | |
551c46ed MI |
450 | qp->swqe_polarity); |
451 | i = 1; | |
452 | } | |
453 | ||
454 | for (byte_off = 32; i < op_info->num_sges; i++) { | |
455 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i], | |
456 | qp->swqe_polarity); | |
457 | byte_off += 16; | |
458 | } | |
459 | ||
460 | /* if not an odd number set valid bit in next fragment */ | |
461 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) && | |
462 | frag_cnt) { | |
463 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, | |
464 | qp->swqe_polarity); | |
465 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) | |
466 | ++addl_frag_cnt; | |
467 | } | |
468 | ||
469 | set_64bit_val(wqe, 16, | |
470 | FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | | |
471 | FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); | |
472 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | | |
473 | FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | | |
474 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, | |
475 | (info->imm_data_valid ? 1 : 0)) | | |
476 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | | |
477 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | | |
478 | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | |
551c46ed MI |
479 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
480 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | |
481 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | |
482 | FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | | |
483 | FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | | |
484 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
485 | ||
486 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
487 | ||
488 | set_64bit_val(wqe, 24, hdr); | |
295c95aa SS |
489 | |
490 | if (post_sq) | |
491 | irdma_uk_qp_post_wr(qp); | |
551c46ed MI |
492 | |
493 | return 0; | |
494 | } | |
495 | ||
496 | /** | |
497 | * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe | |
498 | * @wqe: wqe for setting fragment | |
499 | * @op_info: info for setting bind wqe values | |
500 | */ | |
501 | static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe, | |
502 | struct irdma_bind_window *op_info) | |
503 | { | |
504 | set_64bit_val(wqe, 0, (uintptr_t)op_info->va); | |
505 | set_64bit_val(wqe, 8, | |
506 | FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) | | |
507 | FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag)); | |
508 | set_64bit_val(wqe, 16, op_info->bind_len); | |
509 | } | |
510 | ||
511 | /** | |
512 | * irdma_copy_inline_data_gen_1 - Copy inline data to wqe | |
4f44e519 MI |
513 | * @wqe: pointer to wqe |
514 | * @sge_list: table of pointers to inline data | |
515 | * @num_sges: Total inline data length | |
551c46ed MI |
516 | * @polarity: compatibility parameter |
517 | */ | |
4f44e519 MI |
518 | static void irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list, |
519 | u32 num_sges, u8 polarity) | |
551c46ed | 520 | { |
4f44e519 MI |
521 | u32 quanta_bytes_remaining = 16; |
522 | int i; | |
523 | ||
524 | for (i = 0; i < num_sges; i++) { | |
525 | u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; | |
526 | u32 sge_len = sge_list[i].length; | |
527 | ||
528 | while (sge_len) { | |
529 | u32 bytes_copied; | |
530 | ||
531 | bytes_copied = min(sge_len, quanta_bytes_remaining); | |
532 | memcpy(wqe, cur_sge, bytes_copied); | |
533 | wqe += bytes_copied; | |
534 | cur_sge += bytes_copied; | |
535 | quanta_bytes_remaining -= bytes_copied; | |
536 | sge_len -= bytes_copied; | |
537 | ||
538 | if (!quanta_bytes_remaining) { | |
539 | /* Remaining inline bytes reside after hdr */ | |
540 | wqe += 16; | |
541 | quanta_bytes_remaining = 32; | |
542 | } | |
543 | } | |
551c46ed MI |
544 | } |
545 | } | |
546 | ||
547 | /** | |
548 | * irdma_inline_data_size_to_quanta_gen_1 - based on inline data, quanta | |
549 | * @data_size: data size for inline | |
550 | * | |
551 | * Gets the quanta based on inline and immediate data. | |
552 | */ | |
553 | static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size) | |
554 | { | |
555 | return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2; | |
556 | } | |
557 | ||
558 | /** | |
559 | * irdma_set_mw_bind_wqe - set mw bind in wqe | |
560 | * @wqe: wqe for setting mw bind | |
561 | * @op_info: info for setting wqe values | |
562 | */ | |
563 | static void irdma_set_mw_bind_wqe(__le64 *wqe, | |
564 | struct irdma_bind_window *op_info) | |
565 | { | |
566 | set_64bit_val(wqe, 0, (uintptr_t)op_info->va); | |
567 | set_64bit_val(wqe, 8, | |
568 | FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) | | |
569 | FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag)); | |
570 | set_64bit_val(wqe, 16, op_info->bind_len); | |
571 | } | |
572 | ||
573 | /** | |
574 | * irdma_copy_inline_data - Copy inline data to wqe | |
4f44e519 MI |
575 | * @wqe: pointer to wqe |
576 | * @sge_list: table of pointers to inline data | |
577 | * @num_sges: number of SGE's | |
551c46ed MI |
578 | * @polarity: polarity of wqe valid bit |
579 | */ | |
4f44e519 MI |
580 | static void irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list, |
581 | u32 num_sges, u8 polarity) | |
551c46ed MI |
582 | { |
583 | u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; | |
4f44e519 MI |
584 | u32 quanta_bytes_remaining = 8; |
585 | bool first_quanta = true; | |
586 | int i; | |
587 | ||
588 | wqe += 8; | |
589 | ||
590 | for (i = 0; i < num_sges; i++) { | |
591 | u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; | |
592 | u32 sge_len = sge_list[i].length; | |
593 | ||
594 | while (sge_len) { | |
595 | u32 bytes_copied; | |
596 | ||
597 | bytes_copied = min(sge_len, quanta_bytes_remaining); | |
598 | memcpy(wqe, cur_sge, bytes_copied); | |
599 | wqe += bytes_copied; | |
600 | cur_sge += bytes_copied; | |
601 | quanta_bytes_remaining -= bytes_copied; | |
602 | sge_len -= bytes_copied; | |
603 | ||
604 | if (!quanta_bytes_remaining) { | |
605 | quanta_bytes_remaining = 31; | |
606 | ||
607 | /* Remaining inline bytes reside after hdr */ | |
608 | if (first_quanta) { | |
609 | first_quanta = false; | |
610 | wqe += 16; | |
611 | } else { | |
612 | *wqe = inline_valid; | |
613 | wqe++; | |
614 | } | |
615 | } | |
616 | } | |
551c46ed | 617 | } |
4f44e519 MI |
618 | if (!first_quanta && quanta_bytes_remaining < 31) |
619 | *(wqe + quanta_bytes_remaining) = inline_valid; | |
551c46ed MI |
620 | } |
621 | ||
622 | /** | |
623 | * irdma_inline_data_size_to_quanta - based on inline data, quanta | |
624 | * @data_size: data size for inline | |
625 | * | |
626 | * Gets the quanta based on inline and immediate data. | |
627 | */ | |
628 | static u16 irdma_inline_data_size_to_quanta(u32 data_size) | |
629 | { | |
630 | if (data_size <= 8) | |
631 | return IRDMA_QP_WQE_MIN_QUANTA; | |
632 | else if (data_size <= 39) | |
633 | return 2; | |
634 | else if (data_size <= 70) | |
635 | return 3; | |
636 | else if (data_size <= 101) | |
637 | return 4; | |
638 | else if (data_size <= 132) | |
639 | return 5; | |
640 | else if (data_size <= 163) | |
641 | return 6; | |
642 | else if (data_size <= 194) | |
643 | return 7; | |
644 | else | |
645 | return 8; | |
646 | } | |
647 | ||
648 | /** | |
649 | * irdma_uk_inline_rdma_write - inline rdma write operation | |
650 | * @qp: hw qp ptr | |
651 | * @info: post sq information | |
652 | * @post_sq: flag to post sq | |
653 | */ | |
2c4b14ea SS |
654 | int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, |
655 | struct irdma_post_sq_info *info, bool post_sq) | |
551c46ed MI |
656 | { |
657 | __le64 *wqe; | |
4f44e519 | 658 | struct irdma_rdma_write *op_info; |
551c46ed MI |
659 | u64 hdr = 0; |
660 | u32 wqe_idx; | |
661 | bool read_fence = false; | |
4f44e519 | 662 | u32 i, total_size = 0; |
551c46ed MI |
663 | u16 quanta; |
664 | ||
4f44e519 | 665 | op_info = &info->op.rdma_write; |
551c46ed | 666 | |
4f44e519 | 667 | if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges)) |
2c4b14ea | 668 | return -EINVAL; |
551c46ed | 669 | |
4f44e519 MI |
670 | for (i = 0; i < op_info->num_lo_sges; i++) |
671 | total_size += op_info->lo_sg_list[i].length; | |
672 | ||
673 | if (unlikely(total_size > qp->max_inline_data)) | |
674 | return -EINVAL; | |
675 | ||
676 | quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); | |
677 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | |
551c46ed MI |
678 | info); |
679 | if (!wqe) | |
2c4b14ea | 680 | return -ENOMEM; |
551c46ed MI |
681 | |
682 | irdma_clr_wqes(qp, wqe_idx); | |
683 | ||
684 | read_fence |= info->read_fence; | |
685 | set_64bit_val(wqe, 16, | |
9ed8110c | 686 | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); |
551c46ed | 687 | |
9ed8110c | 688 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | |
551c46ed | 689 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | |
4f44e519 | 690 | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | |
551c46ed MI |
691 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) | |
692 | FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) | | |
693 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) | | |
551c46ed MI |
694 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
695 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | |
696 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | |
697 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
698 | ||
699 | if (info->imm_data_valid) | |
700 | set_64bit_val(wqe, 0, | |
701 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | |
702 | ||
4f44e519 MI |
703 | qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list, |
704 | op_info->num_lo_sges, | |
551c46ed MI |
705 | qp->swqe_polarity); |
706 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
707 | ||
708 | set_64bit_val(wqe, 24, hdr); | |
709 | ||
295c95aa SS |
710 | if (post_sq) |
711 | irdma_uk_qp_post_wr(qp); | |
551c46ed MI |
712 | |
713 | return 0; | |
714 | } | |
715 | ||
716 | /** | |
717 | * irdma_uk_inline_send - inline send operation | |
718 | * @qp: hw qp ptr | |
719 | * @info: post sq information | |
720 | * @post_sq: flag to post sq | |
721 | */ | |
2c4b14ea SS |
722 | int irdma_uk_inline_send(struct irdma_qp_uk *qp, |
723 | struct irdma_post_sq_info *info, bool post_sq) | |
551c46ed MI |
724 | { |
725 | __le64 *wqe; | |
4f44e519 | 726 | struct irdma_post_send *op_info; |
551c46ed MI |
727 | u64 hdr; |
728 | u32 wqe_idx; | |
729 | bool read_fence = false; | |
4f44e519 | 730 | u32 i, total_size = 0; |
551c46ed MI |
731 | u16 quanta; |
732 | ||
4f44e519 MI |
733 | op_info = &info->op.send; |
734 | ||
735 | if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges)) | |
736 | return -EINVAL; | |
737 | ||
738 | for (i = 0; i < op_info->num_sges; i++) | |
739 | total_size += op_info->sg_list[i].length; | |
551c46ed | 740 | |
4f44e519 | 741 | if (unlikely(total_size > qp->max_inline_data)) |
2c4b14ea | 742 | return -EINVAL; |
551c46ed | 743 | |
4f44e519 MI |
744 | quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); |
745 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | |
551c46ed MI |
746 | info); |
747 | if (!wqe) | |
2c4b14ea | 748 | return -ENOMEM; |
551c46ed MI |
749 | |
750 | irdma_clr_wqes(qp, wqe_idx); | |
751 | ||
752 | set_64bit_val(wqe, 16, | |
753 | FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | | |
754 | FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); | |
755 | ||
756 | read_fence |= info->read_fence; | |
757 | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | | |
758 | FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | | |
759 | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | | |
4f44e519 | 760 | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | |
551c46ed MI |
761 | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, |
762 | (info->imm_data_valid ? 1 : 0)) | | |
763 | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | | |
764 | FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) | | |
551c46ed MI |
765 | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | |
766 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | |
767 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | |
768 | FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | | |
769 | FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | | |
770 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
771 | ||
772 | if (info->imm_data_valid) | |
773 | set_64bit_val(wqe, 0, | |
774 | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | |
4f44e519 MI |
775 | qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list, |
776 | op_info->num_sges, qp->swqe_polarity); | |
551c46ed MI |
777 | |
778 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
779 | ||
780 | set_64bit_val(wqe, 24, hdr); | |
781 | ||
295c95aa SS |
782 | if (post_sq) |
783 | irdma_uk_qp_post_wr(qp); | |
551c46ed MI |
784 | |
785 | return 0; | |
786 | } | |
787 | ||
788 | /** | |
789 | * irdma_uk_stag_local_invalidate - stag invalidate operation | |
790 | * @qp: hw qp ptr | |
791 | * @info: post sq information | |
792 | * @post_sq: flag to post sq | |
793 | */ | |
2c4b14ea SS |
794 | int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, |
795 | struct irdma_post_sq_info *info, | |
796 | bool post_sq) | |
551c46ed MI |
797 | { |
798 | __le64 *wqe; | |
799 | struct irdma_inv_local_stag *op_info; | |
800 | u64 hdr; | |
801 | u32 wqe_idx; | |
802 | bool local_fence = false; | |
9ed8110c | 803 | struct ib_sge sge = {}; |
551c46ed | 804 | |
551c46ed MI |
805 | op_info = &info->op.inv_local_stag; |
806 | local_fence = info->local_fence; | |
807 | ||
808 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, | |
809 | 0, info); | |
810 | if (!wqe) | |
2c4b14ea | 811 | return -ENOMEM; |
551c46ed MI |
812 | |
813 | irdma_clr_wqes(qp, wqe_idx); | |
814 | ||
9ed8110c | 815 | sge.lkey = op_info->target_stag; |
551c46ed MI |
816 | qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0); |
817 | ||
818 | set_64bit_val(wqe, 16, 0); | |
819 | ||
820 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) | | |
551c46ed MI |
821 | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | |
822 | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | | |
823 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | |
824 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
825 | ||
826 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
827 | ||
828 | set_64bit_val(wqe, 24, hdr); | |
829 | ||
295c95aa SS |
830 | if (post_sq) |
831 | irdma_uk_qp_post_wr(qp); | |
551c46ed MI |
832 | |
833 | return 0; | |
834 | } | |
835 | ||
551c46ed MI |
836 | /** |
837 | * irdma_uk_post_receive - post receive wqe | |
838 | * @qp: hw qp ptr | |
839 | * @info: post rq information | |
840 | */ | |
2c4b14ea SS |
841 | int irdma_uk_post_receive(struct irdma_qp_uk *qp, |
842 | struct irdma_post_rq_info *info) | |
551c46ed | 843 | { |
6407c69d | 844 | u32 wqe_idx, i, byte_off; |
551c46ed MI |
845 | u32 addl_frag_cnt; |
846 | __le64 *wqe; | |
847 | u64 hdr; | |
848 | ||
849 | if (qp->max_rq_frag_cnt < info->num_sges) | |
2c4b14ea | 850 | return -EINVAL; |
551c46ed | 851 | |
551c46ed MI |
852 | wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx); |
853 | if (!wqe) | |
2c4b14ea | 854 | return -ENOMEM; |
551c46ed MI |
855 | |
856 | qp->rq_wrid_array[wqe_idx] = info->wr_id; | |
857 | addl_frag_cnt = info->num_sges > 1 ? (info->num_sges - 1) : 0; | |
858 | qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list, | |
859 | qp->rwqe_polarity); | |
860 | ||
861 | for (i = 1, byte_off = 32; i < info->num_sges; i++) { | |
862 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i], | |
863 | qp->rwqe_polarity); | |
864 | byte_off += 16; | |
865 | } | |
866 | ||
867 | /* if not an odd number set valid bit in next fragment */ | |
868 | if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) && | |
869 | info->num_sges) { | |
870 | qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL, | |
871 | qp->rwqe_polarity); | |
872 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_2) | |
873 | ++addl_frag_cnt; | |
874 | } | |
875 | ||
876 | set_64bit_val(wqe, 16, 0); | |
877 | hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | |
878 | FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity); | |
879 | ||
880 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
881 | ||
882 | set_64bit_val(wqe, 24, hdr); | |
883 | ||
884 | return 0; | |
885 | } | |
886 | ||
887 | /** | |
888 | * irdma_uk_cq_resize - reset the cq buffer info | |
889 | * @cq: cq to resize | |
890 | * @cq_base: new cq buffer addr | |
891 | * @cq_size: number of cqes | |
892 | */ | |
893 | void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int cq_size) | |
894 | { | |
895 | cq->cq_base = cq_base; | |
896 | cq->cq_size = cq_size; | |
897 | IRDMA_RING_INIT(cq->cq_ring, cq->cq_size); | |
898 | cq->polarity = 1; | |
899 | } | |
900 | ||
901 | /** | |
902 | * irdma_uk_cq_set_resized_cnt - record the count of the resized buffers | |
903 | * @cq: cq to resize | |
904 | * @cq_cnt: the count of the resized cq buffers | |
905 | */ | |
906 | void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *cq, u16 cq_cnt) | |
907 | { | |
908 | u64 temp_val; | |
909 | u16 sw_cq_sel; | |
910 | u8 arm_next_se; | |
911 | u8 arm_next; | |
912 | u8 arm_seq_num; | |
913 | ||
914 | get_64bit_val(cq->shadow_area, 32, &temp_val); | |
915 | ||
916 | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); | |
917 | sw_cq_sel += cq_cnt; | |
918 | ||
919 | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); | |
920 | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); | |
921 | arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val); | |
922 | ||
923 | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | | |
924 | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | | |
925 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | | |
926 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next); | |
927 | ||
928 | set_64bit_val(cq->shadow_area, 32, temp_val); | |
929 | } | |
930 | ||
931 | /** | |
932 | * irdma_uk_cq_request_notification - cq notification request (door bell) | |
933 | * @cq: hw cq | |
934 | * @cq_notify: notification type | |
935 | */ | |
936 | void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq, | |
937 | enum irdma_cmpl_notify cq_notify) | |
938 | { | |
939 | u64 temp_val; | |
940 | u16 sw_cq_sel; | |
941 | u8 arm_next_se = 0; | |
942 | u8 arm_next = 0; | |
943 | u8 arm_seq_num; | |
944 | ||
945 | get_64bit_val(cq->shadow_area, 32, &temp_val); | |
946 | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); | |
947 | arm_seq_num++; | |
948 | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); | |
949 | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); | |
950 | arm_next_se |= 1; | |
951 | if (cq_notify == IRDMA_CQ_COMPL_EVENT) | |
952 | arm_next = 1; | |
953 | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | | |
954 | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | | |
955 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | | |
956 | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next); | |
957 | ||
958 | set_64bit_val(cq->shadow_area, 32, temp_val); | |
959 | ||
960 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
961 | ||
962 | writel(cq->cq_id, cq->cqe_alloc_db); | |
963 | } | |
964 | ||
965 | /** | |
966 | * irdma_uk_cq_poll_cmpl - get cq completion info | |
967 | * @cq: hw cq | |
968 | * @info: cq poll information returned | |
969 | */ | |
2c4b14ea SS |
970 | int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, |
971 | struct irdma_cq_poll_info *info) | |
551c46ed MI |
972 | { |
973 | u64 comp_ctx, qword0, qword2, qword3; | |
974 | __le64 *cqe; | |
975 | struct irdma_qp_uk *qp; | |
976 | struct irdma_ring *pring = NULL; | |
24419777 | 977 | u32 wqe_idx; |
2c4b14ea | 978 | int ret_code; |
551c46ed MI |
979 | bool move_cq_head = true; |
980 | u8 polarity; | |
981 | bool ext_valid; | |
982 | __le64 *ext_cqe; | |
983 | ||
984 | if (cq->avoid_mem_cflct) | |
985 | cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq); | |
986 | else | |
987 | cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq); | |
988 | ||
989 | get_64bit_val(cqe, 24, &qword3); | |
990 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); | |
991 | if (polarity != cq->polarity) | |
2c4b14ea | 992 | return -ENOENT; |
551c46ed MI |
993 | |
994 | /* Ensure CQE contents are read after valid bit is checked */ | |
995 | dma_rmb(); | |
996 | ||
997 | ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3); | |
998 | if (ext_valid) { | |
999 | u64 qword6, qword7; | |
1000 | u32 peek_head; | |
1001 | ||
1002 | if (cq->avoid_mem_cflct) { | |
1003 | ext_cqe = (__le64 *)((u8 *)cqe + 32); | |
1004 | get_64bit_val(ext_cqe, 24, &qword7); | |
e93c7d8e | 1005 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7); |
551c46ed MI |
1006 | } else { |
1007 | peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size; | |
1008 | ext_cqe = cq->cq_base[peek_head].buf; | |
1009 | get_64bit_val(ext_cqe, 24, &qword7); | |
e93c7d8e | 1010 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7); |
551c46ed MI |
1011 | if (!peek_head) |
1012 | polarity ^= 1; | |
1013 | } | |
1014 | if (polarity != cq->polarity) | |
2c4b14ea | 1015 | return -ENOENT; |
551c46ed MI |
1016 | |
1017 | /* Ensure ext CQE contents are read after ext valid bit is checked */ | |
1018 | dma_rmb(); | |
1019 | ||
1020 | info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7); | |
1021 | if (info->imm_valid) { | |
1022 | u64 qword4; | |
1023 | ||
1024 | get_64bit_val(ext_cqe, 0, &qword4); | |
1025 | info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4); | |
1026 | } | |
1027 | info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7); | |
1028 | info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7); | |
1029 | if (info->ud_smac_valid || info->ud_vlan_valid) { | |
1030 | get_64bit_val(ext_cqe, 16, &qword6); | |
1031 | if (info->ud_vlan_valid) | |
1032 | info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6); | |
1033 | if (info->ud_smac_valid) { | |
1034 | info->ud_smac[5] = qword6 & 0xFF; | |
1035 | info->ud_smac[4] = (qword6 >> 8) & 0xFF; | |
1036 | info->ud_smac[3] = (qword6 >> 16) & 0xFF; | |
1037 | info->ud_smac[2] = (qword6 >> 24) & 0xFF; | |
1038 | info->ud_smac[1] = (qword6 >> 32) & 0xFF; | |
1039 | info->ud_smac[0] = (qword6 >> 40) & 0xFF; | |
1040 | } | |
1041 | } | |
1042 | } else { | |
1043 | info->imm_valid = false; | |
1044 | info->ud_smac_valid = false; | |
1045 | info->ud_vlan_valid = false; | |
1046 | } | |
1047 | ||
24419777 | 1048 | info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3); |
551c46ed | 1049 | info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3); |
551c46ed MI |
1050 | info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3); |
1051 | if (info->error) { | |
1052 | info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3); | |
1053 | info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3); | |
1054 | if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) { | |
1055 | info->comp_status = IRDMA_COMPL_STATUS_FLUSHED; | |
1056 | /* Set the min error to standard flush error code for remaining cqes */ | |
1057 | if (info->minor_err != FLUSH_GENERAL_ERR) { | |
1058 | qword3 &= ~IRDMA_CQ_MINERR; | |
1059 | qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR); | |
1060 | set_64bit_val(cqe, 24, qword3); | |
1061 | } | |
1062 | } else { | |
1063 | info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN; | |
1064 | } | |
1065 | } else { | |
1066 | info->comp_status = IRDMA_COMPL_STATUS_SUCCESS; | |
1067 | } | |
1068 | ||
1069 | get_64bit_val(cqe, 0, &qword0); | |
1070 | get_64bit_val(cqe, 16, &qword2); | |
1071 | ||
1072 | info->tcp_seq_num_rtt = (u32)FIELD_GET(IRDMACQ_TCPSEQNUMRTT, qword0); | |
1073 | info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2); | |
1074 | info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2); | |
1075 | ||
1076 | get_64bit_val(cqe, 8, &comp_ctx); | |
1077 | ||
1078 | info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3); | |
1079 | qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx; | |
1080 | if (!qp || qp->destroy_pending) { | |
2c4b14ea | 1081 | ret_code = -EFAULT; |
551c46ed MI |
1082 | goto exit; |
1083 | } | |
1084 | wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3); | |
1085 | info->qp_handle = (irdma_qp_handle)(unsigned long)qp; | |
3bfb25fa | 1086 | info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); |
551c46ed | 1087 | |
24419777 | 1088 | if (info->q_type == IRDMA_CQE_QTYPE_RQ) { |
551c46ed MI |
1089 | u32 array_idx; |
1090 | ||
1091 | array_idx = wqe_idx / qp->rq_wqe_size_multiplier; | |
1092 | ||
1093 | if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED || | |
1094 | info->comp_status == IRDMA_COMPL_STATUS_UNKNOWN) { | |
1095 | if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) { | |
2c4b14ea | 1096 | ret_code = -ENOENT; |
551c46ed MI |
1097 | goto exit; |
1098 | } | |
1099 | ||
1100 | info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail]; | |
1101 | array_idx = qp->rq_ring.tail; | |
1102 | } else { | |
1103 | info->wr_id = qp->rq_wrid_array[array_idx]; | |
1104 | } | |
1105 | ||
1106 | info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0); | |
1107 | ||
551c46ed MI |
1108 | if (qword3 & IRDMACQ_STAG) { |
1109 | info->stag_invalid_set = true; | |
1110 | info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2); | |
1111 | } else { | |
1112 | info->stag_invalid_set = false; | |
1113 | } | |
1114 | IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1); | |
1115 | if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) { | |
1116 | qp->rq_flush_seen = true; | |
1117 | if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) | |
1118 | qp->rq_flush_complete = true; | |
1119 | else | |
1120 | move_cq_head = false; | |
1121 | } | |
1122 | pring = &qp->rq_ring; | |
1123 | } else { /* q_type is IRDMA_CQE_QTYPE_SQ */ | |
1124 | if (qp->first_sq_wq) { | |
1125 | if (wqe_idx + 1 >= qp->conn_wqes) | |
1126 | qp->first_sq_wq = false; | |
1127 | ||
1128 | if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) { | |
1129 | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); | |
1130 | IRDMA_RING_MOVE_TAIL(cq->cq_ring); | |
1131 | set_64bit_val(cq->shadow_area, 0, | |
1132 | IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); | |
1133 | memset(info, 0, | |
1134 | sizeof(struct irdma_cq_poll_info)); | |
1135 | return irdma_uk_cq_poll_cmpl(cq, info); | |
1136 | } | |
1137 | } | |
551c46ed MI |
1138 | if (info->comp_status != IRDMA_COMPL_STATUS_FLUSHED) { |
1139 | info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; | |
1140 | if (!info->comp_status) | |
1141 | info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; | |
1142 | info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); | |
1143 | IRDMA_RING_SET_TAIL(qp->sq_ring, | |
1144 | wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); | |
1145 | } else { | |
1146 | if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) { | |
2c4b14ea | 1147 | ret_code = -ENOENT; |
551c46ed MI |
1148 | goto exit; |
1149 | } | |
1150 | ||
1151 | do { | |
1152 | __le64 *sw_wqe; | |
1153 | u64 wqe_qword; | |
551c46ed MI |
1154 | u32 tail; |
1155 | ||
1156 | tail = qp->sq_ring.tail; | |
1157 | sw_wqe = qp->sq_base[tail].elem; | |
1158 | get_64bit_val(sw_wqe, 24, | |
1159 | &wqe_qword); | |
24419777 MI |
1160 | info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, |
1161 | wqe_qword); | |
551c46ed MI |
1162 | IRDMA_RING_SET_TAIL(qp->sq_ring, |
1163 | tail + qp->sq_wrtrk_array[tail].quanta); | |
24419777 | 1164 | if (info->op_type != IRDMAQP_OP_NOP) { |
551c46ed MI |
1165 | info->wr_id = qp->sq_wrtrk_array[tail].wrid; |
1166 | info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; | |
1167 | break; | |
1168 | } | |
1169 | } while (1); | |
24419777 MI |
1170 | if (info->op_type == IRDMA_OP_TYPE_BIND_MW && |
1171 | info->minor_err == FLUSH_PROT_ERR) | |
dcb23bbb | 1172 | info->minor_err = FLUSH_MW_BIND_ERR; |
551c46ed MI |
1173 | qp->sq_flush_seen = true; |
1174 | if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) | |
1175 | qp->sq_flush_complete = true; | |
1176 | } | |
1177 | pring = &qp->sq_ring; | |
1178 | } | |
1179 | ||
1180 | ret_code = 0; | |
1181 | ||
1182 | exit: | |
1183 | if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) | |
1184 | if (pring && IRDMA_RING_MORE_WORK(*pring)) | |
1185 | move_cq_head = false; | |
1186 | ||
1187 | if (move_cq_head) { | |
1188 | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); | |
1189 | if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring)) | |
1190 | cq->polarity ^= 1; | |
1191 | ||
1192 | if (ext_valid && !cq->avoid_mem_cflct) { | |
1193 | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); | |
1194 | if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring)) | |
1195 | cq->polarity ^= 1; | |
1196 | } | |
1197 | ||
1198 | IRDMA_RING_MOVE_TAIL(cq->cq_ring); | |
1199 | if (!cq->avoid_mem_cflct && ext_valid) | |
1200 | IRDMA_RING_MOVE_TAIL(cq->cq_ring); | |
1201 | set_64bit_val(cq->shadow_area, 0, | |
1202 | IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); | |
1203 | } else { | |
1204 | qword3 &= ~IRDMA_CQ_WQEIDX; | |
1205 | qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail); | |
1206 | set_64bit_val(cqe, 24, qword3); | |
1207 | } | |
1208 | ||
1209 | return ret_code; | |
1210 | } | |
1211 | ||
1212 | /** | |
1213 | * irdma_qp_round_up - return round up qp wq depth | |
1214 | * @wqdepth: wq depth in quanta to round up | |
1215 | */ | |
1216 | static int irdma_qp_round_up(u32 wqdepth) | |
1217 | { | |
1218 | int scount = 1; | |
1219 | ||
1220 | for (wqdepth--; scount <= 16; scount *= 2) | |
1221 | wqdepth |= wqdepth >> scount; | |
1222 | ||
1223 | return ++wqdepth; | |
1224 | } | |
1225 | ||
1226 | /** | |
1227 | * irdma_get_wqe_shift - get shift count for maximum wqe size | |
1228 | * @uk_attrs: qp HW attributes | |
1229 | * @sge: Maximum Scatter Gather Elements wqe | |
1230 | * @inline_data: Maximum inline data size | |
1231 | * @shift: Returns the shift needed based on sge | |
1232 | * | |
1233 | * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size. | |
1234 | * For 1 SGE or inline data <= 8, shift = 0 (wqe size of 32 | |
1235 | * bytes). For 2 or 3 SGEs or inline data <= 39, shift = 1 (wqe | |
1236 | * size of 64 bytes). | |
1237 | * For 4-7 SGE's and inline <= 101 Shift of 2 otherwise (wqe | |
1238 | * size of 256 bytes). | |
1239 | */ | |
1240 | void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, | |
1241 | u32 inline_data, u8 *shift) | |
1242 | { | |
1243 | *shift = 0; | |
1244 | if (uk_attrs->hw_rev >= IRDMA_GEN_2) { | |
1245 | if (sge > 1 || inline_data > 8) { | |
1246 | if (sge < 4 && inline_data <= 39) | |
1247 | *shift = 1; | |
1248 | else if (sge < 8 && inline_data <= 101) | |
1249 | *shift = 2; | |
1250 | else | |
1251 | *shift = 3; | |
1252 | } | |
1253 | } else if (sge > 1 || inline_data > 16) { | |
1254 | *shift = (sge < 4 && inline_data <= 48) ? 1 : 2; | |
1255 | } | |
1256 | } | |
1257 | ||
1258 | /* | |
1259 | * irdma_get_sqdepth - get SQ depth (quanta) | |
1260 | * @uk_attrs: qp HW attributes | |
1261 | * @sq_size: SQ size | |
1262 | * @shift: shift which determines size of WQE | |
1263 | * @sqdepth: depth of SQ | |
1264 | * | |
1265 | */ | |
2c4b14ea SS |
1266 | int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, |
1267 | u32 *sqdepth) | |
551c46ed | 1268 | { |
72d422c2 SD |
1269 | u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; |
1270 | ||
551c46ed MI |
1271 | *sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD); |
1272 | ||
72d422c2 SD |
1273 | if (*sqdepth < min_size) |
1274 | *sqdepth = min_size; | |
551c46ed | 1275 | else if (*sqdepth > uk_attrs->max_hw_wq_quanta) |
2c4b14ea | 1276 | return -EINVAL; |
551c46ed MI |
1277 | |
1278 | return 0; | |
1279 | } | |
1280 | ||
1281 | /* | |
1282 | * irdma_get_rqdepth - get RQ depth (quanta) | |
1283 | * @uk_attrs: qp HW attributes | |
1284 | * @rq_size: RQ size | |
1285 | * @shift: shift which determines size of WQE | |
1286 | * @rqdepth: depth of RQ | |
1287 | */ | |
2c4b14ea SS |
1288 | int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, |
1289 | u32 *rqdepth) | |
551c46ed | 1290 | { |
72d422c2 SD |
1291 | u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; |
1292 | ||
551c46ed MI |
1293 | *rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD); |
1294 | ||
72d422c2 SD |
1295 | if (*rqdepth < min_size) |
1296 | *rqdepth = min_size; | |
551c46ed | 1297 | else if (*rqdepth > uk_attrs->max_hw_rq_quanta) |
2c4b14ea | 1298 | return -EINVAL; |
551c46ed MI |
1299 | |
1300 | return 0; | |
1301 | } | |
1302 | ||
1303 | static const struct irdma_wqe_uk_ops iw_wqe_uk_ops = { | |
1304 | .iw_copy_inline_data = irdma_copy_inline_data, | |
1305 | .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta, | |
1306 | .iw_set_fragment = irdma_set_fragment, | |
1307 | .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe, | |
1308 | }; | |
1309 | ||
1310 | static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = { | |
1311 | .iw_copy_inline_data = irdma_copy_inline_data_gen_1, | |
1312 | .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1, | |
1313 | .iw_set_fragment = irdma_set_fragment_gen_1, | |
1314 | .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1, | |
1315 | }; | |
1316 | ||
1317 | /** | |
1318 | * irdma_setup_connection_wqes - setup WQEs necessary to complete | |
1319 | * connection. | |
1320 | * @qp: hw qp (user and kernel) | |
1321 | * @info: qp initialization info | |
1322 | */ | |
1323 | static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, | |
1324 | struct irdma_qp_uk_init_info *info) | |
1325 | { | |
1326 | u16 move_cnt = 1; | |
1327 | ||
1328 | if (!info->legacy_mode && | |
1329 | (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)) | |
1330 | move_cnt = 3; | |
1331 | ||
1332 | qp->conn_wqes = move_cnt; | |
1333 | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); | |
1334 | IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); | |
1335 | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); | |
1336 | } | |
1337 | ||
3a849872 SD |
1338 | /** |
1339 | * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ | |
1340 | * @ukinfo: qp initialization info | |
1341 | * @sq_shift: Returns shift of SQ | |
1342 | * @rq_shift: Returns shift of RQ | |
1343 | */ | |
1344 | void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift, | |
1345 | u8 *rq_shift) | |
1346 | { | |
1347 | bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2; | |
1348 | ||
1349 | irdma_get_wqe_shift(ukinfo->uk_attrs, | |
1350 | imm_support ? ukinfo->max_sq_frag_cnt + 1 : | |
1351 | ukinfo->max_sq_frag_cnt, | |
1352 | ukinfo->max_inline_data, sq_shift); | |
1353 | ||
1354 | irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0, | |
1355 | rq_shift); | |
1356 | ||
1357 | if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { | |
1358 | if (ukinfo->abi_ver > 4) | |
1359 | *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; | |
1360 | } | |
1361 | } | |
1362 | ||
1363 | /** | |
1364 | * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size. | |
1365 | * @ukinfo: qp initialization info | |
1366 | * @sq_depth: Returns depth of SQ | |
1367 | * @sq_shift: Returns shift of SQ | |
1368 | */ | |
1369 | int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo, | |
1370 | u32 *sq_depth, u8 *sq_shift) | |
1371 | { | |
1372 | bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2; | |
1373 | int status; | |
1374 | ||
1375 | irdma_get_wqe_shift(ukinfo->uk_attrs, | |
1376 | imm_support ? ukinfo->max_sq_frag_cnt + 1 : | |
1377 | ukinfo->max_sq_frag_cnt, | |
1378 | ukinfo->max_inline_data, sq_shift); | |
1379 | status = irdma_get_sqdepth(ukinfo->uk_attrs, ukinfo->sq_size, | |
1380 | *sq_shift, sq_depth); | |
1381 | ||
1382 | return status; | |
1383 | } | |
1384 | ||
1385 | /** | |
1386 | * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size. | |
1387 | * @ukinfo: qp initialization info | |
1388 | * @rq_depth: Returns depth of RQ | |
1389 | * @rq_shift: Returns shift of RQ | |
1390 | */ | |
1391 | int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo, | |
1392 | u32 *rq_depth, u8 *rq_shift) | |
1393 | { | |
1394 | int status; | |
1395 | ||
1396 | irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0, | |
1397 | rq_shift); | |
1398 | ||
1399 | if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { | |
1400 | if (ukinfo->abi_ver > 4) | |
1401 | *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; | |
1402 | } | |
1403 | ||
1404 | status = irdma_get_rqdepth(ukinfo->uk_attrs, ukinfo->rq_size, | |
1405 | *rq_shift, rq_depth); | |
1406 | ||
1407 | return status; | |
1408 | } | |
1409 | ||
551c46ed MI |
1410 | /** |
1411 | * irdma_uk_qp_init - initialize shared qp | |
1412 | * @qp: hw qp (user and kernel) | |
1413 | * @info: qp initialization info | |
1414 | * | |
1415 | * initializes the vars used in both user and kernel mode. | |
1416 | * size of the wqe depends on numbers of max. fragements | |
1417 | * allowed. Then size of wqe * the number of wqes should be the | |
1418 | * amount of memory allocated for sq and rq. | |
1419 | */ | |
2c4b14ea | 1420 | int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) |
551c46ed | 1421 | { |
2c4b14ea | 1422 | int ret_code = 0; |
551c46ed | 1423 | u32 sq_ring_size; |
551c46ed MI |
1424 | |
1425 | qp->uk_attrs = info->uk_attrs; | |
1426 | if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags || | |
1427 | info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags) | |
2c4b14ea | 1428 | return -EINVAL; |
551c46ed | 1429 | |
551c46ed MI |
1430 | qp->qp_caps = info->qp_caps; |
1431 | qp->sq_base = info->sq; | |
1432 | qp->rq_base = info->rq; | |
1433 | qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP; | |
1434 | qp->shadow_area = info->shadow_area; | |
1435 | qp->sq_wrtrk_array = info->sq_wrtrk_array; | |
1436 | ||
1437 | qp->rq_wrid_array = info->rq_wrid_array; | |
1438 | qp->wqe_alloc_db = info->wqe_alloc_db; | |
1439 | qp->qp_id = info->qp_id; | |
1440 | qp->sq_size = info->sq_size; | |
551c46ed | 1441 | qp->max_sq_frag_cnt = info->max_sq_frag_cnt; |
3a849872 | 1442 | sq_ring_size = qp->sq_size << info->sq_shift; |
551c46ed MI |
1443 | IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); |
1444 | IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); | |
1445 | if (info->first_sq_wq) { | |
1446 | irdma_setup_connection_wqes(qp, info); | |
1447 | qp->swqe_polarity = 1; | |
1448 | qp->first_sq_wq = true; | |
1449 | } else { | |
1450 | qp->swqe_polarity = 0; | |
1451 | } | |
1452 | qp->swqe_polarity_deferred = 1; | |
1453 | qp->rwqe_polarity = 0; | |
1454 | qp->rq_size = info->rq_size; | |
1455 | qp->max_rq_frag_cnt = info->max_rq_frag_cnt; | |
1456 | qp->max_inline_data = info->max_inline_data; | |
3a849872 | 1457 | qp->rq_wqe_size = info->rq_shift; |
551c46ed | 1458 | IRDMA_RING_INIT(qp->rq_ring, qp->rq_size); |
3a849872 | 1459 | qp->rq_wqe_size_multiplier = 1 << info->rq_shift; |
551c46ed MI |
1460 | if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) |
1461 | qp->wqe_ops = iw_wqe_uk_ops_gen_1; | |
1462 | else | |
1463 | qp->wqe_ops = iw_wqe_uk_ops; | |
1464 | return ret_code; | |
1465 | } | |
1466 | ||
1467 | /** | |
1468 | * irdma_uk_cq_init - initialize shared cq (user and kernel) | |
1469 | * @cq: hw cq | |
1470 | * @info: hw cq initialization info | |
1471 | */ | |
dede33da ZY |
1472 | void irdma_uk_cq_init(struct irdma_cq_uk *cq, |
1473 | struct irdma_cq_uk_init_info *info) | |
551c46ed MI |
1474 | { |
1475 | cq->cq_base = info->cq_base; | |
1476 | cq->cq_id = info->cq_id; | |
1477 | cq->cq_size = info->cq_size; | |
1478 | cq->cqe_alloc_db = info->cqe_alloc_db; | |
1479 | cq->cq_ack_db = info->cq_ack_db; | |
1480 | cq->shadow_area = info->shadow_area; | |
1481 | cq->avoid_mem_cflct = info->avoid_mem_cflct; | |
1482 | IRDMA_RING_INIT(cq->cq_ring, cq->cq_size); | |
1483 | cq->polarity = 1; | |
551c46ed MI |
1484 | } |
1485 | ||
1486 | /** | |
1487 | * irdma_uk_clean_cq - clean cq entries | |
1488 | * @q: completion context | |
1489 | * @cq: cq to clean | |
1490 | */ | |
1491 | void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq) | |
1492 | { | |
1493 | __le64 *cqe; | |
1494 | u64 qword3, comp_ctx; | |
1495 | u32 cq_head; | |
1496 | u8 polarity, temp; | |
1497 | ||
1498 | cq_head = cq->cq_ring.head; | |
1499 | temp = cq->polarity; | |
1500 | do { | |
1501 | if (cq->avoid_mem_cflct) | |
1502 | cqe = ((struct irdma_extended_cqe *)(cq->cq_base))[cq_head].buf; | |
1503 | else | |
1504 | cqe = cq->cq_base[cq_head].buf; | |
1505 | get_64bit_val(cqe, 24, &qword3); | |
1506 | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); | |
1507 | ||
1508 | if (polarity != temp) | |
1509 | break; | |
1510 | ||
4984eb51 SS |
1511 | /* Ensure CQE contents are read after valid bit is checked */ |
1512 | dma_rmb(); | |
1513 | ||
551c46ed MI |
1514 | get_64bit_val(cqe, 8, &comp_ctx); |
1515 | if ((void *)(unsigned long)comp_ctx == q) | |
1516 | set_64bit_val(cqe, 8, 0); | |
1517 | ||
1518 | cq_head = (cq_head + 1) % cq->cq_ring.size; | |
1519 | if (!cq_head) | |
1520 | temp ^= 1; | |
1521 | } while (true); | |
1522 | } | |
1523 | ||
1524 | /** | |
1525 | * irdma_nop - post a nop | |
1526 | * @qp: hw qp ptr | |
1527 | * @wr_id: work request id | |
1528 | * @signaled: signaled for completion | |
1529 | * @post_sq: ring doorbell | |
1530 | */ | |
2c4b14ea | 1531 | int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) |
551c46ed MI |
1532 | { |
1533 | __le64 *wqe; | |
1534 | u64 hdr; | |
1535 | u32 wqe_idx; | |
1536 | struct irdma_post_sq_info info = {}; | |
1537 | ||
551c46ed MI |
1538 | info.wr_id = wr_id; |
1539 | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, | |
1540 | 0, &info); | |
1541 | if (!wqe) | |
2c4b14ea | 1542 | return -ENOMEM; |
551c46ed MI |
1543 | |
1544 | irdma_clr_wqes(qp, wqe_idx); | |
1545 | ||
1546 | set_64bit_val(wqe, 0, 0); | |
1547 | set_64bit_val(wqe, 8, 0); | |
1548 | set_64bit_val(wqe, 16, 0); | |
1549 | ||
1550 | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | | |
1551 | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | | |
1552 | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |
1553 | ||
1554 | dma_wmb(); /* make sure WQE is populated before valid bit is set */ | |
1555 | ||
1556 | set_64bit_val(wqe, 24, hdr); | |
1557 | if (post_sq) | |
1558 | irdma_uk_qp_post_wr(qp); | |
1559 | ||
1560 | return 0; | |
1561 | } | |
1562 | ||
1563 | /** | |
1564 | * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ | |
1565 | * @frag_cnt: number of fragments | |
1566 | * @quanta: quanta for frag_cnt | |
1567 | */ | |
2c4b14ea | 1568 | int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta) |
551c46ed MI |
1569 | { |
1570 | switch (frag_cnt) { | |
1571 | case 0: | |
1572 | case 1: | |
1573 | *quanta = IRDMA_QP_WQE_MIN_QUANTA; | |
1574 | break; | |
1575 | case 2: | |
1576 | case 3: | |
1577 | *quanta = 2; | |
1578 | break; | |
1579 | case 4: | |
1580 | case 5: | |
1581 | *quanta = 3; | |
1582 | break; | |
1583 | case 6: | |
1584 | case 7: | |
1585 | *quanta = 4; | |
1586 | break; | |
1587 | case 8: | |
1588 | case 9: | |
1589 | *quanta = 5; | |
1590 | break; | |
1591 | case 10: | |
1592 | case 11: | |
1593 | *quanta = 6; | |
1594 | break; | |
1595 | case 12: | |
1596 | case 13: | |
1597 | *quanta = 7; | |
1598 | break; | |
1599 | case 14: | |
1600 | case 15: /* when immediate data is present */ | |
1601 | *quanta = 8; | |
1602 | break; | |
1603 | default: | |
2c4b14ea | 1604 | return -EINVAL; |
551c46ed MI |
1605 | } |
1606 | ||
1607 | return 0; | |
1608 | } | |
1609 | ||
1610 | /** | |
1611 | * irdma_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ | |
1612 | * @frag_cnt: number of fragments | |
1613 | * @wqe_size: size in bytes given frag_cnt | |
1614 | */ | |
2c4b14ea | 1615 | int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size) |
551c46ed MI |
1616 | { |
1617 | switch (frag_cnt) { | |
1618 | case 0: | |
1619 | case 1: | |
1620 | *wqe_size = 32; | |
1621 | break; | |
1622 | case 2: | |
1623 | case 3: | |
1624 | *wqe_size = 64; | |
1625 | break; | |
1626 | case 4: | |
1627 | case 5: | |
1628 | case 6: | |
1629 | case 7: | |
1630 | *wqe_size = 128; | |
1631 | break; | |
1632 | case 8: | |
1633 | case 9: | |
1634 | case 10: | |
1635 | case 11: | |
1636 | case 12: | |
1637 | case 13: | |
1638 | case 14: | |
1639 | *wqe_size = 256; | |
1640 | break; | |
1641 | default: | |
2c4b14ea | 1642 | return -EINVAL; |
551c46ed MI |
1643 | } |
1644 | ||
1645 | return 0; | |
1646 | } |