Commit | Line | Data |
---|---|---|
1f4d4ed6 | 1 | /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ |
fe56b9e6 | 2 | /* QLogic qed NIC Driver |
e8f1cb50 | 3 | * Copyright (c) 2015-2017 QLogic Corporation |
663eacd8 | 4 | * Copyright (c) 2019-2020 Marvell International Ltd. |
fe56b9e6 YM |
5 | */ |
6 | ||
7 | #ifndef _QED_CHAIN_H | |
8 | #define _QED_CHAIN_H | |
9 | ||
10 | #include <linux/types.h> | |
11 | #include <asm/byteorder.h> | |
12 | #include <linux/kernel.h> | |
13 | #include <linux/list.h> | |
14 | #include <linux/slab.h> | |
15 | #include <linux/qed/common_hsi.h> | |
16 | ||
fe56b9e6 YM |
17 | enum qed_chain_mode { |
18 | /* Each Page contains a next pointer at its end */ | |
19 | QED_CHAIN_MODE_NEXT_PTR, | |
20 | ||
21 | /* Chain is a single page (next ptr) is unrequired */ | |
22 | QED_CHAIN_MODE_SINGLE, | |
23 | ||
24 | /* Page pointers are located in a side list */ | |
25 | QED_CHAIN_MODE_PBL, | |
26 | }; | |
27 | ||
28 | enum qed_chain_use_mode { | |
29 | QED_CHAIN_USE_TO_PRODUCE, /* Chain starts empty */ | |
30 | QED_CHAIN_USE_TO_CONSUME, /* Chain starts full */ | |
31 | QED_CHAIN_USE_TO_CONSUME_PRODUCE, /* Chain starts empty */ | |
32 | }; | |
33 | ||
a91eb52a YM |
34 | enum qed_chain_cnt_type { |
35 | /* The chain's size/prod/cons are kept in 16-bit variables */ | |
36 | QED_CHAIN_CNT_TYPE_U16, | |
37 | ||
38 | /* The chain's size/prod/cons are kept in 32-bit variables */ | |
39 | QED_CHAIN_CNT_TYPE_U32, | |
40 | }; | |
41 | ||
fe56b9e6 YM |
42 | struct qed_chain_next { |
43 | struct regpair next_phys; | |
44 | void *next_virt; | |
45 | }; | |
46 | ||
a91eb52a YM |
47 | struct qed_chain_pbl_u16 { |
48 | u16 prod_page_idx; | |
49 | u16 cons_page_idx; | |
50 | }; | |
51 | ||
52 | struct qed_chain_pbl_u32 { | |
53 | u32 prod_page_idx; | |
54 | u32 cons_page_idx; | |
55 | }; | |
56 | ||
1a4a6975 MY |
57 | struct qed_chain_ext_pbl { |
58 | dma_addr_t p_pbl_phys; | |
59 | void *p_pbl_virt; | |
60 | }; | |
61 | ||
a91eb52a YM |
62 | struct qed_chain_u16 { |
63 | /* Cyclic index of next element to produce/consme */ | |
64 | u16 prod_idx; | |
65 | u16 cons_idx; | |
66 | }; | |
67 | ||
68 | struct qed_chain_u32 { | |
69 | /* Cyclic index of next element to produce/consme */ | |
70 | u32 prod_idx; | |
71 | u32 cons_idx; | |
fe56b9e6 YM |
72 | }; |
73 | ||
8063f761 YB |
74 | struct addr_tbl_entry { |
75 | void *virt_addr; | |
76 | dma_addr_t dma_map; | |
77 | }; | |
78 | ||
fe56b9e6 | 79 | struct qed_chain { |
6d937acf MY |
80 | /* fastpath portion of the chain - required for commands such |
81 | * as produce / consume. | |
82 | */ | |
83 | /* Point to next element to produce/consume */ | |
84 | void *p_prod_elem; | |
85 | void *p_cons_elem; | |
86 | ||
87 | /* Fastpath portions of the PBL [if exists] */ | |
88 | struct { | |
8063f761 YB |
89 | /* Table for keeping the virtual and physical addresses of the |
90 | * chain pages, respectively to the physical addresses | |
91 | * in the pbl table. | |
6d937acf | 92 | */ |
8063f761 | 93 | struct addr_tbl_entry *pp_addr_tbl; |
a91eb52a | 94 | |
6d937acf MY |
95 | union { |
96 | struct qed_chain_pbl_u16 u16; | |
97 | struct qed_chain_pbl_u32 u32; | |
98 | } c; | |
99 | } pbl; | |
a91eb52a YM |
100 | |
101 | union { | |
102 | struct qed_chain_u16 chain16; | |
103 | struct qed_chain_u32 chain32; | |
104 | } u; | |
105 | ||
6d937acf MY |
106 | /* Capacity counts only usable elements */ |
107 | u32 capacity; | |
a91eb52a YM |
108 | u32 page_cnt; |
109 | ||
6d937acf MY |
110 | enum qed_chain_mode mode; |
111 | ||
112 | /* Elements information for fast calculations */ | |
113 | u16 elem_per_page; | |
114 | u16 elem_per_page_mask; | |
115 | u16 elem_size; | |
116 | u16 next_page_mask; | |
117 | u16 usable_per_page; | |
118 | u8 elem_unusable; | |
119 | ||
120 | u8 cnt_type; | |
121 | ||
122 | /* Slowpath of the chain - required for initialization and destruction, | |
123 | * but isn't involved in regular functionality. | |
a91eb52a | 124 | */ |
6d937acf MY |
125 | |
126 | /* Base address of a pre-allocated buffer for pbl */ | |
127 | struct { | |
128 | dma_addr_t p_phys_table; | |
129 | void *p_virt_table; | |
130 | } pbl_sp; | |
131 | ||
132 | /* Address of first page of the chain - the address is required | |
133 | * for fastpath operation [consume/produce] but only for the the SINGLE | |
134 | * flavour which isn't considered fastpath [== SPQ]. | |
135 | */ | |
136 | void *p_virt_addr; | |
137 | dma_addr_t p_phys_addr; | |
138 | ||
139 | /* Total number of elements [for entire chain] */ | |
a91eb52a YM |
140 | u32 size; |
141 | ||
6d937acf | 142 | u8 intended_use; |
1a4a6975 MY |
143 | |
144 | bool b_external_pbl; | |
fe56b9e6 YM |
145 | }; |
146 | ||
147 | #define QED_CHAIN_PBL_ENTRY_SIZE (8) | |
148 | #define QED_CHAIN_PAGE_SIZE (0x1000) | |
149 | #define ELEMS_PER_PAGE(elem_size) (QED_CHAIN_PAGE_SIZE / (elem_size)) | |
150 | ||
6d937acf MY |
151 | #define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode) \ |
152 | (((mode) == QED_CHAIN_MODE_NEXT_PTR) ? \ | |
153 | (u8)(1 + ((sizeof(struct qed_chain_next) - 1) / \ | |
154 | (elem_size))) : 0) | |
fe56b9e6 YM |
155 | |
156 | #define USABLE_ELEMS_PER_PAGE(elem_size, mode) \ | |
157 | ((u32)(ELEMS_PER_PAGE(elem_size) - \ | |
158 | UNUSABLE_ELEMS_PER_PAGE(elem_size, mode))) | |
159 | ||
160 | #define QED_CHAIN_PAGE_CNT(elem_cnt, elem_size, mode) \ | |
161 | DIV_ROUND_UP(elem_cnt, USABLE_ELEMS_PER_PAGE(elem_size, mode)) | |
162 | ||
a91eb52a YM |
163 | #define is_chain_u16(p) ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U16) |
164 | #define is_chain_u32(p) ((p)->cnt_type == QED_CHAIN_CNT_TYPE_U32) | |
165 | ||
fe56b9e6 YM |
166 | /* Accessors */ |
167 | static inline u16 qed_chain_get_prod_idx(struct qed_chain *p_chain) | |
168 | { | |
a91eb52a | 169 | return p_chain->u.chain16.prod_idx; |
fe56b9e6 YM |
170 | } |
171 | ||
172 | static inline u16 qed_chain_get_cons_idx(struct qed_chain *p_chain) | |
173 | { | |
a91eb52a YM |
174 | return p_chain->u.chain16.cons_idx; |
175 | } | |
176 | ||
177 | static inline u32 qed_chain_get_cons_idx_u32(struct qed_chain *p_chain) | |
178 | { | |
179 | return p_chain->u.chain32.cons_idx; | |
fe56b9e6 YM |
180 | } |
181 | ||
182 | static inline u16 qed_chain_get_elem_left(struct qed_chain *p_chain) | |
183 | { | |
97dd1abd AL |
184 | u16 elem_per_page = p_chain->elem_per_page; |
185 | u32 prod = p_chain->u.chain16.prod_idx; | |
186 | u32 cons = p_chain->u.chain16.cons_idx; | |
fe56b9e6 YM |
187 | u16 used; |
188 | ||
97dd1abd AL |
189 | if (prod < cons) |
190 | prod += (u32)U16_MAX + 1; | |
191 | ||
192 | used = (u16)(prod - cons); | |
fe56b9e6 | 193 | if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR) |
97dd1abd | 194 | used -= prod / elem_per_page - cons / elem_per_page; |
fe56b9e6 | 195 | |
a91eb52a | 196 | return (u16)(p_chain->capacity - used); |
fe56b9e6 YM |
197 | } |
198 | ||
a91eb52a | 199 | static inline u32 qed_chain_get_elem_left_u32(struct qed_chain *p_chain) |
fe56b9e6 | 200 | { |
97dd1abd AL |
201 | u16 elem_per_page = p_chain->elem_per_page; |
202 | u64 prod = p_chain->u.chain32.prod_idx; | |
203 | u64 cons = p_chain->u.chain32.cons_idx; | |
a91eb52a | 204 | u32 used; |
fe56b9e6 | 205 | |
97dd1abd AL |
206 | if (prod < cons) |
207 | prod += (u64)U32_MAX + 1; | |
208 | ||
209 | used = (u32)(prod - cons); | |
a91eb52a | 210 | if (p_chain->mode == QED_CHAIN_MODE_NEXT_PTR) |
97dd1abd | 211 | used -= (u32)(prod / elem_per_page - cons / elem_per_page); |
fe56b9e6 | 212 | |
a91eb52a | 213 | return p_chain->capacity - used; |
fe56b9e6 YM |
214 | } |
215 | ||
a91eb52a | 216 | static inline u16 qed_chain_get_usable_per_page(struct qed_chain *p_chain) |
fe56b9e6 YM |
217 | { |
218 | return p_chain->usable_per_page; | |
219 | } | |
220 | ||
6d937acf | 221 | static inline u8 qed_chain_get_unusable_per_page(struct qed_chain *p_chain) |
fe56b9e6 YM |
222 | { |
223 | return p_chain->elem_unusable; | |
224 | } | |
225 | ||
a91eb52a | 226 | static inline u32 qed_chain_get_page_cnt(struct qed_chain *p_chain) |
fe56b9e6 | 227 | { |
a91eb52a | 228 | return p_chain->page_cnt; |
fe56b9e6 YM |
229 | } |
230 | ||
a91eb52a | 231 | static inline dma_addr_t qed_chain_get_pbl_phys(struct qed_chain *p_chain) |
fe56b9e6 | 232 | { |
6d937acf | 233 | return p_chain->pbl_sp.p_phys_table; |
fe56b9e6 YM |
234 | } |
235 | ||
236 | /** | |
237 | * @brief qed_chain_advance_page - | |
238 | * | |
239 | * Advance the next element accros pages for a linked chain | |
240 | * | |
241 | * @param p_chain | |
242 | * @param p_next_elem | |
243 | * @param idx_to_inc | |
244 | * @param page_to_inc | |
245 | */ | |
246 | static inline void | |
247 | qed_chain_advance_page(struct qed_chain *p_chain, | |
a91eb52a | 248 | void **p_next_elem, void *idx_to_inc, void *page_to_inc) |
fe56b9e6 | 249 | { |
a91eb52a YM |
250 | struct qed_chain_next *p_next = NULL; |
251 | u32 page_index = 0; | |
6d937acf | 252 | |
fe56b9e6 YM |
253 | switch (p_chain->mode) { |
254 | case QED_CHAIN_MODE_NEXT_PTR: | |
a91eb52a | 255 | p_next = *p_next_elem; |
fe56b9e6 | 256 | *p_next_elem = p_next->next_virt; |
a91eb52a YM |
257 | if (is_chain_u16(p_chain)) |
258 | *(u16 *)idx_to_inc += p_chain->elem_unusable; | |
259 | else | |
260 | *(u32 *)idx_to_inc += p_chain->elem_unusable; | |
fe56b9e6 | 261 | break; |
fe56b9e6 YM |
262 | case QED_CHAIN_MODE_SINGLE: |
263 | *p_next_elem = p_chain->p_virt_addr; | |
264 | break; | |
265 | ||
266 | case QED_CHAIN_MODE_PBL: | |
a91eb52a YM |
267 | if (is_chain_u16(p_chain)) { |
268 | if (++(*(u16 *)page_to_inc) == p_chain->page_cnt) | |
269 | *(u16 *)page_to_inc = 0; | |
270 | page_index = *(u16 *)page_to_inc; | |
271 | } else { | |
272 | if (++(*(u32 *)page_to_inc) == p_chain->page_cnt) | |
273 | *(u32 *)page_to_inc = 0; | |
274 | page_index = *(u32 *)page_to_inc; | |
fe56b9e6 | 275 | } |
8063f761 | 276 | *p_next_elem = p_chain->pbl.pp_addr_tbl[page_index].virt_addr; |
fe56b9e6 YM |
277 | } |
278 | } | |
279 | ||
280 | #define is_unusable_idx(p, idx) \ | |
a91eb52a YM |
281 | (((p)->u.chain16.idx & (p)->elem_per_page_mask) == (p)->usable_per_page) |
282 | ||
283 | #define is_unusable_idx_u32(p, idx) \ | |
284 | (((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page) | |
285 | #define is_unusable_next_idx(p, idx) \ | |
286 | ((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \ | |
287 | (p)->usable_per_page) | |
fe56b9e6 | 288 | |
a91eb52a YM |
289 | #define is_unusable_next_idx_u32(p, idx) \ |
290 | ((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) == \ | |
291 | (p)->usable_per_page) | |
fe56b9e6 | 292 | |
a91eb52a | 293 | #define test_and_skip(p, idx) \ |
fe56b9e6 | 294 | do { \ |
a91eb52a YM |
295 | if (is_chain_u16(p)) { \ |
296 | if (is_unusable_idx(p, idx)) \ | |
297 | (p)->u.chain16.idx += (p)->elem_unusable; \ | |
298 | } else { \ | |
299 | if (is_unusable_idx_u32(p, idx)) \ | |
300 | (p)->u.chain32.idx += (p)->elem_unusable; \ | |
fe56b9e6 YM |
301 | } \ |
302 | } while (0) | |
303 | ||
fe56b9e6 YM |
304 | /** |
305 | * @brief qed_chain_return_produced - | |
306 | * | |
307 | * A chain in which the driver "Produces" elements should use this API | |
308 | * to indicate previous produced elements are now consumed. | |
309 | * | |
310 | * @param p_chain | |
311 | */ | |
312 | static inline void qed_chain_return_produced(struct qed_chain *p_chain) | |
313 | { | |
a91eb52a YM |
314 | if (is_chain_u16(p_chain)) |
315 | p_chain->u.chain16.cons_idx++; | |
316 | else | |
317 | p_chain->u.chain32.cons_idx++; | |
318 | test_and_skip(p_chain, cons_idx); | |
fe56b9e6 YM |
319 | } |
320 | ||
321 | /** | |
322 | * @brief qed_chain_produce - | |
323 | * | |
324 | * A chain in which the driver "Produces" elements should use this to get | |
325 | * a pointer to the next element which can be "Produced". It's driver | |
326 | * responsibility to validate that the chain has room for new element. | |
327 | * | |
328 | * @param p_chain | |
329 | * | |
330 | * @return void*, a pointer to next element | |
331 | */ | |
332 | static inline void *qed_chain_produce(struct qed_chain *p_chain) | |
333 | { | |
a91eb52a YM |
334 | void *p_ret = NULL, *p_prod_idx, *p_prod_page_idx; |
335 | ||
336 | if (is_chain_u16(p_chain)) { | |
337 | if ((p_chain->u.chain16.prod_idx & | |
338 | p_chain->elem_per_page_mask) == p_chain->next_page_mask) { | |
339 | p_prod_idx = &p_chain->u.chain16.prod_idx; | |
6d937acf | 340 | p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx; |
a91eb52a YM |
341 | qed_chain_advance_page(p_chain, &p_chain->p_prod_elem, |
342 | p_prod_idx, p_prod_page_idx); | |
343 | } | |
344 | p_chain->u.chain16.prod_idx++; | |
345 | } else { | |
346 | if ((p_chain->u.chain32.prod_idx & | |
347 | p_chain->elem_per_page_mask) == p_chain->next_page_mask) { | |
348 | p_prod_idx = &p_chain->u.chain32.prod_idx; | |
6d937acf | 349 | p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx; |
a91eb52a YM |
350 | qed_chain_advance_page(p_chain, &p_chain->p_prod_elem, |
351 | p_prod_idx, p_prod_page_idx); | |
352 | } | |
353 | p_chain->u.chain32.prod_idx++; | |
fe56b9e6 YM |
354 | } |
355 | ||
a91eb52a | 356 | p_ret = p_chain->p_prod_elem; |
fe56b9e6 YM |
357 | p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) + |
358 | p_chain->elem_size); | |
359 | ||
a91eb52a | 360 | return p_ret; |
fe56b9e6 YM |
361 | } |
362 | ||
363 | /** | |
364 | * @brief qed_chain_get_capacity - | |
365 | * | |
366 | * Get the maximum number of BDs in chain | |
367 | * | |
368 | * @param p_chain | |
369 | * @param num | |
370 | * | |
a91eb52a | 371 | * @return number of unusable BDs |
fe56b9e6 | 372 | */ |
a91eb52a | 373 | static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain) |
fe56b9e6 YM |
374 | { |
375 | return p_chain->capacity; | |
376 | } | |
377 | ||
378 | /** | |
379 | * @brief qed_chain_recycle_consumed - | |
380 | * | |
381 | * Returns an element which was previously consumed; | |
382 | * Increments producers so they could be written to FW. | |
383 | * | |
384 | * @param p_chain | |
385 | */ | |
a91eb52a | 386 | static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain) |
fe56b9e6 | 387 | { |
a91eb52a YM |
388 | test_and_skip(p_chain, prod_idx); |
389 | if (is_chain_u16(p_chain)) | |
390 | p_chain->u.chain16.prod_idx++; | |
391 | else | |
392 | p_chain->u.chain32.prod_idx++; | |
fe56b9e6 YM |
393 | } |
394 | ||
395 | /** | |
396 | * @brief qed_chain_consume - | |
397 | * | |
398 | * A Chain in which the driver utilizes data written by a different source | |
399 | * (i.e., FW) should use this to access passed buffers. | |
400 | * | |
401 | * @param p_chain | |
402 | * | |
403 | * @return void*, a pointer to the next buffer written | |
404 | */ | |
405 | static inline void *qed_chain_consume(struct qed_chain *p_chain) | |
406 | { | |
a91eb52a YM |
407 | void *p_ret = NULL, *p_cons_idx, *p_cons_page_idx; |
408 | ||
409 | if (is_chain_u16(p_chain)) { | |
410 | if ((p_chain->u.chain16.cons_idx & | |
411 | p_chain->elem_per_page_mask) == p_chain->next_page_mask) { | |
412 | p_cons_idx = &p_chain->u.chain16.cons_idx; | |
6d937acf | 413 | p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx; |
a91eb52a YM |
414 | qed_chain_advance_page(p_chain, &p_chain->p_cons_elem, |
415 | p_cons_idx, p_cons_page_idx); | |
416 | } | |
417 | p_chain->u.chain16.cons_idx++; | |
418 | } else { | |
419 | if ((p_chain->u.chain32.cons_idx & | |
420 | p_chain->elem_per_page_mask) == p_chain->next_page_mask) { | |
421 | p_cons_idx = &p_chain->u.chain32.cons_idx; | |
6d937acf MY |
422 | p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx; |
423 | qed_chain_advance_page(p_chain, &p_chain->p_cons_elem, | |
a91eb52a YM |
424 | p_cons_idx, p_cons_page_idx); |
425 | } | |
426 | p_chain->u.chain32.cons_idx++; | |
fe56b9e6 YM |
427 | } |
428 | ||
a91eb52a | 429 | p_ret = p_chain->p_cons_elem; |
fe56b9e6 YM |
430 | p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) + |
431 | p_chain->elem_size); | |
432 | ||
a91eb52a | 433 | return p_ret; |
fe56b9e6 YM |
434 | } |
435 | ||
436 | /** | |
437 | * @brief qed_chain_reset - Resets the chain to its start state | |
438 | * | |
439 | * @param p_chain pointer to a previously allocted chain | |
440 | */ | |
441 | static inline void qed_chain_reset(struct qed_chain *p_chain) | |
442 | { | |
a91eb52a YM |
443 | u32 i; |
444 | ||
445 | if (is_chain_u16(p_chain)) { | |
446 | p_chain->u.chain16.prod_idx = 0; | |
447 | p_chain->u.chain16.cons_idx = 0; | |
448 | } else { | |
449 | p_chain->u.chain32.prod_idx = 0; | |
450 | p_chain->u.chain32.cons_idx = 0; | |
451 | } | |
452 | p_chain->p_cons_elem = p_chain->p_virt_addr; | |
453 | p_chain->p_prod_elem = p_chain->p_virt_addr; | |
fe56b9e6 YM |
454 | |
455 | if (p_chain->mode == QED_CHAIN_MODE_PBL) { | |
a91eb52a YM |
456 | /* Use (page_cnt - 1) as a reset value for the prod/cons page's |
457 | * indices, to avoid unnecessary page advancing on the first | |
458 | * call to qed_chain_produce/consume. Instead, the indices | |
459 | * will be advanced to page_cnt and then will be wrapped to 0. | |
460 | */ | |
461 | u32 reset_val = p_chain->page_cnt - 1; | |
462 | ||
463 | if (is_chain_u16(p_chain)) { | |
6d937acf MY |
464 | p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val; |
465 | p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val; | |
a91eb52a | 466 | } else { |
6d937acf MY |
467 | p_chain->pbl.c.u32.prod_page_idx = reset_val; |
468 | p_chain->pbl.c.u32.cons_page_idx = reset_val; | |
a91eb52a | 469 | } |
fe56b9e6 YM |
470 | } |
471 | ||
472 | switch (p_chain->intended_use) { | |
fe56b9e6 YM |
473 | case QED_CHAIN_USE_TO_CONSUME: |
474 | /* produce empty elements */ | |
475 | for (i = 0; i < p_chain->capacity; i++) | |
476 | qed_chain_recycle_consumed(p_chain); | |
477 | break; | |
6d937acf MY |
478 | |
479 | case QED_CHAIN_USE_TO_CONSUME_PRODUCE: | |
480 | case QED_CHAIN_USE_TO_PRODUCE: | |
481 | default: | |
482 | /* Do nothing */ | |
483 | break; | |
fe56b9e6 YM |
484 | } |
485 | } | |
486 | ||
487 | /** | |
488 | * @brief qed_chain_init - Initalizes a basic chain struct | |
489 | * | |
490 | * @param p_chain | |
491 | * @param p_virt_addr | |
492 | * @param p_phys_addr physical address of allocated buffer's beginning | |
493 | * @param page_cnt number of pages in the allocated buffer | |
494 | * @param elem_size size of each element in the chain | |
495 | * @param intended_use | |
496 | * @param mode | |
497 | */ | |
a91eb52a YM |
498 | static inline void qed_chain_init_params(struct qed_chain *p_chain, |
499 | u32 page_cnt, | |
500 | u8 elem_size, | |
501 | enum qed_chain_use_mode intended_use, | |
502 | enum qed_chain_mode mode, | |
503 | enum qed_chain_cnt_type cnt_type) | |
fe56b9e6 YM |
504 | { |
505 | /* chain fixed parameters */ | |
a91eb52a YM |
506 | p_chain->p_virt_addr = NULL; |
507 | p_chain->p_phys_addr = 0; | |
fe56b9e6 | 508 | p_chain->elem_size = elem_size; |
6d937acf | 509 | p_chain->intended_use = (u8)intended_use; |
fe56b9e6 | 510 | p_chain->mode = mode; |
6d937acf | 511 | p_chain->cnt_type = (u8)cnt_type; |
fe56b9e6 | 512 | |
6d937acf | 513 | p_chain->elem_per_page = ELEMS_PER_PAGE(elem_size); |
a91eb52a | 514 | p_chain->usable_per_page = USABLE_ELEMS_PER_PAGE(elem_size, mode); |
6d937acf | 515 | p_chain->elem_per_page_mask = p_chain->elem_per_page - 1; |
fe56b9e6 | 516 | p_chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(elem_size, mode); |
fe56b9e6 YM |
517 | p_chain->next_page_mask = (p_chain->usable_per_page & |
518 | p_chain->elem_per_page_mask); | |
519 | ||
a91eb52a YM |
520 | p_chain->page_cnt = page_cnt; |
521 | p_chain->capacity = p_chain->usable_per_page * page_cnt; | |
522 | p_chain->size = p_chain->elem_per_page * page_cnt; | |
fe56b9e6 | 523 | |
6d937acf MY |
524 | p_chain->pbl_sp.p_phys_table = 0; |
525 | p_chain->pbl_sp.p_virt_table = NULL; | |
8063f761 | 526 | p_chain->pbl.pp_addr_tbl = NULL; |
fe56b9e6 YM |
527 | } |
528 | ||
529 | /** | |
a91eb52a YM |
530 | * @brief qed_chain_init_mem - |
531 | * | |
532 | * Initalizes a basic chain struct with its chain buffers | |
533 | * | |
fe56b9e6 YM |
534 | * @param p_chain |
535 | * @param p_virt_addr virtual address of allocated buffer's beginning | |
536 | * @param p_phys_addr physical address of allocated buffer's beginning | |
a91eb52a | 537 | * |
fe56b9e6 | 538 | */ |
a91eb52a YM |
539 | static inline void qed_chain_init_mem(struct qed_chain *p_chain, |
540 | void *p_virt_addr, dma_addr_t p_phys_addr) | |
fe56b9e6 | 541 | { |
a91eb52a YM |
542 | p_chain->p_virt_addr = p_virt_addr; |
543 | p_chain->p_phys_addr = p_phys_addr; | |
544 | } | |
fe56b9e6 | 545 | |
a91eb52a YM |
546 | /** |
547 | * @brief qed_chain_init_pbl_mem - | |
548 | * | |
549 | * Initalizes a basic chain struct with its pbl buffers | |
550 | * | |
551 | * @param p_chain | |
552 | * @param p_virt_pbl pointer to a pre allocated side table which will hold | |
553 | * virtual page addresses. | |
554 | * @param p_phys_pbl pointer to a pre-allocated side table which will hold | |
555 | * physical page addresses. | |
556 | * @param pp_virt_addr_tbl | |
557 | * pointer to a pre-allocated side table which will hold | |
558 | * the virtual addresses of the chain pages. | |
559 | * | |
560 | */ | |
561 | static inline void qed_chain_init_pbl_mem(struct qed_chain *p_chain, | |
562 | void *p_virt_pbl, | |
563 | dma_addr_t p_phys_pbl, | |
8063f761 | 564 | struct addr_tbl_entry *pp_addr_tbl) |
a91eb52a | 565 | { |
6d937acf MY |
566 | p_chain->pbl_sp.p_phys_table = p_phys_pbl; |
567 | p_chain->pbl_sp.p_virt_table = p_virt_pbl; | |
8063f761 | 568 | p_chain->pbl.pp_addr_tbl = pp_addr_tbl; |
fe56b9e6 YM |
569 | } |
570 | ||
571 | /** | |
a91eb52a YM |
572 | * @brief qed_chain_init_next_ptr_elem - |
573 | * | |
574 | * Initalizes a next pointer element | |
575 | * | |
576 | * @param p_chain | |
577 | * @param p_virt_curr virtual address of a chain page of which the next | |
578 | * pointer element is initialized | |
579 | * @param p_virt_next virtual address of the next chain page | |
580 | * @param p_phys_next physical address of the next chain page | |
fe56b9e6 | 581 | * |
fe56b9e6 | 582 | */ |
a91eb52a YM |
583 | static inline void |
584 | qed_chain_init_next_ptr_elem(struct qed_chain *p_chain, | |
585 | void *p_virt_curr, | |
586 | void *p_virt_next, dma_addr_t p_phys_next) | |
fe56b9e6 | 587 | { |
a91eb52a YM |
588 | struct qed_chain_next *p_next; |
589 | u32 size; | |
590 | ||
591 | size = p_chain->elem_size * p_chain->usable_per_page; | |
592 | p_next = (struct qed_chain_next *)((u8 *)p_virt_curr + size); | |
593 | ||
594 | DMA_REGPAIR_LE(p_next->next_phys, p_phys_next); | |
595 | ||
596 | p_next->next_virt = p_virt_next; | |
fe56b9e6 YM |
597 | } |
598 | ||
599 | /** | |
a91eb52a | 600 | * @brief qed_chain_get_last_elem - |
fe56b9e6 | 601 | * |
a91eb52a | 602 | * Returns a pointer to the last element of the chain |
fe56b9e6 YM |
603 | * |
604 | * @param p_chain | |
fe56b9e6 | 605 | * |
a91eb52a | 606 | * @return void* |
fe56b9e6 | 607 | */ |
a91eb52a | 608 | static inline void *qed_chain_get_last_elem(struct qed_chain *p_chain) |
fe56b9e6 | 609 | { |
a91eb52a YM |
610 | struct qed_chain_next *p_next = NULL; |
611 | void *p_virt_addr = NULL; | |
612 | u32 size, last_page_idx; | |
fe56b9e6 | 613 | |
a91eb52a YM |
614 | if (!p_chain->p_virt_addr) |
615 | goto out; | |
fe56b9e6 | 616 | |
a91eb52a YM |
617 | switch (p_chain->mode) { |
618 | case QED_CHAIN_MODE_NEXT_PTR: | |
619 | size = p_chain->elem_size * p_chain->usable_per_page; | |
620 | p_virt_addr = p_chain->p_virt_addr; | |
621 | p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + size); | |
622 | while (p_next->next_virt != p_chain->p_virt_addr) { | |
623 | p_virt_addr = p_next->next_virt; | |
624 | p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + | |
625 | size); | |
626 | } | |
627 | break; | |
628 | case QED_CHAIN_MODE_SINGLE: | |
629 | p_virt_addr = p_chain->p_virt_addr; | |
630 | break; | |
631 | case QED_CHAIN_MODE_PBL: | |
632 | last_page_idx = p_chain->page_cnt - 1; | |
8063f761 | 633 | p_virt_addr = p_chain->pbl.pp_addr_tbl[last_page_idx].virt_addr; |
a91eb52a YM |
634 | break; |
635 | } | |
636 | /* p_virt_addr points at this stage to the last page of the chain */ | |
637 | size = p_chain->elem_size * (p_chain->usable_per_page - 1); | |
638 | p_virt_addr = (u8 *)p_virt_addr + size; | |
639 | out: | |
640 | return p_virt_addr; | |
fe56b9e6 YM |
641 | } |
642 | ||
643 | /** | |
a91eb52a | 644 | * @brief qed_chain_set_prod - sets the prod to the given value |
fe56b9e6 | 645 | * |
a91eb52a YM |
646 | * @param prod_idx |
647 | * @param p_prod_elem | |
648 | */ | |
649 | static inline void qed_chain_set_prod(struct qed_chain *p_chain, | |
650 | u32 prod_idx, void *p_prod_elem) | |
651 | { | |
2d533a92 DB |
652 | if (p_chain->mode == QED_CHAIN_MODE_PBL) { |
653 | u32 cur_prod, page_mask, page_cnt, page_diff; | |
654 | ||
655 | cur_prod = is_chain_u16(p_chain) ? p_chain->u.chain16.prod_idx : | |
656 | p_chain->u.chain32.prod_idx; | |
657 | ||
658 | /* Assume that number of elements in a page is power of 2 */ | |
659 | page_mask = ~p_chain->elem_per_page_mask; | |
660 | ||
661 | /* Use "cur_prod - 1" and "prod_idx - 1" since producer index | |
662 | * reaches the first element of next page before the page index | |
663 | * is incremented. See qed_chain_produce(). | |
664 | * Index wrap around is not a problem because the difference | |
665 | * between current and given producer indices is always | |
666 | * positive and lower than the chain's capacity. | |
667 | */ | |
668 | page_diff = (((cur_prod - 1) & page_mask) - | |
669 | ((prod_idx - 1) & page_mask)) / | |
670 | p_chain->elem_per_page; | |
671 | ||
672 | page_cnt = qed_chain_get_page_cnt(p_chain); | |
673 | if (is_chain_u16(p_chain)) | |
674 | p_chain->pbl.c.u16.prod_page_idx = | |
675 | (p_chain->pbl.c.u16.prod_page_idx - | |
676 | page_diff + page_cnt) % page_cnt; | |
677 | else | |
678 | p_chain->pbl.c.u32.prod_page_idx = | |
679 | (p_chain->pbl.c.u32.prod_page_idx - | |
680 | page_diff + page_cnt) % page_cnt; | |
681 | } | |
682 | ||
a91eb52a YM |
683 | if (is_chain_u16(p_chain)) |
684 | p_chain->u.chain16.prod_idx = (u16) prod_idx; | |
685 | else | |
686 | p_chain->u.chain32.prod_idx = prod_idx; | |
687 | p_chain->p_prod_elem = p_prod_elem; | |
688 | } | |
689 | ||
690 | /** | |
691 | * @brief qed_chain_pbl_zero_mem - set chain memory to 0 | |
fe56b9e6 YM |
692 | * |
693 | * @param p_chain | |
fe56b9e6 | 694 | */ |
a91eb52a | 695 | static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain) |
fe56b9e6 | 696 | { |
a91eb52a YM |
697 | u32 i, page_cnt; |
698 | ||
699 | if (p_chain->mode != QED_CHAIN_MODE_PBL) | |
700 | return; | |
701 | ||
702 | page_cnt = qed_chain_get_page_cnt(p_chain); | |
703 | ||
704 | for (i = 0; i < page_cnt; i++) | |
8063f761 | 705 | memset(p_chain->pbl.pp_addr_tbl[i].virt_addr, 0, |
a91eb52a | 706 | QED_CHAIN_PAGE_SIZE); |
fe56b9e6 YM |
707 | } |
708 | ||
709 | #endif |