[linux-block.git] / include / linux / qed / qed_chain.h

/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
/* QLogic qed NIC Driver
 * Copyright (c) 2015-2017  QLogic Corporation
 * Copyright (c) 2019-2020 Marvell International Ltd.
 */

#ifndef _QED_CHAIN_H
#define _QED_CHAIN_H

#include <linux/types.h>
#include <asm/byteorder.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/qed/common_hsi.h>

enum qed_chain_mode {
	/* Each Page contains a next pointer at its end */
	QED_CHAIN_MODE_NEXT_PTR,

	/* Chain is a single page (next ptr) is not required */
	QED_CHAIN_MODE_SINGLE,

	/* Page pointers are located in a side list */
	QED_CHAIN_MODE_PBL,
};

enum qed_chain_use_mode {
	QED_CHAIN_USE_TO_PRODUCE,			/* Chain starts empty */
	QED_CHAIN_USE_TO_CONSUME,			/* Chain starts full */
	QED_CHAIN_USE_TO_CONSUME_PRODUCE,		/* Chain starts empty */
};

enum qed_chain_cnt_type {
	/* The chain's size/prod/cons are kept in 16-bit variables */
	QED_CHAIN_CNT_TYPE_U16,

	/* The chain's size/prod/cons are kept in 32-bit variables  */
	QED_CHAIN_CNT_TYPE_U32,
};

struct qed_chain_next {
	struct regpair					next_phys;
	void						*next_virt;
};

struct qed_chain_pbl_u16 {
	u16						prod_page_idx;
	u16						cons_page_idx;
};

struct qed_chain_pbl_u32 {
	u32						prod_page_idx;
	u32						cons_page_idx;
};

struct qed_chain_u16 {
	/* Cyclic index of next element to produce/consume */
	u16						prod_idx;
	u16						cons_idx;
};

struct qed_chain_u32 {
	/* Cyclic index of next element to produce/consume */
	u32						prod_idx;
	u32						cons_idx;
};

struct addr_tbl_entry {
	void						*virt_addr;
	dma_addr_t					dma_map;
};

struct qed_chain {
	/* Fastpath portion of the chain - required for commands such
	 * as produce / consume.
	 */

	/* Point to next element to produce/consume */
	void						*p_prod_elem;
	void						*p_cons_elem;

	/* Fastpath portions of the PBL [if exists] */

	struct {
		/* Table for keeping the virtual and physical addresses of the
		 * chain pages, respectively to the physical addresses
		 * in the pbl table.
		 */
		struct addr_tbl_entry			*pp_addr_tbl;

		union {
			struct qed_chain_pbl_u16	u16;
			struct qed_chain_pbl_u32	u32;
		}					c;
	}						pbl;

	union {
		struct qed_chain_u16			chain16;
		struct qed_chain_u32			chain32;
	}						u;

	/* Capacity counts only usable elements */
	u32						capacity;
	u32						page_cnt;

	enum qed_chain_mode				mode;

	/* Elements information for fast calculations */
	u16						elem_per_page;
	u16						elem_per_page_mask;
	u16						elem_size;
	u16						next_page_mask;
	u16						usable_per_page;
	u8						elem_unusable;

	enum qed_chain_cnt_type				cnt_type;

	/* Slowpath of the chain - required for initialization and destruction,
	 * but isn't involved in regular functionality.
	 */

	u32						page_size;

	/* Base address of a pre-allocated buffer for pbl */
	struct {
		__le64					*table_virt;
		dma_addr_t				table_phys;
		size_t					table_size;
	}						pbl_sp;

	/* Address of first page of the chain - the address is required
	 * for fastpath operation [consume/produce] but only for the SINGLE
	 * flavour which isn't considered fastpath [== SPQ].
	 */
	void						*p_virt_addr;
	dma_addr_t					p_phys_addr;

	/* Total number of elements [for entire chain] */
	u32						size;

	enum qed_chain_use_mode				intended_use;

	bool						b_external_pbl;
};

struct qed_chain_init_params {
	enum qed_chain_mode				mode;
	enum qed_chain_use_mode				intended_use;
	enum qed_chain_cnt_type				cnt_type;

	u32						page_size;
	u32						num_elems;
	size_t						elem_size;

	void						*ext_pbl_virt;
	dma_addr_t					ext_pbl_phys;
};

#define QED_CHAIN_PAGE_SIZE				SZ_4K

#define ELEMS_PER_PAGE(elem_size, page_size)				     \
	((page_size) / (elem_size))

#define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode)			     \
	(((mode) == QED_CHAIN_MODE_NEXT_PTR) ?				     \
	 (u8)(1 + ((sizeof(struct qed_chain_next) - 1) / (elem_size))) :     \
	 0)

#define USABLE_ELEMS_PER_PAGE(elem_size, page_size, mode)		     \
	((u32)(ELEMS_PER_PAGE((elem_size), (page_size)) -		     \
	       UNUSABLE_ELEMS_PER_PAGE((elem_size), (mode))))

#define QED_CHAIN_PAGE_CNT(elem_cnt, elem_size, page_size, mode)	     \
	DIV_ROUND_UP((elem_cnt),					     \
		     USABLE_ELEMS_PER_PAGE((elem_size), (page_size), (mode)))

#define is_chain_u16(p)							     \
	((p)->cnt_type == QED_CHAIN_CNT_TYPE_U16)
#define is_chain_u32(p)							     \
	((p)->cnt_type == QED_CHAIN_CNT_TYPE_U32)

/* Accessors */

static inline u16 qed_chain_get_prod_idx(const struct qed_chain *chain)
{
	return chain->u.chain16.prod_idx;
}

static inline u16 qed_chain_get_cons_idx(const struct qed_chain *chain)
{
	return chain->u.chain16.cons_idx;
}

static inline u32 qed_chain_get_prod_idx_u32(const struct qed_chain *chain)
{
	return chain->u.chain32.prod_idx;
}

static inline u32 qed_chain_get_cons_idx_u32(const struct qed_chain *chain)
{
	return chain->u.chain32.cons_idx;
}

static inline u16 qed_chain_get_elem_used(const struct qed_chain *chain)
{
	u32 prod = qed_chain_get_prod_idx(chain);
	u32 cons = qed_chain_get_cons_idx(chain);
	u16 elem_per_page = chain->elem_per_page;
	u16 used;

	if (prod < cons)
		prod += (u32)U16_MAX + 1;

	used = (u16)(prod - cons);
	if (chain->mode == QED_CHAIN_MODE_NEXT_PTR)
		used -= (u16)(prod / elem_per_page - cons / elem_per_page);

	return used;
}

static inline u16 qed_chain_get_elem_left(const struct qed_chain *chain)
{
	return (u16)(chain->capacity - qed_chain_get_elem_used(chain));
}

static inline u32 qed_chain_get_elem_used_u32(const struct qed_chain *chain)
{
	u64 prod = qed_chain_get_prod_idx_u32(chain);
	u64 cons = qed_chain_get_cons_idx_u32(chain);
	u16 elem_per_page = chain->elem_per_page;
	u32 used;

	if (prod < cons)
		prod += (u64)U32_MAX + 1;

	used = (u32)(prod - cons);
	if (chain->mode == QED_CHAIN_MODE_NEXT_PTR)
		used -= (u32)(prod / elem_per_page - cons / elem_per_page);

	return used;
}

static inline u32 qed_chain_get_elem_left_u32(const struct qed_chain *chain)
{
	return chain->capacity - qed_chain_get_elem_used_u32(chain);
}

static inline u16 qed_chain_get_usable_per_page(const struct qed_chain *chain)
{
	return chain->usable_per_page;
}

static inline u8 qed_chain_get_unusable_per_page(const struct qed_chain *chain)
{
	return chain->elem_unusable;
}

static inline u32 qed_chain_get_page_cnt(const struct qed_chain *chain)
{
	return chain->page_cnt;
}

static inline dma_addr_t qed_chain_get_pbl_phys(const struct qed_chain *chain)
{
	return chain->pbl_sp.table_phys;
}

/**
 * qed_chain_advance_page(): Advance the next element across pages for a
 *                           linked chain.
 *
 * @p_chain: P_chain.
 * @p_next_elem: P_next_elem.
 * @idx_to_inc: Idx_to_inc.
 * @page_to_inc: page_to_inc.
 *
 * Return: Void.
 */
static inline void
qed_chain_advance_page(struct qed_chain *p_chain,
		       void **p_next_elem, void *idx_to_inc, void *page_to_inc)
{
	struct qed_chain_next *p_next = NULL;
	u32 page_index = 0;

	switch (p_chain->mode) {
	case QED_CHAIN_MODE_NEXT_PTR:
		p_next = *p_next_elem;
		*p_next_elem = p_next->next_virt;
		if (is_chain_u16(p_chain))
			*(u16 *)idx_to_inc += p_chain->elem_unusable;
		else
			*(u32 *)idx_to_inc += p_chain->elem_unusable;
		break;
	case QED_CHAIN_MODE_SINGLE:
		*p_next_elem = p_chain->p_virt_addr;
		break;

	case QED_CHAIN_MODE_PBL:
		if (is_chain_u16(p_chain)) {
			if (++(*(u16 *)page_to_inc) == p_chain->page_cnt)
				*(u16 *)page_to_inc = 0;
			page_index = *(u16 *)page_to_inc;
		} else {
			if (++(*(u32 *)page_to_inc) == p_chain->page_cnt)
				*(u32 *)page_to_inc = 0;
			page_index = *(u32 *)page_to_inc;
		}
		*p_next_elem = p_chain->pbl.pp_addr_tbl[page_index].virt_addr;
	}
}

#define is_unusable_idx(p, idx)	\
	(((p)->u.chain16.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)

#define is_unusable_idx_u32(p, idx) \
	(((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
#define is_unusable_next_idx(p, idx)				 \
	((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \
	 (p)->usable_per_page)

#define is_unusable_next_idx_u32(p, idx)			 \
	((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) == \
	 (p)->usable_per_page)

#define test_and_skip(p, idx)						   \
	do {						\
		if (is_chain_u16(p)) {					   \
			if (is_unusable_idx(p, idx))			   \
				(p)->u.chain16.idx += (p)->elem_unusable;  \
		} else {						   \
			if (is_unusable_idx_u32(p, idx))		   \
				(p)->u.chain32.idx += (p)->elem_unusable;  \
		}					\
	} while (0)

/**
 * qed_chain_return_produced(): A chain in which the driver "Produces"
 *                              elements should use this API
 *                              to indicate previous produced elements
 *                              are now consumed.
 *
 * @p_chain: Chain.
 *
 * Return: Void.
 */
static inline void qed_chain_return_produced(struct qed_chain *p_chain)
{
	if (is_chain_u16(p_chain))
		p_chain->u.chain16.cons_idx++;
	else
		p_chain->u.chain32.cons_idx++;
	test_and_skip(p_chain, cons_idx);
}

/**
 * qed_chain_produce(): A chain in which the driver "Produces"
 *                      elements should use this to get a pointer to
 *                      the next element which can be "Produced". It's driver
 *                      responsibility to validate that the chain has room for
 *                      new element.
 *
 * @p_chain: Chain.
 *
 * Return: void*, a pointer to next element.
 */
static inline void *qed_chain_produce(struct qed_chain *p_chain)
{
	void *p_ret = NULL, *p_prod_idx, *p_prod_page_idx;

	if (is_chain_u16(p_chain)) {
		if ((p_chain->u.chain16.prod_idx &
		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
			p_prod_idx = &p_chain->u.chain16.prod_idx;
			p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx;
			qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
					       p_prod_idx, p_prod_page_idx);
		}
		p_chain->u.chain16.prod_idx++;
	} else {
		if ((p_chain->u.chain32.prod_idx &
		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
			p_prod_idx = &p_chain->u.chain32.prod_idx;
			p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx;
			qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
					       p_prod_idx, p_prod_page_idx);
		}
		p_chain->u.chain32.prod_idx++;
	}

	p_ret = p_chain->p_prod_elem;
	p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) +
					p_chain->elem_size);

	return p_ret;
}

/**
 * qed_chain_get_capacity(): Get the maximum number of BDs in chain
 *
 * @p_chain: Chain.
 *
 * Return: number of unusable BDs.
 */
static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain)
{
	return p_chain->capacity;
}

/**
 * qed_chain_recycle_consumed(): Returns an element which was
 *                               previously consumed;
 *                               Increments producers so they could
 *                               be written to FW.
 *
 * @p_chain: Chain.
 *
 * Return: Void.
 */
static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain)
{
	test_and_skip(p_chain, prod_idx);
	if (is_chain_u16(p_chain))
		p_chain->u.chain16.prod_idx++;
	else
		p_chain->u.chain32.prod_idx++;
}

/**
 * qed_chain_consume(): A Chain in which the driver utilizes data written
 *                      by a different source (i.e., FW) should use this to
 *                      access passed buffers.
 *
 * @p_chain: Chain.
 *
 * Return: void*, a pointer to the next buffer written.
 */
static inline void *qed_chain_consume(struct qed_chain *p_chain)
{
	void *p_ret = NULL, *p_cons_idx, *p_cons_page_idx;

	if (is_chain_u16(p_chain)) {
		if ((p_chain->u.chain16.cons_idx &
		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
			p_cons_idx = &p_chain->u.chain16.cons_idx;
			p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx;
			qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
					       p_cons_idx, p_cons_page_idx);
		}
		p_chain->u.chain16.cons_idx++;
	} else {
		if ((p_chain->u.chain32.cons_idx &
		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
			p_cons_idx = &p_chain->u.chain32.cons_idx;
			p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx;
			qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
					       p_cons_idx, p_cons_page_idx);
		}
		p_chain->u.chain32.cons_idx++;
	}

	p_ret = p_chain->p_cons_elem;
	p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) +
					p_chain->elem_size);

	return p_ret;
}

/**
 * qed_chain_reset(): Resets the chain to its start state.
 *
 * @p_chain: pointer to a previously allocated chain.
 *
 * Return Void.
 */
static inline void qed_chain_reset(struct qed_chain *p_chain)
{
	u32 i;

	if (is_chain_u16(p_chain)) {
		p_chain->u.chain16.prod_idx = 0;
		p_chain->u.chain16.cons_idx = 0;
	} else {
		p_chain->u.chain32.prod_idx = 0;
		p_chain->u.chain32.cons_idx = 0;
	}
	p_chain->p_cons_elem = p_chain->p_virt_addr;
	p_chain->p_prod_elem = p_chain->p_virt_addr;

	if (p_chain->mode == QED_CHAIN_MODE_PBL) {
		/* Use (page_cnt - 1) as a reset value for the prod/cons page's
		 * indices, to avoid unnecessary page advancing on the first
		 * call to qed_chain_produce/consume. Instead, the indices
		 * will be advanced to page_cnt and then will be wrapped to 0.
		 */
		u32 reset_val = p_chain->page_cnt - 1;

		if (is_chain_u16(p_chain)) {
			p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val;
			p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val;
		} else {
			p_chain->pbl.c.u32.prod_page_idx = reset_val;
			p_chain->pbl.c.u32.cons_page_idx = reset_val;
		}
	}

	switch (p_chain->intended_use) {
	case QED_CHAIN_USE_TO_CONSUME:
		/* produce empty elements */
		for (i = 0; i < p_chain->capacity; i++)
			qed_chain_recycle_consumed(p_chain);
		break;

	case QED_CHAIN_USE_TO_CONSUME_PRODUCE:
	case QED_CHAIN_USE_TO_PRODUCE:
	default:
		/* Do nothing */
		break;
	}
}

/**
 * qed_chain_get_last_elem(): Returns a pointer to the last element of the
 *                            chain.
 *
 * @p_chain: Chain.
 *
 * Return: void*.
 */
static inline void *qed_chain_get_last_elem(struct qed_chain *p_chain)
{
	struct qed_chain_next *p_next = NULL;
	void *p_virt_addr = NULL;
	u32 size, last_page_idx;

	if (!p_chain->p_virt_addr)
		goto out;

	switch (p_chain->mode) {
	case QED_CHAIN_MODE_NEXT_PTR:
		size = p_chain->elem_size * p_chain->usable_per_page;
		p_virt_addr = p_chain->p_virt_addr;
		p_next = (struct qed_chain_next *)((u8 *)p_virt_addr + size);
		while (p_next->next_virt != p_chain->p_virt_addr) {
			p_virt_addr = p_next->next_virt;
			p_next = (struct qed_chain_next *)((u8 *)p_virt_addr +
							   size);
		}
		break;
	case QED_CHAIN_MODE_SINGLE:
		p_virt_addr = p_chain->p_virt_addr;
		break;
	case QED_CHAIN_MODE_PBL:
		last_page_idx = p_chain->page_cnt - 1;
		p_virt_addr = p_chain->pbl.pp_addr_tbl[last_page_idx].virt_addr;
		break;
	}
	/* p_virt_addr points at this stage to the last page of the chain */
	size = p_chain->elem_size * (p_chain->usable_per_page - 1);
	p_virt_addr = (u8 *)p_virt_addr + size;
out:
	return p_virt_addr;
}

/**
 * qed_chain_set_prod(): sets the prod to the given value.
 *
 * @p_chain: Chain.
 * @prod_idx: Prod Idx.
 * @p_prod_elem: Prod elem.
 *
 * Return Void.
 */
static inline void qed_chain_set_prod(struct qed_chain *p_chain,
				      u32 prod_idx, void *p_prod_elem)
{
	if (p_chain->mode == QED_CHAIN_MODE_PBL) {
		u32 cur_prod, page_mask, page_cnt, page_diff;

		cur_prod = is_chain_u16(p_chain) ? p_chain->u.chain16.prod_idx :
			   p_chain->u.chain32.prod_idx;

		/* Assume that number of elements in a page is power of 2 */
		page_mask = ~p_chain->elem_per_page_mask;

		/* Use "cur_prod - 1" and "prod_idx - 1" since producer index
		 * reaches the first element of next page before the page index
		 * is incremented. See qed_chain_produce().
		 * Index wrap around is not a problem because the difference
		 * between current and given producer indices is always
		 * positive and lower than the chain's capacity.
		 */
		page_diff = (((cur_prod - 1) & page_mask) -
			     ((prod_idx - 1) & page_mask)) /
			    p_chain->elem_per_page;

		page_cnt = qed_chain_get_page_cnt(p_chain);
		if (is_chain_u16(p_chain))
			p_chain->pbl.c.u16.prod_page_idx =
				(p_chain->pbl.c.u16.prod_page_idx -
				 page_diff + page_cnt) % page_cnt;
		else
			p_chain->pbl.c.u32.prod_page_idx =
				(p_chain->pbl.c.u32.prod_page_idx -
				 page_diff + page_cnt) % page_cnt;
	}

	if (is_chain_u16(p_chain))
		p_chain->u.chain16.prod_idx = (u16) prod_idx;
	else
		p_chain->u.chain32.prod_idx = prod_idx;
	p_chain->p_prod_elem = p_prod_elem;
}

/**
 * qed_chain_pbl_zero_mem(): set chain memory to 0.
 *
 * @p_chain: Chain.
 *
 * Return: Void.
 */
static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain)
{
	u32 i, page_cnt;

	if (p_chain->mode != QED_CHAIN_MODE_PBL)
		return;

	page_cnt = qed_chain_get_page_cnt(p_chain);

	for (i = 0; i < page_cnt; i++)
		memset(p_chain->pbl.pp_addr_tbl[i].virt_addr, 0,
		       p_chain->page_size);
}

#endif
Commit	Line	Data
1f4d4ed6	1	/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
fe56b9e6	2	/* QLogic qed NIC Driver
e8f1cb50	3	* Copyright (c) 2015-2017 QLogic Corporation
663eacd8	4	* Copyright (c) 2019-2020 Marvell International Ltd.
fe56b9e6 YM	5	*/
	6
	7	#ifndef _QED_CHAIN_H
	8	#define _QED_CHAIN_H
	9
	10	#include <linux/types.h>
	11	#include <asm/byteorder.h>
	12	#include <linux/kernel.h>
	13	#include <linux/list.h>
15506586	14	#include <linux/sizes.h>
fe56b9e6 YM	15	#include <linux/slab.h>
	16	#include <linux/qed/common_hsi.h>
	17
fe56b9e6 YM	18	enum qed_chain_mode {
	19	/* Each Page contains a next pointer at its end */
	20	QED_CHAIN_MODE_NEXT_PTR,
	21
405a129f	22	/* Chain is a single page (next ptr) is not required */
fe56b9e6 YM	23	QED_CHAIN_MODE_SINGLE,
	24
	25	/* Page pointers are located in a side list */
	26	QED_CHAIN_MODE_PBL,
	27	};
	28
	29	enum qed_chain_use_mode {
2cf2f4f5 AL	30	QED_CHAIN_USE_TO_PRODUCE, /* Chain starts empty */
	31	QED_CHAIN_USE_TO_CONSUME, /* Chain starts full */
	32	QED_CHAIN_USE_TO_CONSUME_PRODUCE, /* Chain starts empty */
fe56b9e6 YM	33	};
fe56b9e6 YM	34
a91eb52a YM	35	enum qed_chain_cnt_type {
	36	/* The chain's size/prod/cons are kept in 16-bit variables */
	37	QED_CHAIN_CNT_TYPE_U16,
	38
	39	/* The chain's size/prod/cons are kept in 32-bit variables */
	40	QED_CHAIN_CNT_TYPE_U32,
	41	};
	42
fe56b9e6	43	struct qed_chain_next {
2cf2f4f5 AL	44	struct regpair next_phys;
2cf2f4f5 AL	45	void *next_virt;
fe56b9e6 YM	46	};
fe56b9e6 YM	47
a91eb52a	48	struct qed_chain_pbl_u16 {
2cf2f4f5 AL	49	u16 prod_page_idx;
2cf2f4f5 AL	50	u16 cons_page_idx;
a91eb52a YM	51	};
	52
	53	struct qed_chain_pbl_u32 {
2cf2f4f5 AL	54	u32 prod_page_idx;
2cf2f4f5 AL	55	u32 cons_page_idx;
a91eb52a YM	56	};
a91eb52a YM	57
a91eb52a	58	struct qed_chain_u16 {
405a129f	59	/* Cyclic index of next element to produce/consume */
2cf2f4f5 AL	60	u16 prod_idx;
2cf2f4f5 AL	61	u16 cons_idx;
a91eb52a YM	62	};
	63
	64	struct qed_chain_u32 {
405a129f	65	/* Cyclic index of next element to produce/consume */
2cf2f4f5 AL	66	u32 prod_idx;
2cf2f4f5 AL	67	u32 cons_idx;
fe56b9e6 YM	68	};
fe56b9e6 YM	69
8063f761	70	struct addr_tbl_entry {
2cf2f4f5 AL	71	void *virt_addr;
2cf2f4f5 AL	72	dma_addr_t dma_map;
8063f761 YB	73	};
8063f761 YB	74
fe56b9e6	75	struct qed_chain {
2cf2f4f5	76	/* Fastpath portion of the chain - required for commands such
6d937acf MY	77	* as produce / consume.
6d937acf MY	78	*/
2cf2f4f5	79
6d937acf	80	/* Point to next element to produce/consume */
2cf2f4f5 AL	81	void *p_prod_elem;
2cf2f4f5 AL	82	void *p_cons_elem;
6d937acf MY	83
6d937acf MY	84	/* Fastpath portions of the PBL [if exists] */
2cf2f4f5	85
6d937acf	86	struct {
8063f761 YB	87	/* Table for keeping the virtual and physical addresses of the
	88	* chain pages, respectively to the physical addresses
	89	* in the pbl table.
6d937acf	90	*/
2cf2f4f5	91	struct addr_tbl_entry *pp_addr_tbl;
a91eb52a	92
6d937acf	93	union {
2cf2f4f5 AL	94	struct qed_chain_pbl_u16 u16;
	95	struct qed_chain_pbl_u32 u32;
	96	} c;
	97	} pbl;
a91eb52a YM	98
a91eb52a YM	99	union {
2cf2f4f5 AL	100	struct qed_chain_u16 chain16;
	101	struct qed_chain_u32 chain32;
	102	} u;
a91eb52a	103
6d937acf	104	/* Capacity counts only usable elements */
2cf2f4f5 AL	105	u32 capacity;
2cf2f4f5 AL	106	u32 page_cnt;
a91eb52a	107
2cf2f4f5	108	enum qed_chain_mode mode;
6d937acf MY	109
6d937acf MY	110	/* Elements information for fast calculations */
2cf2f4f5 AL	111	u16 elem_per_page;
	112	u16 elem_per_page_mask;
	113	u16 elem_size;
	114	u16 next_page_mask;
	115	u16 usable_per_page;
	116	u8 elem_unusable;
6d937acf	117
b6db3f71	118	enum qed_chain_cnt_type cnt_type;
6d937acf MY	119
	120	/* Slowpath of the chain - required for initialization and destruction,
	121	* but isn't involved in regular functionality.
a91eb52a	122	*/
6d937acf	123
15506586 AL	124	u32 page_size;
15506586 AL	125
6d937acf MY	126	/* Base address of a pre-allocated buffer for pbl */
6d937acf MY	127	struct {
9b6ee3cf AL	128	__le64 *table_virt;
	129	dma_addr_t table_phys;
	130	size_t table_size;
2cf2f4f5	131	} pbl_sp;
6d937acf MY	132
6d937acf MY	133	/* Address of first page of the chain - the address is required
0d80b761	134	* for fastpath operation [consume/produce] but only for the SINGLE
6d937acf MY	135	* flavour which isn't considered fastpath [== SPQ].
6d937acf MY	136	*/
2cf2f4f5 AL	137	void *p_virt_addr;
2cf2f4f5 AL	138	dma_addr_t p_phys_addr;
6d937acf MY	139
6d937acf MY	140	/* Total number of elements [for entire chain] */
2cf2f4f5	141	u32 size;
a91eb52a	142
b6db3f71	143	enum qed_chain_use_mode intended_use;
1a4a6975	144
2cf2f4f5	145	bool b_external_pbl;
fe56b9e6 YM	146	};
fe56b9e6 YM	147
b6db3f71 AL	148	struct qed_chain_init_params {
	149	enum qed_chain_mode mode;
	150	enum qed_chain_use_mode intended_use;
	151	enum qed_chain_cnt_type cnt_type;
	152
15506586	153	u32 page_size;
b6db3f71 AL	154	u32 num_elems;
	155	size_t elem_size;
	156
	157	void *ext_pbl_virt;
	158	dma_addr_t ext_pbl_phys;
	159	};
	160
15506586	161	#define QED_CHAIN_PAGE_SIZE SZ_4K
2cf2f4f5	162
15506586 AL	163	#define ELEMS_PER_PAGE(elem_size, page_size) \
15506586 AL	164	((page_size) / (elem_size))
fe56b9e6	165
2cf2f4f5 AL	166	#define UNUSABLE_ELEMS_PER_PAGE(elem_size, mode) \
	167	(((mode) == QED_CHAIN_MODE_NEXT_PTR) ? \
	168	(u8)(1 + ((sizeof(struct qed_chain_next) - 1) / (elem_size))) : \
	169	0)
fe56b9e6	170
15506586 AL	171	#define USABLE_ELEMS_PER_PAGE(elem_size, page_size, mode) \
15506586 AL	172	((u32)(ELEMS_PER_PAGE((elem_size), (page_size)) - \
2cf2f4f5	173	UNUSABLE_ELEMS_PER_PAGE((elem_size), (mode))))
fe56b9e6	174
15506586 AL	175	#define QED_CHAIN_PAGE_CNT(elem_cnt, elem_size, page_size, mode) \
	176	DIV_ROUND_UP((elem_cnt), \
	177	USABLE_ELEMS_PER_PAGE((elem_size), (page_size), (mode)))
fe56b9e6	178
2cf2f4f5 AL	179	#define is_chain_u16(p) \
	180	((p)->cnt_type == QED_CHAIN_CNT_TYPE_U16)
	181	#define is_chain_u32(p) \
	182	((p)->cnt_type == QED_CHAIN_CNT_TYPE_U32)
a91eb52a	183
fe56b9e6	184	/* Accessors */
f2aefd20 AL	185
	186	static inline u16 qed_chain_get_prod_idx(const struct qed_chain *chain)
	187	{
	188	return chain->u.chain16.prod_idx;
	189	}
	190
	191	static inline u16 qed_chain_get_cons_idx(const struct qed_chain *chain)
fe56b9e6	192	{
f2aefd20	193	return chain->u.chain16.cons_idx;
fe56b9e6 YM	194	}
fe56b9e6 YM	195
f2aefd20	196	static inline u32 qed_chain_get_prod_idx_u32(const struct qed_chain *chain)
fe56b9e6	197	{
f2aefd20	198	return chain->u.chain32.prod_idx;
a91eb52a YM	199	}
a91eb52a YM	200
f2aefd20	201	static inline u32 qed_chain_get_cons_idx_u32(const struct qed_chain *chain)
a91eb52a	202	{
f2aefd20	203	return chain->u.chain32.cons_idx;
fe56b9e6 YM	204	}
fe56b9e6 YM	205
be0cec6f	206	static inline u16 qed_chain_get_elem_used(const struct qed_chain *chain)
fe56b9e6	207	{
f2aefd20 AL	208	u32 prod = qed_chain_get_prod_idx(chain);
	209	u32 cons = qed_chain_get_cons_idx(chain);
	210	u16 elem_per_page = chain->elem_per_page;
fe56b9e6 YM	211	u16 used;
fe56b9e6 YM	212
97dd1abd AL	213	if (prod < cons)
	214	prod += (u32)U16_MAX + 1;
	215
	216	used = (u16)(prod - cons);
f2aefd20 AL	217	if (chain->mode == QED_CHAIN_MODE_NEXT_PTR)
f2aefd20 AL	218	used -= (u16)(prod / elem_per_page - cons / elem_per_page);
fe56b9e6	219
be0cec6f	220	return used;
fe56b9e6 YM	221	}
fe56b9e6 YM	222
be0cec6f AL	223	static inline u16 qed_chain_get_elem_left(const struct qed_chain *chain)
	224	{
	225	return (u16)(chain->capacity - qed_chain_get_elem_used(chain));
	226	}
	227
	228	static inline u32 qed_chain_get_elem_used_u32(const struct qed_chain *chain)
fe56b9e6	229	{
f2aefd20 AL	230	u64 prod = qed_chain_get_prod_idx_u32(chain);
	231	u64 cons = qed_chain_get_cons_idx_u32(chain);
	232	u16 elem_per_page = chain->elem_per_page;
a91eb52a	233	u32 used;
fe56b9e6	234
97dd1abd AL	235	if (prod < cons)
	236	prod += (u64)U32_MAX + 1;
	237
	238	used = (u32)(prod - cons);
f2aefd20	239	if (chain->mode == QED_CHAIN_MODE_NEXT_PTR)
97dd1abd	240	used -= (u32)(prod / elem_per_page - cons / elem_per_page);
fe56b9e6	241
be0cec6f AL	242	return used;
	243	}
	244
	245	static inline u32 qed_chain_get_elem_left_u32(const struct qed_chain *chain)
	246	{
	247	return chain->capacity - qed_chain_get_elem_used_u32(chain);
fe56b9e6 YM	248	}
fe56b9e6 YM	249
f2aefd20	250	static inline u16 qed_chain_get_usable_per_page(const struct qed_chain *chain)
fe56b9e6	251	{
f2aefd20	252	return chain->usable_per_page;
fe56b9e6 YM	253	}
fe56b9e6 YM	254
f2aefd20	255	static inline u8 qed_chain_get_unusable_per_page(const struct qed_chain *chain)
fe56b9e6	256	{
f2aefd20	257	return chain->elem_unusable;
fe56b9e6 YM	258	}
fe56b9e6 YM	259
f2aefd20	260	static inline u32 qed_chain_get_page_cnt(const struct qed_chain *chain)
fe56b9e6	261	{
f2aefd20	262	return chain->page_cnt;
fe56b9e6 YM	263	}
fe56b9e6 YM	264
f2aefd20	265	static inline dma_addr_t qed_chain_get_pbl_phys(const struct qed_chain *chain)
fe56b9e6	266	{
f2aefd20	267	return chain->pbl_sp.table_phys;
fe56b9e6 YM	268	}
	269
	270	/**
19198e4e PK	271	* qed_chain_advance_page(): Advance the next element across pages for a
19198e4e PK	272	* linked chain.
fe56b9e6	273	*
19198e4e PK	274	* @p_chain: P_chain.
	275	* @p_next_elem: P_next_elem.
	276	* @idx_to_inc: Idx_to_inc.
	277	* @page_to_inc: page_to_inc.
fe56b9e6	278	*
19198e4e	279	* Return: Void.
fe56b9e6 YM	280	*/
	281	static inline void
	282	qed_chain_advance_page(struct qed_chain *p_chain,
a91eb52a	283	void *p_next_elem, void idx_to_inc, void *page_to_inc)
fe56b9e6	284	{
a91eb52a YM	285	struct qed_chain_next *p_next = NULL;
a91eb52a YM	286	u32 page_index = 0;
6d937acf	287
fe56b9e6 YM	288	switch (p_chain->mode) {
fe56b9e6 YM	289	case QED_CHAIN_MODE_NEXT_PTR:
a91eb52a	290	p_next = *p_next_elem;
fe56b9e6	291	*p_next_elem = p_next->next_virt;
a91eb52a YM	292	if (is_chain_u16(p_chain))
	293	(u16 )idx_to_inc += p_chain->elem_unusable;
	294	else
	295	(u32 )idx_to_inc += p_chain->elem_unusable;
fe56b9e6	296	break;
fe56b9e6 YM	297	case QED_CHAIN_MODE_SINGLE:
	298	*p_next_elem = p_chain->p_virt_addr;
	299	break;
	300
	301	case QED_CHAIN_MODE_PBL:
a91eb52a YM	302	if (is_chain_u16(p_chain)) {
	303	if (++((u16 )page_to_inc) == p_chain->page_cnt)
	304	(u16 )page_to_inc = 0;
	305	page_index = (u16 )page_to_inc;
	306	} else {
	307	if (++((u32 )page_to_inc) == p_chain->page_cnt)
	308	(u32 )page_to_inc = 0;
	309	page_index = (u32 )page_to_inc;
fe56b9e6	310	}
8063f761	311	*p_next_elem = p_chain->pbl.pp_addr_tbl[page_index].virt_addr;
fe56b9e6 YM	312	}
	313	}
	314
	315	#define is_unusable_idx(p, idx) \
a91eb52a YM	316	(((p)->u.chain16.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
	317
	318	#define is_unusable_idx_u32(p, idx) \
	319	(((p)->u.chain32.idx & (p)->elem_per_page_mask) == (p)->usable_per_page)
	320	#define is_unusable_next_idx(p, idx) \
	321	((((p)->u.chain16.idx + 1) & (p)->elem_per_page_mask) == \
	322	(p)->usable_per_page)
fe56b9e6	323
a91eb52a YM	324	#define is_unusable_next_idx_u32(p, idx) \
	325	((((p)->u.chain32.idx + 1) & (p)->elem_per_page_mask) == \
	326	(p)->usable_per_page)
fe56b9e6	327
a91eb52a	328	#define test_and_skip(p, idx) \
fe56b9e6	329	do { \
a91eb52a YM	330	if (is_chain_u16(p)) { \
	331	if (is_unusable_idx(p, idx)) \
	332	(p)->u.chain16.idx += (p)->elem_unusable; \
	333	} else { \
	334	if (is_unusable_idx_u32(p, idx)) \
	335	(p)->u.chain32.idx += (p)->elem_unusable; \
fe56b9e6 YM	336	} \
	337	} while (0)
	338
fe56b9e6	339	/**
19198e4e PK	340	* qed_chain_return_produced(): A chain in which the driver "Produces"
	341	* elements should use this API
	342	* to indicate previous produced elements
	343	* are now consumed.
fe56b9e6	344	*
19198e4e	345	* @p_chain: Chain.
fe56b9e6	346	*
19198e4e	347	* Return: Void.
fe56b9e6 YM	348	*/
	349	static inline void qed_chain_return_produced(struct qed_chain *p_chain)
	350	{
a91eb52a YM	351	if (is_chain_u16(p_chain))
	352	p_chain->u.chain16.cons_idx++;
	353	else
	354	p_chain->u.chain32.cons_idx++;
	355	test_and_skip(p_chain, cons_idx);
fe56b9e6 YM	356	}
	357
	358	/**
19198e4e PK	359	* qed_chain_produce(): A chain in which the driver "Produces"
	360	* elements should use this to get a pointer to
	361	* the next element which can be "Produced". It's driver
	362	* responsibility to validate that the chain has room for
	363	* new element.
fe56b9e6	364	*
19198e4e	365	* @p_chain: Chain.
fe56b9e6	366	*
19198e4e	367	* Return: void*, a pointer to next element.
fe56b9e6 YM	368	*/
	369	static inline void qed_chain_produce(struct qed_chain p_chain)
	370	{
a91eb52a YM	371	void p_ret = NULL, p_prod_idx, *p_prod_page_idx;
	372
	373	if (is_chain_u16(p_chain)) {
	374	if ((p_chain->u.chain16.prod_idx &
	375	p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
	376	p_prod_idx = &p_chain->u.chain16.prod_idx;
6d937acf	377	p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx;
a91eb52a YM	378	qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
	379	p_prod_idx, p_prod_page_idx);
	380	}
	381	p_chain->u.chain16.prod_idx++;
	382	} else {
	383	if ((p_chain->u.chain32.prod_idx &
	384	p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
	385	p_prod_idx = &p_chain->u.chain32.prod_idx;
6d937acf	386	p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx;
a91eb52a YM	387	qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,
	388	p_prod_idx, p_prod_page_idx);
	389	}
	390	p_chain->u.chain32.prod_idx++;
fe56b9e6 YM	391	}
fe56b9e6 YM	392
a91eb52a	393	p_ret = p_chain->p_prod_elem;
fe56b9e6 YM	394	p_chain->p_prod_elem = (void )(((u8 )p_chain->p_prod_elem) +
	395	p_chain->elem_size);
	396
a91eb52a	397	return p_ret;
fe56b9e6 YM	398	}
	399
	400	/**
19198e4e	401	* qed_chain_get_capacity(): Get the maximum number of BDs in chain
fe56b9e6	402	*
19198e4e	403	* @p_chain: Chain.
fe56b9e6	404	*
19198e4e	405	* Return: number of unusable BDs.
fe56b9e6	406	*/
a91eb52a	407	static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain)
fe56b9e6 YM	408	{
	409	return p_chain->capacity;
	410	}
	411
	412	/**
19198e4e PK	413	* qed_chain_recycle_consumed(): Returns an element which was
	414	* previously consumed;
	415	* Increments producers so they could
	416	* be written to FW.
fe56b9e6	417	*
19198e4e	418	* @p_chain: Chain.
fe56b9e6	419	*
19198e4e	420	* Return: Void.
fe56b9e6	421	*/
a91eb52a	422	static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain)
fe56b9e6	423	{
a91eb52a YM	424	test_and_skip(p_chain, prod_idx);
	425	if (is_chain_u16(p_chain))
	426	p_chain->u.chain16.prod_idx++;
	427	else
	428	p_chain->u.chain32.prod_idx++;
fe56b9e6 YM	429	}
	430
	431	/**
19198e4e PK	432	* qed_chain_consume(): A Chain in which the driver utilizes data written
	433	* by a different source (i.e., FW) should use this to
	434	* access passed buffers.
fe56b9e6	435	*
19198e4e	436	* @p_chain: Chain.
fe56b9e6	437	*
19198e4e	438	* Return: void*, a pointer to the next buffer written.
fe56b9e6 YM	439	*/
	440	static inline void qed_chain_consume(struct qed_chain p_chain)
	441	{
a91eb52a YM	442	void p_ret = NULL, p_cons_idx, *p_cons_page_idx;
	443
	444	if (is_chain_u16(p_chain)) {
	445	if ((p_chain->u.chain16.cons_idx &
	446	p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
	447	p_cons_idx = &p_chain->u.chain16.cons_idx;
6d937acf	448	p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx;
a91eb52a YM	449	qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
	450	p_cons_idx, p_cons_page_idx);
	451	}
	452	p_chain->u.chain16.cons_idx++;
	453	} else {
	454	if ((p_chain->u.chain32.cons_idx &
	455	p_chain->elem_per_page_mask) == p_chain->next_page_mask) {
	456	p_cons_idx = &p_chain->u.chain32.cons_idx;
6d937acf MY	457	p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx;
6d937acf MY	458	qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,
a91eb52a YM	459	p_cons_idx, p_cons_page_idx);
	460	}
	461	p_chain->u.chain32.cons_idx++;
fe56b9e6 YM	462	}
fe56b9e6 YM	463
a91eb52a	464	p_ret = p_chain->p_cons_elem;
fe56b9e6 YM	465	p_chain->p_cons_elem = (void )(((u8 )p_chain->p_cons_elem) +
	466	p_chain->elem_size);
	467
a91eb52a	468	return p_ret;
fe56b9e6 YM	469	}
	470
	471	/**
19198e4e PK	472	* qed_chain_reset(): Resets the chain to its start state.
	473	*
	474	* @p_chain: pointer to a previously allocated chain.
fe56b9e6	475	*
19198e4e	476	* Return Void.
fe56b9e6 YM	477	*/
	478	static inline void qed_chain_reset(struct qed_chain *p_chain)
	479	{
a91eb52a YM	480	u32 i;
	481
	482	if (is_chain_u16(p_chain)) {
	483	p_chain->u.chain16.prod_idx = 0;
	484	p_chain->u.chain16.cons_idx = 0;
	485	} else {
	486	p_chain->u.chain32.prod_idx = 0;
	487	p_chain->u.chain32.cons_idx = 0;
	488	}
	489	p_chain->p_cons_elem = p_chain->p_virt_addr;
	490	p_chain->p_prod_elem = p_chain->p_virt_addr;
fe56b9e6 YM	491
fe56b9e6 YM	492	if (p_chain->mode == QED_CHAIN_MODE_PBL) {
a91eb52a YM	493	/* Use (page_cnt - 1) as a reset value for the prod/cons page's
	494	* indices, to avoid unnecessary page advancing on the first
	495	* call to qed_chain_produce/consume. Instead, the indices
	496	* will be advanced to page_cnt and then will be wrapped to 0.
	497	*/
	498	u32 reset_val = p_chain->page_cnt - 1;
	499
	500	if (is_chain_u16(p_chain)) {
6d937acf MY	501	p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val;
6d937acf MY	502	p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val;
a91eb52a	503	} else {
6d937acf MY	504	p_chain->pbl.c.u32.prod_page_idx = reset_val;
6d937acf MY	505	p_chain->pbl.c.u32.cons_page_idx = reset_val;
a91eb52a	506	}
fe56b9e6 YM	507	}
	508
	509	switch (p_chain->intended_use) {
fe56b9e6 YM	510	case QED_CHAIN_USE_TO_CONSUME:
	511	/* produce empty elements */
	512	for (i = 0; i < p_chain->capacity; i++)
	513	qed_chain_recycle_consumed(p_chain);
	514	break;
6d937acf MY	515
	516	case QED_CHAIN_USE_TO_CONSUME_PRODUCE:
	517	case QED_CHAIN_USE_TO_PRODUCE:
	518	default:
	519	/* Do nothing */
	520	break;
fe56b9e6 YM	521	}
	522	}
	523
fe56b9e6	524	/**
19198e4e PK	525	* qed_chain_get_last_elem(): Returns a pointer to the last element of the
19198e4e PK	526	* chain.
fe56b9e6	527	*
19198e4e	528	* @p_chain: Chain.
fe56b9e6	529	*
19198e4e	530	* Return: void*.
fe56b9e6	531	*/
a91eb52a	532	static inline void qed_chain_get_last_elem(struct qed_chain p_chain)
fe56b9e6	533	{
a91eb52a YM	534	struct qed_chain_next *p_next = NULL;
	535	void *p_virt_addr = NULL;
	536	u32 size, last_page_idx;
fe56b9e6	537
a91eb52a YM	538	if (!p_chain->p_virt_addr)
a91eb52a YM	539	goto out;
fe56b9e6	540
a91eb52a YM	541	switch (p_chain->mode) {
	542	case QED_CHAIN_MODE_NEXT_PTR:
	543	size = p_chain->elem_size * p_chain->usable_per_page;
	544	p_virt_addr = p_chain->p_virt_addr;
	545	p_next = (struct qed_chain_next )((u8 )p_virt_addr + size);
	546	while (p_next->next_virt != p_chain->p_virt_addr) {
	547	p_virt_addr = p_next->next_virt;
	548	p_next = (struct qed_chain_next )((u8 )p_virt_addr +
	549	size);
	550	}
	551	break;
	552	case QED_CHAIN_MODE_SINGLE:
	553	p_virt_addr = p_chain->p_virt_addr;
	554	break;
	555	case QED_CHAIN_MODE_PBL:
	556	last_page_idx = p_chain->page_cnt - 1;
8063f761	557	p_virt_addr = p_chain->pbl.pp_addr_tbl[last_page_idx].virt_addr;
a91eb52a YM	558	break;
	559	}
	560	/* p_virt_addr points at this stage to the last page of the chain */
	561	size = p_chain->elem_size * (p_chain->usable_per_page - 1);
	562	p_virt_addr = (u8 *)p_virt_addr + size;
	563	out:
	564	return p_virt_addr;
fe56b9e6 YM	565	}
	566
	567	/**
19198e4e PK	568	* qed_chain_set_prod(): sets the prod to the given value.
	569	*
	570	* @p_chain: Chain.
	571	* @prod_idx: Prod Idx.
	572	* @p_prod_elem: Prod elem.
fe56b9e6	573	*
19198e4e	574	* Return Void.
a91eb52a YM	575	*/
	576	static inline void qed_chain_set_prod(struct qed_chain *p_chain,
	577	u32 prod_idx, void *p_prod_elem)
	578	{
2d533a92 DB	579	if (p_chain->mode == QED_CHAIN_MODE_PBL) {
	580	u32 cur_prod, page_mask, page_cnt, page_diff;
	581
	582	cur_prod = is_chain_u16(p_chain) ? p_chain->u.chain16.prod_idx :
	583	p_chain->u.chain32.prod_idx;
	584
	585	/* Assume that number of elements in a page is power of 2 */
	586	page_mask = ~p_chain->elem_per_page_mask;
	587
	588	/* Use "cur_prod - 1" and "prod_idx - 1" since producer index
	589	* reaches the first element of next page before the page index
	590	* is incremented. See qed_chain_produce().
	591	* Index wrap around is not a problem because the difference
	592	* between current and given producer indices is always
	593	* positive and lower than the chain's capacity.
	594	*/
	595	page_diff = (((cur_prod - 1) & page_mask) -
	596	((prod_idx - 1) & page_mask)) /
	597	p_chain->elem_per_page;
	598
	599	page_cnt = qed_chain_get_page_cnt(p_chain);
	600	if (is_chain_u16(p_chain))
	601	p_chain->pbl.c.u16.prod_page_idx =
	602	(p_chain->pbl.c.u16.prod_page_idx -
	603	page_diff + page_cnt) % page_cnt;
	604	else
	605	p_chain->pbl.c.u32.prod_page_idx =
	606	(p_chain->pbl.c.u32.prod_page_idx -
	607	page_diff + page_cnt) % page_cnt;
	608	}
	609
a91eb52a YM	610	if (is_chain_u16(p_chain))
	611	p_chain->u.chain16.prod_idx = (u16) prod_idx;
	612	else
	613	p_chain->u.chain32.prod_idx = prod_idx;
	614	p_chain->p_prod_elem = p_prod_elem;
	615	}
	616
	617	/**
19198e4e PK	618	* qed_chain_pbl_zero_mem(): set chain memory to 0.
	619	*
	620	* @p_chain: Chain.
fe56b9e6	621	*
19198e4e	622	* Return: Void.
fe56b9e6	623	*/
a91eb52a	624	static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain)
fe56b9e6	625	{
a91eb52a YM	626	u32 i, page_cnt;
	627
	628	if (p_chain->mode != QED_CHAIN_MODE_PBL)
	629	return;
	630
	631	page_cnt = qed_chain_get_page_cnt(p_chain);
	632
	633	for (i = 0; i < page_cnt; i++)
8063f761	634	memset(p_chain->pbl.pp_addr_tbl[i].virt_addr, 0,
15506586	635	p_chain->page_size);
fe56b9e6 YM	636	}
	637
	638	#endif