[linux-2.6-block.git] / drivers / net / ethernet / google / gve / gve.h

/* SPDX-License-Identifier: (GPL-2.0 OR MIT)
 * Google virtual Ethernet (gve) driver
 *
 * Copyright (C) 2015-2019 Google, Inc.
 */

#ifndef _GVE_H_
#define _GVE_H_

#include <linux/dma-mapping.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
#include <linux/u64_stats_sync.h>
#include "gve_desc.h"

#ifndef PCI_VENDOR_ID_GOOGLE
#define PCI_VENDOR_ID_GOOGLE	0x1ae0
#endif

#define PCI_DEV_ID_GVNIC	0x0042

#define GVE_REGISTER_BAR	0
#define GVE_DOORBELL_BAR	2

/* Driver can alloc up to 2 segments for the header and 2 for the payload. */
#define GVE_TX_MAX_IOVEC	4
/* 1 for management, 1 for rx, 1 for tx */
#define GVE_MIN_MSIX 3

/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
struct gve_rx_desc_queue {
	struct gve_rx_desc *desc_ring; /* the descriptor ring */
	dma_addr_t bus; /* the bus for the desc_ring */
	u8 seqno; /* the next expected seqno for this desc*/
};

/* The page info for a single slot in the RX data queue */
struct gve_rx_slot_page_info {
	struct page *page;
	void *page_address;
	u32 page_offset; /* offset to write to in page */
};

/* A list of pages registered with the device during setup and used by a queue
 * as buffers
 */
struct gve_queue_page_list {
	u32 id; /* unique id */
	u32 num_entries;
	struct page **pages; /* list of num_entries pages */
	dma_addr_t *page_buses; /* the dma addrs of the pages */
};

/* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */
struct gve_rx_data_queue {
	struct gve_rx_data_slot *data_ring; /* read by NIC */
	dma_addr_t data_bus; /* dma mapping of the slots */
	struct gve_rx_slot_page_info *page_info; /* page info of the buffers */
	struct gve_queue_page_list *qpl; /* qpl assigned to this queue */
};

struct gve_priv;

/* An RX ring that contains a power-of-two sized desc and data ring. */
struct gve_rx_ring {
	struct gve_priv *gve;
	struct gve_rx_desc_queue desc;
	struct gve_rx_data_queue data;
	u64 rbytes; /* free-running bytes received */
	u64 rpackets; /* free-running packets received */
	u32 cnt; /* free-running total number of completed packets */
	u32 fill_cnt; /* free-running total number of descs and buffs posted */
	u32 mask; /* masks the cnt and fill_cnt to the size of the ring */
	u32 q_num; /* queue index */
	u32 ntfy_id; /* notification block index */
	struct gve_queue_resources *q_resources; /* head and tail pointer idx */
	dma_addr_t q_resources_bus; /* dma address for the queue resources */
	struct u64_stats_sync statss; /* sync stats for 32bit archs */
};

/* A TX desc ring entry */
union gve_tx_desc {
	struct gve_tx_pkt_desc pkt; /* first desc for a packet */
	struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
};

/* Tracks the memory in the fifo occupied by a segment of a packet */
struct gve_tx_iovec {
	u32 iov_offset; /* offset into this segment */
	u32 iov_len; /* length */
	u32 iov_padding; /* padding associated with this segment */
};

/* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc
 * ring entry but only used for a pkt_desc not a seg_desc
 */
struct gve_tx_buffer_state {
	struct sk_buff *skb; /* skb for this pkt */
	struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
};

/* A TX buffer - each queue has one */
struct gve_tx_fifo {
	void *base; /* address of base of FIFO */
	u32 size; /* total size */
	atomic_t available; /* how much space is still available */
	u32 head; /* offset to write at */
	struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */
};

/* A TX ring that contains a power-of-two sized desc ring and a FIFO buffer */
struct gve_tx_ring {
	/* Cacheline 0 -- Accessed & dirtied during transmit */
	struct gve_tx_fifo tx_fifo;
	u32 req; /* driver tracked head pointer */
	u32 done; /* driver tracked tail pointer */

	/* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
	__be32 last_nic_done ____cacheline_aligned; /* NIC tail pointer */
	u64 pkt_done; /* free-running - total packets completed */
	u64 bytes_done; /* free-running - total bytes completed */

	/* Cacheline 2 -- Read-mostly fields */
	union gve_tx_desc *desc ____cacheline_aligned;
	struct gve_tx_buffer_state *info; /* Maps 1:1 to a desc */
	struct netdev_queue *netdev_txq;
	struct gve_queue_resources *q_resources; /* head and tail pointer idx */
	u32 mask; /* masks req and done down to queue size */

	/* Slow-path fields */
	u32 q_num ____cacheline_aligned; /* queue idx */
	u32 stop_queue; /* count of queue stops */
	u32 wake_queue; /* count of queue wakes */
	u32 ntfy_id; /* notification block index */
	dma_addr_t bus; /* dma address of the descr ring */
	dma_addr_t q_resources_bus; /* dma address of the queue resources */
	struct u64_stats_sync statss; /* sync stats for 32bit archs */
} ____cacheline_aligned;

/* Wraps the info for one irq including the napi struct and the queues
 * associated with that irq.
 */
struct gve_notify_block {
	__be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
	char name[IFNAMSIZ + 16]; /* name registered with the kernel */
	struct napi_struct napi; /* kernel napi struct for this block */
	struct gve_priv *priv;
	struct gve_tx_ring *tx; /* tx rings on this block */
	struct gve_rx_ring *rx; /* rx rings on this block */
} ____cacheline_aligned;

/* Tracks allowed and current queue settings */
struct gve_queue_config {
	u16 max_queues;
	u16 num_queues; /* current */
};

/* Tracks the available and used qpl IDs */
struct gve_qpl_config {
	u32 qpl_map_size; /* map memory size */
	unsigned long *qpl_id_map; /* bitmap of used qpl ids */
};

struct gve_priv {
	struct net_device *dev;
	struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */
	struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */
	struct gve_queue_page_list *qpls; /* array of num qpls */
	struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */
	dma_addr_t ntfy_block_bus;
	struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */
	char mgmt_msix_name[IFNAMSIZ + 16];
	u32 mgmt_msix_idx;
	__be32 *counter_array; /* array of num_event_counters */
	dma_addr_t counter_array_bus;

	u16 num_event_counters;
	u16 tx_desc_cnt; /* num desc per ring */
	u16 rx_desc_cnt; /* num desc per ring */
	u16 tx_pages_per_qpl; /* tx buffer length */
	u16 rx_pages_per_qpl; /* rx buffer length */
	u64 max_registered_pages;
	u64 num_registered_pages; /* num pages registered with NIC */
	u32 rx_copybreak; /* copy packets smaller than this */
	u16 default_num_queues; /* default num queues to set up */

	struct gve_queue_config tx_cfg;
	struct gve_queue_config rx_cfg;
	struct gve_qpl_config qpl_cfg; /* map used QPL ids */
	u32 num_ntfy_blks; /* spilt between TX and RX so must be even */

	struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
	__be32 __iomem *db_bar2; /* "array" of doorbells */
	u32 msg_enable;	/* level for netif* netdev print macros	*/
	struct pci_dev *pdev;

	/* metrics */
	u32 tx_timeo_cnt;

	/* Admin queue - see gve_adminq.h*/
	union gve_adminq_command *adminq;
	dma_addr_t adminq_bus_addr;
	u32 adminq_mask; /* masks prod_cnt to adminq size */
	u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */

	struct workqueue_struct *gve_wq;
	struct work_struct service_task;
	unsigned long service_task_flags;
	unsigned long state_flags;
};

enum gve_service_task_flags {
	GVE_PRIV_FLAGS_DO_RESET			= BIT(1),
	GVE_PRIV_FLAGS_RESET_IN_PROGRESS	= BIT(2),
	GVE_PRIV_FLAGS_PROBE_IN_PROGRESS	= BIT(3),
};

enum gve_state_flags {
	GVE_PRIV_FLAGS_ADMIN_QUEUE_OK		= BIT(1),
	GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK	= BIT(2),
	GVE_PRIV_FLAGS_DEVICE_RINGS_OK		= BIT(3),
	GVE_PRIV_FLAGS_NAPI_ENABLED		= BIT(4),
};

static inline bool gve_get_do_reset(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
}

static inline void gve_set_do_reset(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
}

static inline void gve_clear_do_reset(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
}

static inline bool gve_get_reset_in_progress(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS,
			&priv->service_task_flags);
}

static inline void gve_set_reset_in_progress(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
}

static inline void gve_clear_reset_in_progress(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
}

static inline bool gve_get_probe_in_progress(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS,
			&priv->service_task_flags);
}

static inline void gve_set_probe_in_progress(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
}

static inline void gve_clear_probe_in_progress(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
}

static inline bool gve_get_admin_queue_ok(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
}

static inline void gve_set_admin_queue_ok(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
}

static inline void gve_clear_admin_queue_ok(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
}

static inline bool gve_get_device_resources_ok(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
}

static inline void gve_set_device_resources_ok(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
}

static inline void gve_clear_device_resources_ok(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
}

static inline bool gve_get_device_rings_ok(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
}

static inline void gve_set_device_rings_ok(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
}

static inline void gve_clear_device_rings_ok(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
}

static inline bool gve_get_napi_enabled(struct gve_priv *priv)
{
	return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
}

static inline void gve_set_napi_enabled(struct gve_priv *priv)
{
	set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
}

static inline void gve_clear_napi_enabled(struct gve_priv *priv)
{
	clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
}

/* Returns the address of the ntfy_blocks irq doorbell
 */
static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv,
					       struct gve_notify_block *block)
{
	return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];
}

/* Returns the index into ntfy_blocks of the given tx ring's block
 */
static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
{
	return queue_idx;
}

/* Returns the index into ntfy_blocks of the given rx ring's block
 */
static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
{
	return (priv->num_ntfy_blks / 2) + queue_idx;
}

/* Returns the number of tx queue page lists
 */
static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
{
	return priv->tx_cfg.num_queues;
}

/* Returns the number of rx queue page lists
 */
static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
{
	return priv->rx_cfg.num_queues;
}

/* Returns a pointer to the next available tx qpl in the list of qpls
 */
static inline
struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv)
{
	int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,
				     priv->qpl_cfg.qpl_map_size);

	/* we are out of tx qpls */
	if (id >= gve_num_tx_qpls(priv))
		return NULL;

	set_bit(id, priv->qpl_cfg.qpl_id_map);
	return &priv->qpls[id];
}

/* Returns a pointer to the next available rx qpl in the list of qpls
 */
static inline
struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv)
{
	int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,
				    priv->qpl_cfg.qpl_map_size,
				    gve_num_tx_qpls(priv));

	/* we are out of rx qpls */
	if (id == priv->qpl_cfg.qpl_map_size)
		return NULL;

	set_bit(id, priv->qpl_cfg.qpl_id_map);
	return &priv->qpls[id];
}

/* Unassigns the qpl with the given id
 */
static inline void gve_unassign_qpl(struct gve_priv *priv, int id)
{
	clear_bit(id, priv->qpl_cfg.qpl_id_map);
}

/* Returns the correct dma direction for tx and rx qpls
 */
static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv,
						      int id)
{
	if (id < gve_num_tx_qpls(priv))
		return DMA_TO_DEVICE;
	else
		return DMA_FROM_DEVICE;
}

/* Returns true if the max mtu allows page recycling */
static inline bool gve_can_recycle_pages(struct net_device *dev)
{
	/* We can't recycle the pages if we can't fit a packet into half a
	 * page.
	 */
	return dev->max_mtu <= PAGE_SIZE / 2;
}

/* buffers */
int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma,
		   enum dma_data_direction);
void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
		   enum dma_data_direction);
/* tx handling */
netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev);
bool gve_tx_poll(struct gve_notify_block *block, int budget);
int gve_tx_alloc_rings(struct gve_priv *priv);
void gve_tx_free_rings(struct gve_priv *priv);
__be32 gve_tx_load_event_counter(struct gve_priv *priv,
				 struct gve_tx_ring *tx);
/* rx handling */
void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx);
bool gve_rx_poll(struct gve_notify_block *block, int budget);
int gve_rx_alloc_rings(struct gve_priv *priv);
void gve_rx_free_rings(struct gve_priv *priv);
bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
		       netdev_features_t feat);
/* Reset */
void gve_schedule_reset(struct gve_priv *priv);
int gve_reset(struct gve_priv *priv, bool attempt_teardown);
int gve_adjust_queues(struct gve_priv *priv,
		      struct gve_queue_config new_rx_config,
		      struct gve_queue_config new_tx_config);
/* exported by ethtool.c */
extern const struct ethtool_ops gve_ethtool_ops;
/* needed by ethtool */
extern const char gve_version_str[];
#endif /* _GVE_H_ */
Commit	Line	Data
893ce44d CS	1	/* SPDX-License-Identifier: (GPL-2.0 OR MIT)
	2	* Google virtual Ethernet (gve) driver
	3	*
	4	* Copyright (C) 2015-2019 Google, Inc.
	5	*/
	6
	7	#ifndef _GVE_H_
	8	#define _GVE_H_
	9
	10	#include <linux/dma-mapping.h>
	11	#include <linux/netdevice.h>
	12	#include <linux/pci.h>
f5cedc84 CS	13	#include <linux/u64_stats_sync.h>
f5cedc84 CS	14	#include "gve_desc.h"
893ce44d CS	15
	16	#ifndef PCI_VENDOR_ID_GOOGLE
	17	#define PCI_VENDOR_ID_GOOGLE 0x1ae0
	18	#endif
	19
	20	#define PCI_DEV_ID_GVNIC 0x0042
	21
	22	#define GVE_REGISTER_BAR 0
	23	#define GVE_DOORBELL_BAR 2
	24
f5cedc84 CS	25	/* Driver can alloc up to 2 segments for the header and 2 for the payload. */
	26	#define GVE_TX_MAX_IOVEC 4
	27	/* 1 for management, 1 for rx, 1 for tx */
893ce44d CS	28	#define GVE_MIN_MSIX 3
893ce44d CS	29
f5cedc84 CS	30	/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
	31	struct gve_rx_desc_queue {
	32	struct gve_rx_desc desc_ring; / the descriptor ring */
	33	dma_addr_t bus; /* the bus for the desc_ring */
f5cedc84 CS	34	u8 seqno; /* the next expected seqno for this desc*/
	35	};
	36
	37	/* The page info for a single slot in the RX data queue */
	38	struct gve_rx_slot_page_info {
	39	struct page *page;
	40	void *page_address;
	41	u32 page_offset; /* offset to write to in page */
	42	};
	43
	44	/* A list of pages registered with the device during setup and used by a queue
	45	* as buffers
	46	*/
	47	struct gve_queue_page_list {
	48	u32 id; /* unique id */
	49	u32 num_entries;
	50	struct page *pages; / list of num_entries pages */
	51	dma_addr_t page_buses; / the dma addrs of the pages */
	52	};
	53
	54	/* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */
	55	struct gve_rx_data_queue {
	56	struct gve_rx_data_slot data_ring; / read by NIC */
	57	dma_addr_t data_bus; /* dma mapping of the slots */
	58	struct gve_rx_slot_page_info page_info; / page info of the buffers */
	59	struct gve_queue_page_list qpl; / qpl assigned to this queue */
f5cedc84 CS	60	};
	61
	62	struct gve_priv;
	63
	64	/* An RX ring that contains a power-of-two sized desc and data ring. */
	65	struct gve_rx_ring {
	66	struct gve_priv *gve;
	67	struct gve_rx_desc_queue desc;
	68	struct gve_rx_data_queue data;
	69	u64 rbytes; /* free-running bytes received */
	70	u64 rpackets; /* free-running packets received */
438b43bd CS	71	u32 cnt; /* free-running total number of completed packets */
	72	u32 fill_cnt; /* free-running total number of descs and buffs posted */
	73	u32 mask; /* masks the cnt and fill_cnt to the size of the ring */
f5cedc84 CS	74	u32 q_num; /* queue index */
	75	u32 ntfy_id; /* notification block index */
	76	struct gve_queue_resources q_resources; / head and tail pointer idx */
	77	dma_addr_t q_resources_bus; /* dma address for the queue resources */
	78	struct u64_stats_sync statss; /* sync stats for 32bit archs */
	79	};
	80
	81	/* A TX desc ring entry */
	82	union gve_tx_desc {
	83	struct gve_tx_pkt_desc pkt; /* first desc for a packet */
	84	struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
	85	};
	86
	87	/* Tracks the memory in the fifo occupied by a segment of a packet */
	88	struct gve_tx_iovec {
	89	u32 iov_offset; /* offset into this segment */
	90	u32 iov_len; /* length */
	91	u32 iov_padding; /* padding associated with this segment */
	92	};
	93
	94	/* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc
	95	* ring entry but only used for a pkt_desc not a seg_desc
	96	*/
	97	struct gve_tx_buffer_state {
	98	struct sk_buff skb; / skb for this pkt */
	99	struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */
	100	};
	101
	102	/* A TX buffer - each queue has one */
	103	struct gve_tx_fifo {
	104	void base; / address of base of FIFO */
	105	u32 size; /* total size */
	106	atomic_t available; /* how much space is still available */
	107	u32 head; /* offset to write at */
	108	struct gve_queue_page_list qpl; / QPL mapped into this FIFO */
	109	};
	110
	111	/* A TX ring that contains a power-of-two sized desc ring and a FIFO buffer */
	112	struct gve_tx_ring {
	113	/* Cacheline 0 -- Accessed & dirtied during transmit */
	114	struct gve_tx_fifo tx_fifo;
	115	u32 req; /* driver tracked head pointer */
	116	u32 done; /* driver tracked tail pointer */
	117
	118	/* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */
	119	__be32 last_nic_done ____cacheline_aligned; /* NIC tail pointer */
	120	u64 pkt_done; /* free-running - total packets completed */
	121	u64 bytes_done; /* free-running - total bytes completed */
	122
	123	/* Cacheline 2 -- Read-mostly fields */
	124	union gve_tx_desc *desc ____cacheline_aligned;
	125	struct gve_tx_buffer_state info; / Maps 1:1 to a desc */
	126	struct netdev_queue *netdev_txq;
	127	struct gve_queue_resources q_resources; / head and tail pointer idx */
	128	u32 mask; /* masks req and done down to queue size */
	129
	130	/* Slow-path fields */
	131	u32 q_num ____cacheline_aligned; /* queue idx */
	132	u32 stop_queue; /* count of queue stops */
	133	u32 wake_queue; /* count of queue wakes */
	134	u32 ntfy_id; /* notification block index */
	135	dma_addr_t bus; /* dma address of the descr ring */
	136	dma_addr_t q_resources_bus; /* dma address of the queue resources */
	137	struct u64_stats_sync statss; /* sync stats for 32bit archs */
138	} ____cacheline_aligned;
139
140	/* Wraps the info for one irq including the napi struct and the queues
141	* associated with that irq.
142	*/
893ce44d CS	143	struct gve_notify_block {
	144	__be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */
	145	char name[IFNAMSIZ + 16]; /* name registered with the kernel */
	146	struct napi_struct napi; /* kernel napi struct for this block */
	147	struct gve_priv *priv;
f5cedc84 CS	148	struct gve_tx_ring tx; / tx rings on this block */
f5cedc84 CS	149	struct gve_rx_ring rx; / rx rings on this block */
893ce44d CS	150	} ____cacheline_aligned;
893ce44d CS	151
f5cedc84 CS	152	/* Tracks allowed and current queue settings */
	153	struct gve_queue_config {
	154	u16 max_queues;
	155	u16 num_queues; /* current */
	156	};
	157
	158	/* Tracks the available and used qpl IDs */
	159	struct gve_qpl_config {
	160	u32 qpl_map_size; /* map memory size */
	161	unsigned long qpl_id_map; / bitmap of used qpl ids */
	162	};
	163
893ce44d CS	164	struct gve_priv {
893ce44d CS	165	struct net_device *dev;
f5cedc84 CS	166	struct gve_tx_ring tx; / array of tx_cfg.num_queues */
	167	struct gve_rx_ring rx; / array of rx_cfg.num_queues */
	168	struct gve_queue_page_list qpls; / array of num qpls */
893ce44d CS	169	struct gve_notify_block ntfy_blocks; / array of num_ntfy_blks */
	170	dma_addr_t ntfy_block_bus;
	171	struct msix_entry msix_vectors; / array of num_ntfy_blks + 1 */
	172	char mgmt_msix_name[IFNAMSIZ + 16];
	173	u32 mgmt_msix_idx;
	174	__be32 counter_array; / array of num_event_counters */
	175	dma_addr_t counter_array_bus;
	176
	177	u16 num_event_counters;
f5cedc84 CS	178	u16 tx_desc_cnt; /* num desc per ring */
	179	u16 rx_desc_cnt; /* num desc per ring */
	180	u16 tx_pages_per_qpl; /* tx buffer length */
	181	u16 rx_pages_per_qpl; /* rx buffer length */
	182	u64 max_registered_pages;
	183	u64 num_registered_pages; /* num pages registered with NIC */
	184	u32 rx_copybreak; /* copy packets smaller than this */
	185	u16 default_num_queues; /* default num queues to set up */
893ce44d	186
f5cedc84 CS	187	struct gve_queue_config tx_cfg;
	188	struct gve_queue_config rx_cfg;
	189	struct gve_qpl_config qpl_cfg; /* map used QPL ids */
893ce44d CS	190	u32 num_ntfy_blks; /* spilt between TX and RX so must be even */
	191
	192	struct gve_registers __iomem reg_bar0; / see gve_register.h */
	193	__be32 __iomem db_bar2; / "array" of doorbells */
	194	u32 msg_enable; /* level for netif* netdev print macros */
	195	struct pci_dev *pdev;
	196
f5cedc84 CS	197	/* metrics */
	198	u32 tx_timeo_cnt;
	199
893ce44d CS	200	/* Admin queue - see gve_adminq.h*/
	201	union gve_adminq_command *adminq;
	202	dma_addr_t adminq_bus_addr;
	203	u32 adminq_mask; /* masks prod_cnt to adminq size */
	204	u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */
	205
9e5f7d26 CS	206	struct workqueue_struct *gve_wq;
	207	struct work_struct service_task;
	208	unsigned long service_task_flags;
893ce44d CS	209	unsigned long state_flags;
	210	};
	211
9e5f7d26 CS	212	enum gve_service_task_flags {
	213	GVE_PRIV_FLAGS_DO_RESET = BIT(1),
	214	GVE_PRIV_FLAGS_RESET_IN_PROGRESS = BIT(2),
	215	GVE_PRIV_FLAGS_PROBE_IN_PROGRESS = BIT(3),
	216	};
	217
893ce44d CS	218	enum gve_state_flags {
	219	GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = BIT(1),
	220	GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = BIT(2),
	221	GVE_PRIV_FLAGS_DEVICE_RINGS_OK = BIT(3),
	222	GVE_PRIV_FLAGS_NAPI_ENABLED = BIT(4),
	223	};
	224
9e5f7d26 CS	225	static inline bool gve_get_do_reset(struct gve_priv *priv)
	226	{
	227	return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
	228	}
	229
	230	static inline void gve_set_do_reset(struct gve_priv *priv)
	231	{
	232	set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
	233	}
	234
	235	static inline void gve_clear_do_reset(struct gve_priv *priv)
	236	{
	237	clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags);
	238	}
	239
	240	static inline bool gve_get_reset_in_progress(struct gve_priv *priv)
	241	{
	242	return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS,
	243	&priv->service_task_flags);
	244	}
	245
	246	static inline void gve_set_reset_in_progress(struct gve_priv *priv)
	247	{
	248	set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
	249	}
	250
	251	static inline void gve_clear_reset_in_progress(struct gve_priv *priv)
	252	{
	253	clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags);
	254	}
	255
	256	static inline bool gve_get_probe_in_progress(struct gve_priv *priv)
	257	{
	258	return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS,
	259	&priv->service_task_flags);
	260	}
	261
	262	static inline void gve_set_probe_in_progress(struct gve_priv *priv)
	263	{
	264	set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
	265	}
	266
	267	static inline void gve_clear_probe_in_progress(struct gve_priv *priv)
	268	{
	269	clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags);
	270	}
	271
893ce44d CS	272	static inline bool gve_get_admin_queue_ok(struct gve_priv *priv)
	273	{
	274	return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
	275	}
	276
	277	static inline void gve_set_admin_queue_ok(struct gve_priv *priv)
	278	{
	279	set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
	280	}
	281
	282	static inline void gve_clear_admin_queue_ok(struct gve_priv *priv)
	283	{
	284	clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags);
	285	}
	286
	287	static inline bool gve_get_device_resources_ok(struct gve_priv *priv)
	288	{
	289	return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
	290	}
	291
	292	static inline void gve_set_device_resources_ok(struct gve_priv *priv)
	293	{
	294	set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
	295	}
	296
	297	static inline void gve_clear_device_resources_ok(struct gve_priv *priv)
	298	{
	299	clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags);
	300	}
	301
	302	static inline bool gve_get_device_rings_ok(struct gve_priv *priv)
	303	{
	304	return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
	305	}
	306
	307	static inline void gve_set_device_rings_ok(struct gve_priv *priv)
	308	{
	309	set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
	310	}
	311
	312	static inline void gve_clear_device_rings_ok(struct gve_priv *priv)
	313	{
	314	clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags);
	315	}
	316
	317	static inline bool gve_get_napi_enabled(struct gve_priv *priv)
	318	{
	319	return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
	320	}
	321
	322	static inline void gve_set_napi_enabled(struct gve_priv *priv)
	323	{
	324	set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
	325	}
	326
	327	static inline void gve_clear_napi_enabled(struct gve_priv *priv)
	328	{
	329	clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags);
	330	}
	331
	332	/* Returns the address of the ntfy_blocks irq doorbell
	333	*/
	334	static inline __be32 __iomem gve_irq_doorbell(struct gve_priv priv,
	335	struct gve_notify_block *block)
336	{
337	return &priv->db_bar2[be32_to_cpu(block->irq_db_index)];
338	}
f5cedc84 CS	339
	340	/* Returns the index into ntfy_blocks of the given tx ring's block
	341	*/
	342	static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
	343	{
	344	return queue_idx;
	345	}
	346
	347	/* Returns the index into ntfy_blocks of the given rx ring's block
	348	*/
	349	static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
	350	{
	351	return (priv->num_ntfy_blks / 2) + queue_idx;
	352	}
	353
	354	/* Returns the number of tx queue page lists
	355	*/
	356	static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
	357	{
	358	return priv->tx_cfg.num_queues;
	359	}
	360
	361	/* Returns the number of rx queue page lists
	362	*/
	363	static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
	364	{
	365	return priv->rx_cfg.num_queues;
	366	}
	367
	368	/* Returns a pointer to the next available tx qpl in the list of qpls
	369	*/
	370	static inline
	371	struct gve_queue_page_list gve_assign_tx_qpl(struct gve_priv priv)
	372	{
	373	int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map,
	374	priv->qpl_cfg.qpl_map_size);
	375
	376	/* we are out of tx qpls */
	377	if (id >= gve_num_tx_qpls(priv))
	378	return NULL;
	379
	380	set_bit(id, priv->qpl_cfg.qpl_id_map);
	381	return &priv->qpls[id];
	382	}
	383
	384	/* Returns a pointer to the next available rx qpl in the list of qpls
	385	*/
	386	static inline
	387	struct gve_queue_page_list gve_assign_rx_qpl(struct gve_priv priv)
	388	{
	389	int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map,
	390	priv->qpl_cfg.qpl_map_size,
	391	gve_num_tx_qpls(priv));
	392
	393	/* we are out of rx qpls */
	394	if (id == priv->qpl_cfg.qpl_map_size)
	395	return NULL;
	396
	397	set_bit(id, priv->qpl_cfg.qpl_id_map);
	398	return &priv->qpls[id];
	399	}
	400
	401	/* Unassigns the qpl with the given id
	402	*/
403	static inline void gve_unassign_qpl(struct gve_priv *priv, int id)
404	{
405	clear_bit(id, priv->qpl_cfg.qpl_id_map);
406	}
407
408	/* Returns the correct dma direction for tx and rx qpls
409	*/
410	static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv,
411	int id)
412	{
413	if (id < gve_num_tx_qpls(priv))
414	return DMA_TO_DEVICE;
415	else
416	return DMA_FROM_DEVICE;
417	}
418
419	/* Returns true if the max mtu allows page recycling */
420	static inline bool gve_can_recycle_pages(struct net_device *dev)
421	{
422	/* We can't recycle the pages if we can't fit a packet into half a
423	* page.
424	*/
425	return dev->max_mtu <= PAGE_SIZE / 2;
426	}
427
428	/* buffers */
429	int gve_alloc_page(struct device dev, struct page page, dma_addr_t dma,
430	enum dma_data_direction);
431	void gve_free_page(struct device dev, struct page page, dma_addr_t dma,
432	enum dma_data_direction);
433	/* tx handling */
434	netdev_tx_t gve_tx(struct sk_buff skb, struct net_device dev);
435	bool gve_tx_poll(struct gve_notify_block *block, int budget);
436	int gve_tx_alloc_rings(struct gve_priv *priv);
437	void gve_tx_free_rings(struct gve_priv *priv);
438	__be32 gve_tx_load_event_counter(struct gve_priv *priv,
439	struct gve_tx_ring *tx);
440	/* rx handling */
441	void gve_rx_write_doorbell(struct gve_priv priv, struct gve_rx_ring rx);
442	bool gve_rx_poll(struct gve_notify_block *block, int budget);
443	int gve_rx_alloc_rings(struct gve_priv *priv);
444	void gve_rx_free_rings(struct gve_priv *priv);
445	bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
446	netdev_features_t feat);
9e5f7d26 CS	447	/* Reset */
	448	void gve_schedule_reset(struct gve_priv *priv);
	449	int gve_reset(struct gve_priv *priv, bool attempt_teardown);
	450	int gve_adjust_queues(struct gve_priv *priv,
	451	struct gve_queue_config new_rx_config,
	452	struct gve_queue_config new_tx_config);
e5b845dc CS	453	/* exported by ethtool.c */
	454	extern const struct ethtool_ops gve_ethtool_ops;
	455	/* needed by ethtool */
	456	extern const char gve_version_str[];
893ce44d	457	#endif /* _GVE_H_ */