[linux-block.git] / net / packet / internal.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __PACKET_INTERNAL_H__
#define __PACKET_INTERNAL_H__

#include <linux/refcount.h>

struct packet_mclist {
	struct packet_mclist	*next;
	int			ifindex;
	int			count;
	unsigned short		type;
	unsigned short		alen;
	unsigned char		addr[MAX_ADDR_LEN];
	struct list_head	remove_list;
};

/* kbdq - kernel block descriptor queue */
struct tpacket_kbdq_core {
	struct pgv	*pkbdq;
	unsigned int	feature_req_word;
	unsigned int	hdrlen;
	unsigned char	reset_pending_on_curr_blk;
	unsigned char   delete_blk_timer;
	unsigned short	kactive_blk_num;
	unsigned short	blk_sizeof_priv;

	/* last_kactive_blk_num:
	 * trick to see if user-space has caught up
	 * in order to avoid refreshing timer when every single pkt arrives.
	 */
	unsigned short	last_kactive_blk_num;

	char		*pkblk_start;
	char		*pkblk_end;
	int		kblk_size;
	unsigned int	max_frame_len;
	unsigned int	knum_blocks;
	uint64_t	knxt_seq_num;
	char		*prev;
	char		*nxt_offset;
	struct sk_buff	*skb;

	rwlock_t	blk_fill_in_prog_lock;

	/* Default is set to 8ms */
#define DEFAULT_PRB_RETIRE_TOV	(8)

	unsigned short  retire_blk_tov;
	unsigned short  version;
	unsigned long	tov_in_jiffies;

	/* timer to retire an outstanding block */
	struct timer_list retire_blk_timer;
};

struct pgv {
	char *buffer;
};

struct packet_ring_buffer {
	struct pgv		*pg_vec;

	unsigned int		head;
	unsigned int		frames_per_block;
	unsigned int		frame_size;
	unsigned int		frame_max;

	unsigned int		pg_vec_order;
	unsigned int		pg_vec_pages;
	unsigned int		pg_vec_len;

	unsigned int __percpu	*pending_refcnt;

	union {
		unsigned long			*rx_owner_map;
		struct tpacket_kbdq_core	prb_bdqc;
	};
};

extern struct mutex fanout_mutex;
#define PACKET_FANOUT_MAX	(1 << 16)

struct packet_fanout {
	possible_net_t		net;
	unsigned int		num_members;
	u32			max_num_members;
	u16			id;
	u8			type;
	u8			flags;
	union {
		atomic_t		rr_cur;
		struct bpf_prog __rcu	*bpf_prog;
	};
	struct list_head	list;
	spinlock_t		lock;
	refcount_t		sk_ref;
	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
	struct sock	__rcu	*arr[] __counted_by(max_num_members);
};

struct packet_rollover {
	int			sock;
	atomic_long_t		num;
	atomic_long_t		num_huge;
	atomic_long_t		num_failed;
#define ROLLOVER_HLEN	(L1_CACHE_BYTES / sizeof(u32))
	u32			history[ROLLOVER_HLEN] ____cacheline_aligned;
} ____cacheline_aligned_in_smp;

struct packet_sock {
	/* struct sock has to be the first member of packet_sock */
	struct sock		sk;
	struct packet_fanout	*fanout;
	union  tpacket_stats_u	stats;
	struct packet_ring_buffer	rx_ring;
	struct packet_ring_buffer	tx_ring;
	int			copy_thresh;
	spinlock_t		bind_lock;
	struct mutex		pg_vec_lock;
	unsigned long		flags;
	int			ifindex;	/* bound device		*/
	u8			vnet_hdr_sz;
	__be16			num;
	struct packet_rollover	*rollover;
	struct packet_mclist	*mclist;
	atomic_long_t		mapped;
	enum tpacket_versions	tp_version;
	unsigned int		tp_hdrlen;
	unsigned int		tp_reserve;
	unsigned int		tp_tstamp;
	struct completion	skb_completion;
	struct net_device __rcu	*cached_dev;
	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
	atomic_t		tp_drops ____cacheline_aligned_in_smp;
};

#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk)

enum packet_sock_flags {
	PACKET_SOCK_ORIGDEV,
	PACKET_SOCK_AUXDATA,
	PACKET_SOCK_TX_HAS_OFF,
	PACKET_SOCK_TP_LOSS,
	PACKET_SOCK_RUNNING,
	PACKET_SOCK_PRESSURE,
	PACKET_SOCK_QDISC_BYPASS,
};

static inline void packet_sock_flag_set(struct packet_sock *po,
					enum packet_sock_flags flag,
					bool val)
{
	if (val)
		set_bit(flag, &po->flags);
	else
		clear_bit(flag, &po->flags);
}

static inline bool packet_sock_flag(const struct packet_sock *po,
				    enum packet_sock_flags flag)
{
	return test_bit(flag, &po->flags);
}

#endif
Commit	Line	Data
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __PACKET_INTERNAL_H__
	3	#define __PACKET_INTERNAL_H__
	4
	5	#include <linux/refcount.h>
	6
	7	struct packet_mclist {
	8	struct packet_mclist *next;
	9	int ifindex;
	10	int count;
	11	unsigned short type;
	12	unsigned short alen;
	13	unsigned char addr[MAX_ADDR_LEN];
	14	struct list_head remove_list;
	15	};
	16
	17	/* kbdq - kernel block descriptor queue */
	18	struct tpacket_kbdq_core {
	19	struct pgv *pkbdq;
	20	unsigned int feature_req_word;
	21	unsigned int hdrlen;
	22	unsigned char reset_pending_on_curr_blk;
	23	unsigned char delete_blk_timer;
	24	unsigned short kactive_blk_num;
	25	unsigned short blk_sizeof_priv;
	26
	27	/* last_kactive_blk_num:
	28	* trick to see if user-space has caught up
	29	* in order to avoid refreshing timer when every single pkt arrives.
	30	*/
	31	unsigned short last_kactive_blk_num;
	32
	33	char *pkblk_start;
	34	char *pkblk_end;
	35	int kblk_size;
	36	unsigned int max_frame_len;
	37	unsigned int knum_blocks;
	38	uint64_t knxt_seq_num;
	39	char *prev;
	40	char *nxt_offset;
	41	struct sk_buff *skb;
	42
	43	rwlock_t blk_fill_in_prog_lock;
	44
	45	/* Default is set to 8ms */
	46	#define DEFAULT_PRB_RETIRE_TOV (8)
	47
	48	unsigned short retire_blk_tov;
	49	unsigned short version;
	50	unsigned long tov_in_jiffies;
	51
	52	/* timer to retire an outstanding block */
	53	struct timer_list retire_blk_timer;
	54	};
	55
	56	struct pgv {
	57	char *buffer;
	58	};
	59
	60	struct packet_ring_buffer {
	61	struct pgv *pg_vec;
	62
	63	unsigned int head;
	64	unsigned int frames_per_block;
	65	unsigned int frame_size;
	66	unsigned int frame_max;
	67
	68	unsigned int pg_vec_order;
	69	unsigned int pg_vec_pages;
	70	unsigned int pg_vec_len;
	71
	72	unsigned int __percpu *pending_refcnt;
	73
	74	union {
	75	unsigned long *rx_owner_map;
	76	struct tpacket_kbdq_core prb_bdqc;
	77	};
	78	};
	79
	80	extern struct mutex fanout_mutex;
	81	#define PACKET_FANOUT_MAX (1 << 16)
	82
	83	struct packet_fanout {
	84	possible_net_t net;
	85	unsigned int num_members;
	86	u32 max_num_members;
	87	u16 id;
	88	u8 type;
	89	u8 flags;
	90	union {
	91	atomic_t rr_cur;
	92	struct bpf_prog __rcu *bpf_prog;
	93	};
	94	struct list_head list;
	95	spinlock_t lock;
	96	refcount_t sk_ref;
	97	struct packet_type prot_hook ____cacheline_aligned_in_smp;
	98	struct sock __rcu *arr[] __counted_by(max_num_members);
	99	};
	100
	101	struct packet_rollover {
	102	int sock;
	103	atomic_long_t num;
	104	atomic_long_t num_huge;
	105	atomic_long_t num_failed;
	106	#define ROLLOVER_HLEN (L1_CACHE_BYTES / sizeof(u32))
	107	u32 history[ROLLOVER_HLEN] ____cacheline_aligned;
	108	} ____cacheline_aligned_in_smp;
	109
	110	struct packet_sock {
	111	/* struct sock has to be the first member of packet_sock */
	112	struct sock sk;
	113	struct packet_fanout *fanout;
	114	union tpacket_stats_u stats;
	115	struct packet_ring_buffer rx_ring;
	116	struct packet_ring_buffer tx_ring;
	117	int copy_thresh;
	118	spinlock_t bind_lock;
	119	struct mutex pg_vec_lock;
	120	unsigned long flags;
	121	int ifindex; /* bound device */
	122	u8 vnet_hdr_sz;
	123	__be16 num;
	124	struct packet_rollover *rollover;
	125	struct packet_mclist *mclist;
	126	atomic_long_t mapped;
	127	enum tpacket_versions tp_version;
	128	unsigned int tp_hdrlen;
	129	unsigned int tp_reserve;
	130	unsigned int tp_tstamp;
	131	struct completion skb_completion;
	132	struct net_device __rcu *cached_dev;
	133	struct packet_type prot_hook ____cacheline_aligned_in_smp;
	134	atomic_t tp_drops ____cacheline_aligned_in_smp;
	135	};
	136
	137	#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk)
	138
	139	enum packet_sock_flags {
	140	PACKET_SOCK_ORIGDEV,
	141	PACKET_SOCK_AUXDATA,
	142	PACKET_SOCK_TX_HAS_OFF,
	143	PACKET_SOCK_TP_LOSS,
	144	PACKET_SOCK_RUNNING,
	145	PACKET_SOCK_PRESSURE,
	146	PACKET_SOCK_QDISC_BYPASS,
	147	};
	148
	149	static inline void packet_sock_flag_set(struct packet_sock *po,
	150	enum packet_sock_flags flag,
	151	bool val)
	152	{
	153	if (val)
	154	set_bit(flag, &po->flags);
	155	else
	156	clear_bit(flag, &po->flags);
	157	}
	158
	159	static inline bool packet_sock_flag(const struct packet_sock *po,
	160	enum packet_sock_flags flag)
	161	{
	162	return test_bit(flag, &po->flags);
	163	}
	164
	165	#endif