Merge tag 'mm-hotfixes-stable-2025-07-11-16-16' of git://git.kernel.org/pub/scm/linux...
[linux-block.git] / net / packet / internal.h
CommitLineData
b2441318 1/* SPDX-License-Identifier: GPL-2.0 */
2787b04b
PE
2#ifndef __PACKET_INTERNAL_H__
3#define __PACKET_INTERNAL_H__
4
fb5c2c17
RE
5#include <linux/refcount.h>
6
2787b04b
PE
7struct packet_mclist {
8 struct packet_mclist *next;
9 int ifindex;
10 int count;
11 unsigned short type;
12 unsigned short alen;
13 unsigned char addr[MAX_ADDR_LEN];
d8d85ef0 14 struct list_head remove_list;
2787b04b
PE
15};
16
17/* kbdq - kernel block descriptor queue */
18struct tpacket_kbdq_core {
19 struct pgv *pkbdq;
20 unsigned int feature_req_word;
21 unsigned int hdrlen;
22 unsigned char reset_pending_on_curr_blk;
23 unsigned char delete_blk_timer;
24 unsigned short kactive_blk_num;
25 unsigned short blk_sizeof_priv;
26
27 /* last_kactive_blk_num:
28 * trick to see if user-space has caught up
29 * in order to avoid refreshing timer when every single pkt arrives.
30 */
31 unsigned short last_kactive_blk_num;
32
33 char *pkblk_start;
34 char *pkblk_end;
35 int kblk_size;
dc808110 36 unsigned int max_frame_len;
2787b04b
PE
37 unsigned int knum_blocks;
38 uint64_t knxt_seq_num;
39 char *prev;
40 char *nxt_offset;
41 struct sk_buff *skb;
42
632ca50f 43 rwlock_t blk_fill_in_prog_lock;
2787b04b
PE
44
45 /* Default is set to 8ms */
46#define DEFAULT_PRB_RETIRE_TOV (8)
47
48 unsigned short retire_blk_tov;
49 unsigned short version;
50 unsigned long tov_in_jiffies;
51
52 /* timer to retire an outstanding block */
53 struct timer_list retire_blk_timer;
54};
55
56struct pgv {
57 char *buffer;
58};
59
60struct packet_ring_buffer {
61 struct pgv *pg_vec;
0578edc5 62
2787b04b
PE
63 unsigned int head;
64 unsigned int frames_per_block;
65 unsigned int frame_size;
66 unsigned int frame_max;
67
3a7ad063 68 unsigned int pg_vec_order;
2787b04b
PE
69 unsigned int pg_vec_pages;
70 unsigned int pg_vec_len;
71
b0138408 72 unsigned int __percpu *pending_refcnt;
0578edc5 73
61fad681
WB
74 union {
75 unsigned long *rx_owner_map;
76 struct tpacket_kbdq_core prb_bdqc;
77 };
2787b04b
PE
78};
79
fff3321d 80extern struct mutex fanout_mutex;
9c661b0b 81#define PACKET_FANOUT_MAX (1 << 16)
fff3321d
PE
82
83struct packet_fanout {
0c5c9fb5 84 possible_net_t net;
fff3321d 85 unsigned int num_members;
9c661b0b 86 u32 max_num_members;
fff3321d
PE
87 u16 id;
88 u8 type;
77f65ebd 89 u8 flags;
47dceb8e
WB
90 union {
91 atomic_t rr_cur;
92 struct bpf_prog __rcu *bpf_prog;
93 };
fff3321d 94 struct list_head list;
fff3321d 95 spinlock_t lock;
fb5c2c17 96 refcount_t sk_ref;
fff3321d 97 struct packet_type prot_hook ____cacheline_aligned_in_smp;
b3783e5e 98 struct sock __rcu *arr[] __counted_by(max_num_members);
fff3321d
PE
99};
100
0648ab70
WB
101struct packet_rollover {
102 int sock;
a9b63918
WB
103 atomic_long_t num;
104 atomic_long_t num_huge;
105 atomic_long_t num_failed;
3b3a5b0a
WB
106#define ROLLOVER_HLEN (L1_CACHE_BYTES / sizeof(u32))
107 u32 history[ROLLOVER_HLEN] ____cacheline_aligned;
0648ab70
WB
108} ____cacheline_aligned_in_smp;
109
2787b04b
PE
110struct packet_sock {
111 /* struct sock has to be the first member of packet_sock */
112 struct sock sk;
113 struct packet_fanout *fanout;
ee80fbf3 114 union tpacket_stats_u stats;
2787b04b
PE
115 struct packet_ring_buffer rx_ring;
116 struct packet_ring_buffer tx_ring;
117 int copy_thresh;
118 spinlock_t bind_lock;
119 struct mutex pg_vec_lock;
ee5675ec 120 unsigned long flags;
2787b04b 121 int ifindex; /* bound device */
dfc39d40 122 u8 vnet_hdr_sz;
2787b04b 123 __be16 num;
0648ab70 124 struct packet_rollover *rollover;
2787b04b 125 struct packet_mclist *mclist;
db3fadac 126 atomic_long_t mapped;
2787b04b
PE
127 enum tpacket_versions tp_version;
128 unsigned int tp_hdrlen;
129 unsigned int tp_reserve;
2787b04b 130 unsigned int tp_tstamp;
89ed5b51 131 struct completion skb_completion;
e40526cb 132 struct net_device __rcu *cached_dev;
2787b04b 133 struct packet_type prot_hook ____cacheline_aligned_in_smp;
8e8e2951 134 atomic_t tp_drops ____cacheline_aligned_in_smp;
2787b04b
PE
135};
136
68ac9a8b 137#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk)
2787b04b 138
ee5675ec
ED
139enum packet_sock_flags {
140 PACKET_SOCK_ORIGDEV,
fd53c297 141 PACKET_SOCK_AUXDATA,
74383446 142 PACKET_SOCK_TX_HAS_OFF,
164bddac 143 PACKET_SOCK_TP_LOSS,
61edf479 144 PACKET_SOCK_RUNNING,
791a3e9f 145 PACKET_SOCK_PRESSURE,
105a201e 146 PACKET_SOCK_QDISC_BYPASS,
ee5675ec
ED
147};
148
149static inline void packet_sock_flag_set(struct packet_sock *po,
150 enum packet_sock_flags flag,
151 bool val)
152{
153 if (val)
154 set_bit(flag, &po->flags);
155 else
156 clear_bit(flag, &po->flags);
157}
158
159static inline bool packet_sock_flag(const struct packet_sock *po,
160 enum packet_sock_flags flag)
161{
162 return test_bit(flag, &po->flags);
163}
164
2787b04b 165#endif