projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/packet: constify prb_lookup_block() and __tpacket_v3_has_room()
[linux-block.git] / net / packet / af_packet.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * PACKET - implements raw packet sockets.
8 *
02c30a84 9 * Authors: Ross Biro
1da177e4
LT		 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Alan Cox, <gw4pts@gw4pts.ampr.org>
12 *
1ce4f28b 13 * Fixes:
1da177e4
LT		 14 * Alan Cox : verify_area() now used correctly
15 * Alan Cox : new skbuff lists, look ma no backlogs!
16 * Alan Cox : tidied skbuff lists.
17 * Alan Cox : Now uses generic datagram routines I
18 * added. Also fixed the peek/read crash
19 * from all old Linux datagram code.
20 * Alan Cox : Uses the improved datagram code.
21 * Alan Cox : Added NULL's for socket options.
22 * Alan Cox : Re-commented the code.
23 * Alan Cox : Use new kernel side addressing
24 * Rob Janssen : Correct MTU usage.
25 * Dave Platt : Counter leaks caused by incorrect
26 * interrupt locking and some slightly
27 * dubious gcc output. Can you read
28 * compiler: it said _VOLATILE_
29 * Richard Kooijman : Timestamp fixes.
30 * Alan Cox : New buffers. Use sk->mac.raw.
31 * Alan Cox : sendmsg/recvmsg support.
32 * Alan Cox : Protocol setting support
33 * Alexey Kuznetsov : Untied from IPv4 stack.
34 * Cyrus Durgin : Fixed kerneld for kmod.
35 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 36 * Ulises Alonso : Frame number limit removal and
1da177e4 37 * packet_set_ring memory leak.
0fb375fb
EB		 38 * Eric Biederman : Allow for > 8 byte hardware addresses.
39 * The convention is that longer addresses
40 * will simply extend the hardware address
1ce4f28b 41 * byte arrays at the end of sockaddr_ll
0fb375fb 42 * and packet_mreq.
69e3c75f 43 * Johann Baudy : Added TX RING.
f6fb8f10 44 * Chetan Loke : Implemented TPACKET_V3 block abstraction
45 * layer.
46 * Copyright (C) 2011, <lokec@ccs.neu.edu>
1da177e4 47 */
1ce4f28b 48
1da177e4 49#include <linux/types.h>
1da177e4 50#include <linux/mm.h>
4fc268d2 51#include <linux/capability.h>
1da177e4
LT		 52#include <linux/fcntl.h>
53#include <linux/socket.h>
54#include <linux/in.h>
55#include <linux/inet.h>
56#include <linux/netdevice.h>
57#include <linux/if_packet.h>
58#include <linux/wireless.h>
ffbc6111 59#include <linux/kernel.h>
1da177e4 60#include <linux/kmod.h>
5a0e3ad6 61#include <linux/slab.h>
0e3125c7 62#include <linux/vmalloc.h>
457c4cbc 63#include <net/net_namespace.h>
1da177e4
LT		 64#include <net/ip.h>
65#include <net/protocol.h>
66#include <linux/skbuff.h>
67#include <net/sock.h>
68#include <linux/errno.h>
69#include <linux/timer.h>
7c0f6ba6 70#include <linux/uaccess.h>
1da177e4
LT		 71#include <asm/ioctls.h>
72#include <asm/page.h>
a1f8e7f7 73#include <asm/cacheflush.h>
1da177e4
LT		 74#include <asm/io.h>
75#include <linux/proc_fs.h>
76#include <linux/seq_file.h>
77#include <linux/poll.h>
78#include <linux/module.h>
79#include <linux/init.h>
905db440 80#include <linux/mutex.h>
05423b24 81#include <linux/if_vlan.h>
bfd5f4a3 82#include <linux/virtio_net.h>
ed85b565 83#include <linux/errqueue.h>
614f60fa 84#include <linux/net_tstamp.h>
b0138408 85#include <linux/percpu.h>
1da177e4
LT		 86#ifdef CONFIG_INET
87#include <net/inet_common.h>
88#endif
47dceb8e 89#include <linux/bpf.h>
719c44d3 90#include <net/compat.h>
1da177e4 91
2787b04b
PE		 92#include "internal.h"
93
1da177e4
LT		 94/*
95 Assumptions:
96 - if device has no dev->hard_header routine, it adds and removes ll header
97 inside itself. In this case ll header is invisible outside of device,
98 but higher levels still should reserve dev->hard_header_len.
99 Some devices are enough clever to reallocate skb, when header
100 will not fit to reserved space (tunnel), another ones are silly
101 (PPP).
102 - packet socket receives packets with pulled ll header,
103 so that SOCK_RAW should push it back.
104
105On receive:
106-----------
107
108Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 109 mac_header -> ll header
110 data -> data
1da177e4
LT		 111
112Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> ll header
1da177e4
LT		 115
116Incoming, dev->hard_header==NULL
b0e380b1
ACM		 117 mac_header -> UNKNOWN position. It is very likely, that it points to ll
118 header. PPP makes it, that is wrong, because introduce
db0c58f9 119 assymetry between rx and tx paths.
b0e380b1 120 data -> data
1da177e4
LT		 121
122Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 123 mac_header -> data. ll header is still not built!
124 data -> data
1da177e4
LT		 125
126Resume
127 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
128
129
130On transmit:
131------------
132
133dev->hard_header != NULL
b0e380b1
ACM		 134 mac_header -> ll header
135 data -> ll header
1da177e4
LT		 136
137dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 138 mac_header -> data
139 data -> data
1da177e4
LT		 140
141 We should set nh.raw on output to correct posistion,
142 packet classifier depends on it.
143 */
144
1da177e4
LT		 145/* Private packet socket structures. */
146
0fb375fb
EB		 147/* identical to struct packet_mreq except it has
148 * a longer address field.
149 */
40d4e3df 150struct packet_mreq_max {
0fb375fb
EB		 151 int mr_ifindex;
152 unsigned short mr_type;
153 unsigned short mr_alen;
154 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 155};
a2efcfa0 156
184f489e
DB		 157union tpacket_uhdr {
158 struct tpacket_hdr *h1;
159 struct tpacket2_hdr *h2;
160 struct tpacket3_hdr *h3;
161 void *raw;
162};
163
f6fb8f10 164static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 165 int closing, int tx_ring);
166
f6fb8f10 167#define V3_ALIGNMENT (8)
168
bc59ba39 169#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 170
171#define BLK_PLUS_PRIV(sz_of_priv) \
172 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
173
f6fb8f10 174#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
175#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
176#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
177#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
178#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
179#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
180#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
181
69e3c75f 182struct packet_sock;
77f65ebd
WB		 183static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
184 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 185
f6fb8f10 186static void *packet_previous_frame(struct packet_sock *po,
187 struct packet_ring_buffer *rb,
188 int status);
189static void packet_increment_head(struct packet_ring_buffer *buff);
878cd3ba 190static int prb_curr_blk_in_use(struct tpacket_block_desc *);
bc59ba39 191static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 192 struct packet_sock *);
bc59ba39 193static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 194 struct packet_sock *, unsigned int status);
bc59ba39 195static int prb_queue_frozen(struct tpacket_kbdq_core *);
196static void prb_open_block(struct tpacket_kbdq_core *,
197 struct tpacket_block_desc *);
17bfd8c8 198static void prb_retire_rx_blk_timer_expired(struct timer_list *);
bc59ba39 199static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
bc59ba39 200static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
201static void prb_clear_rxhash(struct tpacket_kbdq_core *,
202 struct tpacket3_hdr *);
203static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
204 struct tpacket3_hdr *);
1da177e4 205static void packet_flush_mclist(struct sock *sk);
865b03f2 206static u16 packet_pick_tx_queue(struct sk_buff *skb);
1da177e4 207
ffbc6111 208struct packet_skb_cb {
ffbc6111
HX		 209 union {
210 struct sockaddr_pkt pkt;
2472d761
EB		 211 union {
212 /* Trick: alias skb original length with
213 * ll.sll_family and ll.protocol in order
214 * to save room.
215 */
216 unsigned int origlen;
217 struct sockaddr_ll ll;
218 };
ffbc6111
HX		 219 } sa;
220};
221
d3869efe
DW		 222#define vio_le() virtio_legacy_is_little_endian()
223
ffbc6111 224#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 225
bc59ba39 226#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 227#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 228 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 229#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 231#define GET_NEXT_PRB_BLK_NUM(x) \
232 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
233 ((x)->kactive_blk_num+1) : 0)
234
dc99f600
DM		 235static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
236static void __fanout_link(struct sock *sk, struct packet_sock *po);
237
d346a3fa
DB		 238static int packet_direct_xmit(struct sk_buff *skb)
239{
865b03f2 240 return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
d346a3fa
DB		 241}
242
66e56cd4
DB		 243static struct net_device *packet_cached_dev_get(struct packet_sock *po)
244{
245 struct net_device *dev;
246
247 rcu_read_lock();
248 dev = rcu_dereference(po->cached_dev);
249 if (likely(dev))
250 dev_hold(dev);
251 rcu_read_unlock();
252
253 return dev;
254}
255
256static void packet_cached_dev_assign(struct packet_sock *po,
257 struct net_device *dev)
258{
259 rcu_assign_pointer(po->cached_dev, dev);
260}
261
262static void packet_cached_dev_reset(struct packet_sock *po)
263{
264 RCU_INIT_POINTER(po->cached_dev, NULL);
265}
266
d346a3fa
DB		 267static bool packet_use_direct_xmit(const struct packet_sock *po)
268{
269 return po->xmit == packet_direct_xmit;
270}
271
865b03f2 272static u16 packet_pick_tx_queue(struct sk_buff *skb)
0fd5d57b 273{
865b03f2 274 struct net_device *dev = skb->dev;
0fd5d57b 275 const struct net_device_ops *ops = dev->netdev_ops;
b71b5837 276 int cpu = raw_smp_processor_id();
0fd5d57b
DB		 277 u16 queue_index;
278
b71b5837
PA		 279#ifdef CONFIG_XPS
280 skb->sender_cpu = cpu + 1;
281#endif
282 skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
0fd5d57b 283 if (ops->ndo_select_queue) {
a350ecce 284 queue_index = ops->ndo_select_queue(dev, skb, NULL);
0fd5d57b
DB		 285 queue_index = netdev_cap_txqueue(dev, queue_index);
286 } else {
b71b5837 287 queue_index = netdev_pick_tx(dev, skb, NULL);
0fd5d57b
DB		 288 }
289
865b03f2 290 return queue_index;
0fd5d57b
DB		 291}
292
a6361f0c 293/* __register_prot_hook must be invoked through register_prot_hook
ce06b03e
DM		 294 * or from a context in which asynchronous accesses to the packet
295 * socket is not possible (packet_create()).
296 */
a6361f0c 297static void __register_prot_hook(struct sock *sk)
ce06b03e
DM		 298{
299 struct packet_sock *po = pkt_sk(sk);
e40526cb 300
ce06b03e 301 if (!po->running) {
66e56cd4 302 if (po->fanout)
dc99f600 303 __fanout_link(sk, po);
66e56cd4 304 else
dc99f600 305 dev_add_pack(&po->prot_hook);
e40526cb 306
ce06b03e
DM		 307 sock_hold(sk);
308 po->running = 1;
309 }
310}
311
a6361f0c
WB		 312static void register_prot_hook(struct sock *sk)
313{
314 lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
315 __register_prot_hook(sk);
316}
317
318/* If the sync parameter is true, we will temporarily drop
ce06b03e
DM		 319 * the po->bind_lock and do a synchronize_net to make sure no
320 * asynchronous packet processing paths still refer to the elements
321 * of po->prot_hook. If the sync parameter is false, it is the
322 * callers responsibility to take care of this.
323 */
324static void __unregister_prot_hook(struct sock *sk, bool sync)
325{
326 struct packet_sock *po = pkt_sk(sk);
327
a6361f0c
WB		 328 lockdep_assert_held_once(&po->bind_lock);
329
ce06b03e 330 po->running = 0;
66e56cd4
DB		 331
332 if (po->fanout)
dc99f600 333 __fanout_unlink(sk, po);
66e56cd4 334 else
dc99f600 335 __dev_remove_pack(&po->prot_hook);
e40526cb 336
ce06b03e
DM		 337 __sock_put(sk);
338
339 if (sync) {
340 spin_unlock(&po->bind_lock);
341 synchronize_net();
342 spin_lock(&po->bind_lock);
343 }
344}
345
346static void unregister_prot_hook(struct sock *sk, bool sync)
347{
348 struct packet_sock *po = pkt_sk(sk);
349
350 if (po->running)
351 __unregister_prot_hook(sk, sync);
352}
353
6e58040b 354static inline struct page * __pure pgv_to_page(void *addr)
0af55bb5
CG		 355{
356 if (is_vmalloc_addr(addr))
357 return vmalloc_to_page(addr);
358 return virt_to_page(addr);
359}
360
69e3c75f 361static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 362{
184f489e 363 union tpacket_uhdr h;
1da177e4 364
69e3c75f 365 h.raw = frame;
bbd6ef87
PM		 366 switch (po->tp_version) {
367 case TPACKET_V1:
69e3c75f 368 h.h1->tp_status = status;
0af55bb5 369 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 370 break;
371 case TPACKET_V2:
69e3c75f 372 h.h2->tp_status = status;
0af55bb5 373 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 374 break;
f6fb8f10 375 case TPACKET_V3:
7f953ab2
SV		 376 h.h3->tp_status = status;
377 flush_dcache_page(pgv_to_page(&h.h3->tp_status));
378 break;
69e3c75f 379 default:
f6fb8f10 380 WARN(1, "TPACKET version not supported.\n");
69e3c75f 381 BUG();
bbd6ef87 382 }
69e3c75f
JB		 383
384 smp_wmb();
bbd6ef87
PM		 385}
386
96f657e6 387static int __packet_get_status(const struct packet_sock *po, void *frame)
bbd6ef87 388{
184f489e 389 union tpacket_uhdr h;
bbd6ef87 390
69e3c75f
JB		 391 smp_rmb();
392
bbd6ef87
PM		 393 h.raw = frame;
394 switch (po->tp_version) {
395 case TPACKET_V1:
0af55bb5 396 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 397 return h.h1->tp_status;
bbd6ef87 398 case TPACKET_V2:
0af55bb5 399 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 400 return h.h2->tp_status;
f6fb8f10 401 case TPACKET_V3:
7f953ab2
SV		 402 flush_dcache_page(pgv_to_page(&h.h3->tp_status));
403 return h.h3->tp_status;
69e3c75f 404 default:
f6fb8f10 405 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 406 BUG();
407 return 0;
bbd6ef87 408 }
1da177e4 409}
69e3c75f 410
b9c32fb2
DB		 411static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
412 unsigned int flags)
7a51384c
DB		 413{
414 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
415
68a360e8
WB		 416 if (shhwtstamps &&
417 (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
418 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
419 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 420
421 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 422 return TP_STATUS_TS_SOFTWARE;
7a51384c 423
b9c32fb2 424 return 0;
7a51384c
DB		 425}
426
b9c32fb2
DB		 427static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
428 struct sk_buff *skb)
2e31396f
WB		 429{
430 union tpacket_uhdr h;
431 struct timespec ts;
b9c32fb2 432 __u32 ts_status;
2e31396f 433
b9c32fb2
DB		 434 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
435 return 0;
2e31396f
WB		 436
437 h.raw = frame;
438 switch (po->tp_version) {
439 case TPACKET_V1:
440 h.h1->tp_sec = ts.tv_sec;
441 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
442 break;
443 case TPACKET_V2:
444 h.h2->tp_sec = ts.tv_sec;
445 h.h2->tp_nsec = ts.tv_nsec;
446 break;
447 case TPACKET_V3:
57ea884b
DB		 448 h.h3->tp_sec = ts.tv_sec;
449 h.h3->tp_nsec = ts.tv_nsec;
450 break;
2e31396f
WB		 451 default:
452 WARN(1, "TPACKET version not supported.\n");
453 BUG();
454 }
455
456 /* one flush is safe, as both fields always lie on the same cacheline */
457 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
458 smp_wmb();
b9c32fb2
DB		 459
460 return ts_status;
2e31396f
WB		 461}
462
d4b5bd98
ED		 463static void *packet_lookup_frame(const struct packet_sock *po,
464 const struct packet_ring_buffer *rb,
465 unsigned int position,
466 int status)
69e3c75f
JB		 467{
468 unsigned int pg_vec_pos, frame_offset;
184f489e 469 union tpacket_uhdr h;
69e3c75f
JB		 470
471 pg_vec_pos = position / rb->frames_per_block;
472 frame_offset = position % rb->frames_per_block;
473
0e3125c7
NH		 474 h.raw = rb->pg_vec[pg_vec_pos].buffer +
475 (frame_offset * rb->frame_size);
69e3c75f
JB		 476
477 if (status != __packet_get_status(po, h.raw))
478 return NULL;
479
480 return h.raw;
481}
482
eea49cc9 483static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 484 struct packet_ring_buffer *rb,
485 int status)
486{
487 return packet_lookup_frame(po, rb, rb->head, status);
488}
489
bc59ba39 490static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 491{
492 del_timer_sync(&pkc->retire_blk_timer);
493}
494
495static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
f6fb8f10 496 struct sk_buff_head *rb_queue)
497{
bc59ba39 498 struct tpacket_kbdq_core *pkc;
f6fb8f10 499
73d0fcf2 500 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 501
ec6f809f 502 spin_lock_bh(&rb_queue->lock);
f6fb8f10 503 pkc->delete_blk_timer = 1;
ec6f809f 504 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 505
506 prb_del_retire_blk_timer(pkc);
507}
508
e8e85cc5 509static void prb_setup_retire_blk_timer(struct packet_sock *po)
f6fb8f10 510{
bc59ba39 511 struct tpacket_kbdq_core *pkc;
f6fb8f10 512
e8e85cc5 513 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
17bfd8c8
KC		 514 timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
515 0);
516 pkc->retire_blk_timer.expires = jiffies;
f6fb8f10 517}
518
519static int prb_calc_retire_blk_tmo(struct packet_sock *po,
520 int blk_size_in_bytes)
521{
522 struct net_device *dev;
523 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
7cad1bac 524 struct ethtool_link_ksettings ecmd;
4bc71cb9 525 int err;
f6fb8f10 526
4bc71cb9
JP		 527 rtnl_lock();
528 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
529 if (unlikely(!dev)) {
530 rtnl_unlock();
f6fb8f10 531 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9 532 }
7cad1bac 533 err = __ethtool_get_link_ksettings(dev, &ecmd);
4bc71cb9
JP		 534 rtnl_unlock();
535 if (!err) {
4bc71cb9
JP		 536 /*
537 * If the link speed is so slow you don't really
538 * need to worry about perf anyways
539 */
7cad1bac
DD		 540 if (ecmd.base.speed < SPEED_1000 ||
541 ecmd.base.speed == SPEED_UNKNOWN) {
4bc71cb9 542 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 543 } else {
544 msec = 1;
7cad1bac 545 div = ecmd.base.speed / 1000;
f6fb8f10 546 }
547 }
548
549 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
550
551 if (div)
552 mbits /= div;
553
554 tmo = mbits * msec;
555
556 if (div)
557 return tmo+1;
558 return tmo;
559}
560
bc59ba39 561static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 562 union tpacket_req_u *req_u)
563{
564 p1->feature_req_word = req_u->req3.tp_feature_req_word;
565}
566
567static void init_prb_bdqc(struct packet_sock *po,
568 struct packet_ring_buffer *rb,
569 struct pgv *pg_vec,
e8e85cc5 570 union tpacket_req_u *req_u)
f6fb8f10 571{
22781a5b 572 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 573 struct tpacket_block_desc *pbd;
f6fb8f10 574
575 memset(p1, 0x0, sizeof(*p1));
576
577 p1->knxt_seq_num = 1;
578 p1->pkbdq = pg_vec;
bc59ba39 579 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 580 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 581 p1->kblk_size = req_u->req3.tp_block_size;
582 p1->knum_blocks = req_u->req3.tp_block_nr;
583 p1->hdrlen = po->tp_hdrlen;
584 p1->version = po->tp_version;
585 p1->last_kactive_blk_num = 0;
ee80fbf3 586 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 587 if (req_u->req3.tp_retire_blk_tov)
588 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
589 else
590 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
591 req_u->req3.tp_block_size);
592 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
593 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
594
dc808110 595 p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
f6fb8f10 596 prb_init_ft_ops(p1, req_u);
e8e85cc5 597 prb_setup_retire_blk_timer(po);
f6fb8f10 598 prb_open_block(p1, pbd);
599}
600
601/* Do NOT update the last_blk_num first.
602 * Assumes sk_buff_head lock is held.
603 */
bc59ba39 604static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 605{
606 mod_timer(&pkc->retire_blk_timer,
607 jiffies + pkc->tov_in_jiffies);
608 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
609}
610
611/*
612 * Timer logic:
613 * 1) We refresh the timer only when we open a block.
614 * By doing this we don't waste cycles refreshing the timer
615 * on packet-by-packet basis.
616 *
617 * With a 1MB block-size, on a 1Gbps line, it will take
618 * i) ~8 ms to fill a block + ii) memcpy etc.
619 * In this cut we are not accounting for the memcpy time.
620 *
621 * So, if the user sets the 'tmo' to 10ms then the timer
622 * will never fire while the block is still getting filled
623 * (which is what we want). However, the user could choose
624 * to close a block early and that's fine.
625 *
626 * But when the timer does fire, we check whether or not to refresh it.
627 * Since the tmo granularity is in msecs, it is not too expensive
628 * to refresh the timer, lets say every '8' msecs.
629 * Either the user can set the 'tmo' or we can derive it based on
630 * a) line-speed and b) block-size.
631 * prb_calc_retire_blk_tmo() calculates the tmo.
632 *
633 */
17bfd8c8 634static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
f6fb8f10 635{
17bfd8c8
KC		 636 struct packet_sock *po =
637 from_timer(po, t, rx_ring.prb_bdqc.retire_blk_timer);
22781a5b 638 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 639 unsigned int frozen;
bc59ba39 640 struct tpacket_block_desc *pbd;
f6fb8f10 641
642 spin_lock(&po->sk.sk_receive_queue.lock);
643
644 frozen = prb_queue_frozen(pkc);
645 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
646
647 if (unlikely(pkc->delete_blk_timer))
648 goto out;
649
650 /* We only need to plug the race when the block is partially filled.
651 * tpacket_rcv:
652 * lock(); increment BLOCK_NUM_PKTS; unlock()
653 * copy_bits() is in progress ...
654 * timer fires on other cpu:
655 * we can't retire the current block because copy_bits
656 * is in progress.
657 *
658 */
659 if (BLOCK_NUM_PKTS(pbd)) {
660 while (atomic_read(&pkc->blk_fill_in_prog)) {
661 /* Waiting for skb_copy_bits to finish... */
662 cpu_relax();
663 }
664 }
665
666 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
667 if (!frozen) {
41a50d62
AD		 668 if (!BLOCK_NUM_PKTS(pbd)) {
669 /* An empty block. Just refresh the timer. */
670 goto refresh_timer;
671 }
f6fb8f10 672 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
673 if (!prb_dispatch_next_block(pkc, po))
674 goto refresh_timer;
675 else
676 goto out;
677 } else {
678 /* Case 1. Queue was frozen because user-space was
679 * lagging behind.
680 */
878cd3ba 681 if (prb_curr_blk_in_use(pbd)) {
f6fb8f10 682 /*
683 * Ok, user-space is still behind.
684 * So just refresh the timer.
685 */
686 goto refresh_timer;
687 } else {
688 /* Case 2. queue was frozen,user-space caught up,
689 * now the link went idle && the timer fired.
690 * We don't have a block to close.So we open this
691 * block and restart the timer.
692 * opening a block thaws the queue,restarts timer
693 * Thawing/timer-refresh is a side effect.
694 */
695 prb_open_block(pkc, pbd);
696 goto out;
697 }
698 }
699 }
700
701refresh_timer:
702 _prb_refresh_rx_retire_blk_timer(pkc);
703
704out:
705 spin_unlock(&po->sk.sk_receive_queue.lock);
706}
707
eea49cc9 708static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 709 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 710{
711 /* Flush everything minus the block header */
712
713#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
714 u8 *start, *end;
715
716 start = (u8 *)pbd1;
717
718 /* Skip the block header(we know header WILL fit in 4K) */
719 start += PAGE_SIZE;
720
721 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
722 for (; start < end; start += PAGE_SIZE)
723 flush_dcache_page(pgv_to_page(start));
724
725 smp_wmb();
726#endif
727
728 /* Now update the block status. */
729
730 BLOCK_STATUS(pbd1) = status;
731
732 /* Flush the block header */
733
734#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
735 start = (u8 *)pbd1;
736 flush_dcache_page(pgv_to_page(start));
737
738 smp_wmb();
739#endif
740}
741
742/*
743 * Side effect:
744 *
745 * 1) flush the block
746 * 2) Increment active_blk_num
747 *
748 * Note:We DONT refresh the timer on purpose.
749 * Because almost always the next block will be opened.
750 */
bc59ba39 751static void prb_close_block(struct tpacket_kbdq_core *pkc1,
752 struct tpacket_block_desc *pbd1,
f6fb8f10 753 struct packet_sock *po, unsigned int stat)
754{
755 __u32 status = TP_STATUS_USER | stat;
756
757 struct tpacket3_hdr *last_pkt;
bc59ba39 758 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
da413eec 759 struct sock *sk = &po->sk;
f6fb8f10 760
ee80fbf3 761 if (po->stats.stats3.tp_drops)
f6fb8f10 762 status |= TP_STATUS_LOSING;
763
764 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
765 last_pkt->tp_next_offset = 0;
766
767 /* Get the ts of the last pkt */
768 if (BLOCK_NUM_PKTS(pbd1)) {
769 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
770 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
771 } else {
41a50d62
AD		 772 /* Ok, we tmo'd - so get the current time.
773 *
774 * It shouldn't really happen as we don't close empty
775 * blocks. See prb_retire_rx_blk_timer_expired().
776 */
f6fb8f10 777 struct timespec ts;
778 getnstimeofday(&ts);
779 h1->ts_last_pkt.ts_sec = ts.tv_sec;
780 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
781 }
782
783 smp_wmb();
784
785 /* Flush the block */
786 prb_flush_block(pkc1, pbd1, status);
787
da413eec
DC		 788 sk->sk_data_ready(sk);
789
f6fb8f10 790 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
791}
792
eea49cc9 793static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 794{
795 pkc->reset_pending_on_curr_blk = 0;
796}
797
798/*
799 * Side effect of opening a block:
800 *
801 * 1) prb_queue is thawed.
802 * 2) retire_blk_timer is refreshed.
803 *
804 */
bc59ba39 805static void prb_open_block(struct tpacket_kbdq_core *pkc1,
806 struct tpacket_block_desc *pbd1)
f6fb8f10 807{
808 struct timespec ts;
bc59ba39 809 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 810
811 smp_rmb();
812
8da3056c
DB		 813 /* We could have just memset this but we will lose the
814 * flexibility of making the priv area sticky
815 */
f6fb8f10 816
8da3056c
DB		 817 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
818 BLOCK_NUM_PKTS(pbd1) = 0;
819 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 820
8da3056c
DB		 821 getnstimeofday(&ts);
822
823 h1->ts_first_pkt.ts_sec = ts.tv_sec;
824 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 825
8da3056c
DB		 826 pkc1->pkblk_start = (char *)pbd1;
827 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
828
829 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
830 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
831
832 pbd1->version = pkc1->version;
833 pkc1->prev = pkc1->nxt_offset;
834 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
835
836 prb_thaw_queue(pkc1);
837 _prb_refresh_rx_retire_blk_timer(pkc1);
838
839 smp_wmb();
f6fb8f10 840}
841
842/*
843 * Queue freeze logic:
844 * 1) Assume tp_block_nr = 8 blocks.
845 * 2) At time 't0', user opens Rx ring.
846 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
847 * 4) user-space is either sleeping or processing block '0'.
848 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
849 * it will close block-7,loop around and try to fill block '0'.
850 * call-flow:
851 * __packet_lookup_frame_in_block
852 * prb_retire_current_block()
853 * prb_dispatch_next_block()
854 * |->(BLOCK_STATUS == USER) evaluates to true
855 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
856 * 6) Now there are two cases:
857 * 6.1) Link goes idle right after the queue is frozen.
858 * But remember, the last open_block() refreshed the timer.
859 * When this timer expires,it will refresh itself so that we can
860 * re-open block-0 in near future.
861 * 6.2) Link is busy and keeps on receiving packets. This is a simple
862 * case and __packet_lookup_frame_in_block will check if block-0
863 * is free and can now be re-used.
864 */
eea49cc9 865static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 866 struct packet_sock *po)
867{
868 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 869 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 870}
871
872#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
873
874/*
875 * If the next block is free then we will dispatch it
876 * and return a good offset.
877 * Else, we will freeze the queue.
878 * So, caller must check the return value.
879 */
bc59ba39 880static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 881 struct packet_sock *po)
882{
bc59ba39 883 struct tpacket_block_desc *pbd;
f6fb8f10 884
885 smp_rmb();
886
887 /* 1. Get current block num */
888 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
889
890 /* 2. If this block is currently in_use then freeze the queue */
891 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
892 prb_freeze_queue(pkc, po);
893 return NULL;
894 }
895
896 /*
897 * 3.
898 * open this block and return the offset where the first packet
899 * needs to get stored.
900 */
901 prb_open_block(pkc, pbd);
902 return (void *)pkc->nxt_offset;
903}
904
bc59ba39 905static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 906 struct packet_sock *po, unsigned int status)
907{
bc59ba39 908 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 909
910 /* retire/close the current block */
911 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
912 /*
913 * Plug the case where copy_bits() is in progress on
914 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
915 * have space to copy the pkt in the current block and
916 * called prb_retire_current_block()
917 *
918 * We don't need to worry about the TMO case because
919 * the timer-handler already handled this case.
920 */
921 if (!(status & TP_STATUS_BLK_TMO)) {
922 while (atomic_read(&pkc->blk_fill_in_prog)) {
923 /* Waiting for skb_copy_bits to finish... */
924 cpu_relax();
925 }
926 }
927 prb_close_block(pkc, pbd, po, status);
928 return;
929 }
f6fb8f10 930}
931
878cd3ba 932static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
f6fb8f10 933{
934 return TP_STATUS_USER & BLOCK_STATUS(pbd);
935}
936
eea49cc9 937static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 938{
939 return pkc->reset_pending_on_curr_blk;
940}
941
eea49cc9 942static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 943{
bc59ba39 944 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 945 atomic_dec(&pkc->blk_fill_in_prog);
946}
947
eea49cc9 948static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 949 struct tpacket3_hdr *ppd)
950{
3958afa1 951 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 952}
953
eea49cc9 954static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 955 struct tpacket3_hdr *ppd)
956{
957 ppd->hv1.tp_rxhash = 0;
958}
959
eea49cc9 960static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 961 struct tpacket3_hdr *ppd)
962{
df8a39de
JP		 963 if (skb_vlan_tag_present(pkc->skb)) {
964 ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
a0cdfcf3
AW		 965 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
966 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 967 } else {
9e67030a 968 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 969 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 970 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 971 }
972}
973
bc59ba39 974static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 975 struct tpacket3_hdr *ppd)
976{
a0cdfcf3 977 ppd->hv1.tp_padding = 0;
f6fb8f10 978 prb_fill_vlan_info(pkc, ppd);
979
980 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
981 prb_fill_rxhash(pkc, ppd);
982 else
983 prb_clear_rxhash(pkc, ppd);
984}
985
eea49cc9 986static void prb_fill_curr_block(char *curr,
bc59ba39 987 struct tpacket_kbdq_core *pkc,
988 struct tpacket_block_desc *pbd,
f6fb8f10 989 unsigned int len)
990{
991 struct tpacket3_hdr *ppd;
992
993 ppd = (struct tpacket3_hdr *)curr;
994 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
995 pkc->prev = curr;
996 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
997 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
998 BLOCK_NUM_PKTS(pbd) += 1;
999 atomic_inc(&pkc->blk_fill_in_prog);
1000 prb_run_all_ft_ops(pkc, ppd);
1001}
1002
1003/* Assumes caller has the sk->rx_queue.lock */
1004static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1005 struct sk_buff *skb,
f6fb8f10 1006 unsigned int len
1007 )
1008{
bc59ba39 1009 struct tpacket_kbdq_core *pkc;
1010 struct tpacket_block_desc *pbd;
f6fb8f10 1011 char *curr, *end;
1012
e3192690 1013 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1014 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1015
1016 /* Queue is frozen when user space is lagging behind */
1017 if (prb_queue_frozen(pkc)) {
1018 /*
1019 * Check if that last block which caused the queue to freeze,
1020 * is still in_use by user-space.
1021 */
878cd3ba 1022 if (prb_curr_blk_in_use(pbd)) {
f6fb8f10 1023 /* Can't record this packet */
1024 return NULL;
1025 } else {
1026 /*
1027 * Ok, the block was released by user-space.
1028 * Now let's open that block.
1029 * opening a block also thaws the queue.
1030 * Thawing is a side effect.
1031 */
1032 prb_open_block(pkc, pbd);
1033 }
1034 }
1035
1036 smp_mb();
1037 curr = pkc->nxt_offset;
1038 pkc->skb = skb;
e3192690 1039 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1040
1041 /* first try the current block */
1042 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1043 prb_fill_curr_block(curr, pkc, pbd, len);
1044 return (void *)curr;
1045 }
1046
1047 /* Ok, close the current block */
1048 prb_retire_current_block(pkc, po, 0);
1049
1050 /* Now, try to dispatch the next block */
1051 curr = (char *)prb_dispatch_next_block(pkc, po);
1052 if (curr) {
1053 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1054 prb_fill_curr_block(curr, pkc, pbd, len);
1055 return (void *)curr;
1056 }
1057
1058 /*
1059 * No free blocks are available.user_space hasn't caught up yet.
1060 * Queue was just frozen and now this packet will get dropped.
1061 */
1062 return NULL;
1063}
1064
eea49cc9 1065static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1066 struct sk_buff *skb,
1067 int status, unsigned int len)
1068{
1069 char *curr = NULL;
1070 switch (po->tp_version) {
1071 case TPACKET_V1:
1072 case TPACKET_V2:
1073 curr = packet_lookup_frame(po, &po->rx_ring,
1074 po->rx_ring.head, status);
1075 return curr;
1076 case TPACKET_V3:
46088059 1077 return __packet_lookup_frame_in_block(po, skb, len);
f6fb8f10 1078 default:
1079 WARN(1, "TPACKET version not supported\n");
1080 BUG();
99aa3473 1081 return NULL;
f6fb8f10 1082 }
1083}
1084
dcf70cef
ED		 1085static void *prb_lookup_block(const struct packet_sock *po,
1086 const struct packet_ring_buffer *rb,
1087 unsigned int idx,
1088 int status)
f6fb8f10 1089{
bc59ba39 1090 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1091 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1092
1093 if (status != BLOCK_STATUS(pbd))
1094 return NULL;
1095 return pbd;
1096}
1097
eea49cc9 1098static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1099{
1100 unsigned int prev;
1101 if (rb->prb_bdqc.kactive_blk_num)
1102 prev = rb->prb_bdqc.kactive_blk_num-1;
1103 else
1104 prev = rb->prb_bdqc.knum_blocks-1;
1105 return prev;
1106}
1107
1108/* Assumes caller has held the rx_queue.lock */
eea49cc9 1109static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1110 struct packet_ring_buffer *rb,
1111 int status)
1112{
1113 unsigned int previous = prb_previous_blk_num(rb);
1114 return prb_lookup_block(po, rb, previous, status);
1115}
1116
eea49cc9 1117static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1118 struct packet_ring_buffer *rb,
1119 int status)
1120{
1121 if (po->tp_version <= TPACKET_V2)
1122 return packet_previous_frame(po, rb, status);
1123
1124 return __prb_previous_block(po, rb, status);
1125}
1126
eea49cc9 1127static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1128 struct packet_ring_buffer *rb)
1129{
1130 switch (po->tp_version) {
1131 case TPACKET_V1:
1132 case TPACKET_V2:
1133 return packet_increment_head(rb);
1134 case TPACKET_V3:
1135 default:
1136 WARN(1, "TPACKET version not supported.\n");
1137 BUG();
1138 return;
1139 }
1140}
1141
eea49cc9 1142static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1143 struct packet_ring_buffer *rb,
1144 int status)
1145{
1146 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1147 return packet_lookup_frame(po, rb, previous, status);
1148}
1149
eea49cc9 1150static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1151{
1152 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1153}
1154
b0138408
DB		 1155static void packet_inc_pending(struct packet_ring_buffer *rb)
1156{
1157 this_cpu_inc(*rb->pending_refcnt);
1158}
1159
1160static void packet_dec_pending(struct packet_ring_buffer *rb)
1161{
1162 this_cpu_dec(*rb->pending_refcnt);
1163}
1164
1165static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1166{
1167 unsigned int refcnt = 0;
1168 int cpu;
1169
1170 /* We don't use pending refcount in rx_ring. */
1171 if (rb->pending_refcnt == NULL)
1172 return 0;
1173
1174 for_each_possible_cpu(cpu)
1175 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1176
1177 return refcnt;
1178}
1179
1180static int packet_alloc_pending(struct packet_sock *po)
1181{
1182 po->rx_ring.pending_refcnt = NULL;
1183
1184 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1185 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1186 return -ENOBUFS;
1187
1188 return 0;
1189}
1190
1191static void packet_free_pending(struct packet_sock *po)
1192{
1193 free_percpu(po->tx_ring.pending_refcnt);
1194}
1195
9954729b
WB		 1196#define ROOM_POW_OFF 2
1197#define ROOM_NONE 0x0
1198#define ROOM_LOW 0x1
1199#define ROOM_NORMAL 0x2
1200
d4b5bd98 1201static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
77f65ebd 1202{
9954729b
WB		 1203 int idx, len;
1204
d4b5bd98
ED		 1205 len = READ_ONCE(po->rx_ring.frame_max) + 1;
1206 idx = READ_ONCE(po->rx_ring.head);
9954729b
WB		 1207 if (pow_off)
1208 idx += len >> pow_off;
1209 if (idx >= len)
1210 idx -= len;
1211 return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1212}
1213
dcf70cef 1214static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
9954729b
WB		 1215{
1216 int idx, len;
1217
dcf70cef
ED		 1218 len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
1219 idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
9954729b
WB		 1220 if (pow_off)
1221 idx += len >> pow_off;
1222 if (idx >= len)
1223 idx -= len;
1224 return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
1225}
77f65ebd 1226
2ccdbaa6 1227static int __packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
9954729b
WB		 1228{
1229 struct sock *sk = &po->sk;
1230 int ret = ROOM_NONE;
1231
1232 if (po->prot_hook.func != tpacket_rcv) {
1233 int avail = sk->sk_rcvbuf - atomic_read(&sk->sk_rmem_alloc)
2ccdbaa6 1234 - (skb ? skb->truesize : 0);
9954729b
WB		 1235 if (avail > (sk->sk_rcvbuf >> ROOM_POW_OFF))
1236 return ROOM_NORMAL;
1237 else if (avail > 0)
1238 return ROOM_LOW;
1239 else
1240 return ROOM_NONE;
1241 }
77f65ebd 1242
9954729b
WB		 1243 if (po->tp_version == TPACKET_V3) {
1244 if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1245 ret = ROOM_NORMAL;
1246 else if (__tpacket_v3_has_room(po, 0))
1247 ret = ROOM_LOW;
1248 } else {
1249 if (__tpacket_has_room(po, ROOM_POW_OFF))
1250 ret = ROOM_NORMAL;
1251 else if (__tpacket_has_room(po, 0))
1252 ret = ROOM_LOW;
1253 }
2ccdbaa6
WB		 1254
1255 return ret;
1256}
1257
1258static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1259{
1260 int ret;
1261 bool has_room;
1262
54d7c01d
WB		 1263 spin_lock_bh(&po->sk.sk_receive_queue.lock);
1264 ret = __packet_rcv_has_room(po, skb);
2ccdbaa6
WB		 1265 has_room = ret == ROOM_NORMAL;
1266 if (po->pressure == has_room)
54d7c01d
WB		 1267 po->pressure = !has_room;
1268 spin_unlock_bh(&po->sk.sk_receive_queue.lock);
77f65ebd 1269
9954729b 1270 return ret;
77f65ebd
WB		 1271}
1272
1da177e4
LT		 1273static void packet_sock_destruct(struct sock *sk)
1274{
ed85b565
RC		 1275 skb_queue_purge(&sk->sk_error_queue);
1276
547b792c 1277 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
14afee4b 1278 WARN_ON(refcount_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1279
1280 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1281 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1282 return;
1283 }
1284
17ab56a2 1285 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1286}
1287
3b3a5b0a
WB		 1288static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1289{
1290 u32 rxhash;
1291 int i, count = 0;
1292
1293 rxhash = skb_get_hash(skb);
1294 for (i = 0; i < ROLLOVER_HLEN; i++)
1295 if (po->rollover->history[i] == rxhash)
1296 count++;
1297
1298 po->rollover->history[prandom_u32() % ROLLOVER_HLEN] = rxhash;
1299 return count > (ROLLOVER_HLEN >> 1);
1300}
1301
77f65ebd
WB		 1302static unsigned int fanout_demux_hash(struct packet_fanout *f,
1303 struct sk_buff *skb,
1304 unsigned int num)
dc99f600 1305{
eb70db87 1306 return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
dc99f600
DM		 1307}
1308
77f65ebd
WB		 1309static unsigned int fanout_demux_lb(struct packet_fanout *f,
1310 struct sk_buff *skb,
1311 unsigned int num)
dc99f600 1312{
468479e6 1313 unsigned int val = atomic_inc_return(&f->rr_cur);
dc99f600 1314
468479e6 1315 return val % num;
77f65ebd
WB		 1316}
1317
1318static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1319 struct sk_buff *skb,
1320 unsigned int num)
1321{
1322 return smp_processor_id() % num;
dc99f600
DM		 1323}
1324
5df0ddfb
DB		 1325static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1326 struct sk_buff *skb,
1327 unsigned int num)
1328{
f337db64 1329 return prandom_u32_max(num);
5df0ddfb
DB		 1330}
1331
77f65ebd
WB		 1332static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1333 struct sk_buff *skb,
ad377cab 1334 unsigned int idx, bool try_self,
77f65ebd 1335 unsigned int num)
95ec3eb4 1336{
4633c9e0 1337 struct packet_sock *po, *po_next, *po_skip = NULL;
a9b63918 1338 unsigned int i, j, room = ROOM_NONE;
95ec3eb4 1339
0648ab70 1340 po = pkt_sk(f->arr[idx]);
3b3a5b0a
WB		 1341
1342 if (try_self) {
1343 room = packet_rcv_has_room(po, skb);
1344 if (room == ROOM_NORMAL ||
1345 (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1346 return idx;
4633c9e0 1347 po_skip = po;
3b3a5b0a 1348 }
ad377cab 1349
0648ab70 1350 i = j = min_t(int, po->rollover->sock, num - 1);
77f65ebd 1351 do {
2ccdbaa6 1352 po_next = pkt_sk(f->arr[i]);
4633c9e0 1353 if (po_next != po_skip && !po_next->pressure &&
2ccdbaa6 1354 packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
77f65ebd 1355 if (i != j)
0648ab70 1356 po->rollover->sock = i;
a9b63918
WB		 1357 atomic_long_inc(&po->rollover->num);
1358 if (room == ROOM_LOW)
1359 atomic_long_inc(&po->rollover->num_huge);
77f65ebd
WB		 1360 return i;
1361 }
ad377cab 1362
77f65ebd
WB		 1363 if (++i == num)
1364 i = 0;
1365 } while (i != j);
1366
a9b63918 1367 atomic_long_inc(&po->rollover->num_failed);
77f65ebd
WB		 1368 return idx;
1369}
1370
2d36097d
NH		 1371static unsigned int fanout_demux_qm(struct packet_fanout *f,
1372 struct sk_buff *skb,
1373 unsigned int num)
1374{
1375 return skb_get_queue_mapping(skb) % num;
1376}
1377
47dceb8e
WB		 1378static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1379 struct sk_buff *skb,
1380 unsigned int num)
1381{
1382 struct bpf_prog *prog;
1383 unsigned int ret = 0;
1384
1385 rcu_read_lock();
1386 prog = rcu_dereference(f->bpf_prog);
1387 if (prog)
ff936a04 1388 ret = bpf_prog_run_clear_cb(prog, skb) % num;
47dceb8e
WB		 1389 rcu_read_unlock();
1390
1391 return ret;
1392}
1393
77f65ebd
WB		 1394static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1395{
1396 return f->flags & (flag >> 8);
95ec3eb4
DM		 1397}
1398
95ec3eb4
DM		 1399static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1400 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1401{
1402 struct packet_fanout *f = pt->af_packet_priv;
f98f4514 1403 unsigned int num = READ_ONCE(f->num_members);
19bcf9f2 1404 struct net *net = read_pnet(&f->net);
dc99f600 1405 struct packet_sock *po;
77f65ebd 1406 unsigned int idx;
dc99f600 1407
19bcf9f2 1408 if (!net_eq(dev_net(dev), net) || !num) {
dc99f600
DM		 1409 kfree_skb(skb);
1410 return 0;
1411 }
1412
3f34b24a 1413 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
19bcf9f2 1414 skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
3f34b24a
AD		 1415 if (!skb)
1416 return 0;
1417 }
95ec3eb4
DM		 1418 switch (f->type) {
1419 case PACKET_FANOUT_HASH:
1420 default:
77f65ebd 1421 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1422 break;
1423 case PACKET_FANOUT_LB:
77f65ebd 1424 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1425 break;
1426 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1427 idx = fanout_demux_cpu(f, skb, num);
1428 break;
5df0ddfb
DB		 1429 case PACKET_FANOUT_RND:
1430 idx = fanout_demux_rnd(f, skb, num);
1431 break;
2d36097d
NH		 1432 case PACKET_FANOUT_QM:
1433 idx = fanout_demux_qm(f, skb, num);
1434 break;
77f65ebd 1435 case PACKET_FANOUT_ROLLOVER:
ad377cab 1436 idx = fanout_demux_rollover(f, skb, 0, false, num);
95ec3eb4 1437 break;
47dceb8e 1438 case PACKET_FANOUT_CBPF:
f2e52095 1439 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1440 idx = fanout_demux_bpf(f, skb, num);
1441 break;
dc99f600
DM		 1442 }
1443
ad377cab
WB		 1444 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1445 idx = fanout_demux_rollover(f, skb, idx, true, num);
dc99f600 1446
ad377cab 1447 po = pkt_sk(f->arr[idx]);
dc99f600
DM		 1448 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1449}
1450
fff3321d
PE		 1451DEFINE_MUTEX(fanout_mutex);
1452EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600 1453static LIST_HEAD(fanout_list);
4a69a864 1454static u16 fanout_next_id;
dc99f600
DM		 1455
1456static void __fanout_link(struct sock *sk, struct packet_sock *po)
1457{
1458 struct packet_fanout *f = po->fanout;
1459
1460 spin_lock(&f->lock);
1461 f->arr[f->num_members] = sk;
1462 smp_wmb();
1463 f->num_members++;
2bd624b4
AS		 1464 if (f->num_members == 1)
1465 dev_add_pack(&f->prot_hook);
dc99f600
DM		 1466 spin_unlock(&f->lock);
1467}
1468
1469static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1470{
1471 struct packet_fanout *f = po->fanout;
1472 int i;
1473
1474 spin_lock(&f->lock);
1475 for (i = 0; i < f->num_members; i++) {
1476 if (f->arr[i] == sk)
1477 break;
1478 }
1479 BUG_ON(i >= f->num_members);
1480 f->arr[i] = f->arr[f->num_members - 1];
1481 f->num_members--;
2bd624b4
AS		 1482 if (f->num_members == 0)
1483 __dev_remove_pack(&f->prot_hook);
dc99f600
DM		 1484 spin_unlock(&f->lock);
1485}
1486
d4dd8aee 1487static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1488{
161642e2
ED		 1489 if (sk->sk_family != PF_PACKET)
1490 return false;
c0de08d0 1491
161642e2 1492 return ptype->af_packet_priv == pkt_sk(sk)->fanout;
c0de08d0
EL		 1493}
1494
47dceb8e
WB		 1495static void fanout_init_data(struct packet_fanout *f)
1496{
1497 switch (f->type) {
1498 case PACKET_FANOUT_LB:
1499 atomic_set(&f->rr_cur, 0);
1500 break;
1501 case PACKET_FANOUT_CBPF:
f2e52095 1502 case PACKET_FANOUT_EBPF:
47dceb8e
WB		 1503 RCU_INIT_POINTER(f->bpf_prog, NULL);
1504 break;
1505 }
1506}
1507
1508static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1509{
1510 struct bpf_prog *old;
1511
1512 spin_lock(&f->lock);
1513 old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1514 rcu_assign_pointer(f->bpf_prog, new);
1515 spin_unlock(&f->lock);
1516
1517 if (old) {
1518 synchronize_net();
1519 bpf_prog_destroy(old);
1520 }
1521}
1522
1523static int fanout_set_data_cbpf(struct packet_sock *po, char __user *data,
1524 unsigned int len)
1525{
1526 struct bpf_prog *new;
1527 struct sock_fprog fprog;
1528 int ret;
1529
1530 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1531 return -EPERM;
1532 if (len != sizeof(fprog))
1533 return -EINVAL;
1534 if (copy_from_user(&fprog, data, len))
1535 return -EFAULT;
1536
bab18991 1537 ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
47dceb8e
WB		 1538 if (ret)
1539 return ret;
1540
1541 __fanout_set_data_bpf(po->fanout, new);
1542 return 0;
1543}
1544
f2e52095
WB		 1545static int fanout_set_data_ebpf(struct packet_sock *po, char __user *data,
1546 unsigned int len)
1547{
1548 struct bpf_prog *new;
1549 u32 fd;
1550
1551 if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
1552 return -EPERM;
1553 if (len != sizeof(fd))
1554 return -EINVAL;
1555 if (copy_from_user(&fd, data, len))
1556 return -EFAULT;
1557
113214be 1558 new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
f2e52095
WB		 1559 if (IS_ERR(new))
1560 return PTR_ERR(new);
f2e52095
WB		 1561
1562 __fanout_set_data_bpf(po->fanout, new);
1563 return 0;
1564}
1565
47dceb8e
WB		 1566static int fanout_set_data(struct packet_sock *po, char __user *data,
1567 unsigned int len)
1568{
1569 switch (po->fanout->type) {
1570 case PACKET_FANOUT_CBPF:
1571 return fanout_set_data_cbpf(po, data, len);
f2e52095
WB		 1572 case PACKET_FANOUT_EBPF:
1573 return fanout_set_data_ebpf(po, data, len);
47dceb8e
WB		 1574 default:
1575 return -EINVAL;
07d53ae4 1576 }
47dceb8e
WB		 1577}
1578
1579static void fanout_release_data(struct packet_fanout *f)
1580{
1581 switch (f->type) {
1582 case PACKET_FANOUT_CBPF:
f2e52095 1583 case PACKET_FANOUT_EBPF:
47dceb8e 1584 __fanout_set_data_bpf(f, NULL);
07d53ae4 1585 }
47dceb8e
WB		 1586}
1587
4a69a864
MM		 1588static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
1589{
1590 struct packet_fanout *f;
1591
1592 list_for_each_entry(f, &fanout_list, list) {
1593 if (f->id == candidate_id &&
1594 read_pnet(&f->net) == sock_net(sk)) {
1595 return false;
1596 }
1597 }
1598 return true;
1599}
1600
1601static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
1602{
1603 u16 id = fanout_next_id;
1604
1605 do {
1606 if (__fanout_id_is_free(sk, id)) {
1607 *new_id = id;
1608 fanout_next_id = id + 1;
1609 return true;
1610 }
1611
1612 id++;
1613 } while (id != fanout_next_id);
1614
1615 return false;
1616}
1617
7736d33f 1618static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600 1619{
d199fab6 1620 struct packet_rollover *rollover = NULL;
dc99f600
DM		 1621 struct packet_sock *po = pkt_sk(sk);
1622 struct packet_fanout *f, *match;
7736d33f 1623 u8 type = type_flags & 0xff;
77f65ebd 1624 u8 flags = type_flags >> 8;
dc99f600
DM		 1625 int err;
1626
1627 switch (type) {
77f65ebd
WB		 1628 case PACKET_FANOUT_ROLLOVER:
1629 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1630 return -EINVAL;
dc99f600
DM		 1631 case PACKET_FANOUT_HASH:
1632 case PACKET_FANOUT_LB:
95ec3eb4 1633 case PACKET_FANOUT_CPU:
5df0ddfb 1634 case PACKET_FANOUT_RND:
2d36097d 1635 case PACKET_FANOUT_QM:
47dceb8e 1636 case PACKET_FANOUT_CBPF:
f2e52095 1637 case PACKET_FANOUT_EBPF:
dc99f600
DM		 1638 break;
1639 default:
1640 return -EINVAL;
1641 }
1642
d199fab6
ED		 1643 mutex_lock(&fanout_mutex);
1644
d199fab6 1645 err = -EALREADY;
dc99f600 1646 if (po->fanout)
d199fab6 1647 goto out;
dc99f600 1648
4633c9e0
WB		 1649 if (type == PACKET_FANOUT_ROLLOVER ||
1650 (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
d199fab6
ED		 1651 err = -ENOMEM;
1652 rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1653 if (!rollover)
1654 goto out;
1655 atomic_long_set(&rollover->num, 0);
1656 atomic_long_set(&rollover->num_huge, 0);
1657 atomic_long_set(&rollover->num_failed, 0);
0648ab70
WB		 1658 }
1659
4a69a864
MM		 1660 if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
1661 if (id != 0) {
1662 err = -EINVAL;
1663 goto out;
1664 }
1665 if (!fanout_find_new_id(sk, &id)) {
1666 err = -ENOMEM;
1667 goto out;
1668 }
1669 /* ephemeral flag for the first socket in the group: drop it */
1670 flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
1671 }
1672
dc99f600
DM		 1673 match = NULL;
1674 list_for_each_entry(f, &fanout_list, list) {
1675 if (f->id == id &&
1676 read_pnet(&f->net) == sock_net(sk)) {
1677 match = f;
1678 break;
1679 }
1680 }
afe62c68 1681 err = -EINVAL;
77f65ebd 1682 if (match && match->flags != flags)
afe62c68 1683 goto out;
dc99f600 1684 if (!match) {
afe62c68 1685 err = -ENOMEM;
dc99f600 1686 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1687 if (!match)
1688 goto out;
1689 write_pnet(&match->net, sock_net(sk));
1690 match->id = id;
1691 match->type = type;
77f65ebd 1692 match->flags = flags;
afe62c68
ED		 1693 INIT_LIST_HEAD(&match->list);
1694 spin_lock_init(&match->lock);
fb5c2c17 1695 refcount_set(&match->sk_ref, 0);
47dceb8e 1696 fanout_init_data(match);
afe62c68
ED		 1697 match->prot_hook.type = po->prot_hook.type;
1698 match->prot_hook.dev = po->prot_hook.dev;
1699 match->prot_hook.func = packet_rcv_fanout;
1700 match->prot_hook.af_packet_priv = match;
c0de08d0 1701 match->prot_hook.id_match = match_fanout_group;
afe62c68 1702 list_add(&match->list, &fanout_list);
dc99f600 1703 }
afe62c68 1704 err = -EINVAL;
008ba2a1
WB		 1705
1706 spin_lock(&po->bind_lock);
1707 if (po->running &&
1708 match->type == type &&
afe62c68
ED		 1709 match->prot_hook.type == po->prot_hook.type &&
1710 match->prot_hook.dev == po->prot_hook.dev) {
1711 err = -ENOSPC;
fb5c2c17 1712 if (refcount_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
afe62c68
ED		 1713 __dev_remove_pack(&po->prot_hook);
1714 po->fanout = match;
57f015f5
MM		 1715 po->rollover = rollover;
1716 rollover = NULL;
fb5c2c17 1717 refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
afe62c68
ED		 1718 __fanout_link(sk, po);
1719 err = 0;
dc99f600
DM		 1720 }
1721 }
008ba2a1
WB		 1722 spin_unlock(&po->bind_lock);
1723
1724 if (err && !refcount_read(&match->sk_ref)) {
1725 list_del(&match->list);
1726 kfree(match);
1727 }
1728
afe62c68 1729out:
57f015f5 1730 kfree(rollover);
d199fab6 1731 mutex_unlock(&fanout_mutex);
dc99f600
DM		 1732 return err;
1733}
1734
2bd624b4
AS		 1735/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1736 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1737 * It is the responsibility of the caller to call fanout_release_data() and
1738 * free the returned packet_fanout (after synchronize_net())
1739 */
1740static struct packet_fanout *fanout_release(struct sock *sk)
dc99f600
DM		 1741{
1742 struct packet_sock *po = pkt_sk(sk);
1743 struct packet_fanout *f;
1744
fff3321d 1745 mutex_lock(&fanout_mutex);
d199fab6
ED		 1746 f = po->fanout;
1747 if (f) {
1748 po->fanout = NULL;
1749
fb5c2c17 1750 if (refcount_dec_and_test(&f->sk_ref))
d199fab6 1751 list_del(&f->list);
2bd624b4
AS		 1752 else
1753 f = NULL;
dc99f600
DM		 1754 }
1755 mutex_unlock(&fanout_mutex);
2bd624b4
AS		 1756
1757 return f;
dc99f600 1758}
1da177e4 1759
3c70c132
DB		 1760static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1761 struct sk_buff *skb)
1762{
1763 /* Earlier code assumed this would be a VLAN pkt, double-check
1764 * this now that we have the actual packet in hand. We can only
1765 * do this check on Ethernet devices.
1766 */
1767 if (unlikely(dev->type != ARPHRD_ETHER))
1768 return false;
1769
1770 skb_reset_mac_header(skb);
1771 return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1772}
1773
90ddc4f0 1774static const struct proto_ops packet_ops;
1da177e4 1775
90ddc4f0 1776static const struct proto_ops packet_ops_spkt;
1da177e4 1777
40d4e3df
ED		 1778static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1779 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1780{
1781 struct sock *sk;
1782 struct sockaddr_pkt *spkt;
1783
1784 /*
1785 * When we registered the protocol we saved the socket in the data
1786 * field for just this event.
1787 */
1788
1789 sk = pt->af_packet_priv;
1ce4f28b 1790
1da177e4
LT		 1791 /*
1792 * Yank back the headers [hope the device set this
1793 * right or kerboom...]
1794 *
1795 * Incoming packets have ll header pulled,
1796 * push it back.
1797 *
98e399f8 1798 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1799 * so that this procedure is noop.
1800 */
1801
1802 if (skb->pkt_type == PACKET_LOOPBACK)
1803 goto out;
1804
09ad9bc7 1805 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1806 goto out;
1807
40d4e3df
ED		 1808 skb = skb_share_check(skb, GFP_ATOMIC);
1809 if (skb == NULL)
1da177e4
LT		 1810 goto oom;
1811
1812 /* drop any routing info */
adf30907 1813 skb_dst_drop(skb);
1da177e4 1814
84531c24
PO		 1815 /* drop conntrack reference */
1816 nf_reset(skb);
1817
ffbc6111 1818 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1819
98e399f8 1820 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1821
1822 /*
1823 * The SOCK_PACKET socket receives _all_ frames.
1824 */
1825
1826 spkt->spkt_family = dev->type;
1827 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1828 spkt->spkt_protocol = skb->protocol;
1829
1830 /*
1831 * Charge the memory to the socket. This is done specifically
1832 * to prevent sockets using all the memory up.
1833 */
1834
40d4e3df 1835 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1836 return 0;
1837
1838out:
1839 kfree_skb(skb);
1840oom:
1841 return 0;
1842}
1843
75c65772
MM		 1844static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
1845{
18bed891
YK		 1846 if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) &&
1847 sock->type == SOCK_RAW) {
75c65772
MM		 1848 skb_reset_mac_header(skb);
1849 skb->protocol = dev_parse_header_protocol(skb);
1850 }
1851
1852 skb_probe_transport_header(skb);
1853}
1da177e4
LT		 1854
1855/*
1856 * Output a raw packet to a device layer. This bypasses all the other
1857 * protocol layers and you must therefore supply it with a complete frame
1858 */
1ce4f28b 1859
1b784140
YX		 1860static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1861 size_t len)
1da177e4
LT		 1862{
1863 struct sock *sk = sock->sk;
342dfc30 1864 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1865 struct sk_buff *skb = NULL;
1da177e4 1866 struct net_device *dev;
c14ac945 1867 struct sockcm_cookie sockc;
40d4e3df 1868 __be16 proto = 0;
1da177e4 1869 int err;
3bdc0eba 1870 int extra_len = 0;
1ce4f28b 1871
1da177e4 1872 /*
1ce4f28b 1873 * Get and verify the address.
1da177e4
LT		 1874 */
1875
40d4e3df 1876 if (saddr) {
1da177e4 1877 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1878 return -EINVAL;
1879 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1880 proto = saddr->spkt_protocol;
1881 } else
1882 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1883
1884 /*
1ce4f28b 1885 * Find the device first to size check it
1da177e4
LT		 1886 */
1887
de74e92a 1888 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1889retry:
654d1f8a
ED		 1890 rcu_read_lock();
1891 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1892 err = -ENODEV;
1893 if (dev == NULL)
1894 goto out_unlock;
1ce4f28b 1895
d5e76b0a
DM		 1896 err = -ENETDOWN;
1897 if (!(dev->flags & IFF_UP))
1898 goto out_unlock;
1899
1da177e4 1900 /*
40d4e3df
ED		 1901 * You may not queue a frame bigger than the mtu. This is the lowest level
1902 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1903 */
1ce4f28b 1904
3bdc0eba
BG		 1905 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1906 if (!netif_supports_nofcs(dev)) {
1907 err = -EPROTONOSUPPORT;
1908 goto out_unlock;
1909 }
1910 extra_len = 4; /* We're doing our own CRC */
1911 }
1912
1da177e4 1913 err = -EMSGSIZE;
3bdc0eba 1914 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1915 goto out_unlock;
1916
1a35ca80
ED		 1917 if (!skb) {
1918 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1919 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1920 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1921
1922 rcu_read_unlock();
4ce40912 1923 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1924 if (skb == NULL)
1925 return -ENOBUFS;
1926 /* FIXME: Save some space for broken drivers that write a hard
1927 * header at transmission time by themselves. PPP is the notable
1928 * one here. This should really be fixed at the driver level.
1929 */
1930 skb_reserve(skb, reserved);
1931 skb_reset_network_header(skb);
1932
1933 /* Try to align data part correctly */
1934 if (hhlen) {
1935 skb->data -= hhlen;
1936 skb->tail -= hhlen;
1937 if (len < hhlen)
1938 skb_reset_network_header(skb);
1939 }
6ce8e9ce 1940 err = memcpy_from_msg(skb_put(skb, len), msg, len);
1a35ca80
ED		 1941 if (err)
1942 goto out_free;
1943 goto retry;
1da177e4
LT		 1944 }
1945
9ed988cd
WB		 1946 if (!dev_validate_header(dev, skb->data, len)) {
1947 err = -EINVAL;
1948 goto out_unlock;
1949 }
3c70c132
DB		 1950 if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
1951 !packet_extra_vlan_len_allowed(dev, skb)) {
1952 err = -EMSGSIZE;
1953 goto out_unlock;
57f89bfa 1954 }
1a35ca80 1955
657a0667 1956 sockcm_init(&sockc, sk);
c14ac945
SHY		 1957 if (msg->msg_controllen) {
1958 err = sock_cmsg_send(sk, msg, &sockc);
f8e7718c 1959 if (unlikely(err))
c14ac945 1960 goto out_unlock;
c14ac945
SHY		 1961 }
1962
1da177e4
LT		 1963 skb->protocol = proto;
1964 skb->dev = dev;
1965 skb->priority = sk->sk_priority;
2d37a186 1966 skb->mark = sk->sk_mark;
3d0ba8c0 1967 skb->tstamp = sockc.transmit_time;
bf84a010 1968
8f932f76 1969 skb_setup_tx_timestamp(skb, sockc.tsflags);
1da177e4 1970
3bdc0eba
BG		 1971 if (unlikely(extra_len == 4))
1972 skb->no_fcs = 1;
1973
75c65772 1974 packet_parse_headers(skb, sock);
c1aad275 1975
1da177e4 1976 dev_queue_xmit(skb);
654d1f8a 1977 rcu_read_unlock();
40d4e3df 1978 return len;
1da177e4 1979
1da177e4 1980out_unlock:
654d1f8a 1981 rcu_read_unlock();
1a35ca80
ED		 1982out_free:
1983 kfree_skb(skb);
1da177e4
LT		 1984 return err;
1985}
1da177e4 1986
ff936a04
AS		 1987static unsigned int run_filter(struct sk_buff *skb,
1988 const struct sock *sk,
1989 unsigned int res)
1da177e4
LT		 1990{
1991 struct sk_filter *filter;
fda9ef5d 1992
80f8f102
ED		 1993 rcu_read_lock();
1994 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1995 if (filter != NULL)
ff936a04 1996 res = bpf_prog_run_clear_cb(filter->prog, skb);
80f8f102 1997 rcu_read_unlock();
1da177e4 1998
dbcb5855 1999 return res;
1da177e4
LT		 2000}
2001
16cc1400
WB		 2002static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2003 size_t *len)
2004{
2005 struct virtio_net_hdr vnet_hdr;
2006
2007 if (*len < sizeof(vnet_hdr))
2008 return -EINVAL;
2009 *len -= sizeof(vnet_hdr);
2010
fd3a8862 2011 if (virtio_net_hdr_from_skb(skb, &vnet_hdr, vio_le(), true, 0))
16cc1400
WB		 2012 return -EINVAL;
2013
2014 return memcpy_to_msg(msg, (void *)&vnet_hdr, sizeof(vnet_hdr));
2015}
2016
1da177e4 2017/*
62ab0812
ED		 2018 * This function makes lazy skb cloning in hope that most of packets
2019 * are discarded by BPF.
2020 *
2021 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
2022 * and skb->cb are mangled. It works because (and until) packets
2023 * falling here are owned by current CPU. Output packets are cloned
2024 * by dev_queue_xmit_nit(), input packets are processed by net_bh
2025 * sequencially, so that if we return skb to original state on exit,
2026 * we will not harm anyone.
1da177e4
LT		 2027 */
2028
40d4e3df
ED		 2029static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2030 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 2031{
2032 struct sock *sk;
2033 struct sockaddr_ll *sll;
2034 struct packet_sock *po;
40d4e3df 2035 u8 *skb_head = skb->data;
1da177e4 2036 int skb_len = skb->len;
dbcb5855 2037 unsigned int snaplen, res;
da37845f 2038 bool is_drop_n_account = false;
1da177e4
LT		 2039
2040 if (skb->pkt_type == PACKET_LOOPBACK)
2041 goto drop;
2042
2043 sk = pt->af_packet_priv;
2044 po = pkt_sk(sk);
2045
09ad9bc7 2046 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2047 goto drop;
2048
1da177e4
LT		 2049 skb->dev = dev;
2050
3b04ddde 2051 if (dev->header_ops) {
1da177e4 2052 /* The device has an explicit notion of ll header,
62ab0812
ED		 2053 * exported to higher levels.
2054 *
2055 * Otherwise, the device hides details of its frame
2056 * structure, so that corresponding packet head is
2057 * never delivered to user.
1da177e4
LT		 2058 */
2059 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2060 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2061 else if (skb->pkt_type == PACKET_OUTGOING) {
2062 /* Special case: outgoing packets have ll header at head */
bbe735e4 2063 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2064 }
2065 }
2066
2067 snaplen = skb->len;
2068
dbcb5855
DM		 2069 res = run_filter(skb, sk, snaplen);
2070 if (!res)
fda9ef5d 2071 goto drop_n_restore;
dbcb5855
DM		 2072 if (snaplen > res)
2073 snaplen = res;
1da177e4 2074
0fd7bac6 2075 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 2076 goto drop_n_acct;
2077
2078 if (skb_shared(skb)) {
2079 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2080 if (nskb == NULL)
2081 goto drop_n_acct;
2082
2083 if (skb_head != skb->data) {
2084 skb->data = skb_head;
2085 skb->len = skb_len;
2086 }
abc4e4fa 2087 consume_skb(skb);
1da177e4
LT		 2088 skb = nskb;
2089 }
2090
b4772ef8 2091 sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
ffbc6111
HX		 2092
2093 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4 2094 sll->sll_hatype = dev->type;
1da177e4 2095 sll->sll_pkttype = skb->pkt_type;
8032b464 2096 if (unlikely(po->origdev))
80feaacb
PWJ		 2097 sll->sll_ifindex = orig_dev->ifindex;
2098 else
2099 sll->sll_ifindex = dev->ifindex;
1da177e4 2100
b95cce35 2101 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 2102
2472d761
EB		 2103 /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2104 * Use their space for storing the original skb length.
2105 */
2106 PACKET_SKB_CB(skb)->sa.origlen = skb->len;
8dc41944 2107
1da177e4
LT		 2108 if (pskb_trim(skb, snaplen))
2109 goto drop_n_acct;
2110
2111 skb_set_owner_r(skb, sk);
2112 skb->dev = NULL;
adf30907 2113 skb_dst_drop(skb);
1da177e4 2114
84531c24
PO		 2115 /* drop conntrack reference */
2116 nf_reset(skb);
2117
1da177e4 2118 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 2119 po->stats.stats1.tp_packets++;
3bc3b96f 2120 sock_skb_set_dropcount(sk, skb);
1da177e4
LT		 2121 __skb_queue_tail(&sk->sk_receive_queue, skb);
2122 spin_unlock(&sk->sk_receive_queue.lock);
676d2369 2123 sk->sk_data_ready(sk);
1da177e4
LT		 2124 return 0;
2125
2126drop_n_acct:
da37845f 2127 is_drop_n_account = true;
7091fbd8 2128 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 2129 po->stats.stats1.tp_drops++;
7091fbd8
WB		 2130 atomic_inc(&sk->sk_drops);
2131 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 2132
2133drop_n_restore:
2134 if (skb_head != skb->data && skb_shared(skb)) {
2135 skb->data = skb_head;
2136 skb->len = skb_len;
2137 }
2138drop:
da37845f
WJ		 2139 if (!is_drop_n_account)
2140 consume_skb(skb);
2141 else
2142 kfree_skb(skb);
1da177e4
LT		 2143 return 0;
2144}
2145
40d4e3df
ED		 2146static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2147 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 2148{
2149 struct sock *sk;
2150 struct packet_sock *po;
2151 struct sockaddr_ll *sll;
184f489e 2152 union tpacket_uhdr h;
40d4e3df 2153 u8 *skb_head = skb->data;
1da177e4 2154 int skb_len = skb->len;
dbcb5855 2155 unsigned int snaplen, res;
f6fb8f10 2156 unsigned long status = TP_STATUS_USER;
bbd6ef87 2157 unsigned short macoff, netoff, hdrlen;
1da177e4 2158 struct sk_buff *copy_skb = NULL;
bbd6ef87 2159 struct timespec ts;
b9c32fb2 2160 __u32 ts_status;
da37845f 2161 bool is_drop_n_account = false;
edbd58be 2162 bool do_vnet = false;
1da177e4 2163
51846355
AW		 2164 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2165 * We may add members to them until current aligned size without forcing
2166 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2167 */
2168 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2169 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2170
1da177e4
LT		 2171 if (skb->pkt_type == PACKET_LOOPBACK)
2172 goto drop;
2173
2174 sk = pt->af_packet_priv;
2175 po = pkt_sk(sk);
2176
09ad9bc7 2177 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 2178 goto drop;
2179
3b04ddde 2180 if (dev->header_ops) {
1da177e4 2181 if (sk->sk_type != SOCK_DGRAM)
98e399f8 2182 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 2183 else if (skb->pkt_type == PACKET_OUTGOING) {
2184 /* Special case: outgoing packets have ll header at head */
bbe735e4 2185 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 2186 }
2187 }
2188
2189 snaplen = skb->len;
2190
dbcb5855
DM		 2191 res = run_filter(skb, sk, snaplen);
2192 if (!res)
fda9ef5d 2193 goto drop_n_restore;
68c2e5de
AD		 2194
2195 if (skb->ip_summed == CHECKSUM_PARTIAL)
2196 status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 2197 else if (skb->pkt_type != PACKET_OUTGOING &&
2198 (skb->ip_summed == CHECKSUM_COMPLETE ||
2199 skb_csum_unnecessary(skb)))
2200 status |= TP_STATUS_CSUM_VALID;
68c2e5de 2201
dbcb5855
DM		 2202 if (snaplen > res)
2203 snaplen = res;
1da177e4
LT		 2204
2205 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 2206 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2207 po->tp_reserve;
1da177e4 2208 } else {
95c96174 2209 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 2210 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a 2211 (maclen < 16 ? 16 : maclen)) +
58d19b19 2212 po->tp_reserve;
edbd58be 2213 if (po->has_vnet_hdr) {
58d19b19 2214 netoff += sizeof(struct virtio_net_hdr);
edbd58be
BP		 2215 do_vnet = true;
2216 }
1da177e4
LT		 2217 macoff = netoff - maclen;
2218 }
f6fb8f10 2219 if (po->tp_version <= TPACKET_V2) {
2220 if (macoff + snaplen > po->rx_ring.frame_size) {
2221 if (po->copy_thresh &&
0fd7bac6 2222 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 2223 if (skb_shared(skb)) {
2224 copy_skb = skb_clone(skb, GFP_ATOMIC);
2225 } else {
2226 copy_skb = skb_get(skb);
2227 skb_head = skb->data;
2228 }
2229 if (copy_skb)
2230 skb_set_owner_r(copy_skb, sk);
1da177e4 2231 }
f6fb8f10 2232 snaplen = po->rx_ring.frame_size - macoff;
edbd58be 2233 if ((int)snaplen < 0) {
f6fb8f10 2234 snaplen = 0;
edbd58be
BP		 2235 do_vnet = false;
2236 }
1da177e4 2237 }
dc808110
ED		 2238 } else if (unlikely(macoff + snaplen >
2239 GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2240 u32 nval;
2241
2242 nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2243 pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2244 snaplen, nval, macoff);
2245 snaplen = nval;
2246 if (unlikely((int)snaplen < 0)) {
2247 snaplen = 0;
2248 macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
edbd58be 2249 do_vnet = false;
dc808110 2250 }
1da177e4 2251 }
1da177e4 2252 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 2253 h.raw = packet_current_rx_frame(po, skb,
2254 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 2255 if (!h.raw)
58d19b19 2256 goto drop_n_account;
f6fb8f10 2257 if (po->tp_version <= TPACKET_V2) {
2258 packet_increment_rx_head(po, &po->rx_ring);
2259 /*
2260 * LOSING will be reported till you read the stats,
2261 * because it's COR - Clear On Read.
2262 * Anyways, moving it for V1/V2 only as V3 doesn't need this
2263 * at packet level.
2264 */
ee80fbf3 2265 if (po->stats.stats1.tp_drops)
f6fb8f10 2266 status |= TP_STATUS_LOSING;
2267 }
945d015e
ED		 2268
2269 if (do_vnet &&
2270 virtio_net_hdr_from_skb(skb, h.raw + macoff -
2271 sizeof(struct virtio_net_hdr),
2272 vio_le(), true, 0))
2273 goto drop_n_account;
2274
ee80fbf3 2275 po->stats.stats1.tp_packets++;
1da177e4
LT		 2276 if (copy_skb) {
2277 status |= TP_STATUS_COPY;
2278 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
2279 }
1da177e4
LT		 2280 spin_unlock(&sk->sk_receive_queue.lock);
2281
bbd6ef87 2282 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 2283
2284 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 2285 getnstimeofday(&ts);
1da177e4 2286
b9c32fb2
DB		 2287 status |= ts_status;
2288
bbd6ef87
PM		 2289 switch (po->tp_version) {
2290 case TPACKET_V1:
2291 h.h1->tp_len = skb->len;
2292 h.h1->tp_snaplen = snaplen;
2293 h.h1->tp_mac = macoff;
2294 h.h1->tp_net = netoff;
4b457bdf
DB		 2295 h.h1->tp_sec = ts.tv_sec;
2296 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 2297 hdrlen = sizeof(*h.h1);
2298 break;
2299 case TPACKET_V2:
2300 h.h2->tp_len = skb->len;
2301 h.h2->tp_snaplen = snaplen;
2302 h.h2->tp_mac = macoff;
2303 h.h2->tp_net = netoff;
bbd6ef87
PM		 2304 h.h2->tp_sec = ts.tv_sec;
2305 h.h2->tp_nsec = ts.tv_nsec;
df8a39de
JP		 2306 if (skb_vlan_tag_present(skb)) {
2307 h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 2308 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2309 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2310 } else {
2311 h.h2->tp_vlan_tci = 0;
a0cdfcf3 2312 h.h2->tp_vlan_tpid = 0;
a3bcc23e 2313 }
e4d26f4b 2314 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 2315 hdrlen = sizeof(*h.h2);
2316 break;
f6fb8f10 2317 case TPACKET_V3:
2318 /* tp_nxt_offset,vlan are already populated above.
2319 * So DONT clear those fields here
2320 */
2321 h.h3->tp_status |= status;
2322 h.h3->tp_len = skb->len;
2323 h.h3->tp_snaplen = snaplen;
2324 h.h3->tp_mac = macoff;
2325 h.h3->tp_net = netoff;
f6fb8f10 2326 h.h3->tp_sec = ts.tv_sec;
2327 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2328 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2329 hdrlen = sizeof(*h.h3);
2330 break;
bbd6ef87
PM		 2331 default:
2332 BUG();
2333 }
1da177e4 2334
bbd6ef87 2335 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2336 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2337 sll->sll_family = AF_PACKET;
2338 sll->sll_hatype = dev->type;
2339 sll->sll_protocol = skb->protocol;
2340 sll->sll_pkttype = skb->pkt_type;
8032b464 2341 if (unlikely(po->origdev))
80feaacb
PWJ		 2342 sll->sll_ifindex = orig_dev->ifindex;
2343 else
2344 sll->sll_ifindex = dev->ifindex;
1da177e4 2345
e16aa207 2346 smp_mb();
f0d4eb29 2347
f6dafa95 2348#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2349 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2350 u8 *start, *end;
2351
f0d4eb29
DB		 2352 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2353 macoff + snaplen);
2354
2355 for (start = h.raw; start < end; start += PAGE_SIZE)
2356 flush_dcache_page(pgv_to_page(start));
1da177e4 2357 }
f0d4eb29 2358 smp_wmb();
f6dafa95 2359#endif
f0d4eb29 2360
da413eec 2361 if (po->tp_version <= TPACKET_V2) {
f6fb8f10 2362 __packet_set_status(po, h.raw, status);
da413eec
DC		 2363 sk->sk_data_ready(sk);
2364 } else {
f6fb8f10 2365 prb_clear_blk_fill_status(&po->rx_ring);
da413eec 2366 }
1da177e4
LT		 2367
2368drop_n_restore:
2369 if (skb_head != skb->data && skb_shared(skb)) {
2370 skb->data = skb_head;
2371 skb->len = skb_len;
2372 }
2373drop:
da37845f
WJ		 2374 if (!is_drop_n_account)
2375 consume_skb(skb);
2376 else
2377 kfree_skb(skb);
1da177e4
LT		 2378 return 0;
2379
58d19b19 2380drop_n_account:
da37845f 2381 is_drop_n_account = true;
ee80fbf3 2382 po->stats.stats1.tp_drops++;
1da177e4
LT		 2383 spin_unlock(&sk->sk_receive_queue.lock);
2384
676d2369 2385 sk->sk_data_ready(sk);
acb5d75b 2386 kfree_skb(copy_skb);
1da177e4
LT		 2387 goto drop_n_restore;
2388}
2389
69e3c75f
JB		 2390static void tpacket_destruct_skb(struct sk_buff *skb)
2391{
2392 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2393
69e3c75f 2394 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2395 void *ph;
b9c32fb2
DB		 2396 __u32 ts;
2397
5cd8d46e 2398 ph = skb_zcopy_get_nouarg(skb);
b0138408 2399 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2400
2401 ts = __packet_set_timestamp(po, ph, skb);
2402 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2403 }
2404
2405 sock_wfree(skb);
2406}
2407
16cc1400
WB		 2408static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2409{
16cc1400
WB		 2410 if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2411 (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2412 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
2413 __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
2414 vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2415 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
2416 __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);
2417
2418 if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
2419 return -EINVAL;
2420
16cc1400
WB		 2421 return 0;
2422}
2423
2424static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2425 struct virtio_net_hdr *vnet_hdr)
2426{
16cc1400
WB		 2427 if (*len < sizeof(*vnet_hdr))
2428 return -EINVAL;
2429 *len -= sizeof(*vnet_hdr);
2430
cbbd26b8 2431 if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
16cc1400
WB		 2432 return -EFAULT;
2433
2434 return __packet_snd_vnet_parse(vnet_hdr, *len);
2435}
2436
40d4e3df 2437static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
8d39b4a6 2438 void *frame, struct net_device *dev, void *data, int tp_len,
c14ac945
SHY		 2439 __be16 proto, unsigned char *addr, int hlen, int copylen,
2440 const struct sockcm_cookie *sockc)
69e3c75f 2441{
184f489e 2442 union tpacket_uhdr ph;
8d39b4a6 2443 int to_write, offset, len, nr_frags, len_max;
69e3c75f
JB		 2444 struct socket *sock = po->sk.sk_socket;
2445 struct page *page;
69e3c75f
JB		 2446 int err;
2447
2448 ph.raw = frame;
2449
2450 skb->protocol = proto;
2451 skb->dev = dev;
2452 skb->priority = po->sk.sk_priority;
2d37a186 2453 skb->mark = po->sk.sk_mark;
3d0ba8c0 2454 skb->tstamp = sockc->transmit_time;
8f932f76 2455 skb_setup_tx_timestamp(skb, sockc->tsflags);
5cd8d46e 2456 skb_zcopy_set_nouarg(skb, ph.raw);
69e3c75f 2457
ae641949 2458 skb_reserve(skb, hlen);
69e3c75f 2459 skb_reset_network_header(skb);
c1aad275 2460
69e3c75f
JB		 2461 to_write = tp_len;
2462
2463 if (sock->type == SOCK_DGRAM) {
2464 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2465 NULL, tp_len);
2466 if (unlikely(err < 0))
2467 return -EINVAL;
1d036d25 2468 } else if (copylen) {
9ed988cd
WB		 2469 int hdrlen = min_t(int, copylen, tp_len);
2470
69e3c75f 2471 skb_push(skb, dev->hard_header_len);
1d036d25 2472 skb_put(skb, copylen - dev->hard_header_len);
9ed988cd 2473 err = skb_store_bits(skb, 0, data, hdrlen);
69e3c75f
JB		 2474 if (unlikely(err))
2475 return err;
9ed988cd
WB		 2476 if (!dev_validate_header(dev, skb->data, hdrlen))
2477 return -EINVAL;
69e3c75f 2478
9ed988cd
WB		 2479 data += hdrlen;
2480 to_write -= hdrlen;
69e3c75f
JB		 2481 }
2482
69e3c75f
JB		 2483 offset = offset_in_page(data);
2484 len_max = PAGE_SIZE - offset;
2485 len = ((to_write > len_max) ? len_max : to_write);
2486
2487 skb->data_len = to_write;
2488 skb->len += to_write;
2489 skb->truesize += to_write;
14afee4b 2490 refcount_add(to_write, &po->sk.sk_wmem_alloc);
69e3c75f
JB		 2491
2492 while (likely(to_write)) {
2493 nr_frags = skb_shinfo(skb)->nr_frags;
2494
2495 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2496 pr_err("Packet exceed the number of skb frags(%lu)\n",
2497 MAX_SKB_FRAGS);
69e3c75f
JB		 2498 return -EFAULT;
2499 }
2500
0af55bb5
CG		 2501 page = pgv_to_page(data);
2502 data += len;
69e3c75f
JB		 2503 flush_dcache_page(page);
2504 get_page(page);
0af55bb5 2505 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2506 to_write -= len;
2507 offset = 0;
2508 len_max = PAGE_SIZE;
2509 len = ((to_write > len_max) ? len_max : to_write);
2510 }
2511
75c65772 2512 packet_parse_headers(skb, sock);
efdfa2f7 2513
69e3c75f
JB		 2514 return tp_len;
2515}
2516
8d39b4a6
WB		 2517static int tpacket_parse_header(struct packet_sock *po, void *frame,
2518 int size_max, void **data)
2519{
2520 union tpacket_uhdr ph;
2521 int tp_len, off;
2522
2523 ph.raw = frame;
2524
2525 switch (po->tp_version) {
7f953ab2
SV		 2526 case TPACKET_V3:
2527 if (ph.h3->tp_next_offset != 0) {
2528 pr_warn_once("variable sized slot not supported");
2529 return -EINVAL;
2530 }
2531 tp_len = ph.h3->tp_len;
2532 break;
8d39b4a6
WB		 2533 case TPACKET_V2:
2534 tp_len = ph.h2->tp_len;
2535 break;
2536 default:
2537 tp_len = ph.h1->tp_len;
2538 break;
2539 }
2540 if (unlikely(tp_len > size_max)) {
2541 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2542 return -EMSGSIZE;
2543 }
2544
2545 if (unlikely(po->tp_tx_has_off)) {
2546 int off_min, off_max;
2547
2548 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2549 off_max = po->tx_ring.frame_size - tp_len;
2550 if (po->sk.sk_type == SOCK_DGRAM) {
2551 switch (po->tp_version) {
7f953ab2
SV		 2552 case TPACKET_V3:
2553 off = ph.h3->tp_net;
2554 break;
8d39b4a6
WB		 2555 case TPACKET_V2:
2556 off = ph.h2->tp_net;
2557 break;
2558 default:
2559 off = ph.h1->tp_net;
2560 break;
2561 }
2562 } else {
2563 switch (po->tp_version) {
7f953ab2
SV		 2564 case TPACKET_V3:
2565 off = ph.h3->tp_mac;
2566 break;
8d39b4a6
WB		 2567 case TPACKET_V2:
2568 off = ph.h2->tp_mac;
2569 break;
2570 default:
2571 off = ph.h1->tp_mac;
2572 break;
2573 }
2574 }
2575 if (unlikely((off < off_min) || (off_max < off)))
2576 return -EINVAL;
2577 } else {
2578 off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2579 }
2580
2581 *data = frame + off;
2582 return tp_len;
2583}
2584
69e3c75f
JB		 2585static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2586{
69e3c75f
JB		 2587 struct sk_buff *skb;
2588 struct net_device *dev;
1d036d25 2589 struct virtio_net_hdr *vnet_hdr = NULL;
c14ac945 2590 struct sockcm_cookie sockc;
69e3c75f 2591 __be16 proto;
09effa67 2592 int err, reserve = 0;
40d4e3df 2593 void *ph;
342dfc30 2594 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2595 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
486efdc8 2596 unsigned char *addr = NULL;
69e3c75f 2597 int tp_len, size_max;
8d39b4a6 2598 void *data;
69e3c75f 2599 int len_sum = 0;
9e67030a 2600 int status = TP_STATUS_AVAILABLE;
1d036d25 2601 int hlen, tlen, copylen = 0;
69e3c75f 2602
69e3c75f
JB		 2603 mutex_lock(&po->pg_vec_lock);
2604
66e56cd4 2605 if (likely(saddr == NULL)) {
e40526cb 2606 dev = packet_cached_dev_get(po);
69e3c75f 2607 proto = po->num;
69e3c75f
JB		 2608 } else {
2609 err = -EINVAL;
2610 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2611 goto out;
2612 if (msg->msg_namelen < (saddr->sll_halen
2613 + offsetof(struct sockaddr_ll,
2614 sll_addr)))
2615 goto out;
69e3c75f 2616 proto = saddr->sll_protocol;
827d9780 2617 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
486efdc8
WB		 2618 if (po->sk.sk_socket->type == SOCK_DGRAM) {
2619 if (dev && msg->msg_namelen < dev->addr_len +
2620 offsetof(struct sockaddr_ll, sll_addr))
2621 goto out_put;
2622 addr = saddr->sll_addr;
2623 }
69e3c75f
JB		 2624 }
2625
69e3c75f
JB		 2626 err = -ENXIO;
2627 if (unlikely(dev == NULL))
2628 goto out;
69e3c75f
JB		 2629 err = -ENETDOWN;
2630 if (unlikely(!(dev->flags & IFF_UP)))
2631 goto out_put;
2632
657a0667 2633 sockcm_init(&sockc, &po->sk);
d19b183c
DCS		 2634 if (msg->msg_controllen) {
2635 err = sock_cmsg_send(&po->sk, msg, &sockc);
2636 if (unlikely(err))
2637 goto out_put;
2638 }
2639
5cfb4c8d
DB		 2640 if (po->sk.sk_socket->type == SOCK_RAW)
2641 reserve = dev->hard_header_len;
69e3c75f 2642 size_max = po->tx_ring.frame_size
b5dd884e 2643 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2644
1d036d25 2645 if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !po->has_vnet_hdr)
5cfb4c8d 2646 size_max = dev->mtu + reserve + VLAN_HLEN;
09effa67 2647
69e3c75f
JB		 2648 do {
2649 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2650 TP_STATUS_SEND_REQUEST);
69e3c75f 2651 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2652 if (need_wait && need_resched())
2653 schedule();
69e3c75f
JB		 2654 continue;
2655 }
2656
8d39b4a6
WB		 2657 skb = NULL;
2658 tp_len = tpacket_parse_header(po, ph, size_max, &data);
2659 if (tp_len < 0)
2660 goto tpacket_error;
2661
69e3c75f 2662 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2663 hlen = LL_RESERVED_SPACE(dev);
2664 tlen = dev->needed_tailroom;
1d036d25
WB		 2665 if (po->has_vnet_hdr) {
2666 vnet_hdr = data;
2667 data += sizeof(*vnet_hdr);
2668 tp_len -= sizeof(*vnet_hdr);
2669 if (tp_len < 0 ||
2670 __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
2671 tp_len = -EINVAL;
2672 goto tpacket_error;
2673 }
2674 copylen = __virtio16_to_cpu(vio_le(),
2675 vnet_hdr->hdr_len);
2676 }
9ed988cd 2677 copylen = max_t(int, copylen, dev->hard_header_len);
69e3c75f 2678 skb = sock_alloc_send_skb(&po->sk,
1d036d25
WB		 2679 hlen + tlen + sizeof(struct sockaddr_ll) +
2680 (copylen - dev->hard_header_len),
fbf33a28 2681 !need_wait, &err);
69e3c75f 2682
fbf33a28
KM		 2683 if (unlikely(skb == NULL)) {
2684 /* we assume the socket was initially writeable ... */
2685 if (likely(len_sum > 0))
2686 err = len_sum;
69e3c75f 2687 goto out_status;
fbf33a28 2688 }
8d39b4a6 2689 tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
c14ac945 2690 addr, hlen, copylen, &sockc);
dbd46ab4 2691 if (likely(tp_len >= 0) &&
5cfb4c8d 2692 tp_len > dev->mtu + reserve &&
1d036d25 2693 !po->has_vnet_hdr &&
3c70c132
DB		 2694 !packet_extra_vlan_len_allowed(dev, skb))
2695 tp_len = -EMSGSIZE;
69e3c75f
JB		 2696
2697 if (unlikely(tp_len < 0)) {
8d39b4a6 2698tpacket_error:
69e3c75f
JB		 2699 if (po->tp_loss) {
2700 __packet_set_status(po, ph,
2701 TP_STATUS_AVAILABLE);
2702 packet_increment_head(&po->tx_ring);
2703 kfree_skb(skb);
2704 continue;
2705 } else {
2706 status = TP_STATUS_WRONG_FORMAT;
2707 err = tp_len;
2708 goto out_status;
2709 }
2710 }
2711
9d2f67e4
JT		 2712 if (po->has_vnet_hdr) {
2713 if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) {
2714 tp_len = -EINVAL;
2715 goto tpacket_error;
2716 }
2717 virtio_net_hdr_set_proto(skb, vnet_hdr);
1d036d25
WB		 2718 }
2719
69e3c75f
JB		 2720 skb->destructor = tpacket_destruct_skb;
2721 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2722 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2723
2724 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2725 err = po->xmit(skb);
eb70df13
JP		 2726 if (unlikely(err > 0)) {
2727 err = net_xmit_errno(err);
2728 if (err && __packet_get_status(po, ph) ==
2729 TP_STATUS_AVAILABLE) {
2730 /* skb was destructed already */
2731 skb = NULL;
2732 goto out_status;
2733 }
2734 /*
2735 * skb was dropped but not destructed yet;
2736 * let's treat it like congestion or err < 0
2737 */
2738 err = 0;
2739 }
69e3c75f
JB		 2740 packet_increment_head(&po->tx_ring);
2741 len_sum += tp_len;
b0138408
DB		 2742 } while (likely((ph != NULL) ||
2743 /* Note: packet_read_pending() might be slow if we have
2744 * to call it as it's per_cpu variable, but in fast-path
2745 * we already short-circuit the loop with the first
2746 * condition, and luckily don't have to go that path
2747 * anyway.
2748 */
2749 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2750
2751 err = len_sum;
2752 goto out_put;
2753
69e3c75f
JB		 2754out_status:
2755 __packet_set_status(po, ph, status);
2756 kfree_skb(skb);
2757out_put:
e40526cb 2758 dev_put(dev);
69e3c75f
JB		 2759out:
2760 mutex_unlock(&po->pg_vec_lock);
2761 return err;
2762}
69e3c75f 2763
eea49cc9
OJ		 2764static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2765 size_t reserve, size_t len,
2766 size_t linear, int noblock,
2767 int *err)
bfd5f4a3
SS		 2768{
2769 struct sk_buff *skb;
2770
2771 /* Under a page? Don't bother with paged skb. */
2772 if (prepad + len < PAGE_SIZE || !linear)
2773 linear = len;
2774
2775 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2776 err, 0);
bfd5f4a3
SS		 2777 if (!skb)
2778 return NULL;
2779
2780 skb_reserve(skb, reserve);
2781 skb_put(skb, linear);
2782 skb->data_len = len - linear;
2783 skb->len += len - linear;
2784
2785 return skb;
2786}
2787
d346a3fa 2788static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2789{
2790 struct sock *sk = sock->sk;
342dfc30 2791 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2792 struct sk_buff *skb;
2793 struct net_device *dev;
0e11c91e 2794 __be16 proto;
486efdc8 2795 unsigned char *addr = NULL;
827d9780 2796 int err, reserve = 0;
c7d39e32 2797 struct sockcm_cookie sockc;
bfd5f4a3
SS		 2798 struct virtio_net_hdr vnet_hdr = { 0 };
2799 int offset = 0;
bfd5f4a3 2800 struct packet_sock *po = pkt_sk(sk);
da7c9561 2801 bool has_vnet_hdr = false;
57031eb7 2802 int hlen, tlen, linear;
3bdc0eba 2803 int extra_len = 0;
1da177e4
LT		 2804
2805 /*
1ce4f28b 2806 * Get and verify the address.
1da177e4 2807 */
1ce4f28b 2808
66e56cd4 2809 if (likely(saddr == NULL)) {
e40526cb 2810 dev = packet_cached_dev_get(po);
1da177e4 2811 proto = po->num;
1da177e4
LT		 2812 } else {
2813 err = -EINVAL;
2814 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2815 goto out;
0fb375fb
EB		 2816 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2817 goto out;
1da177e4 2818 proto = saddr->sll_protocol;
827d9780 2819 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
486efdc8
WB		 2820 if (sock->type == SOCK_DGRAM) {
2821 if (dev && msg->msg_namelen < dev->addr_len +
2822 offsetof(struct sockaddr_ll, sll_addr))
2823 goto out_unlock;
2824 addr = saddr->sll_addr;
2825 }
1da177e4
LT		 2826 }
2827
1da177e4 2828 err = -ENXIO;
e40526cb 2829 if (unlikely(dev == NULL))
1da177e4 2830 goto out_unlock;
d5e76b0a 2831 err = -ENETDOWN;
e40526cb 2832 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2833 goto out_unlock;
2834
657a0667 2835 sockcm_init(&sockc, sk);
c7d39e32
EJ		 2836 sockc.mark = sk->sk_mark;
2837 if (msg->msg_controllen) {
2838 err = sock_cmsg_send(sk, msg, &sockc);
2839 if (unlikely(err))
2840 goto out_unlock;
2841 }
2842
e40526cb
DB		 2843 if (sock->type == SOCK_RAW)
2844 reserve = dev->hard_header_len;
bfd5f4a3 2845 if (po->has_vnet_hdr) {
16cc1400
WB		 2846 err = packet_snd_vnet_parse(msg, &len, &vnet_hdr);
2847 if (err)
bfd5f4a3 2848 goto out_unlock;
da7c9561 2849 has_vnet_hdr = true;
bfd5f4a3
SS		 2850 }
2851
3bdc0eba
BG		 2852 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2853 if (!netif_supports_nofcs(dev)) {
2854 err = -EPROTONOSUPPORT;
2855 goto out_unlock;
2856 }
2857 extra_len = 4; /* We're doing our own CRC */
2858 }
2859
1da177e4 2860 err = -EMSGSIZE;
16cc1400
WB		 2861 if (!vnet_hdr.gso_type &&
2862 (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2863 goto out_unlock;
2864
bfd5f4a3 2865 err = -ENOBUFS;
ae641949
HX		 2866 hlen = LL_RESERVED_SPACE(dev);
2867 tlen = dev->needed_tailroom;
57031eb7
WB		 2868 linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
2869 linear = max(linear, min_t(int, len, dev->hard_header_len));
2870 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
bfd5f4a3 2871 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2872 if (skb == NULL)
1da177e4
LT		 2873 goto out_unlock;
2874
b84bbaf7 2875 skb_reset_network_header(skb);
1da177e4 2876
0c4e8581 2877 err = -EINVAL;
9c707762
WB		 2878 if (sock->type == SOCK_DGRAM) {
2879 offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
46d2cfb1 2880 if (unlikely(offset < 0))
9c707762 2881 goto out_free;
b84bbaf7 2882 } else if (reserve) {
9aad13b0 2883 skb_reserve(skb, -reserve);
88a8121d
ND		 2884 if (len < reserve + sizeof(struct ipv6hdr) &&
2885 dev->min_header_len != dev->hard_header_len)
993675a3 2886 skb_reset_network_header(skb);
9c707762 2887 }
1da177e4
LT		 2888
2889 /* Returns -EFAULT on error */
c0371da6 2890 err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
1da177e4
LT		 2891 if (err)
2892 goto out_free;
bf84a010 2893
9ed988cd
WB		 2894 if (sock->type == SOCK_RAW &&
2895 !dev_validate_header(dev, skb->data, len)) {
2896 err = -EINVAL;
2897 goto out_free;
2898 }
2899
8f932f76 2900 skb_setup_tx_timestamp(skb, sockc.tsflags);
1da177e4 2901
16cc1400 2902 if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
3c70c132
DB		 2903 !packet_extra_vlan_len_allowed(dev, skb)) {
2904 err = -EMSGSIZE;
2905 goto out_free;
57f89bfa
BG		 2906 }
2907
09effa67
DM		 2908 skb->protocol = proto;
2909 skb->dev = dev;
1da177e4 2910 skb->priority = sk->sk_priority;
c7d39e32 2911 skb->mark = sockc.mark;
3d0ba8c0 2912 skb->tstamp = sockc.transmit_time;
0fd5d57b 2913
da7c9561 2914 if (has_vnet_hdr) {
db60eb5f 2915 err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
16cc1400
WB		 2916 if (err)
2917 goto out_free;
2918 len += sizeof(vnet_hdr);
9d2f67e4 2919 virtio_net_hdr_set_proto(skb, &vnet_hdr);
bfd5f4a3
SS		 2920 }
2921
75c65772 2922 packet_parse_headers(skb, sock);
8fd6c80d 2923
3bdc0eba
BG		 2924 if (unlikely(extra_len == 4))
2925 skb->no_fcs = 1;
2926
d346a3fa 2927 err = po->xmit(skb);
1da177e4
LT		 2928 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2929 goto out_unlock;
2930
e40526cb 2931 dev_put(dev);
1da177e4 2932
40d4e3df 2933 return len;
1da177e4
LT		 2934
2935out_free:
2936 kfree_skb(skb);
2937out_unlock:
e40526cb 2938 if (dev)
1da177e4
LT		 2939 dev_put(dev);
2940out:
2941 return err;
2942}
2943
1b784140 2944static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
69e3c75f 2945{
69e3c75f
JB		 2946 struct sock *sk = sock->sk;
2947 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2948
69e3c75f
JB		 2949 if (po->tx_ring.pg_vec)
2950 return tpacket_snd(po, msg);
2951 else
69e3c75f
JB		 2952 return packet_snd(sock, msg, len);
2953}
2954
1da177e4
LT		 2955/*
2956 * Close a PACKET socket. This is fairly simple. We immediately go
2957 * to 'closed' state and remove our protocol entry in the device list.
2958 */
2959
2960static int packet_release(struct socket *sock)
2961{
2962 struct sock *sk = sock->sk;
2963 struct packet_sock *po;
2bd624b4 2964 struct packet_fanout *f;
d12d01d6 2965 struct net *net;
f6fb8f10 2966 union tpacket_req_u req_u;
1da177e4
LT		 2967
2968 if (!sk)
2969 return 0;
2970
3b1e0a65 2971 net = sock_net(sk);
1da177e4
LT		 2972 po = pkt_sk(sk);
2973
0fa7fa98 2974 mutex_lock(&net->packet.sklist_lock);
808f5114 2975 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2976 mutex_unlock(&net->packet.sklist_lock);
2977
2978 preempt_disable();
920de804 2979 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2980 preempt_enable();
1da177e4 2981
808f5114 2982 spin_lock(&po->bind_lock);
ce06b03e 2983 unregister_prot_hook(sk, false);
66e56cd4
DB		 2984 packet_cached_dev_reset(po);
2985
160ff18a
BG		 2986 if (po->prot_hook.dev) {
2987 dev_put(po->prot_hook.dev);
2988 po->prot_hook.dev = NULL;
2989 }
808f5114 2990 spin_unlock(&po->bind_lock);
1da177e4 2991
1da177e4 2992 packet_flush_mclist(sk);
1da177e4 2993
5171b37d 2994 lock_sock(sk);
9665d5d6
PS		 2995 if (po->rx_ring.pg_vec) {
2996 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2997 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2998 }
69e3c75f 2999
9665d5d6
PS		 3000 if (po->tx_ring.pg_vec) {
3001 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 3002 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 3003 }
5171b37d 3004 release_sock(sk);
1da177e4 3005
2bd624b4 3006 f = fanout_release(sk);
dc99f600 3007
808f5114 3008 synchronize_net();
2bd624b4 3009
afa0925c 3010 kfree(po->rollover);
2bd624b4
AS		 3011 if (f) {
3012 fanout_release_data(f);
3013 kfree(f);
3014 }
1da177e4
LT		 3015 /*
3016 * Now the socket is dead. No more input will appear.
3017 */
1da177e4
LT		 3018 sock_orphan(sk);
3019 sock->sk = NULL;
3020
3021 /* Purge queues */
3022
3023 skb_queue_purge(&sk->sk_receive_queue);
b0138408 3024 packet_free_pending(po);
17ab56a2 3025 sk_refcnt_debug_release(sk);
1da177e4
LT		 3026
3027 sock_put(sk);
3028 return 0;
3029}
3030
3031/*
3032 * Attach a packet hook.
3033 */
3034
30f7ea1c
FR		 3035static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3036 __be16 proto)
1da177e4
LT		 3037{
3038 struct packet_sock *po = pkt_sk(sk);
158cd4af 3039 struct net_device *dev_curr;
902fefb8
DB		 3040 __be16 proto_curr;
3041 bool need_rehook;
30f7ea1c
FR		 3042 struct net_device *dev = NULL;
3043 int ret = 0;
3044 bool unlisted = false;
dc99f600 3045
1da177e4 3046 lock_sock(sk);
1da177e4 3047 spin_lock(&po->bind_lock);
30f7ea1c
FR		 3048 rcu_read_lock();
3049
4971613c
WB		 3050 if (po->fanout) {
3051 ret = -EINVAL;
3052 goto out_unlock;
3053 }
3054
30f7ea1c
FR		 3055 if (name) {
3056 dev = dev_get_by_name_rcu(sock_net(sk), name);
3057 if (!dev) {
3058 ret = -ENODEV;
3059 goto out_unlock;
3060 }
3061 } else if (ifindex) {
3062 dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
3063 if (!dev) {
3064 ret = -ENODEV;
3065 goto out_unlock;
3066 }
3067 }
3068
3069 if (dev)
3070 dev_hold(dev);
66e56cd4 3071
902fefb8
DB		 3072 proto_curr = po->prot_hook.type;
3073 dev_curr = po->prot_hook.dev;
3074
3075 need_rehook = proto_curr != proto || dev_curr != dev;
3076
3077 if (need_rehook) {
30f7ea1c
FR		 3078 if (po->running) {
3079 rcu_read_unlock();
15fe076e
ED		 3080 /* prevents packet_notifier() from calling
3081 * register_prot_hook()
3082 */
3083 po->num = 0;
30f7ea1c
FR		 3084 __unregister_prot_hook(sk, true);
3085 rcu_read_lock();
3086 dev_curr = po->prot_hook.dev;
3087 if (dev)
3088 unlisted = !dev_get_by_index_rcu(sock_net(sk),
3089 dev->ifindex);
3090 }
1da177e4 3091
15fe076e 3092 BUG_ON(po->running);
902fefb8
DB		 3093 po->num = proto;
3094 po->prot_hook.type = proto;
902fefb8 3095
30f7ea1c
FR		 3096 if (unlikely(unlisted)) {
3097 dev_put(dev);
3098 po->prot_hook.dev = NULL;
3099 po->ifindex = -1;
3100 packet_cached_dev_reset(po);
3101 } else {
3102 po->prot_hook.dev = dev;
3103 po->ifindex = dev ? dev->ifindex : 0;
3104 packet_cached_dev_assign(po, dev);
3105 }
902fefb8 3106 }
158cd4af
LW		 3107 if (dev_curr)
3108 dev_put(dev_curr);
66e56cd4 3109
902fefb8 3110 if (proto == 0 || !need_rehook)
1da177e4
LT		 3111 goto out_unlock;
3112
30f7ea1c 3113 if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
ce06b03e 3114 register_prot_hook(sk);
be85d4ad
UT		 3115 } else {
3116 sk->sk_err = ENETDOWN;
3117 if (!sock_flag(sk, SOCK_DEAD))
3118 sk->sk_error_report(sk);
1da177e4
LT		 3119 }
3120
3121out_unlock:
30f7ea1c 3122 rcu_read_unlock();
1da177e4
LT		 3123 spin_unlock(&po->bind_lock);
3124 release_sock(sk);
30f7ea1c 3125 return ret;
1da177e4
LT		 3126}
3127
3128/*
3129 * Bind a packet socket to a device
3130 */
3131
40d4e3df
ED		 3132static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
3133 int addr_len)
1da177e4 3134{
40d4e3df 3135 struct sock *sk = sock->sk;
540e2894 3136 char name[sizeof(uaddr->sa_data) + 1];
1ce4f28b 3137
1da177e4
LT		 3138 /*
3139 * Check legality
3140 */
1ce4f28b 3141
8ae55f04 3142 if (addr_len != sizeof(struct sockaddr))
1da177e4 3143 return -EINVAL;
540e2894
AP		 3144 /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3145 * zero-terminated.
3146 */
3147 memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data));
3148 name[sizeof(uaddr->sa_data)] = 0;
1da177e4 3149
30f7ea1c 3150 return packet_do_bind(sk, name, 0, pkt_sk(sk)->num);
1da177e4 3151}
1da177e4
LT		 3152
3153static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3154{
40d4e3df
ED		 3155 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3156 struct sock *sk = sock->sk;
1da177e4
LT		 3157
3158 /*
3159 * Check legality
3160 */
1ce4f28b 3161
1da177e4
LT		 3162 if (addr_len < sizeof(struct sockaddr_ll))
3163 return -EINVAL;
3164 if (sll->sll_family != AF_PACKET)
3165 return -EINVAL;
3166
30f7ea1c
FR		 3167 return packet_do_bind(sk, NULL, sll->sll_ifindex,
3168 sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 3169}
3170
3171static struct proto packet_proto = {
3172 .name = "PACKET",
3173 .owner = THIS_MODULE,
3174 .obj_size = sizeof(struct packet_sock),
3175};
3176
3177/*
1ce4f28b 3178 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 3179 */
3180
3f378b68
EP		 3181static int packet_create(struct net *net, struct socket *sock, int protocol,
3182 int kern)
1da177e4
LT		 3183{
3184 struct sock *sk;
3185 struct packet_sock *po;
0e11c91e 3186 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 3187 int err;
3188
df008c91 3189 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 3190 return -EPERM;
be02097c
DM		 3191 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3192 sock->type != SOCK_PACKET)
1da177e4
LT		 3193 return -ESOCKTNOSUPPORT;
3194
3195 sock->state = SS_UNCONNECTED;
3196
3197 err = -ENOBUFS;
11aa9c28 3198 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
1da177e4
LT		 3199 if (sk == NULL)
3200 goto out;
3201
3202 sock->ops = &packet_ops;
1da177e4
LT		 3203 if (sock->type == SOCK_PACKET)
3204 sock->ops = &packet_ops_spkt;
be02097c 3205
1da177e4
LT		 3206 sock_init_data(sock, sk);
3207
3208 po = pkt_sk(sk);
3209 sk->sk_family = PF_PACKET;
0e11c91e 3210 po->num = proto;
d346a3fa 3211 po->xmit = dev_queue_xmit;
66e56cd4 3212
b0138408
DB		 3213 err = packet_alloc_pending(po);
3214 if (err)
3215 goto out2;
3216
66e56cd4 3217 packet_cached_dev_reset(po);
1da177e4
LT		 3218
3219 sk->sk_destruct = packet_sock_destruct;
17ab56a2 3220 sk_refcnt_debug_inc(sk);
1da177e4
LT		 3221
3222 /*
3223 * Attach a protocol block
3224 */
3225
3226 spin_lock_init(&po->bind_lock);
905db440 3227 mutex_init(&po->pg_vec_lock);
0648ab70 3228 po->rollover = NULL;
1da177e4 3229 po->prot_hook.func = packet_rcv;
be02097c 3230
1da177e4
LT		 3231 if (sock->type == SOCK_PACKET)
3232 po->prot_hook.func = packet_rcv_spkt;
be02097c 3233
1da177e4
LT		 3234 po->prot_hook.af_packet_priv = sk;
3235
0e11c91e
AV		 3236 if (proto) {
3237 po->prot_hook.type = proto;
a6361f0c 3238 __register_prot_hook(sk);
1da177e4
LT		 3239 }
3240
0fa7fa98 3241 mutex_lock(&net->packet.sklist_lock);
a4dc6a49 3242 sk_add_node_tail_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 3243 mutex_unlock(&net->packet.sklist_lock);
3244
3245 preempt_disable();
3680453c 3246 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 3247 preempt_enable();
808f5114 3248
40d4e3df 3249 return 0;
b0138408
DB		 3250out2:
3251 sk_free(sk);
1da177e4
LT		 3252out:
3253 return err;
3254}
3255
3256/*
3257 * Pull a packet from our receive queue and hand it to the user.
3258 * If necessary we block.
3259 */
3260
1b784140
YX		 3261static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3262 int flags)
1da177e4
LT		 3263{
3264 struct sock *sk = sock->sk;
3265 struct sk_buff *skb;
3266 int copied, err;
bfd5f4a3 3267 int vnet_hdr_len = 0;
2472d761 3268 unsigned int origlen = 0;
1da177e4
LT		 3269
3270 err = -EINVAL;
ed85b565 3271 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 3272 goto out;
3273
3274#if 0
3275 /* What error should we return now? EUNATTACH? */
3276 if (pkt_sk(sk)->ifindex < 0)
3277 return -ENODEV;
3278#endif
3279
ed85b565 3280 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 3281 err = sock_recv_errqueue(sk, msg, len,
3282 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 3283 goto out;
3284 }
3285
1da177e4
LT		 3286 /*
3287 * Call the generic datagram receiver. This handles all sorts
3288 * of horrible races and re-entrancy so we can forget about it
3289 * in the protocol layers.
3290 *
3291 * Now it will return ENETDOWN, if device have just gone down,
3292 * but then it will block.
3293 */
3294
40d4e3df 3295 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 3296
3297 /*
1ce4f28b 3298 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 3299 * handles the blocking we don't see and worry about blocking
3300 * retries.
3301 */
3302
8ae55f04 3303 if (skb == NULL)
1da177e4
LT		 3304 goto out;
3305
2ccdbaa6
WB		 3306 if (pkt_sk(sk)->pressure)
3307 packet_rcv_has_room(pkt_sk(sk), NULL);
3308
bfd5f4a3 3309 if (pkt_sk(sk)->has_vnet_hdr) {
16cc1400
WB		 3310 err = packet_rcv_vnet(msg, skb, &len);
3311 if (err)
bfd5f4a3 3312 goto out_free;
16cc1400 3313 vnet_hdr_len = sizeof(struct virtio_net_hdr);
bfd5f4a3
SS		 3314 }
3315
f3d33426
HFS		 3316 /* You lose any data beyond the buffer you gave. If it worries
3317 * a user program they can ask the device for its MTU
3318 * anyway.
1da177e4 3319 */
1da177e4 3320 copied = skb->len;
40d4e3df
ED		 3321 if (copied > len) {
3322 copied = len;
3323 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 3324 }
3325
51f3d02b 3326 err = skb_copy_datagram_msg(skb, 0, msg, copied);
1da177e4
LT		 3327 if (err)
3328 goto out_free;
3329
2472d761
EB		 3330 if (sock->type != SOCK_PACKET) {
3331 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3332
3333 /* Original length was stored in sockaddr_ll fields */
3334 origlen = PACKET_SKB_CB(skb)->sa.origlen;
3335 sll->sll_family = AF_PACKET;
3336 sll->sll_protocol = skb->protocol;
3337 }
3338
3b885787 3339 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 3340
f3d33426 3341 if (msg->msg_name) {
b2cf86e1
WB		 3342 int copy_len;
3343
f3d33426
HFS		 3344 /* If the address length field is there to be filled
3345 * in, we fill it in now.
3346 */
3347 if (sock->type == SOCK_PACKET) {
342dfc30 3348 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426 3349 msg->msg_namelen = sizeof(struct sockaddr_pkt);
b2cf86e1 3350 copy_len = msg->msg_namelen;
f3d33426
HFS		 3351 } else {
3352 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2472d761 3353
f3d33426
HFS		 3354 msg->msg_namelen = sll->sll_halen +
3355 offsetof(struct sockaddr_ll, sll_addr);
b2cf86e1
WB		 3356 copy_len = msg->msg_namelen;
3357 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
3358 memset(msg->msg_name +
3359 offsetof(struct sockaddr_ll, sll_addr),
3360 0, sizeof(sll->sll_addr));
3361 msg->msg_namelen = sizeof(struct sockaddr_ll);
3362 }
f3d33426 3363 }
b2cf86e1 3364 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
f3d33426 3365 }
1da177e4 3366
8dc41944 3367 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 3368 struct tpacket_auxdata aux;
3369
3370 aux.tp_status = TP_STATUS_USER;
3371 if (skb->ip_summed == CHECKSUM_PARTIAL)
3372 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
682f048b
AD		 3373 else if (skb->pkt_type != PACKET_OUTGOING &&
3374 (skb->ip_summed == CHECKSUM_COMPLETE ||
3375 skb_csum_unnecessary(skb)))
3376 aux.tp_status |= TP_STATUS_CSUM_VALID;
3377
2472d761 3378 aux.tp_len = origlen;
ffbc6111
HX		 3379 aux.tp_snaplen = skb->len;
3380 aux.tp_mac = 0;
bbe735e4 3381 aux.tp_net = skb_network_offset(skb);
df8a39de
JP		 3382 if (skb_vlan_tag_present(skb)) {
3383 aux.tp_vlan_tci = skb_vlan_tag_get(skb);
a0cdfcf3
AW		 3384 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3385 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 3386 } else {
3387 aux.tp_vlan_tci = 0;
a0cdfcf3 3388 aux.tp_vlan_tpid = 0;
a3bcc23e 3389 }
ffbc6111 3390 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 3391 }
3392
1da177e4
LT		 3393 /*
3394 * Free or return the buffer as appropriate. Again this
3395 * hides all the races and re-entrancy issues from us.
3396 */
bfd5f4a3 3397 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 3398
3399out_free:
3400 skb_free_datagram(sk, skb);
3401out:
3402 return err;
3403}
3404
1da177e4 3405static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
9b2c45d4 3406 int peer)
1da177e4
LT		 3407{
3408 struct net_device *dev;
3409 struct sock *sk = sock->sk;
3410
3411 if (peer)
3412 return -EOPNOTSUPP;
3413
3414 uaddr->sa_family = AF_PACKET;
2dc85bf3 3415 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 3416 rcu_read_lock();
3417 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
3418 if (dev)
2dc85bf3 3419 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 3420 rcu_read_unlock();
1da177e4 3421
9b2c45d4 3422 return sizeof(*uaddr);
1da177e4 3423}
1da177e4
LT		 3424
3425static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
9b2c45d4 3426 int peer)
1da177e4
LT		 3427{
3428 struct net_device *dev;
3429 struct sock *sk = sock->sk;
3430 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3431 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3432
3433 if (peer)
3434 return -EOPNOTSUPP;
3435
3436 sll->sll_family = AF_PACKET;
3437 sll->sll_ifindex = po->ifindex;
3438 sll->sll_protocol = po->num;
67286640 3439 sll->sll_pkttype = 0;
654d1f8a
ED		 3440 rcu_read_lock();
3441 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3442 if (dev) {
3443 sll->sll_hatype = dev->type;
3444 sll->sll_halen = dev->addr_len;
3445 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3446 } else {
3447 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3448 sll->sll_halen = 0;
3449 }
654d1f8a 3450 rcu_read_unlock();
1da177e4 3451
9b2c45d4 3452 return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3453}
3454
2aeb0b88
WC		 3455static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3456 int what)
1da177e4
LT		 3457{
3458 switch (i->type) {
3459 case PACKET_MR_MULTICAST:
1162563f
JP		 3460 if (i->alen != dev->addr_len)
3461 return -EINVAL;
1da177e4 3462 if (what > 0)
22bedad3 3463 return dev_mc_add(dev, i->addr);
1da177e4 3464 else
22bedad3 3465 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3466 break;
3467 case PACKET_MR_PROMISC:
2aeb0b88 3468 return dev_set_promiscuity(dev, what);
1da177e4 3469 case PACKET_MR_ALLMULTI:
2aeb0b88 3470 return dev_set_allmulti(dev, what);
d95ed927 3471 case PACKET_MR_UNICAST:
1162563f
JP		 3472 if (i->alen != dev->addr_len)
3473 return -EINVAL;
d95ed927 3474 if (what > 0)
a748ee24 3475 return dev_uc_add(dev, i->addr);
d95ed927 3476 else
a748ee24 3477 return dev_uc_del(dev, i->addr);
d95ed927 3478 break;
40d4e3df
ED		 3479 default:
3480 break;
1da177e4 3481 }
2aeb0b88 3482 return 0;
1da177e4
LT		 3483}
3484
82f17091
FR		 3485static void packet_dev_mclist_delete(struct net_device *dev,
3486 struct packet_mclist **mlp)
1da177e4 3487{
82f17091
FR		 3488 struct packet_mclist *ml;
3489
3490 while ((ml = *mlp) != NULL) {
3491 if (ml->ifindex == dev->ifindex) {
3492 packet_dev_mc(dev, ml, -1);
3493 *mlp = ml->next;
3494 kfree(ml);
3495 } else
3496 mlp = &ml->next;
1da177e4
LT		 3497 }
3498}
3499
0fb375fb 3500static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3501{
3502 struct packet_sock *po = pkt_sk(sk);
3503 struct packet_mclist *ml, *i;
3504 struct net_device *dev;
3505 int err;
3506
3507 rtnl_lock();
3508
3509 err = -ENODEV;
3b1e0a65 3510 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3511 if (!dev)
3512 goto done;
3513
3514 err = -EINVAL;
1162563f 3515 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3516 goto done;
3517
3518 err = -ENOBUFS;
8b3a7005 3519 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3520 if (i == NULL)
3521 goto done;
3522
3523 err = 0;
3524 for (ml = po->mclist; ml; ml = ml->next) {
3525 if (ml->ifindex == mreq->mr_ifindex &&
3526 ml->type == mreq->mr_type &&
3527 ml->alen == mreq->mr_alen &&
3528 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3529 ml->count++;
3530 /* Free the new element ... */
3531 kfree(i);
3532 goto done;
3533 }
3534 }
3535
3536 i->type = mreq->mr_type;
3537 i->ifindex = mreq->mr_ifindex;
3538 i->alen = mreq->mr_alen;
3539 memcpy(i->addr, mreq->mr_address, i->alen);
309cf37f 3540 memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
1da177e4
LT		 3541 i->count = 1;
3542 i->next = po->mclist;
3543 po->mclist = i;
2aeb0b88
WC		 3544 err = packet_dev_mc(dev, i, 1);
3545 if (err) {
3546 po->mclist = i->next;
3547 kfree(i);
3548 }
1da177e4
LT		 3549
3550done:
3551 rtnl_unlock();
3552 return err;
3553}
3554
0fb375fb 3555static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3556{
3557 struct packet_mclist *ml, **mlp;
3558
3559 rtnl_lock();
3560
3561 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3562 if (ml->ifindex == mreq->mr_ifindex &&
3563 ml->type == mreq->mr_type &&
3564 ml->alen == mreq->mr_alen &&
3565 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3566 if (--ml->count == 0) {
3567 struct net_device *dev;
3568 *mlp = ml->next;
ad959e76
ED		 3569 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3570 if (dev)
1da177e4 3571 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3572 kfree(ml);
3573 }
82f17091 3574 break;
1da177e4
LT		 3575 }
3576 }
3577 rtnl_unlock();
82f17091 3578 return 0;
1da177e4
LT		 3579}
3580
3581static void packet_flush_mclist(struct sock *sk)
3582{
3583 struct packet_sock *po = pkt_sk(sk);
3584 struct packet_mclist *ml;
3585
3586 if (!po->mclist)
3587 return;
3588
3589 rtnl_lock();
3590 while ((ml = po->mclist) != NULL) {
3591 struct net_device *dev;
3592
3593 po->mclist = ml->next;
ad959e76
ED		 3594 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3595 if (dev != NULL)
1da177e4 3596 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3597 kfree(ml);
3598 }
3599 rtnl_unlock();
3600}
1da177e4
LT		 3601
3602static int
b7058842 3603packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3604{
3605 struct sock *sk = sock->sk;
8dc41944 3606 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3607 int ret;
3608
3609 if (level != SOL_PACKET)
3610 return -ENOPROTOOPT;
3611
69e3c75f 3612 switch (optname) {
1ce4f28b 3613 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3614 case PACKET_DROP_MEMBERSHIP:
3615 {
0fb375fb
EB		 3616 struct packet_mreq_max mreq;
3617 int len = optlen;
3618 memset(&mreq, 0, sizeof(mreq));
3619 if (len < sizeof(struct packet_mreq))
1da177e4 3620 return -EINVAL;
0fb375fb
EB		 3621 if (len > sizeof(mreq))
3622 len = sizeof(mreq);
40d4e3df 3623 if (copy_from_user(&mreq, optval, len))
1da177e4 3624 return -EFAULT;
0fb375fb
EB		 3625 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3626 return -EINVAL;
1da177e4
LT		 3627 if (optname == PACKET_ADD_MEMBERSHIP)
3628 ret = packet_mc_add(sk, &mreq);
3629 else
3630 ret = packet_mc_drop(sk, &mreq);
3631 return ret;
3632 }
a2efcfa0 3633
1da177e4 3634 case PACKET_RX_RING:
69e3c75f 3635 case PACKET_TX_RING:
1da177e4 3636 {
f6fb8f10 3637 union tpacket_req_u req_u;
3638 int len;
1da177e4 3639
5171b37d 3640 lock_sock(sk);
f6fb8f10 3641 switch (po->tp_version) {
3642 case TPACKET_V1:
3643 case TPACKET_V2:
3644 len = sizeof(req_u.req);
3645 break;
3646 case TPACKET_V3:
3647 default:
3648 len = sizeof(req_u.req3);
3649 break;
3650 }
5171b37d
ED		 3651 if (optlen < len) {
3652 ret = -EINVAL;
3653 } else {
3654 if (copy_from_user(&req_u.req, optval, len))
3655 ret = -EFAULT;
3656 else
3657 ret = packet_set_ring(sk, &req_u, 0,
3658 optname == PACKET_TX_RING);
3659 }
3660 release_sock(sk);
3661 return ret;
1da177e4
LT		 3662 }
3663 case PACKET_COPY_THRESH:
3664 {
3665 int val;
3666
40d4e3df 3667 if (optlen != sizeof(val))
1da177e4 3668 return -EINVAL;
40d4e3df 3669 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3670 return -EFAULT;
3671
3672 pkt_sk(sk)->copy_thresh = val;
3673 return 0;
3674 }
bbd6ef87
PM		 3675 case PACKET_VERSION:
3676 {
3677 int val;
3678
3679 if (optlen != sizeof(val))
3680 return -EINVAL;
bbd6ef87
PM		 3681 if (copy_from_user(&val, optval, sizeof(val)))
3682 return -EFAULT;
3683 switch (val) {
3684 case TPACKET_V1:
3685 case TPACKET_V2:
f6fb8f10 3686 case TPACKET_V3:
84ac7260 3687 break;
bbd6ef87
PM		 3688 default:
3689 return -EINVAL;
3690 }
84ac7260
PP		 3691 lock_sock(sk);
3692 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3693 ret = -EBUSY;
3694 } else {
3695 po->tp_version = val;
3696 ret = 0;
3697 }
3698 release_sock(sk);
3699 return ret;
bbd6ef87 3700 }
8913336a
PM		 3701 case PACKET_RESERVE:
3702 {
3703 unsigned int val;
3704
3705 if (optlen != sizeof(val))
3706 return -EINVAL;
8913336a
PM		 3707 if (copy_from_user(&val, optval, sizeof(val)))
3708 return -EFAULT;
bcc5364b
AK		 3709 if (val > INT_MAX)
3710 return -EINVAL;
c27927e3
WB		 3711 lock_sock(sk);
3712 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3713 ret = -EBUSY;
3714 } else {
3715 po->tp_reserve = val;
3716 ret = 0;
3717 }
3718 release_sock(sk);
3719 return ret;
8913336a 3720 }
69e3c75f
JB		 3721 case PACKET_LOSS:
3722 {
3723 unsigned int val;
3724
3725 if (optlen != sizeof(val))
3726 return -EINVAL;
69e3c75f
JB		 3727 if (copy_from_user(&val, optval, sizeof(val)))
3728 return -EFAULT;
a6361f0c
WB		 3729
3730 lock_sock(sk);
3731 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3732 ret = -EBUSY;
3733 } else {
3734 po->tp_loss = !!val;
3735 ret = 0;
3736 }
3737 release_sock(sk);
3738 return ret;
69e3c75f 3739 }
8dc41944
HX		 3740 case PACKET_AUXDATA:
3741 {
3742 int val;
3743
3744 if (optlen < sizeof(val))
3745 return -EINVAL;
3746 if (copy_from_user(&val, optval, sizeof(val)))
3747 return -EFAULT;
3748
a6361f0c 3749 lock_sock(sk);
8dc41944 3750 po->auxdata = !!val;
a6361f0c 3751 release_sock(sk);
8dc41944
HX		 3752 return 0;
3753 }
80feaacb
PWJ		 3754 case PACKET_ORIGDEV:
3755 {
3756 int val;
3757
3758 if (optlen < sizeof(val))
3759 return -EINVAL;
3760 if (copy_from_user(&val, optval, sizeof(val)))
3761 return -EFAULT;
3762
a6361f0c 3763 lock_sock(sk);
80feaacb 3764 po->origdev = !!val;
a6361f0c 3765 release_sock(sk);
80feaacb
PWJ		 3766 return 0;
3767 }
bfd5f4a3
SS		 3768 case PACKET_VNET_HDR:
3769 {
3770 int val;
3771
3772 if (sock->type != SOCK_RAW)
3773 return -EINVAL;
bfd5f4a3
SS		 3774 if (optlen < sizeof(val))
3775 return -EINVAL;
3776 if (copy_from_user(&val, optval, sizeof(val)))
3777 return -EFAULT;
3778
a6361f0c
WB		 3779 lock_sock(sk);
3780 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3781 ret = -EBUSY;
3782 } else {
3783 po->has_vnet_hdr = !!val;
3784 ret = 0;
3785 }
3786 release_sock(sk);
3787 return ret;
bfd5f4a3 3788 }
614f60fa
SM		 3789 case PACKET_TIMESTAMP:
3790 {
3791 int val;
3792
3793 if (optlen != sizeof(val))
3794 return -EINVAL;
3795 if (copy_from_user(&val, optval, sizeof(val)))
3796 return -EFAULT;
3797
3798 po->tp_tstamp = val;
3799 return 0;
3800 }
dc99f600
DM		 3801 case PACKET_FANOUT:
3802 {
3803 int val;
3804
3805 if (optlen != sizeof(val))
3806 return -EINVAL;
3807 if (copy_from_user(&val, optval, sizeof(val)))
3808 return -EFAULT;
3809
3810 return fanout_add(sk, val & 0xffff, val >> 16);
3811 }
47dceb8e
WB		 3812 case PACKET_FANOUT_DATA:
3813 {
3814 if (!po->fanout)
3815 return -EINVAL;
3816
3817 return fanout_set_data(po, optval, optlen);
3818 }
fa788d98
VW		 3819 case PACKET_IGNORE_OUTGOING:
3820 {
3821 int val;
3822
3823 if (optlen != sizeof(val))
3824 return -EINVAL;
3825 if (copy_from_user(&val, optval, sizeof(val)))
3826 return -EFAULT;
3827 if (val < 0 || val > 1)
3828 return -EINVAL;
3829
3830 po->prot_hook.ignore_outgoing = !!val;
3831 return 0;
3832 }
5920cd3a
PC		 3833 case PACKET_TX_HAS_OFF:
3834 {
3835 unsigned int val;
3836
3837 if (optlen != sizeof(val))
3838 return -EINVAL;
5920cd3a
PC		 3839 if (copy_from_user(&val, optval, sizeof(val)))
3840 return -EFAULT;
a6361f0c
WB		 3841
3842 lock_sock(sk);
3843 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3844 ret = -EBUSY;
3845 } else {
3846 po->tp_tx_has_off = !!val;
3847 ret = 0;
3848 }
3849 release_sock(sk);
5920cd3a
PC		 3850 return 0;
3851 }
d346a3fa
DB		 3852 case PACKET_QDISC_BYPASS:
3853 {
3854 int val;
3855
3856 if (optlen != sizeof(val))
3857 return -EINVAL;
3858 if (copy_from_user(&val, optval, sizeof(val)))
3859 return -EFAULT;
3860
3861 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3862 return 0;
3863 }
1da177e4
LT		 3864 default:
3865 return -ENOPROTOOPT;
3866 }
3867}
3868
3869static int packet_getsockopt(struct socket *sock, int level, int optname,
3870 char __user *optval, int __user *optlen)
3871{
3872 int len;
c06fff6e 3873 int val, lv = sizeof(val);
1da177e4
LT		 3874 struct sock *sk = sock->sk;
3875 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3876 void *data = &val;
ee80fbf3 3877 union tpacket_stats_u st;
a9b63918 3878 struct tpacket_rollover_stats rstats;
1da177e4
LT		 3879
3880 if (level != SOL_PACKET)
3881 return -ENOPROTOOPT;
3882
8ae55f04
KK		 3883 if (get_user(len, optlen))
3884 return -EFAULT;
1da177e4
LT		 3885
3886 if (len < 0)
3887 return -EINVAL;
1ce4f28b 3888
69e3c75f 3889 switch (optname) {
1da177e4 3890 case PACKET_STATISTICS:
1da177e4 3891 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3892 memcpy(&st, &po->stats, sizeof(st));
3893 memset(&po->stats, 0, sizeof(po->stats));
3894 spin_unlock_bh(&sk->sk_receive_queue.lock);
3895
f6fb8f10 3896 if (po->tp_version == TPACKET_V3) {
c06fff6e 3897 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3898 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3899 data = &st.stats3;
f6fb8f10 3900 } else {
c06fff6e 3901 lv = sizeof(struct tpacket_stats);
8bcdeaff 3902 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3903 data = &st.stats1;
f6fb8f10 3904 }
ee80fbf3 3905
8dc41944
HX		 3906 break;
3907 case PACKET_AUXDATA:
8dc41944 3908 val = po->auxdata;
80feaacb
PWJ		 3909 break;
3910 case PACKET_ORIGDEV:
80feaacb 3911 val = po->origdev;
bfd5f4a3
SS		 3912 break;
3913 case PACKET_VNET_HDR:
bfd5f4a3 3914 val = po->has_vnet_hdr;
1da177e4 3915 break;
bbd6ef87 3916 case PACKET_VERSION:
bbd6ef87 3917 val = po->tp_version;
bbd6ef87
PM		 3918 break;
3919 case PACKET_HDRLEN:
3920 if (len > sizeof(int))
3921 len = sizeof(int);
fd2c83b3
AP		 3922 if (len < sizeof(int))
3923 return -EINVAL;
bbd6ef87
PM		 3924 if (copy_from_user(&val, optval, len))
3925 return -EFAULT;
3926 switch (val) {
3927 case TPACKET_V1:
3928 val = sizeof(struct tpacket_hdr);
3929 break;
3930 case TPACKET_V2:
3931 val = sizeof(struct tpacket2_hdr);
3932 break;
f6fb8f10 3933 case TPACKET_V3:
3934 val = sizeof(struct tpacket3_hdr);
3935 break;
bbd6ef87
PM		 3936 default:
3937 return -EINVAL;
3938 }
bbd6ef87 3939 break;
8913336a 3940 case PACKET_RESERVE:
8913336a 3941 val = po->tp_reserve;
8913336a 3942 break;
69e3c75f 3943 case PACKET_LOSS:
69e3c75f 3944 val = po->tp_loss;
69e3c75f 3945 break;
614f60fa 3946 case PACKET_TIMESTAMP:
614f60fa 3947 val = po->tp_tstamp;
614f60fa 3948 break;
dc99f600 3949 case PACKET_FANOUT:
dc99f600
DM		 3950 val = (po->fanout ?
3951 ((u32)po->fanout->id |
77f65ebd
WB		 3952 ((u32)po->fanout->type << 16) |
3953 ((u32)po->fanout->flags << 24)) :
dc99f600 3954 0);
dc99f600 3955 break;
fa788d98
VW		 3956 case PACKET_IGNORE_OUTGOING:
3957 val = po->prot_hook.ignore_outgoing;
3958 break;
a9b63918 3959 case PACKET_ROLLOVER_STATS:
57f015f5 3960 if (!po->rollover)
a9b63918 3961 return -EINVAL;
57f015f5
MM		 3962 rstats.tp_all = atomic_long_read(&po->rollover->num);
3963 rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
3964 rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
3965 data = &rstats;
3966 lv = sizeof(rstats);
a9b63918 3967 break;
5920cd3a
PC		 3968 case PACKET_TX_HAS_OFF:
3969 val = po->tp_tx_has_off;
3970 break;
d346a3fa
DB		 3971 case PACKET_QDISC_BYPASS:
3972 val = packet_use_direct_xmit(po);
3973 break;
1da177e4
LT		 3974 default:
3975 return -ENOPROTOOPT;
3976 }
3977
c06fff6e
ED		 3978 if (len > lv)
3979 len = lv;
8ae55f04
KK		 3980 if (put_user(len, optlen))
3981 return -EFAULT;
8dc41944
HX		 3982 if (copy_to_user(optval, data, len))
3983 return -EFAULT;
8ae55f04 3984 return 0;
1da177e4
LT		 3985}
3986
3987
719c44d3
WB		 3988#ifdef CONFIG_COMPAT
3989static int compat_packet_setsockopt(struct socket *sock, int level, int optname,
3990 char __user *optval, unsigned int optlen)
3991{
3992 struct packet_sock *po = pkt_sk(sock->sk);
3993
3994 if (level != SOL_PACKET)
3995 return -ENOPROTOOPT;
3996
3997 if (optname == PACKET_FANOUT_DATA &&
3998 po->fanout && po->fanout->type == PACKET_FANOUT_CBPF) {
3999 optval = (char __user *)get_compat_bpf_fprog(optval);
4000 if (!optval)
4001 return -EFAULT;
4002 optlen = sizeof(struct sock_fprog);
4003 }
4004
4005 return packet_setsockopt(sock, level, optname, optval, optlen);
4006}
4007#endif
4008
351638e7
JP		 4009static int packet_notifier(struct notifier_block *this,
4010 unsigned long msg, void *ptr)
1da177e4
LT		 4011{
4012 struct sock *sk;
351638e7 4013 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 4014 struct net *net = dev_net(dev);
1da177e4 4015
808f5114 4016 rcu_read_lock();
b67bfe0d 4017 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 4018 struct packet_sock *po = pkt_sk(sk);
4019
4020 switch (msg) {
4021 case NETDEV_UNREGISTER:
1da177e4 4022 if (po->mclist)
82f17091 4023 packet_dev_mclist_delete(dev, &po->mclist);
a2efcfa0
DM		 4024 /* fallthrough */
4025
1da177e4
LT		 4026 case NETDEV_DOWN:
4027 if (dev->ifindex == po->ifindex) {
4028 spin_lock(&po->bind_lock);
4029 if (po->running) {
ce06b03e 4030 __unregister_prot_hook(sk, false);
1da177e4
LT		 4031 sk->sk_err = ENETDOWN;
4032 if (!sock_flag(sk, SOCK_DEAD))
4033 sk->sk_error_report(sk);
4034 }
4035 if (msg == NETDEV_UNREGISTER) {
66e56cd4 4036 packet_cached_dev_reset(po);
1da177e4 4037 po->ifindex = -1;
160ff18a
BG		 4038 if (po->prot_hook.dev)
4039 dev_put(po->prot_hook.dev);
1da177e4
LT		 4040 po->prot_hook.dev = NULL;
4041 }
4042 spin_unlock(&po->bind_lock);
4043 }
4044 break;
4045 case NETDEV_UP:
808f5114 4046 if (dev->ifindex == po->ifindex) {
4047 spin_lock(&po->bind_lock);
ce06b03e
DM		 4048 if (po->num)
4049 register_prot_hook(sk);
808f5114 4050 spin_unlock(&po->bind_lock);
1da177e4 4051 }
1da177e4
LT		 4052 break;
4053 }
4054 }
808f5114 4055 rcu_read_unlock();
1da177e4
LT		 4056 return NOTIFY_DONE;
4057}
4058
4059
4060static int packet_ioctl(struct socket *sock, unsigned int cmd,
4061 unsigned long arg)
4062{
4063 struct sock *sk = sock->sk;
4064
69e3c75f 4065 switch (cmd) {
40d4e3df
ED		 4066 case SIOCOUTQ:
4067 {
4068 int amount = sk_wmem_alloc_get(sk);
31e6d363 4069
40d4e3df
ED		 4070 return put_user(amount, (int __user *)arg);
4071 }
4072 case SIOCINQ:
4073 {
4074 struct sk_buff *skb;
4075 int amount = 0;
4076
4077 spin_lock_bh(&sk->sk_receive_queue.lock);
4078 skb = skb_peek(&sk->sk_receive_queue);
4079 if (skb)
4080 amount = skb->len;
4081 spin_unlock_bh(&sk->sk_receive_queue.lock);
4082 return put_user(amount, (int __user *)arg);
4083 }
1da177e4 4084#ifdef CONFIG_INET
40d4e3df
ED		 4085 case SIOCADDRT:
4086 case SIOCDELRT:
4087 case SIOCDARP:
4088 case SIOCGARP:
4089 case SIOCSARP:
4090 case SIOCGIFADDR:
4091 case SIOCSIFADDR:
4092 case SIOCGIFBRDADDR:
4093 case SIOCSIFBRDADDR:
4094 case SIOCGIFNETMASK:
4095 case SIOCSIFNETMASK:
4096 case SIOCGIFDSTADDR:
4097 case SIOCSIFDSTADDR:
4098 case SIOCSIFFLAGS:
40d4e3df 4099 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 4100#endif
4101
40d4e3df
ED		 4102 default:
4103 return -ENOIOCTLCMD;
1da177e4
LT		 4104 }
4105 return 0;
4106}
4107
a11e1d43
LT		 4108static __poll_t packet_poll(struct file *file, struct socket *sock,
4109 poll_table *wait)
1da177e4
LT		 4110{
4111 struct sock *sk = sock->sk;
4112 struct packet_sock *po = pkt_sk(sk);
a11e1d43 4113 __poll_t mask = datagram_poll(file, sock, wait);
1da177e4
LT		 4114
4115 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 4116 if (po->rx_ring.pg_vec) {
f6fb8f10 4117 if (!packet_previous_rx_frame(po, &po->rx_ring,
4118 TP_STATUS_KERNEL))
a9a08845 4119 mask |= EPOLLIN | EPOLLRDNORM;
1da177e4 4120 }
2ccdbaa6 4121 if (po->pressure && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
54d7c01d 4122 po->pressure = 0;
1da177e4 4123 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 4124 spin_lock_bh(&sk->sk_write_queue.lock);
4125 if (po->tx_ring.pg_vec) {
4126 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
a9a08845 4127 mask |= EPOLLOUT | EPOLLWRNORM;
69e3c75f
JB		 4128 }
4129 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 4130 return mask;
4131}
4132
4133
4134/* Dirty? Well, I still did not learn better way to account
4135 * for user mmaps.
4136 */
4137
4138static void packet_mm_open(struct vm_area_struct *vma)
4139{
4140 struct file *file = vma->vm_file;
40d4e3df 4141 struct socket *sock = file->private_data;
1da177e4 4142 struct sock *sk = sock->sk;
1ce4f28b 4143
1da177e4
LT		 4144 if (sk)
4145 atomic_inc(&pkt_sk(sk)->mapped);
4146}
4147
4148static void packet_mm_close(struct vm_area_struct *vma)
4149{
4150 struct file *file = vma->vm_file;
40d4e3df 4151 struct socket *sock = file->private_data;
1da177e4 4152 struct sock *sk = sock->sk;
1ce4f28b 4153
1da177e4
LT		 4154 if (sk)
4155 atomic_dec(&pkt_sk(sk)->mapped);
4156}
4157
f0f37e2f 4158static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 4159 .open = packet_mm_open,
4160 .close = packet_mm_close,
1da177e4
LT		 4161};
4162
3a7ad063
ED		 4163static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4164 unsigned int len)
1da177e4
LT		 4165{
4166 int i;
4167
4ebf0ae2 4168 for (i = 0; i < len; i++) {
0e3125c7 4169 if (likely(pg_vec[i].buffer)) {
3a7ad063
ED		 4170 if (is_vmalloc_addr(pg_vec[i].buffer))
4171 vfree(pg_vec[i].buffer);
4172 else
4173 free_pages((unsigned long)pg_vec[i].buffer,
4174 order);
0e3125c7
NH		 4175 pg_vec[i].buffer = NULL;
4176 }
1da177e4
LT		 4177 }
4178 kfree(pg_vec);
4179}
4180
3a7ad063 4181static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 4182{
f0d4eb29 4183 char *buffer;
3a7ad063
ED		 4184 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4185 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
0e3125c7 4186
3a7ad063 4187 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 4188 if (buffer)
4189 return buffer;
4190
3a7ad063
ED		 4191 /* __get_free_pages failed, fall back to vmalloc */
4192 buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
4193 if (buffer)
4194 return buffer;
0e3125c7 4195
3a7ad063
ED		 4196 /* vmalloc failed, lets dig into swap here */
4197 gfp_flags &= ~__GFP_NORETRY;
4198 buffer = (char *) __get_free_pages(gfp_flags, order);
4199 if (buffer)
4200 return buffer;
4201
4202 /* complete and utter failure */
4203 return NULL;
4ebf0ae2
DM		 4204}
4205
3a7ad063 4206static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 4207{
4208 unsigned int block_nr = req->tp_block_nr;
0e3125c7 4209 struct pgv *pg_vec;
4ebf0ae2
DM		 4210 int i;
4211
398f0132 4212 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN);
4ebf0ae2
DM		 4213 if (unlikely(!pg_vec))
4214 goto out;
4215
4216 for (i = 0; i < block_nr; i++) {
3a7ad063 4217 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 4218 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 4219 goto out_free_pgvec;
4220 }
4221
4222out:
4223 return pg_vec;
4224
4225out_free_pgvec:
3a7ad063 4226 free_pg_vec(pg_vec, order, block_nr);
4ebf0ae2
DM		 4227 pg_vec = NULL;
4228 goto out;
4229}
1da177e4 4230
f6fb8f10 4231static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 4232 int closing, int tx_ring)
1da177e4 4233{
0e3125c7 4234 struct pgv *pg_vec = NULL;
1da177e4 4235 struct packet_sock *po = pkt_sk(sk);
3a7ad063 4236 int was_running, order = 0;
69e3c75f
JB		 4237 struct packet_ring_buffer *rb;
4238 struct sk_buff_head *rb_queue;
0e11c91e 4239 __be16 num;
f6fb8f10 4240 int err = -EINVAL;
4241 /* Added to avoid minimal code churn */
4242 struct tpacket_req *req = &req_u->req;
4243
69e3c75f
JB		 4244 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4245 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 4246
69e3c75f
JB		 4247 err = -EBUSY;
4248 if (!closing) {
4249 if (atomic_read(&po->mapped))
4250 goto out;
b0138408 4251 if (packet_read_pending(rb))
69e3c75f
JB		 4252 goto out;
4253 }
1da177e4 4254
69e3c75f 4255 if (req->tp_block_nr) {
4576cd46
WB		 4256 unsigned int min_frame_size;
4257
69e3c75f
JB		 4258 /* Sanity tests and some calculations */
4259 err = -EBUSY;
4260 if (unlikely(rb->pg_vec))
4261 goto out;
1da177e4 4262
bbd6ef87
PM		 4263 switch (po->tp_version) {
4264 case TPACKET_V1:
4265 po->tp_hdrlen = TPACKET_HDRLEN;
4266 break;
4267 case TPACKET_V2:
4268 po->tp_hdrlen = TPACKET2_HDRLEN;
4269 break;
f6fb8f10 4270 case TPACKET_V3:
4271 po->tp_hdrlen = TPACKET3_HDRLEN;
4272 break;
bbd6ef87
PM		 4273 }
4274
69e3c75f 4275 err = -EINVAL;
4ebf0ae2 4276 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 4277 goto out;
90836b67 4278 if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
69e3c75f 4279 goto out;
4576cd46 4280 min_frame_size = po->tp_hdrlen + po->tp_reserve;
dc808110 4281 if (po->tp_version >= TPACKET_V3 &&
4576cd46
WB		 4282 req->tp_block_size <
4283 BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
dc808110 4284 goto out;
4576cd46 4285 if (unlikely(req->tp_frame_size < min_frame_size))
69e3c75f 4286 goto out;
4ebf0ae2 4287 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 4288 goto out;
1da177e4 4289
4194b491
TK		 4290 rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4291 if (unlikely(rb->frames_per_block == 0))
69e3c75f 4292 goto out;
fc62814d 4293 if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
8f8d28e4 4294 goto out;
69e3c75f
JB		 4295 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4296 req->tp_frame_nr))
4297 goto out;
1da177e4
LT		 4298
4299 err = -ENOMEM;
3a7ad063
ED		 4300 order = get_order(req->tp_block_size);
4301 pg_vec = alloc_pg_vec(req, order);
4ebf0ae2 4302 if (unlikely(!pg_vec))
1da177e4 4303 goto out;
f6fb8f10 4304 switch (po->tp_version) {
4305 case TPACKET_V3:
7f953ab2
SV		 4306 /* Block transmit is not supported yet */
4307 if (!tx_ring) {
e8e85cc5 4308 init_prb_bdqc(po, rb, pg_vec, req_u);
7f953ab2
SV		 4309 } else {
4310 struct tpacket_req3 *req3 = &req_u->req3;
4311
4312 if (req3->tp_retire_blk_tov ||
4313 req3->tp_sizeof_priv ||
4314 req3->tp_feature_req_word) {
4315 err = -EINVAL;
4316 goto out;
4317 }
4318 }
d7cf0c34 4319 break;
f6fb8f10 4320 default:
4321 break;
4322 }
69e3c75f
JB		 4323 }
4324 /* Done */
4325 else {
4326 err = -EINVAL;
4ebf0ae2 4327 if (unlikely(req->tp_frame_nr))
69e3c75f 4328 goto out;
1da177e4
LT		 4329 }
4330
1da177e4
LT		 4331
4332 /* Detach socket from network */
4333 spin_lock(&po->bind_lock);
4334 was_running = po->running;
4335 num = po->num;
4336 if (was_running) {
1da177e4 4337 po->num = 0;
ce06b03e 4338 __unregister_prot_hook(sk, false);
1da177e4
LT		 4339 }
4340 spin_unlock(&po->bind_lock);
1ce4f28b 4341
1da177e4
LT		 4342 synchronize_net();
4343
4344 err = -EBUSY;
905db440 4345 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 4346 if (closing || atomic_read(&po->mapped) == 0) {
4347 err = 0;
69e3c75f 4348 spin_lock_bh(&rb_queue->lock);
c053fd96 4349 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 4350 rb->frame_max = (req->tp_frame_nr - 1);
4351 rb->head = 0;
4352 rb->frame_size = req->tp_frame_size;
4353 spin_unlock_bh(&rb_queue->lock);
4354
3a7ad063 4355 swap(rb->pg_vec_order, order);
c053fd96 4356 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 4357
4358 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4359 po->prot_hook.func = (po->rx_ring.pg_vec) ?
4360 tpacket_rcv : packet_rcv;
4361 skb_queue_purge(rb_queue);
1da177e4 4362 if (atomic_read(&po->mapped))
40d4e3df
ED		 4363 pr_err("packet_mmap: vma is busy: %d\n",
4364 atomic_read(&po->mapped));
1da177e4 4365 }
905db440 4366 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4367
4368 spin_lock(&po->bind_lock);
ce06b03e 4369 if (was_running) {
1da177e4 4370 po->num = num;
ce06b03e 4371 register_prot_hook(sk);
1da177e4
LT		 4372 }
4373 spin_unlock(&po->bind_lock);
c800aaf8 4374 if (pg_vec && (po->tp_version > TPACKET_V2)) {
f6fb8f10 4375 /* Because we don't support block-based V3 on tx-ring */
4376 if (!tx_ring)
73d0fcf2 4377 prb_shutdown_retire_blk_timer(po, rb_queue);
f6fb8f10 4378 }
1da177e4 4379
1da177e4 4380 if (pg_vec)
3a7ad063 4381 free_pg_vec(pg_vec, order, req->tp_block_nr);
1da177e4
LT		 4382out:
4383 return err;
4384}
4385
69e3c75f
JB		 4386static int packet_mmap(struct file *file, struct socket *sock,
4387 struct vm_area_struct *vma)
1da177e4
LT		 4388{
4389 struct sock *sk = sock->sk;
4390 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 4391 unsigned long size, expected_size;
4392 struct packet_ring_buffer *rb;
1da177e4
LT		 4393 unsigned long start;
4394 int err = -EINVAL;
4395 int i;
4396
4397 if (vma->vm_pgoff)
4398 return -EINVAL;
4399
905db440 4400 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 4401
4402 expected_size = 0;
4403 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4404 if (rb->pg_vec) {
4405 expected_size += rb->pg_vec_len
4406 * rb->pg_vec_pages
4407 * PAGE_SIZE;
4408 }
4409 }
4410
4411 if (expected_size == 0)
1da177e4 4412 goto out;
69e3c75f
JB		 4413
4414 size = vma->vm_end - vma->vm_start;
4415 if (size != expected_size)
1da177e4
LT		 4416 goto out;
4417
1da177e4 4418 start = vma->vm_start;
69e3c75f
JB		 4419 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4420 if (rb->pg_vec == NULL)
4421 continue;
4422
4423 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 4424 struct page *page;
4425 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 4426 int pg_num;
4427
c56b4d90
CG		 4428 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4429 page = pgv_to_page(kaddr);
69e3c75f
JB		 4430 err = vm_insert_page(vma, start, page);
4431 if (unlikely(err))
4432 goto out;
4433 start += PAGE_SIZE;
0e3125c7 4434 kaddr += PAGE_SIZE;
69e3c75f 4435 }
4ebf0ae2 4436 }
1da177e4 4437 }
69e3c75f 4438
4ebf0ae2 4439 atomic_inc(&po->mapped);
1da177e4
LT		 4440 vma->vm_ops = &packet_mmap_ops;
4441 err = 0;
4442
4443out:
905db440 4444 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 4445 return err;
4446}
1da177e4 4447
90ddc4f0 4448static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 4449 .family = PF_PACKET,
4450 .owner = THIS_MODULE,
4451 .release = packet_release,
4452 .bind = packet_bind_spkt,
4453 .connect = sock_no_connect,
4454 .socketpair = sock_no_socketpair,
4455 .accept = sock_no_accept,
4456 .getname = packet_getname_spkt,
a11e1d43 4457 .poll = datagram_poll,
1da177e4 4458 .ioctl = packet_ioctl,
c7cbdbf2 4459 .gettstamp = sock_gettstamp,
1da177e4
LT		 4460 .listen = sock_no_listen,
4461 .shutdown = sock_no_shutdown,
4462 .setsockopt = sock_no_setsockopt,
4463 .getsockopt = sock_no_getsockopt,
4464 .sendmsg = packet_sendmsg_spkt,
4465 .recvmsg = packet_recvmsg,
4466 .mmap = sock_no_mmap,
4467 .sendpage = sock_no_sendpage,
4468};
1da177e4 4469
90ddc4f0 4470static const struct proto_ops packet_ops = {
1da177e4
LT		 4471 .family = PF_PACKET,
4472 .owner = THIS_MODULE,
4473 .release = packet_release,
4474 .bind = packet_bind,
4475 .connect = sock_no_connect,
4476 .socketpair = sock_no_socketpair,
4477 .accept = sock_no_accept,
1ce4f28b 4478 .getname = packet_getname,
a11e1d43 4479 .poll = packet_poll,
1da177e4 4480 .ioctl = packet_ioctl,
c7cbdbf2 4481 .gettstamp = sock_gettstamp,
1da177e4
LT		 4482 .listen = sock_no_listen,
4483 .shutdown = sock_no_shutdown,
4484 .setsockopt = packet_setsockopt,
4485 .getsockopt = packet_getsockopt,
719c44d3
WB		 4486#ifdef CONFIG_COMPAT
4487 .compat_setsockopt = compat_packet_setsockopt,
4488#endif
1da177e4
LT		 4489 .sendmsg = packet_sendmsg,
4490 .recvmsg = packet_recvmsg,
4491 .mmap = packet_mmap,
4492 .sendpage = sock_no_sendpage,
4493};
4494
ec1b4cf7 4495static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 4496 .family = PF_PACKET,
4497 .create = packet_create,
4498 .owner = THIS_MODULE,
4499};
4500
4501static struct notifier_block packet_netdev_notifier = {
40d4e3df 4502 .notifier_call = packet_notifier,
1da177e4
LT		 4503};
4504
4505#ifdef CONFIG_PROC_FS
1da177e4
LT		 4506
4507static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 4508 __acquires(RCU)
1da177e4 4509{
e372c414 4510 struct net *net = seq_file_net(seq);
808f5114 4511
4512 rcu_read_lock();
4513 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 4514}
4515
4516static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4517{
1bf40954 4518 struct net *net = seq_file_net(seq);
808f5114 4519 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 4520}
4521
4522static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 4523 __releases(RCU)
1da177e4 4524{
808f5114 4525 rcu_read_unlock();
1da177e4
LT		 4526}
4527
1ce4f28b 4528static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 4529{
4530 if (v == SEQ_START_TOKEN)
4531 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
4532 else {
b7ceabd9 4533 struct sock *s = sk_entry(v);
1da177e4
LT		 4534 const struct packet_sock *po = pkt_sk(s);
4535
4536 seq_printf(seq,
71338aa7 4537 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4 4538 s,
41c6d650 4539 refcount_read(&s->sk_refcnt),
1da177e4
LT		 4540 s->sk_type,
4541 ntohs(po->num),
4542 po->ifindex,
4543 po->running,
4544 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4545 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4546 sock_i_ino(s));
1da177e4
LT		 4547 }
4548
4549 return 0;
4550}
4551
56b3d975 4552static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4553 .start = packet_seq_start,
4554 .next = packet_seq_next,
4555 .stop = packet_seq_stop,
4556 .show = packet_seq_show,
4557};
1da177e4
LT		 4558#endif
4559
2c8c1e72 4560static int __net_init packet_net_init(struct net *net)
d12d01d6 4561{
0fa7fa98 4562 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4563 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4564
c3506372
CH		 4565 if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
4566 sizeof(struct seq_net_private)))
d12d01d6
DL		 4567 return -ENOMEM;
4568
4569 return 0;
4570}
4571
2c8c1e72 4572static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4573{
ece31ffd 4574 remove_proc_entry("packet", net->proc_net);
669f8f1a 4575 WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
d12d01d6
DL		 4576}
4577
4578static struct pernet_operations packet_net_ops = {
4579 .init = packet_net_init,
4580 .exit = packet_net_exit,
4581};
4582
4583
1da177e4
LT		 4584static void __exit packet_exit(void)
4585{
1da177e4 4586 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4587 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4588 sock_unregister(PF_PACKET);
4589 proto_unregister(&packet_proto);
4590}
4591
4592static int __init packet_init(void)
4593{
36096f2f 4594 int rc;
1da177e4 4595
36096f2f
Y		 4596 rc = proto_register(&packet_proto, 0);
4597 if (rc)
1da177e4 4598 goto out;
36096f2f
Y		 4599 rc = sock_register(&packet_family_ops);
4600 if (rc)
4601 goto out_proto;
4602 rc = register_pernet_subsys(&packet_net_ops);
4603 if (rc)
4604 goto out_sock;
4605 rc = register_netdevice_notifier(&packet_netdev_notifier);
4606 if (rc)
4607 goto out_pernet;
1da177e4 4608
36096f2f
Y		 4609 return 0;
4610
4611out_pernet:
4612 unregister_pernet_subsys(&packet_net_ops);
4613out_sock:
4614 sock_unregister(PF_PACKET);
4615out_proto:
4616 proto_unregister(&packet_proto);
1da177e4
LT		 4617out:
4618 return rc;
4619}
4620
4621module_init(packet_init);
4622module_exit(packet_exit);
4623MODULE_LICENSE("GPL");
4624MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux 6.x block layer and io_uring tree(s)