projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'misc' of git://git.kernel.org/pub/scm/linux/kernel/git/mmarek/kbuild
[linux-2.6-block.git] / net / packet / af_packet.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
1ce4f28b 12 * Fixes:
1da177e4
LT		 13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
1ce4f28b 35 * Ulises Alonso : Frame number limit removal and
1da177e4 36 * packet_set_ring memory leak.
0fb375fb
EB		 37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
1ce4f28b 40 * byte arrays at the end of sockaddr_ll
0fb375fb 41 * and packet_mreq.
69e3c75f 42 * Johann Baudy : Added TX RING.
f6fb8f10 43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
44 * layer.
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
46 *
1da177e4
LT		 47 *
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
52 *
53 */
1ce4f28b 54
1da177e4 55#include <linux/types.h>
1da177e4 56#include <linux/mm.h>
4fc268d2 57#include <linux/capability.h>
1da177e4
LT		 58#include <linux/fcntl.h>
59#include <linux/socket.h>
60#include <linux/in.h>
61#include <linux/inet.h>
62#include <linux/netdevice.h>
63#include <linux/if_packet.h>
64#include <linux/wireless.h>
ffbc6111 65#include <linux/kernel.h>
1da177e4 66#include <linux/kmod.h>
5a0e3ad6 67#include <linux/slab.h>
0e3125c7 68#include <linux/vmalloc.h>
457c4cbc 69#include <net/net_namespace.h>
1da177e4
LT		 70#include <net/ip.h>
71#include <net/protocol.h>
72#include <linux/skbuff.h>
73#include <net/sock.h>
74#include <linux/errno.h>
75#include <linux/timer.h>
1da177e4
LT		 76#include <asm/uaccess.h>
77#include <asm/ioctls.h>
78#include <asm/page.h>
a1f8e7f7 79#include <asm/cacheflush.h>
1da177e4
LT		 80#include <asm/io.h>
81#include <linux/proc_fs.h>
82#include <linux/seq_file.h>
83#include <linux/poll.h>
84#include <linux/module.h>
85#include <linux/init.h>
905db440 86#include <linux/mutex.h>
05423b24 87#include <linux/if_vlan.h>
bfd5f4a3 88#include <linux/virtio_net.h>
ed85b565 89#include <linux/errqueue.h>
614f60fa 90#include <linux/net_tstamp.h>
b0138408 91#include <linux/percpu.h>
1da177e4
LT		 92#ifdef CONFIG_INET
93#include <net/inet_common.h>
94#endif
95
2787b04b
PE		 96#include "internal.h"
97
1da177e4
LT		 98/*
99 Assumptions:
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
105 (PPP).
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
108
109On receive:
110-----------
111
112Incoming, dev->hard_header!=NULL
b0e380b1
ACM		 113 mac_header -> ll header
114 data -> data
1da177e4
LT		 115
116Outgoing, dev->hard_header!=NULL
b0e380b1
ACM		 117 mac_header -> ll header
118 data -> ll header
1da177e4
LT		 119
120Incoming, dev->hard_header==NULL
b0e380b1
ACM		 121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
db0c58f9 123 assymetry between rx and tx paths.
b0e380b1 124 data -> data
1da177e4
LT		 125
126Outgoing, dev->hard_header==NULL
b0e380b1
ACM		 127 mac_header -> data. ll header is still not built!
128 data -> data
1da177e4
LT		 129
130Resume
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132
133
134On transmit:
135------------
136
137dev->hard_header != NULL
b0e380b1
ACM		 138 mac_header -> ll header
139 data -> ll header
1da177e4
LT		 140
141dev->hard_header == NULL (ll header is added by device, we cannot control it)
b0e380b1
ACM		 142 mac_header -> data
143 data -> data
1da177e4
LT		 144
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
147 */
148
1da177e4
LT		 149/* Private packet socket structures. */
150
0fb375fb
EB		 151/* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
40d4e3df 154struct packet_mreq_max {
0fb375fb
EB		 155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
1da177e4 159};
a2efcfa0 160
184f489e
DB		 161union tpacket_uhdr {
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
165 void *raw;
166};
167
f6fb8f10 168static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f
JB		 169 int closing, int tx_ring);
170
f6fb8f10 171#define V3_ALIGNMENT (8)
172
bc59ba39 173#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
f6fb8f10 174
175#define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
177
f6fb8f10 178#define PGV_FROM_VMALLOC 1
69e3c75f 179
f6fb8f10 180#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186#define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
187
69e3c75f
JB		 188struct packet_sock;
189static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
77f65ebd
WB		 190static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
1da177e4 192
f6fb8f10 193static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
195 int status);
196static void packet_increment_head(struct packet_ring_buffer *buff);
bc59ba39 197static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
f6fb8f10 200 struct packet_sock *);
bc59ba39 201static void prb_retire_current_block(struct tpacket_kbdq_core *,
f6fb8f10 202 struct packet_sock *, unsigned int status);
bc59ba39 203static int prb_queue_frozen(struct tpacket_kbdq_core *);
204static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
f6fb8f10 206static void prb_retire_rx_blk_timer_expired(unsigned long);
bc59ba39 207static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
1da177e4
LT		 216static void packet_flush_mclist(struct sock *sk);
217
ffbc6111
HX		 218struct packet_skb_cb {
219 unsigned int origlen;
220 union {
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
223 } sa;
224};
225
226#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
8dc41944 227
bc59ba39 228#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
f6fb8f10 229#define GET_PBLOCK_DESC(x, bid) \
bc59ba39 230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
f6fb8f10 231#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
bc59ba39 232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
f6fb8f10 233#define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
236
dc99f600
DM		 237static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238static void __fanout_link(struct sock *sk, struct packet_sock *po);
239
d346a3fa
DB		 240static int packet_direct_xmit(struct sk_buff *skb)
241{
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
246 u16 queue_map;
247 int ret;
248
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
251 kfree_skb(skb);
252 return NET_XMIT_DROP;
253 }
254
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
258 kfree_skb(skb);
259 return NET_XMIT_DROP;
260 }
261
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
264
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
268 kfree_skb(skb);
269 goto out;
270 }
271
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
275 else
276 kfree_skb(skb);
277out:
278 __netif_tx_unlock_bh(txq);
279 return ret;
280}
281
66e56cd4
DB		 282static struct net_device *packet_cached_dev_get(struct packet_sock *po)
283{
284 struct net_device *dev;
285
286 rcu_read_lock();
287 dev = rcu_dereference(po->cached_dev);
288 if (likely(dev))
289 dev_hold(dev);
290 rcu_read_unlock();
291
292 return dev;
293}
294
295static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
297{
298 rcu_assign_pointer(po->cached_dev, dev);
299}
300
301static void packet_cached_dev_reset(struct packet_sock *po)
302{
303 RCU_INIT_POINTER(po->cached_dev, NULL);
304}
305
d346a3fa
DB		 306static bool packet_use_direct_xmit(const struct packet_sock *po)
307{
308 return po->xmit == packet_direct_xmit;
309}
310
311static u16 packet_pick_tx_queue(struct net_device *dev)
312{
1cbac010 313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
d346a3fa
DB		 314}
315
ce06b03e
DM		 316/* register_prot_hook must be invoked with the po->bind_lock held,
317 * or from a context in which asynchronous accesses to the packet
318 * socket is not possible (packet_create()).
319 */
320static void register_prot_hook(struct sock *sk)
321{
322 struct packet_sock *po = pkt_sk(sk);
e40526cb 323
ce06b03e 324 if (!po->running) {
66e56cd4 325 if (po->fanout)
dc99f600 326 __fanout_link(sk, po);
66e56cd4 327 else
dc99f600 328 dev_add_pack(&po->prot_hook);
e40526cb 329
ce06b03e
DM		 330 sock_hold(sk);
331 po->running = 1;
332 }
333}
334
335/* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
336 * held. If the sync parameter is true, we will temporarily drop
337 * the po->bind_lock and do a synchronize_net to make sure no
338 * asynchronous packet processing paths still refer to the elements
339 * of po->prot_hook. If the sync parameter is false, it is the
340 * callers responsibility to take care of this.
341 */
342static void __unregister_prot_hook(struct sock *sk, bool sync)
343{
344 struct packet_sock *po = pkt_sk(sk);
345
346 po->running = 0;
66e56cd4
DB		 347
348 if (po->fanout)
dc99f600 349 __fanout_unlink(sk, po);
66e56cd4 350 else
dc99f600 351 __dev_remove_pack(&po->prot_hook);
e40526cb 352
ce06b03e
DM		 353 __sock_put(sk);
354
355 if (sync) {
356 spin_unlock(&po->bind_lock);
357 synchronize_net();
358 spin_lock(&po->bind_lock);
359 }
360}
361
362static void unregister_prot_hook(struct sock *sk, bool sync)
363{
364 struct packet_sock *po = pkt_sk(sk);
365
366 if (po->running)
367 __unregister_prot_hook(sk, sync);
368}
369
f6dafa95 370static inline __pure struct page *pgv_to_page(void *addr)
0af55bb5
CG		 371{
372 if (is_vmalloc_addr(addr))
373 return vmalloc_to_page(addr);
374 return virt_to_page(addr);
375}
376
69e3c75f 377static void __packet_set_status(struct packet_sock *po, void *frame, int status)
1da177e4 378{
184f489e 379 union tpacket_uhdr h;
1da177e4 380
69e3c75f 381 h.raw = frame;
bbd6ef87
PM		 382 switch (po->tp_version) {
383 case TPACKET_V1:
69e3c75f 384 h.h1->tp_status = status;
0af55bb5 385 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
bbd6ef87
PM		 386 break;
387 case TPACKET_V2:
69e3c75f 388 h.h2->tp_status = status;
0af55bb5 389 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
bbd6ef87 390 break;
f6fb8f10 391 case TPACKET_V3:
69e3c75f 392 default:
f6fb8f10 393 WARN(1, "TPACKET version not supported.\n");
69e3c75f 394 BUG();
bbd6ef87 395 }
69e3c75f
JB		 396
397 smp_wmb();
bbd6ef87
PM		 398}
399
69e3c75f 400static int __packet_get_status(struct packet_sock *po, void *frame)
bbd6ef87 401{
184f489e 402 union tpacket_uhdr h;
bbd6ef87 403
69e3c75f
JB		 404 smp_rmb();
405
bbd6ef87
PM		 406 h.raw = frame;
407 switch (po->tp_version) {
408 case TPACKET_V1:
0af55bb5 409 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
69e3c75f 410 return h.h1->tp_status;
bbd6ef87 411 case TPACKET_V2:
0af55bb5 412 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
69e3c75f 413 return h.h2->tp_status;
f6fb8f10 414 case TPACKET_V3:
69e3c75f 415 default:
f6fb8f10 416 WARN(1, "TPACKET version not supported.\n");
69e3c75f
JB		 417 BUG();
418 return 0;
bbd6ef87 419 }
1da177e4 420}
69e3c75f 421
b9c32fb2
DB		 422static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
423 unsigned int flags)
7a51384c
DB		 424{
425 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
426
427 if (shhwtstamps) {
428 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
429 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
b9c32fb2 430 return TP_STATUS_TS_SYS_HARDWARE;
7a51384c
DB		 431 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
432 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
b9c32fb2 433 return TP_STATUS_TS_RAW_HARDWARE;
7a51384c
DB		 434 }
435
436 if (ktime_to_timespec_cond(skb->tstamp, ts))
b9c32fb2 437 return TP_STATUS_TS_SOFTWARE;
7a51384c 438
b9c32fb2 439 return 0;
7a51384c
DB		 440}
441
b9c32fb2
DB		 442static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
443 struct sk_buff *skb)
2e31396f
WB		 444{
445 union tpacket_uhdr h;
446 struct timespec ts;
b9c32fb2 447 __u32 ts_status;
2e31396f 448
b9c32fb2
DB		 449 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
450 return 0;
2e31396f
WB		 451
452 h.raw = frame;
453 switch (po->tp_version) {
454 case TPACKET_V1:
455 h.h1->tp_sec = ts.tv_sec;
456 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
457 break;
458 case TPACKET_V2:
459 h.h2->tp_sec = ts.tv_sec;
460 h.h2->tp_nsec = ts.tv_nsec;
461 break;
462 case TPACKET_V3:
463 default:
464 WARN(1, "TPACKET version not supported.\n");
465 BUG();
466 }
467
468 /* one flush is safe, as both fields always lie on the same cacheline */
469 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
470 smp_wmb();
b9c32fb2
DB		 471
472 return ts_status;
2e31396f
WB		 473}
474
69e3c75f
JB		 475static void *packet_lookup_frame(struct packet_sock *po,
476 struct packet_ring_buffer *rb,
477 unsigned int position,
478 int status)
479{
480 unsigned int pg_vec_pos, frame_offset;
184f489e 481 union tpacket_uhdr h;
69e3c75f
JB		 482
483 pg_vec_pos = position / rb->frames_per_block;
484 frame_offset = position % rb->frames_per_block;
485
0e3125c7
NH		 486 h.raw = rb->pg_vec[pg_vec_pos].buffer +
487 (frame_offset * rb->frame_size);
69e3c75f
JB		 488
489 if (status != __packet_get_status(po, h.raw))
490 return NULL;
491
492 return h.raw;
493}
494
eea49cc9 495static void *packet_current_frame(struct packet_sock *po,
69e3c75f
JB		 496 struct packet_ring_buffer *rb,
497 int status)
498{
499 return packet_lookup_frame(po, rb, rb->head, status);
500}
501
bc59ba39 502static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 503{
504 del_timer_sync(&pkc->retire_blk_timer);
505}
506
507static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
508 int tx_ring,
509 struct sk_buff_head *rb_queue)
510{
bc59ba39 511 struct tpacket_kbdq_core *pkc;
f6fb8f10 512
22781a5b
DJ		 513 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
514 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 515
ec6f809f 516 spin_lock_bh(&rb_queue->lock);
f6fb8f10 517 pkc->delete_blk_timer = 1;
ec6f809f 518 spin_unlock_bh(&rb_queue->lock);
f6fb8f10 519
520 prb_del_retire_blk_timer(pkc);
521}
522
523static void prb_init_blk_timer(struct packet_sock *po,
bc59ba39 524 struct tpacket_kbdq_core *pkc,
f6fb8f10 525 void (*func) (unsigned long))
526{
527 init_timer(&pkc->retire_blk_timer);
528 pkc->retire_blk_timer.data = (long)po;
529 pkc->retire_blk_timer.function = func;
530 pkc->retire_blk_timer.expires = jiffies;
531}
532
533static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
534{
bc59ba39 535 struct tpacket_kbdq_core *pkc;
f6fb8f10 536
537 if (tx_ring)
538 BUG();
539
22781a5b
DJ		 540 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
541 GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 542 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
543}
544
545static int prb_calc_retire_blk_tmo(struct packet_sock *po,
546 int blk_size_in_bytes)
547{
548 struct net_device *dev;
549 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
4bc71cb9
JP		 550 struct ethtool_cmd ecmd;
551 int err;
e440cf2c 552 u32 speed;
f6fb8f10 553
4bc71cb9
JP		 554 rtnl_lock();
555 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
556 if (unlikely(!dev)) {
557 rtnl_unlock();
f6fb8f10 558 return DEFAULT_PRB_RETIRE_TOV;
4bc71cb9
JP		 559 }
560 err = __ethtool_get_settings(dev, &ecmd);
e440cf2c 561 speed = ethtool_cmd_speed(&ecmd);
4bc71cb9
JP		 562 rtnl_unlock();
563 if (!err) {
4bc71cb9
JP		 564 /*
565 * If the link speed is so slow you don't really
566 * need to worry about perf anyways
567 */
e440cf2c 568 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
4bc71cb9 569 return DEFAULT_PRB_RETIRE_TOV;
e440cf2c 570 } else {
571 msec = 1;
572 div = speed / 1000;
f6fb8f10 573 }
574 }
575
576 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
577
578 if (div)
579 mbits /= div;
580
581 tmo = mbits * msec;
582
583 if (div)
584 return tmo+1;
585 return tmo;
586}
587
bc59ba39 588static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
f6fb8f10 589 union tpacket_req_u *req_u)
590{
591 p1->feature_req_word = req_u->req3.tp_feature_req_word;
592}
593
594static void init_prb_bdqc(struct packet_sock *po,
595 struct packet_ring_buffer *rb,
596 struct pgv *pg_vec,
597 union tpacket_req_u *req_u, int tx_ring)
598{
22781a5b 599 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
bc59ba39 600 struct tpacket_block_desc *pbd;
f6fb8f10 601
602 memset(p1, 0x0, sizeof(*p1));
603
604 p1->knxt_seq_num = 1;
605 p1->pkbdq = pg_vec;
bc59ba39 606 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
e3192690 607 p1->pkblk_start = pg_vec[0].buffer;
f6fb8f10 608 p1->kblk_size = req_u->req3.tp_block_size;
609 p1->knum_blocks = req_u->req3.tp_block_nr;
610 p1->hdrlen = po->tp_hdrlen;
611 p1->version = po->tp_version;
612 p1->last_kactive_blk_num = 0;
ee80fbf3 613 po->stats.stats3.tp_freeze_q_cnt = 0;
f6fb8f10 614 if (req_u->req3.tp_retire_blk_tov)
615 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
616 else
617 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
618 req_u->req3.tp_block_size);
619 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
620 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
621
622 prb_init_ft_ops(p1, req_u);
623 prb_setup_retire_blk_timer(po, tx_ring);
624 prb_open_block(p1, pbd);
625}
626
627/* Do NOT update the last_blk_num first.
628 * Assumes sk_buff_head lock is held.
629 */
bc59ba39 630static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
f6fb8f10 631{
632 mod_timer(&pkc->retire_blk_timer,
633 jiffies + pkc->tov_in_jiffies);
634 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
635}
636
637/*
638 * Timer logic:
639 * 1) We refresh the timer only when we open a block.
640 * By doing this we don't waste cycles refreshing the timer
641 * on packet-by-packet basis.
642 *
643 * With a 1MB block-size, on a 1Gbps line, it will take
644 * i) ~8 ms to fill a block + ii) memcpy etc.
645 * In this cut we are not accounting for the memcpy time.
646 *
647 * So, if the user sets the 'tmo' to 10ms then the timer
648 * will never fire while the block is still getting filled
649 * (which is what we want). However, the user could choose
650 * to close a block early and that's fine.
651 *
652 * But when the timer does fire, we check whether or not to refresh it.
653 * Since the tmo granularity is in msecs, it is not too expensive
654 * to refresh the timer, lets say every '8' msecs.
655 * Either the user can set the 'tmo' or we can derive it based on
656 * a) line-speed and b) block-size.
657 * prb_calc_retire_blk_tmo() calculates the tmo.
658 *
659 */
660static void prb_retire_rx_blk_timer_expired(unsigned long data)
661{
662 struct packet_sock *po = (struct packet_sock *)data;
22781a5b 663 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 664 unsigned int frozen;
bc59ba39 665 struct tpacket_block_desc *pbd;
f6fb8f10 666
667 spin_lock(&po->sk.sk_receive_queue.lock);
668
669 frozen = prb_queue_frozen(pkc);
670 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
671
672 if (unlikely(pkc->delete_blk_timer))
673 goto out;
674
675 /* We only need to plug the race when the block is partially filled.
676 * tpacket_rcv:
677 * lock(); increment BLOCK_NUM_PKTS; unlock()
678 * copy_bits() is in progress ...
679 * timer fires on other cpu:
680 * we can't retire the current block because copy_bits
681 * is in progress.
682 *
683 */
684 if (BLOCK_NUM_PKTS(pbd)) {
685 while (atomic_read(&pkc->blk_fill_in_prog)) {
686 /* Waiting for skb_copy_bits to finish... */
687 cpu_relax();
688 }
689 }
690
691 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
692 if (!frozen) {
693 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
694 if (!prb_dispatch_next_block(pkc, po))
695 goto refresh_timer;
696 else
697 goto out;
698 } else {
699 /* Case 1. Queue was frozen because user-space was
700 * lagging behind.
701 */
702 if (prb_curr_blk_in_use(pkc, pbd)) {
703 /*
704 * Ok, user-space is still behind.
705 * So just refresh the timer.
706 */
707 goto refresh_timer;
708 } else {
709 /* Case 2. queue was frozen,user-space caught up,
710 * now the link went idle && the timer fired.
711 * We don't have a block to close.So we open this
712 * block and restart the timer.
713 * opening a block thaws the queue,restarts timer
714 * Thawing/timer-refresh is a side effect.
715 */
716 prb_open_block(pkc, pbd);
717 goto out;
718 }
719 }
720 }
721
722refresh_timer:
723 _prb_refresh_rx_retire_blk_timer(pkc);
724
725out:
726 spin_unlock(&po->sk.sk_receive_queue.lock);
727}
728
eea49cc9 729static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
bc59ba39 730 struct tpacket_block_desc *pbd1, __u32 status)
f6fb8f10 731{
732 /* Flush everything minus the block header */
733
734#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
735 u8 *start, *end;
736
737 start = (u8 *)pbd1;
738
739 /* Skip the block header(we know header WILL fit in 4K) */
740 start += PAGE_SIZE;
741
742 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
743 for (; start < end; start += PAGE_SIZE)
744 flush_dcache_page(pgv_to_page(start));
745
746 smp_wmb();
747#endif
748
749 /* Now update the block status. */
750
751 BLOCK_STATUS(pbd1) = status;
752
753 /* Flush the block header */
754
755#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
756 start = (u8 *)pbd1;
757 flush_dcache_page(pgv_to_page(start));
758
759 smp_wmb();
760#endif
761}
762
763/*
764 * Side effect:
765 *
766 * 1) flush the block
767 * 2) Increment active_blk_num
768 *
769 * Note:We DONT refresh the timer on purpose.
770 * Because almost always the next block will be opened.
771 */
bc59ba39 772static void prb_close_block(struct tpacket_kbdq_core *pkc1,
773 struct tpacket_block_desc *pbd1,
f6fb8f10 774 struct packet_sock *po, unsigned int stat)
775{
776 __u32 status = TP_STATUS_USER | stat;
777
778 struct tpacket3_hdr *last_pkt;
bc59ba39 779 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 780
ee80fbf3 781 if (po->stats.stats3.tp_drops)
f6fb8f10 782 status |= TP_STATUS_LOSING;
783
784 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
785 last_pkt->tp_next_offset = 0;
786
787 /* Get the ts of the last pkt */
788 if (BLOCK_NUM_PKTS(pbd1)) {
789 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
790 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
791 } else {
792 /* Ok, we tmo'd - so get the current time */
793 struct timespec ts;
794 getnstimeofday(&ts);
795 h1->ts_last_pkt.ts_sec = ts.tv_sec;
796 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
797 }
798
799 smp_wmb();
800
801 /* Flush the block */
802 prb_flush_block(pkc1, pbd1, status);
803
804 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
805}
806
eea49cc9 807static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
f6fb8f10 808{
809 pkc->reset_pending_on_curr_blk = 0;
810}
811
812/*
813 * Side effect of opening a block:
814 *
815 * 1) prb_queue is thawed.
816 * 2) retire_blk_timer is refreshed.
817 *
818 */
bc59ba39 819static void prb_open_block(struct tpacket_kbdq_core *pkc1,
820 struct tpacket_block_desc *pbd1)
f6fb8f10 821{
822 struct timespec ts;
bc59ba39 823 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
f6fb8f10 824
825 smp_rmb();
826
8da3056c
DB		 827 /* We could have just memset this but we will lose the
828 * flexibility of making the priv area sticky
829 */
f6fb8f10 830
8da3056c
DB		 831 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
832 BLOCK_NUM_PKTS(pbd1) = 0;
833 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
f6fb8f10 834
8da3056c
DB		 835 getnstimeofday(&ts);
836
837 h1->ts_first_pkt.ts_sec = ts.tv_sec;
838 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
f6fb8f10 839
8da3056c
DB		 840 pkc1->pkblk_start = (char *)pbd1;
841 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
842
843 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
844 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
845
846 pbd1->version = pkc1->version;
847 pkc1->prev = pkc1->nxt_offset;
848 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
849
850 prb_thaw_queue(pkc1);
851 _prb_refresh_rx_retire_blk_timer(pkc1);
852
853 smp_wmb();
f6fb8f10 854}
855
856/*
857 * Queue freeze logic:
858 * 1) Assume tp_block_nr = 8 blocks.
859 * 2) At time 't0', user opens Rx ring.
860 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
861 * 4) user-space is either sleeping or processing block '0'.
862 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
863 * it will close block-7,loop around and try to fill block '0'.
864 * call-flow:
865 * __packet_lookup_frame_in_block
866 * prb_retire_current_block()
867 * prb_dispatch_next_block()
868 * |->(BLOCK_STATUS == USER) evaluates to true
869 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
870 * 6) Now there are two cases:
871 * 6.1) Link goes idle right after the queue is frozen.
872 * But remember, the last open_block() refreshed the timer.
873 * When this timer expires,it will refresh itself so that we can
874 * re-open block-0 in near future.
875 * 6.2) Link is busy and keeps on receiving packets. This is a simple
876 * case and __packet_lookup_frame_in_block will check if block-0
877 * is free and can now be re-used.
878 */
eea49cc9 879static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
f6fb8f10 880 struct packet_sock *po)
881{
882 pkc->reset_pending_on_curr_blk = 1;
ee80fbf3 883 po->stats.stats3.tp_freeze_q_cnt++;
f6fb8f10 884}
885
886#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
887
888/*
889 * If the next block is free then we will dispatch it
890 * and return a good offset.
891 * Else, we will freeze the queue.
892 * So, caller must check the return value.
893 */
bc59ba39 894static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 895 struct packet_sock *po)
896{
bc59ba39 897 struct tpacket_block_desc *pbd;
f6fb8f10 898
899 smp_rmb();
900
901 /* 1. Get current block num */
902 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
903
904 /* 2. If this block is currently in_use then freeze the queue */
905 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
906 prb_freeze_queue(pkc, po);
907 return NULL;
908 }
909
910 /*
911 * 3.
912 * open this block and return the offset where the first packet
913 * needs to get stored.
914 */
915 prb_open_block(pkc, pbd);
916 return (void *)pkc->nxt_offset;
917}
918
bc59ba39 919static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
f6fb8f10 920 struct packet_sock *po, unsigned int status)
921{
bc59ba39 922 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
f6fb8f10 923
924 /* retire/close the current block */
925 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
926 /*
927 * Plug the case where copy_bits() is in progress on
928 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
929 * have space to copy the pkt in the current block and
930 * called prb_retire_current_block()
931 *
932 * We don't need to worry about the TMO case because
933 * the timer-handler already handled this case.
934 */
935 if (!(status & TP_STATUS_BLK_TMO)) {
936 while (atomic_read(&pkc->blk_fill_in_prog)) {
937 /* Waiting for skb_copy_bits to finish... */
938 cpu_relax();
939 }
940 }
941 prb_close_block(pkc, pbd, po, status);
942 return;
943 }
f6fb8f10 944}
945
eea49cc9 946static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
bc59ba39 947 struct tpacket_block_desc *pbd)
f6fb8f10 948{
949 return TP_STATUS_USER & BLOCK_STATUS(pbd);
950}
951
eea49cc9 952static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
f6fb8f10 953{
954 return pkc->reset_pending_on_curr_blk;
955}
956
eea49cc9 957static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
f6fb8f10 958{
bc59ba39 959 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
f6fb8f10 960 atomic_dec(&pkc->blk_fill_in_prog);
961}
962
eea49cc9 963static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 964 struct tpacket3_hdr *ppd)
965{
3958afa1 966 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
f6fb8f10 967}
968
eea49cc9 969static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
f6fb8f10 970 struct tpacket3_hdr *ppd)
971{
972 ppd->hv1.tp_rxhash = 0;
973}
974
eea49cc9 975static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
f6fb8f10 976 struct tpacket3_hdr *ppd)
977{
978 if (vlan_tx_tag_present(pkc->skb)) {
979 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
a0cdfcf3
AW		 980 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
981 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
f6fb8f10 982 } else {
9e67030a 983 ppd->hv1.tp_vlan_tci = 0;
a0cdfcf3 984 ppd->hv1.tp_vlan_tpid = 0;
9e67030a 985 ppd->tp_status = TP_STATUS_AVAILABLE;
f6fb8f10 986 }
987}
988
bc59ba39 989static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
f6fb8f10 990 struct tpacket3_hdr *ppd)
991{
a0cdfcf3 992 ppd->hv1.tp_padding = 0;
f6fb8f10 993 prb_fill_vlan_info(pkc, ppd);
994
995 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
996 prb_fill_rxhash(pkc, ppd);
997 else
998 prb_clear_rxhash(pkc, ppd);
999}
1000
eea49cc9 1001static void prb_fill_curr_block(char *curr,
bc59ba39 1002 struct tpacket_kbdq_core *pkc,
1003 struct tpacket_block_desc *pbd,
f6fb8f10 1004 unsigned int len)
1005{
1006 struct tpacket3_hdr *ppd;
1007
1008 ppd = (struct tpacket3_hdr *)curr;
1009 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1010 pkc->prev = curr;
1011 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1012 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1013 BLOCK_NUM_PKTS(pbd) += 1;
1014 atomic_inc(&pkc->blk_fill_in_prog);
1015 prb_run_all_ft_ops(pkc, ppd);
1016}
1017
1018/* Assumes caller has the sk->rx_queue.lock */
1019static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1020 struct sk_buff *skb,
1021 int status,
1022 unsigned int len
1023 )
1024{
bc59ba39 1025 struct tpacket_kbdq_core *pkc;
1026 struct tpacket_block_desc *pbd;
f6fb8f10 1027 char *curr, *end;
1028
e3192690 1029 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
f6fb8f10 1030 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1031
1032 /* Queue is frozen when user space is lagging behind */
1033 if (prb_queue_frozen(pkc)) {
1034 /*
1035 * Check if that last block which caused the queue to freeze,
1036 * is still in_use by user-space.
1037 */
1038 if (prb_curr_blk_in_use(pkc, pbd)) {
1039 /* Can't record this packet */
1040 return NULL;
1041 } else {
1042 /*
1043 * Ok, the block was released by user-space.
1044 * Now let's open that block.
1045 * opening a block also thaws the queue.
1046 * Thawing is a side effect.
1047 */
1048 prb_open_block(pkc, pbd);
1049 }
1050 }
1051
1052 smp_mb();
1053 curr = pkc->nxt_offset;
1054 pkc->skb = skb;
e3192690 1055 end = (char *)pbd + pkc->kblk_size;
f6fb8f10 1056
1057 /* first try the current block */
1058 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1059 prb_fill_curr_block(curr, pkc, pbd, len);
1060 return (void *)curr;
1061 }
1062
1063 /* Ok, close the current block */
1064 prb_retire_current_block(pkc, po, 0);
1065
1066 /* Now, try to dispatch the next block */
1067 curr = (char *)prb_dispatch_next_block(pkc, po);
1068 if (curr) {
1069 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1070 prb_fill_curr_block(curr, pkc, pbd, len);
1071 return (void *)curr;
1072 }
1073
1074 /*
1075 * No free blocks are available.user_space hasn't caught up yet.
1076 * Queue was just frozen and now this packet will get dropped.
1077 */
1078 return NULL;
1079}
1080
eea49cc9 1081static void *packet_current_rx_frame(struct packet_sock *po,
f6fb8f10 1082 struct sk_buff *skb,
1083 int status, unsigned int len)
1084{
1085 char *curr = NULL;
1086 switch (po->tp_version) {
1087 case TPACKET_V1:
1088 case TPACKET_V2:
1089 curr = packet_lookup_frame(po, &po->rx_ring,
1090 po->rx_ring.head, status);
1091 return curr;
1092 case TPACKET_V3:
1093 return __packet_lookup_frame_in_block(po, skb, status, len);
1094 default:
1095 WARN(1, "TPACKET version not supported\n");
1096 BUG();
99aa3473 1097 return NULL;
f6fb8f10 1098 }
1099}
1100
eea49cc9 1101static void *prb_lookup_block(struct packet_sock *po,
f6fb8f10 1102 struct packet_ring_buffer *rb,
77f65ebd 1103 unsigned int idx,
f6fb8f10 1104 int status)
1105{
bc59ba39 1106 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
77f65ebd 1107 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
f6fb8f10 1108
1109 if (status != BLOCK_STATUS(pbd))
1110 return NULL;
1111 return pbd;
1112}
1113
eea49cc9 1114static int prb_previous_blk_num(struct packet_ring_buffer *rb)
f6fb8f10 1115{
1116 unsigned int prev;
1117 if (rb->prb_bdqc.kactive_blk_num)
1118 prev = rb->prb_bdqc.kactive_blk_num-1;
1119 else
1120 prev = rb->prb_bdqc.knum_blocks-1;
1121 return prev;
1122}
1123
1124/* Assumes caller has held the rx_queue.lock */
eea49cc9 1125static void *__prb_previous_block(struct packet_sock *po,
f6fb8f10 1126 struct packet_ring_buffer *rb,
1127 int status)
1128{
1129 unsigned int previous = prb_previous_blk_num(rb);
1130 return prb_lookup_block(po, rb, previous, status);
1131}
1132
eea49cc9 1133static void *packet_previous_rx_frame(struct packet_sock *po,
f6fb8f10 1134 struct packet_ring_buffer *rb,
1135 int status)
1136{
1137 if (po->tp_version <= TPACKET_V2)
1138 return packet_previous_frame(po, rb, status);
1139
1140 return __prb_previous_block(po, rb, status);
1141}
1142
eea49cc9 1143static void packet_increment_rx_head(struct packet_sock *po,
f6fb8f10 1144 struct packet_ring_buffer *rb)
1145{
1146 switch (po->tp_version) {
1147 case TPACKET_V1:
1148 case TPACKET_V2:
1149 return packet_increment_head(rb);
1150 case TPACKET_V3:
1151 default:
1152 WARN(1, "TPACKET version not supported.\n");
1153 BUG();
1154 return;
1155 }
1156}
1157
eea49cc9 1158static void *packet_previous_frame(struct packet_sock *po,
69e3c75f
JB		 1159 struct packet_ring_buffer *rb,
1160 int status)
1161{
1162 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1163 return packet_lookup_frame(po, rb, previous, status);
1164}
1165
eea49cc9 1166static void packet_increment_head(struct packet_ring_buffer *buff)
69e3c75f
JB		 1167{
1168 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1169}
1170
b0138408
DB		 1171static void packet_inc_pending(struct packet_ring_buffer *rb)
1172{
1173 this_cpu_inc(*rb->pending_refcnt);
1174}
1175
1176static void packet_dec_pending(struct packet_ring_buffer *rb)
1177{
1178 this_cpu_dec(*rb->pending_refcnt);
1179}
1180
1181static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1182{
1183 unsigned int refcnt = 0;
1184 int cpu;
1185
1186 /* We don't use pending refcount in rx_ring. */
1187 if (rb->pending_refcnt == NULL)
1188 return 0;
1189
1190 for_each_possible_cpu(cpu)
1191 refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1192
1193 return refcnt;
1194}
1195
1196static int packet_alloc_pending(struct packet_sock *po)
1197{
1198 po->rx_ring.pending_refcnt = NULL;
1199
1200 po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1201 if (unlikely(po->tx_ring.pending_refcnt == NULL))
1202 return -ENOBUFS;
1203
1204 return 0;
1205}
1206
1207static void packet_free_pending(struct packet_sock *po)
1208{
1209 free_percpu(po->tx_ring.pending_refcnt);
1210}
1211
77f65ebd
WB		 1212static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1213{
1214 struct sock *sk = &po->sk;
1215 bool has_room;
1216
1217 if (po->prot_hook.func != tpacket_rcv)
1218 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1219 <= sk->sk_rcvbuf;
1220
1221 spin_lock(&sk->sk_receive_queue.lock);
1222 if (po->tp_version == TPACKET_V3)
1223 has_room = prb_lookup_block(po, &po->rx_ring,
1224 po->rx_ring.prb_bdqc.kactive_blk_num,
1225 TP_STATUS_KERNEL);
1226 else
1227 has_room = packet_lookup_frame(po, &po->rx_ring,
1228 po->rx_ring.head,
1229 TP_STATUS_KERNEL);
1230 spin_unlock(&sk->sk_receive_queue.lock);
1231
1232 return has_room;
1233}
1234
1da177e4
LT		 1235static void packet_sock_destruct(struct sock *sk)
1236{
ed85b565
RC		 1237 skb_queue_purge(&sk->sk_error_queue);
1238
547b792c
IJ		 1239 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1240 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1da177e4
LT		 1241
1242 if (!sock_flag(sk, SOCK_DEAD)) {
40d4e3df 1243 pr_err("Attempt to release alive packet socket: %p\n", sk);
1da177e4
LT		 1244 return;
1245 }
1246
17ab56a2 1247 sk_refcnt_debug_dec(sk);
1da177e4
LT		 1248}
1249
dc99f600
DM		 1250static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1251{
1252 int x = atomic_read(&f->rr_cur) + 1;
1253
1254 if (x >= num)
1255 x = 0;
1256
1257 return x;
1258}
1259
77f65ebd
WB		 1260static unsigned int fanout_demux_hash(struct packet_fanout *f,
1261 struct sk_buff *skb,
1262 unsigned int num)
dc99f600 1263{
89770b0a 1264 return reciprocal_scale(skb->rxhash, num);
dc99f600
DM		 1265}
1266
77f65ebd
WB		 1267static unsigned int fanout_demux_lb(struct packet_fanout *f,
1268 struct sk_buff *skb,
1269 unsigned int num)
dc99f600
DM		 1270{
1271 int cur, old;
1272
1273 cur = atomic_read(&f->rr_cur);
1274 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1275 fanout_rr_next(f, num))) != cur)
1276 cur = old;
77f65ebd
WB		 1277 return cur;
1278}
1279
1280static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1281 struct sk_buff *skb,
1282 unsigned int num)
1283{
1284 return smp_processor_id() % num;
dc99f600
DM		 1285}
1286
5df0ddfb
DB		 1287static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1288 struct sk_buff *skb,
1289 unsigned int num)
1290{
f337db64 1291 return prandom_u32_max(num);
5df0ddfb
DB		 1292}
1293
77f65ebd
WB		 1294static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1295 struct sk_buff *skb,
1296 unsigned int idx, unsigned int skip,
1297 unsigned int num)
95ec3eb4 1298{
77f65ebd 1299 unsigned int i, j;
95ec3eb4 1300
77f65ebd
WB		 1301 i = j = min_t(int, f->next[idx], num - 1);
1302 do {
1303 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1304 if (i != j)
1305 f->next[idx] = i;
1306 return i;
1307 }
1308 if (++i == num)
1309 i = 0;
1310 } while (i != j);
1311
1312 return idx;
1313}
1314
2d36097d
NH		 1315static unsigned int fanout_demux_qm(struct packet_fanout *f,
1316 struct sk_buff *skb,
1317 unsigned int num)
1318{
1319 return skb_get_queue_mapping(skb) % num;
1320}
1321
77f65ebd
WB		 1322static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1323{
1324 return f->flags & (flag >> 8);
95ec3eb4
DM		 1325}
1326
95ec3eb4
DM		 1327static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1328 struct packet_type *pt, struct net_device *orig_dev)
dc99f600
DM		 1329{
1330 struct packet_fanout *f = pt->af_packet_priv;
1331 unsigned int num = f->num_members;
1332 struct packet_sock *po;
77f65ebd 1333 unsigned int idx;
dc99f600
DM		 1334
1335 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1336 !num) {
1337 kfree_skb(skb);
1338 return 0;
1339 }
1340
95ec3eb4
DM		 1341 switch (f->type) {
1342 case PACKET_FANOUT_HASH:
1343 default:
77f65ebd 1344 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
bc416d97 1345 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
95ec3eb4
DM		 1346 if (!skb)
1347 return 0;
1348 }
3958afa1 1349 skb_get_hash(skb);
77f65ebd 1350 idx = fanout_demux_hash(f, skb, num);
95ec3eb4
DM		 1351 break;
1352 case PACKET_FANOUT_LB:
77f65ebd 1353 idx = fanout_demux_lb(f, skb, num);
95ec3eb4
DM		 1354 break;
1355 case PACKET_FANOUT_CPU:
77f65ebd
WB		 1356 idx = fanout_demux_cpu(f, skb, num);
1357 break;
5df0ddfb
DB		 1358 case PACKET_FANOUT_RND:
1359 idx = fanout_demux_rnd(f, skb, num);
1360 break;
2d36097d
NH		 1361 case PACKET_FANOUT_QM:
1362 idx = fanout_demux_qm(f, skb, num);
1363 break;
77f65ebd
WB		 1364 case PACKET_FANOUT_ROLLOVER:
1365 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
95ec3eb4 1366 break;
dc99f600
DM		 1367 }
1368
77f65ebd
WB		 1369 po = pkt_sk(f->arr[idx]);
1370 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1371 unlikely(!packet_rcv_has_room(po, skb))) {
1372 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1373 po = pkt_sk(f->arr[idx]);
1374 }
dc99f600
DM		 1375
1376 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1377}
1378
fff3321d
PE		 1379DEFINE_MUTEX(fanout_mutex);
1380EXPORT_SYMBOL_GPL(fanout_mutex);
dc99f600
DM		 1381static LIST_HEAD(fanout_list);
1382
1383static void __fanout_link(struct sock *sk, struct packet_sock *po)
1384{
1385 struct packet_fanout *f = po->fanout;
1386
1387 spin_lock(&f->lock);
1388 f->arr[f->num_members] = sk;
1389 smp_wmb();
1390 f->num_members++;
1391 spin_unlock(&f->lock);
1392}
1393
1394static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1395{
1396 struct packet_fanout *f = po->fanout;
1397 int i;
1398
1399 spin_lock(&f->lock);
1400 for (i = 0; i < f->num_members; i++) {
1401 if (f->arr[i] == sk)
1402 break;
1403 }
1404 BUG_ON(i >= f->num_members);
1405 f->arr[i] = f->arr[f->num_members - 1];
1406 f->num_members--;
1407 spin_unlock(&f->lock);
1408}
1409
d4dd8aee 1410static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
c0de08d0 1411{
d4dd8aee 1412 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
c0de08d0
EL		 1413 return true;
1414
1415 return false;
1416}
1417
7736d33f 1418static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
dc99f600
DM		 1419{
1420 struct packet_sock *po = pkt_sk(sk);
1421 struct packet_fanout *f, *match;
7736d33f 1422 u8 type = type_flags & 0xff;
77f65ebd 1423 u8 flags = type_flags >> 8;
dc99f600
DM		 1424 int err;
1425
1426 switch (type) {
77f65ebd
WB		 1427 case PACKET_FANOUT_ROLLOVER:
1428 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1429 return -EINVAL;
dc99f600
DM		 1430 case PACKET_FANOUT_HASH:
1431 case PACKET_FANOUT_LB:
95ec3eb4 1432 case PACKET_FANOUT_CPU:
5df0ddfb 1433 case PACKET_FANOUT_RND:
2d36097d 1434 case PACKET_FANOUT_QM:
dc99f600
DM		 1435 break;
1436 default:
1437 return -EINVAL;
1438 }
1439
1440 if (!po->running)
1441 return -EINVAL;
1442
1443 if (po->fanout)
1444 return -EALREADY;
1445
1446 mutex_lock(&fanout_mutex);
1447 match = NULL;
1448 list_for_each_entry(f, &fanout_list, list) {
1449 if (f->id == id &&
1450 read_pnet(&f->net) == sock_net(sk)) {
1451 match = f;
1452 break;
1453 }
1454 }
afe62c68 1455 err = -EINVAL;
77f65ebd 1456 if (match && match->flags != flags)
afe62c68 1457 goto out;
dc99f600 1458 if (!match) {
afe62c68 1459 err = -ENOMEM;
dc99f600 1460 match = kzalloc(sizeof(*match), GFP_KERNEL);
afe62c68
ED		 1461 if (!match)
1462 goto out;
1463 write_pnet(&match->net, sock_net(sk));
1464 match->id = id;
1465 match->type = type;
77f65ebd 1466 match->flags = flags;
afe62c68
ED		 1467 atomic_set(&match->rr_cur, 0);
1468 INIT_LIST_HEAD(&match->list);
1469 spin_lock_init(&match->lock);
1470 atomic_set(&match->sk_ref, 0);
1471 match->prot_hook.type = po->prot_hook.type;
1472 match->prot_hook.dev = po->prot_hook.dev;
1473 match->prot_hook.func = packet_rcv_fanout;
1474 match->prot_hook.af_packet_priv = match;
c0de08d0 1475 match->prot_hook.id_match = match_fanout_group;
afe62c68
ED		 1476 dev_add_pack(&match->prot_hook);
1477 list_add(&match->list, &fanout_list);
dc99f600 1478 }
afe62c68
ED		 1479 err = -EINVAL;
1480 if (match->type == type &&
1481 match->prot_hook.type == po->prot_hook.type &&
1482 match->prot_hook.dev == po->prot_hook.dev) {
1483 err = -ENOSPC;
1484 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1485 __dev_remove_pack(&po->prot_hook);
1486 po->fanout = match;
1487 atomic_inc(&match->sk_ref);
1488 __fanout_link(sk, po);
1489 err = 0;
dc99f600
DM		 1490 }
1491 }
afe62c68 1492out:
dc99f600
DM		 1493 mutex_unlock(&fanout_mutex);
1494 return err;
1495}
1496
1497static void fanout_release(struct sock *sk)
1498{
1499 struct packet_sock *po = pkt_sk(sk);
1500 struct packet_fanout *f;
1501
1502 f = po->fanout;
1503 if (!f)
1504 return;
1505
fff3321d 1506 mutex_lock(&fanout_mutex);
dc99f600
DM		 1507 po->fanout = NULL;
1508
dc99f600
DM		 1509 if (atomic_dec_and_test(&f->sk_ref)) {
1510 list_del(&f->list);
1511 dev_remove_pack(&f->prot_hook);
1512 kfree(f);
1513 }
1514 mutex_unlock(&fanout_mutex);
1515}
1da177e4 1516
90ddc4f0 1517static const struct proto_ops packet_ops;
1da177e4 1518
90ddc4f0 1519static const struct proto_ops packet_ops_spkt;
1da177e4 1520
40d4e3df
ED		 1521static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1522 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1523{
1524 struct sock *sk;
1525 struct sockaddr_pkt *spkt;
1526
1527 /*
1528 * When we registered the protocol we saved the socket in the data
1529 * field for just this event.
1530 */
1531
1532 sk = pt->af_packet_priv;
1ce4f28b 1533
1da177e4
LT		 1534 /*
1535 * Yank back the headers [hope the device set this
1536 * right or kerboom...]
1537 *
1538 * Incoming packets have ll header pulled,
1539 * push it back.
1540 *
98e399f8 1541 * For outgoing ones skb->data == skb_mac_header(skb)
1da177e4
LT		 1542 * so that this procedure is noop.
1543 */
1544
1545 if (skb->pkt_type == PACKET_LOOPBACK)
1546 goto out;
1547
09ad9bc7 1548 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1549 goto out;
1550
40d4e3df
ED		 1551 skb = skb_share_check(skb, GFP_ATOMIC);
1552 if (skb == NULL)
1da177e4
LT		 1553 goto oom;
1554
1555 /* drop any routing info */
adf30907 1556 skb_dst_drop(skb);
1da177e4 1557
84531c24
PO		 1558 /* drop conntrack reference */
1559 nf_reset(skb);
1560
ffbc6111 1561 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1da177e4 1562
98e399f8 1563 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1564
1565 /*
1566 * The SOCK_PACKET socket receives _all_ frames.
1567 */
1568
1569 spkt->spkt_family = dev->type;
1570 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1571 spkt->spkt_protocol = skb->protocol;
1572
1573 /*
1574 * Charge the memory to the socket. This is done specifically
1575 * to prevent sockets using all the memory up.
1576 */
1577
40d4e3df 1578 if (sock_queue_rcv_skb(sk, skb) == 0)
1da177e4
LT		 1579 return 0;
1580
1581out:
1582 kfree_skb(skb);
1583oom:
1584 return 0;
1585}
1586
1587
1588/*
1589 * Output a raw packet to a device layer. This bypasses all the other
1590 * protocol layers and you must therefore supply it with a complete frame
1591 */
1ce4f28b 1592
1da177e4
LT		 1593static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1594 struct msghdr *msg, size_t len)
1595{
1596 struct sock *sk = sock->sk;
342dfc30 1597 DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1a35ca80 1598 struct sk_buff *skb = NULL;
1da177e4 1599 struct net_device *dev;
40d4e3df 1600 __be16 proto = 0;
1da177e4 1601 int err;
3bdc0eba 1602 int extra_len = 0;
1ce4f28b 1603
1da177e4 1604 /*
1ce4f28b 1605 * Get and verify the address.
1da177e4
LT		 1606 */
1607
40d4e3df 1608 if (saddr) {
1da177e4 1609 if (msg->msg_namelen < sizeof(struct sockaddr))
40d4e3df
ED		 1610 return -EINVAL;
1611 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1612 proto = saddr->spkt_protocol;
1613 } else
1614 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1da177e4
LT		 1615
1616 /*
1ce4f28b 1617 * Find the device first to size check it
1da177e4
LT		 1618 */
1619
de74e92a 1620 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1a35ca80 1621retry:
654d1f8a
ED		 1622 rcu_read_lock();
1623 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1da177e4
LT		 1624 err = -ENODEV;
1625 if (dev == NULL)
1626 goto out_unlock;
1ce4f28b 1627
d5e76b0a
DM		 1628 err = -ENETDOWN;
1629 if (!(dev->flags & IFF_UP))
1630 goto out_unlock;
1631
1da177e4 1632 /*
40d4e3df
ED		 1633 * You may not queue a frame bigger than the mtu. This is the lowest level
1634 * raw protocol and you must do your own fragmentation at this level.
1da177e4 1635 */
1ce4f28b 1636
3bdc0eba
BG		 1637 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1638 if (!netif_supports_nofcs(dev)) {
1639 err = -EPROTONOSUPPORT;
1640 goto out_unlock;
1641 }
1642 extra_len = 4; /* We're doing our own CRC */
1643 }
1644
1da177e4 1645 err = -EMSGSIZE;
3bdc0eba 1646 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1da177e4
LT		 1647 goto out_unlock;
1648
1a35ca80
ED		 1649 if (!skb) {
1650 size_t reserved = LL_RESERVED_SPACE(dev);
4ce40912 1651 int tlen = dev->needed_tailroom;
1a35ca80
ED		 1652 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1653
1654 rcu_read_unlock();
4ce40912 1655 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1a35ca80
ED		 1656 if (skb == NULL)
1657 return -ENOBUFS;
1658 /* FIXME: Save some space for broken drivers that write a hard
1659 * header at transmission time by themselves. PPP is the notable
1660 * one here. This should really be fixed at the driver level.
1661 */
1662 skb_reserve(skb, reserved);
1663 skb_reset_network_header(skb);
1664
1665 /* Try to align data part correctly */
1666 if (hhlen) {
1667 skb->data -= hhlen;
1668 skb->tail -= hhlen;
1669 if (len < hhlen)
1670 skb_reset_network_header(skb);
1671 }
1672 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1673 if (err)
1674 goto out_free;
1675 goto retry;
1da177e4
LT		 1676 }
1677
3bdc0eba 1678 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
57f89bfa
BG		 1679 /* Earlier code assumed this would be a VLAN pkt,
1680 * double-check this now that we have the actual
1681 * packet in hand.
1682 */
1683 struct ethhdr *ehdr;
1684 skb_reset_mac_header(skb);
1685 ehdr = eth_hdr(skb);
1686 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1687 err = -EMSGSIZE;
1688 goto out_unlock;
1689 }
1690 }
1a35ca80 1691
1da177e4
LT		 1692 skb->protocol = proto;
1693 skb->dev = dev;
1694 skb->priority = sk->sk_priority;
2d37a186 1695 skb->mark = sk->sk_mark;
bf84a010
DB		 1696
1697 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 1698
3bdc0eba
BG		 1699 if (unlikely(extra_len == 4))
1700 skb->no_fcs = 1;
1701
40893fd0 1702 skb_probe_transport_header(skb, 0);
c1aad275 1703
1da177e4 1704 dev_queue_xmit(skb);
654d1f8a 1705 rcu_read_unlock();
40d4e3df 1706 return len;
1da177e4 1707
1da177e4 1708out_unlock:
654d1f8a 1709 rcu_read_unlock();
1a35ca80
ED		 1710out_free:
1711 kfree_skb(skb);
1da177e4
LT		 1712 return err;
1713}
1da177e4 1714
eea49cc9 1715static unsigned int run_filter(const struct sk_buff *skb,
62ab0812 1716 const struct sock *sk,
dbcb5855 1717 unsigned int res)
1da177e4
LT		 1718{
1719 struct sk_filter *filter;
fda9ef5d 1720
80f8f102
ED		 1721 rcu_read_lock();
1722 filter = rcu_dereference(sk->sk_filter);
dbcb5855 1723 if (filter != NULL)
0a14842f 1724 res = SK_RUN_FILTER(filter, skb);
80f8f102 1725 rcu_read_unlock();
1da177e4 1726
dbcb5855 1727 return res;
1da177e4
LT		 1728}
1729
1730/*
62ab0812
ED		 1731 * This function makes lazy skb cloning in hope that most of packets
1732 * are discarded by BPF.
1733 *
1734 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1735 * and skb->cb are mangled. It works because (and until) packets
1736 * falling here are owned by current CPU. Output packets are cloned
1737 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1738 * sequencially, so that if we return skb to original state on exit,
1739 * we will not harm anyone.
1da177e4
LT		 1740 */
1741
40d4e3df
ED		 1742static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1743 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1744{
1745 struct sock *sk;
1746 struct sockaddr_ll *sll;
1747 struct packet_sock *po;
40d4e3df 1748 u8 *skb_head = skb->data;
1da177e4 1749 int skb_len = skb->len;
dbcb5855 1750 unsigned int snaplen, res;
1da177e4
LT		 1751
1752 if (skb->pkt_type == PACKET_LOOPBACK)
1753 goto drop;
1754
1755 sk = pt->af_packet_priv;
1756 po = pkt_sk(sk);
1757
09ad9bc7 1758 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1759 goto drop;
1760
1da177e4
LT		 1761 skb->dev = dev;
1762
3b04ddde 1763 if (dev->header_ops) {
1da177e4 1764 /* The device has an explicit notion of ll header,
62ab0812
ED		 1765 * exported to higher levels.
1766 *
1767 * Otherwise, the device hides details of its frame
1768 * structure, so that corresponding packet head is
1769 * never delivered to user.
1da177e4
LT		 1770 */
1771 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1772 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1773 else if (skb->pkt_type == PACKET_OUTGOING) {
1774 /* Special case: outgoing packets have ll header at head */
bbe735e4 1775 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1776 }
1777 }
1778
1779 snaplen = skb->len;
1780
dbcb5855
DM		 1781 res = run_filter(skb, sk, snaplen);
1782 if (!res)
fda9ef5d 1783 goto drop_n_restore;
dbcb5855
DM		 1784 if (snaplen > res)
1785 snaplen = res;
1da177e4 1786
0fd7bac6 1787 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1da177e4
LT		 1788 goto drop_n_acct;
1789
1790 if (skb_shared(skb)) {
1791 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1792 if (nskb == NULL)
1793 goto drop_n_acct;
1794
1795 if (skb_head != skb->data) {
1796 skb->data = skb_head;
1797 skb->len = skb_len;
1798 }
abc4e4fa 1799 consume_skb(skb);
1da177e4
LT		 1800 skb = nskb;
1801 }
1802
ffbc6111
HX		 1803 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1804 sizeof(skb->cb));
1805
1806 sll = &PACKET_SKB_CB(skb)->sa.ll;
1da177e4
LT		 1807 sll->sll_family = AF_PACKET;
1808 sll->sll_hatype = dev->type;
1809 sll->sll_protocol = skb->protocol;
1810 sll->sll_pkttype = skb->pkt_type;
8032b464 1811 if (unlikely(po->origdev))
80feaacb
PWJ		 1812 sll->sll_ifindex = orig_dev->ifindex;
1813 else
1814 sll->sll_ifindex = dev->ifindex;
1da177e4 1815
b95cce35 1816 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4 1817
ffbc6111 1818 PACKET_SKB_CB(skb)->origlen = skb->len;
8dc41944 1819
1da177e4
LT		 1820 if (pskb_trim(skb, snaplen))
1821 goto drop_n_acct;
1822
1823 skb_set_owner_r(skb, sk);
1824 skb->dev = NULL;
adf30907 1825 skb_dst_drop(skb);
1da177e4 1826
84531c24
PO		 1827 /* drop conntrack reference */
1828 nf_reset(skb);
1829
1da177e4 1830 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1831 po->stats.stats1.tp_packets++;
3b885787 1832 skb->dropcount = atomic_read(&sk->sk_drops);
1da177e4
LT		 1833 __skb_queue_tail(&sk->sk_receive_queue, skb);
1834 spin_unlock(&sk->sk_receive_queue.lock);
1835 sk->sk_data_ready(sk, skb->len);
1836 return 0;
1837
1838drop_n_acct:
7091fbd8 1839 spin_lock(&sk->sk_receive_queue.lock);
ee80fbf3 1840 po->stats.stats1.tp_drops++;
7091fbd8
WB		 1841 atomic_inc(&sk->sk_drops);
1842 spin_unlock(&sk->sk_receive_queue.lock);
1da177e4
LT		 1843
1844drop_n_restore:
1845 if (skb_head != skb->data && skb_shared(skb)) {
1846 skb->data = skb_head;
1847 skb->len = skb_len;
1848 }
1849drop:
ead2ceb0 1850 consume_skb(skb);
1da177e4
LT		 1851 return 0;
1852}
1853
40d4e3df
ED		 1854static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1855 struct packet_type *pt, struct net_device *orig_dev)
1da177e4
LT		 1856{
1857 struct sock *sk;
1858 struct packet_sock *po;
1859 struct sockaddr_ll *sll;
184f489e 1860 union tpacket_uhdr h;
40d4e3df 1861 u8 *skb_head = skb->data;
1da177e4 1862 int skb_len = skb->len;
dbcb5855 1863 unsigned int snaplen, res;
f6fb8f10 1864 unsigned long status = TP_STATUS_USER;
bbd6ef87 1865 unsigned short macoff, netoff, hdrlen;
1da177e4 1866 struct sk_buff *copy_skb = NULL;
bbd6ef87 1867 struct timespec ts;
b9c32fb2 1868 __u32 ts_status;
1da177e4 1869
51846355
AW		 1870 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1871 * We may add members to them until current aligned size without forcing
1872 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1873 */
1874 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1875 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1876
1da177e4
LT		 1877 if (skb->pkt_type == PACKET_LOOPBACK)
1878 goto drop;
1879
1880 sk = pt->af_packet_priv;
1881 po = pkt_sk(sk);
1882
09ad9bc7 1883 if (!net_eq(dev_net(dev), sock_net(sk)))
d12d01d6
DL		 1884 goto drop;
1885
3b04ddde 1886 if (dev->header_ops) {
1da177e4 1887 if (sk->sk_type != SOCK_DGRAM)
98e399f8 1888 skb_push(skb, skb->data - skb_mac_header(skb));
1da177e4
LT		 1889 else if (skb->pkt_type == PACKET_OUTGOING) {
1890 /* Special case: outgoing packets have ll header at head */
bbe735e4 1891 skb_pull(skb, skb_network_offset(skb));
1da177e4
LT		 1892 }
1893 }
1894
8dc41944
HX		 1895 if (skb->ip_summed == CHECKSUM_PARTIAL)
1896 status |= TP_STATUS_CSUMNOTREADY;
1897
1da177e4
LT		 1898 snaplen = skb->len;
1899
dbcb5855
DM		 1900 res = run_filter(skb, sk, snaplen);
1901 if (!res)
fda9ef5d 1902 goto drop_n_restore;
dbcb5855
DM		 1903 if (snaplen > res)
1904 snaplen = res;
1da177e4
LT		 1905
1906 if (sk->sk_type == SOCK_DGRAM) {
8913336a
PM		 1907 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1908 po->tp_reserve;
1da177e4 1909 } else {
95c96174 1910 unsigned int maclen = skb_network_offset(skb);
bbd6ef87 1911 netoff = TPACKET_ALIGN(po->tp_hdrlen +
8913336a
PM		 1912 (maclen < 16 ? 16 : maclen)) +
1913 po->tp_reserve;
1da177e4
LT		 1914 macoff = netoff - maclen;
1915 }
f6fb8f10 1916 if (po->tp_version <= TPACKET_V2) {
1917 if (macoff + snaplen > po->rx_ring.frame_size) {
1918 if (po->copy_thresh &&
0fd7bac6 1919 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
f6fb8f10 1920 if (skb_shared(skb)) {
1921 copy_skb = skb_clone(skb, GFP_ATOMIC);
1922 } else {
1923 copy_skb = skb_get(skb);
1924 skb_head = skb->data;
1925 }
1926 if (copy_skb)
1927 skb_set_owner_r(copy_skb, sk);
1da177e4 1928 }
f6fb8f10 1929 snaplen = po->rx_ring.frame_size - macoff;
1930 if ((int)snaplen < 0)
1931 snaplen = 0;
1da177e4 1932 }
1da177e4 1933 }
1da177e4 1934 spin_lock(&sk->sk_receive_queue.lock);
f6fb8f10 1935 h.raw = packet_current_rx_frame(po, skb,
1936 TP_STATUS_KERNEL, (macoff+snaplen));
bbd6ef87 1937 if (!h.raw)
1da177e4 1938 goto ring_is_full;
f6fb8f10 1939 if (po->tp_version <= TPACKET_V2) {
1940 packet_increment_rx_head(po, &po->rx_ring);
1941 /*
1942 * LOSING will be reported till you read the stats,
1943 * because it's COR - Clear On Read.
1944 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1945 * at packet level.
1946 */
ee80fbf3 1947 if (po->stats.stats1.tp_drops)
f6fb8f10 1948 status |= TP_STATUS_LOSING;
1949 }
ee80fbf3 1950 po->stats.stats1.tp_packets++;
1da177e4
LT		 1951 if (copy_skb) {
1952 status |= TP_STATUS_COPY;
1953 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1954 }
1da177e4
LT		 1955 spin_unlock(&sk->sk_receive_queue.lock);
1956
bbd6ef87 1957 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
b9c32fb2
DB		 1958
1959 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
7a51384c 1960 getnstimeofday(&ts);
1da177e4 1961
b9c32fb2
DB		 1962 status |= ts_status;
1963
bbd6ef87
PM		 1964 switch (po->tp_version) {
1965 case TPACKET_V1:
1966 h.h1->tp_len = skb->len;
1967 h.h1->tp_snaplen = snaplen;
1968 h.h1->tp_mac = macoff;
1969 h.h1->tp_net = netoff;
4b457bdf
DB		 1970 h.h1->tp_sec = ts.tv_sec;
1971 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
bbd6ef87
PM		 1972 hdrlen = sizeof(*h.h1);
1973 break;
1974 case TPACKET_V2:
1975 h.h2->tp_len = skb->len;
1976 h.h2->tp_snaplen = snaplen;
1977 h.h2->tp_mac = macoff;
1978 h.h2->tp_net = netoff;
bbd6ef87
PM		 1979 h.h2->tp_sec = ts.tv_sec;
1980 h.h2->tp_nsec = ts.tv_nsec;
a3bcc23e
BG		 1981 if (vlan_tx_tag_present(skb)) {
1982 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 1983 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
1984 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 1985 } else {
1986 h.h2->tp_vlan_tci = 0;
a0cdfcf3 1987 h.h2->tp_vlan_tpid = 0;
a3bcc23e 1988 }
e4d26f4b 1989 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
bbd6ef87
PM		 1990 hdrlen = sizeof(*h.h2);
1991 break;
f6fb8f10 1992 case TPACKET_V3:
1993 /* tp_nxt_offset,vlan are already populated above.
1994 * So DONT clear those fields here
1995 */
1996 h.h3->tp_status |= status;
1997 h.h3->tp_len = skb->len;
1998 h.h3->tp_snaplen = snaplen;
1999 h.h3->tp_mac = macoff;
2000 h.h3->tp_net = netoff;
f6fb8f10 2001 h.h3->tp_sec = ts.tv_sec;
2002 h.h3->tp_nsec = ts.tv_nsec;
e4d26f4b 2003 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
f6fb8f10 2004 hdrlen = sizeof(*h.h3);
2005 break;
bbd6ef87
PM		 2006 default:
2007 BUG();
2008 }
1da177e4 2009
bbd6ef87 2010 sll = h.raw + TPACKET_ALIGN(hdrlen);
b95cce35 2011 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1da177e4
LT		 2012 sll->sll_family = AF_PACKET;
2013 sll->sll_hatype = dev->type;
2014 sll->sll_protocol = skb->protocol;
2015 sll->sll_pkttype = skb->pkt_type;
8032b464 2016 if (unlikely(po->origdev))
80feaacb
PWJ		 2017 sll->sll_ifindex = orig_dev->ifindex;
2018 else
2019 sll->sll_ifindex = dev->ifindex;
1da177e4 2020
e16aa207 2021 smp_mb();
f0d4eb29 2022
f6dafa95 2023#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
f0d4eb29 2024 if (po->tp_version <= TPACKET_V2) {
0af55bb5
CG		 2025 u8 *start, *end;
2026
f0d4eb29
DB		 2027 end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2028 macoff + snaplen);
2029
2030 for (start = h.raw; start < end; start += PAGE_SIZE)
2031 flush_dcache_page(pgv_to_page(start));
1da177e4 2032 }
f0d4eb29 2033 smp_wmb();
f6dafa95 2034#endif
f0d4eb29 2035
f6fb8f10 2036 if (po->tp_version <= TPACKET_V2)
2037 __packet_set_status(po, h.raw, status);
2038 else
2039 prb_clear_blk_fill_status(&po->rx_ring);
1da177e4
LT		 2040
2041 sk->sk_data_ready(sk, 0);
2042
2043drop_n_restore:
2044 if (skb_head != skb->data && skb_shared(skb)) {
2045 skb->data = skb_head;
2046 skb->len = skb_len;
2047 }
2048drop:
1ce4f28b 2049 kfree_skb(skb);
1da177e4
LT		 2050 return 0;
2051
2052ring_is_full:
ee80fbf3 2053 po->stats.stats1.tp_drops++;
1da177e4
LT		 2054 spin_unlock(&sk->sk_receive_queue.lock);
2055
2056 sk->sk_data_ready(sk, 0);
acb5d75b 2057 kfree_skb(copy_skb);
1da177e4
LT		 2058 goto drop_n_restore;
2059}
2060
69e3c75f
JB		 2061static void tpacket_destruct_skb(struct sk_buff *skb)
2062{
2063 struct packet_sock *po = pkt_sk(skb->sk);
1da177e4 2064
69e3c75f 2065 if (likely(po->tx_ring.pg_vec)) {
f0d4eb29 2066 void *ph;
b9c32fb2
DB		 2067 __u32 ts;
2068
69e3c75f 2069 ph = skb_shinfo(skb)->destructor_arg;
b0138408 2070 packet_dec_pending(&po->tx_ring);
b9c32fb2
DB		 2071
2072 ts = __packet_set_timestamp(po, ph, skb);
2073 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
69e3c75f
JB		 2074 }
2075
2076 sock_wfree(skb);
2077}
2078
40d4e3df
ED		 2079static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2080 void *frame, struct net_device *dev, int size_max,
ae641949 2081 __be16 proto, unsigned char *addr, int hlen)
69e3c75f 2082{
184f489e 2083 union tpacket_uhdr ph;
09effa67 2084 int to_write, offset, len, tp_len, nr_frags, len_max;
69e3c75f
JB		 2085 struct socket *sock = po->sk.sk_socket;
2086 struct page *page;
2087 void *data;
2088 int err;
2089
2090 ph.raw = frame;
2091
2092 skb->protocol = proto;
2093 skb->dev = dev;
2094 skb->priority = po->sk.sk_priority;
2d37a186 2095 skb->mark = po->sk.sk_mark;
2e31396f 2096 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
69e3c75f
JB		 2097 skb_shinfo(skb)->destructor_arg = ph.raw;
2098
2099 switch (po->tp_version) {
2100 case TPACKET_V2:
2101 tp_len = ph.h2->tp_len;
2102 break;
2103 default:
2104 tp_len = ph.h1->tp_len;
2105 break;
2106 }
09effa67
DM		 2107 if (unlikely(tp_len > size_max)) {
2108 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2109 return -EMSGSIZE;
2110 }
69e3c75f 2111
ae641949 2112 skb_reserve(skb, hlen);
69e3c75f 2113 skb_reset_network_header(skb);
c1aad275 2114
d346a3fa
DB		 2115 if (!packet_use_direct_xmit(po))
2116 skb_probe_transport_header(skb, 0);
2117 if (unlikely(po->tp_tx_has_off)) {
5920cd3a
PC		 2118 int off_min, off_max, off;
2119 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2120 off_max = po->tx_ring.frame_size - tp_len;
2121 if (sock->type == SOCK_DGRAM) {
2122 switch (po->tp_version) {
2123 case TPACKET_V2:
2124 off = ph.h2->tp_net;
2125 break;
2126 default:
2127 off = ph.h1->tp_net;
2128 break;
2129 }
2130 } else {
2131 switch (po->tp_version) {
2132 case TPACKET_V2:
2133 off = ph.h2->tp_mac;
2134 break;
2135 default:
2136 off = ph.h1->tp_mac;
2137 break;
2138 }
2139 }
2140 if (unlikely((off < off_min) || (off_max < off)))
2141 return -EINVAL;
2142 data = ph.raw + off;
2143 } else {
2144 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2145 }
69e3c75f
JB		 2146 to_write = tp_len;
2147
2148 if (sock->type == SOCK_DGRAM) {
2149 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2150 NULL, tp_len);
2151 if (unlikely(err < 0))
2152 return -EINVAL;
40d4e3df 2153 } else if (dev->hard_header_len) {
69e3c75f
JB		 2154 /* net device doesn't like empty head */
2155 if (unlikely(tp_len <= dev->hard_header_len)) {
40d4e3df
ED		 2156 pr_err("packet size is too short (%d < %d)\n",
2157 tp_len, dev->hard_header_len);
69e3c75f
JB		 2158 return -EINVAL;
2159 }
2160
2161 skb_push(skb, dev->hard_header_len);
2162 err = skb_store_bits(skb, 0, data,
2163 dev->hard_header_len);
2164 if (unlikely(err))
2165 return err;
2166
2167 data += dev->hard_header_len;
2168 to_write -= dev->hard_header_len;
2169 }
2170
69e3c75f
JB		 2171 offset = offset_in_page(data);
2172 len_max = PAGE_SIZE - offset;
2173 len = ((to_write > len_max) ? len_max : to_write);
2174
2175 skb->data_len = to_write;
2176 skb->len += to_write;
2177 skb->truesize += to_write;
2178 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2179
2180 while (likely(to_write)) {
2181 nr_frags = skb_shinfo(skb)->nr_frags;
2182
2183 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
40d4e3df
ED		 2184 pr_err("Packet exceed the number of skb frags(%lu)\n",
2185 MAX_SKB_FRAGS);
69e3c75f
JB		 2186 return -EFAULT;
2187 }
2188
0af55bb5
CG		 2189 page = pgv_to_page(data);
2190 data += len;
69e3c75f
JB		 2191 flush_dcache_page(page);
2192 get_page(page);
0af55bb5 2193 skb_fill_page_desc(skb, nr_frags, page, offset, len);
69e3c75f
JB		 2194 to_write -= len;
2195 offset = 0;
2196 len_max = PAGE_SIZE;
2197 len = ((to_write > len_max) ? len_max : to_write);
2198 }
2199
2200 return tp_len;
2201}
2202
2203static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2204{
69e3c75f
JB		 2205 struct sk_buff *skb;
2206 struct net_device *dev;
2207 __be16 proto;
09effa67 2208 int err, reserve = 0;
40d4e3df 2209 void *ph;
342dfc30 2210 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
87a2fd28 2211 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
69e3c75f
JB		 2212 int tp_len, size_max;
2213 unsigned char *addr;
2214 int len_sum = 0;
9e67030a 2215 int status = TP_STATUS_AVAILABLE;
ae641949 2216 int hlen, tlen;
69e3c75f 2217
69e3c75f
JB		 2218 mutex_lock(&po->pg_vec_lock);
2219
66e56cd4 2220 if (likely(saddr == NULL)) {
e40526cb 2221 dev = packet_cached_dev_get(po);
69e3c75f
JB		 2222 proto = po->num;
2223 addr = NULL;
2224 } else {
2225 err = -EINVAL;
2226 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2227 goto out;
2228 if (msg->msg_namelen < (saddr->sll_halen
2229 + offsetof(struct sockaddr_ll,
2230 sll_addr)))
2231 goto out;
69e3c75f
JB		 2232 proto = saddr->sll_protocol;
2233 addr = saddr->sll_addr;
827d9780 2234 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
69e3c75f
JB		 2235 }
2236
69e3c75f
JB		 2237 err = -ENXIO;
2238 if (unlikely(dev == NULL))
2239 goto out;
69e3c75f
JB		 2240 err = -ENETDOWN;
2241 if (unlikely(!(dev->flags & IFF_UP)))
2242 goto out_put;
2243
e40526cb
DB		 2244 reserve = dev->hard_header_len;
2245
69e3c75f 2246 size_max = po->tx_ring.frame_size
b5dd884e 2247 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
69e3c75f 2248
09effa67
DM		 2249 if (size_max > dev->mtu + reserve)
2250 size_max = dev->mtu + reserve;
2251
69e3c75f
JB		 2252 do {
2253 ph = packet_current_frame(po, &po->tx_ring,
87a2fd28 2254 TP_STATUS_SEND_REQUEST);
69e3c75f 2255 if (unlikely(ph == NULL)) {
87a2fd28
DB		 2256 if (need_wait && need_resched())
2257 schedule();
69e3c75f
JB		 2258 continue;
2259 }
2260
2261 status = TP_STATUS_SEND_REQUEST;
ae641949
HX		 2262 hlen = LL_RESERVED_SPACE(dev);
2263 tlen = dev->needed_tailroom;
69e3c75f 2264 skb = sock_alloc_send_skb(&po->sk,
ae641949 2265 hlen + tlen + sizeof(struct sockaddr_ll),
69e3c75f
JB		 2266 0, &err);
2267
2268 if (unlikely(skb == NULL))
2269 goto out_status;
2270
2271 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
ae641949 2272 addr, hlen);
69e3c75f
JB		 2273
2274 if (unlikely(tp_len < 0)) {
2275 if (po->tp_loss) {
2276 __packet_set_status(po, ph,
2277 TP_STATUS_AVAILABLE);
2278 packet_increment_head(&po->tx_ring);
2279 kfree_skb(skb);
2280 continue;
2281 } else {
2282 status = TP_STATUS_WRONG_FORMAT;
2283 err = tp_len;
2284 goto out_status;
2285 }
2286 }
2287
d346a3fa 2288 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
69e3c75f
JB		 2289 skb->destructor = tpacket_destruct_skb;
2290 __packet_set_status(po, ph, TP_STATUS_SENDING);
b0138408 2291 packet_inc_pending(&po->tx_ring);
69e3c75f
JB		 2292
2293 status = TP_STATUS_SEND_REQUEST;
d346a3fa 2294 err = po->xmit(skb);
eb70df13
JP		 2295 if (unlikely(err > 0)) {
2296 err = net_xmit_errno(err);
2297 if (err && __packet_get_status(po, ph) ==
2298 TP_STATUS_AVAILABLE) {
2299 /* skb was destructed already */
2300 skb = NULL;
2301 goto out_status;
2302 }
2303 /*
2304 * skb was dropped but not destructed yet;
2305 * let's treat it like congestion or err < 0
2306 */
2307 err = 0;
2308 }
69e3c75f
JB		 2309 packet_increment_head(&po->tx_ring);
2310 len_sum += tp_len;
b0138408
DB		 2311 } while (likely((ph != NULL) ||
2312 /* Note: packet_read_pending() might be slow if we have
2313 * to call it as it's per_cpu variable, but in fast-path
2314 * we already short-circuit the loop with the first
2315 * condition, and luckily don't have to go that path
2316 * anyway.
2317 */
2318 (need_wait && packet_read_pending(&po->tx_ring))));
69e3c75f
JB		 2319
2320 err = len_sum;
2321 goto out_put;
2322
69e3c75f
JB		 2323out_status:
2324 __packet_set_status(po, ph, status);
2325 kfree_skb(skb);
2326out_put:
e40526cb 2327 dev_put(dev);
69e3c75f
JB		 2328out:
2329 mutex_unlock(&po->pg_vec_lock);
2330 return err;
2331}
69e3c75f 2332
eea49cc9
OJ		 2333static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2334 size_t reserve, size_t len,
2335 size_t linear, int noblock,
2336 int *err)
bfd5f4a3
SS		 2337{
2338 struct sk_buff *skb;
2339
2340 /* Under a page? Don't bother with paged skb. */
2341 if (prepad + len < PAGE_SIZE || !linear)
2342 linear = len;
2343
2344 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
28d64271 2345 err, 0);
bfd5f4a3
SS		 2346 if (!skb)
2347 return NULL;
2348
2349 skb_reserve(skb, reserve);
2350 skb_put(skb, linear);
2351 skb->data_len = len - linear;
2352 skb->len += len - linear;
2353
2354 return skb;
2355}
2356
d346a3fa 2357static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
1da177e4
LT		 2358{
2359 struct sock *sk = sock->sk;
342dfc30 2360 DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
1da177e4
LT		 2361 struct sk_buff *skb;
2362 struct net_device *dev;
0e11c91e 2363 __be16 proto;
1da177e4 2364 unsigned char *addr;
827d9780 2365 int err, reserve = 0;
bfd5f4a3
SS		 2366 struct virtio_net_hdr vnet_hdr = { 0 };
2367 int offset = 0;
2368 int vnet_hdr_len;
2369 struct packet_sock *po = pkt_sk(sk);
2370 unsigned short gso_type = 0;
ae641949 2371 int hlen, tlen;
3bdc0eba 2372 int extra_len = 0;
1da177e4
LT		 2373
2374 /*
1ce4f28b 2375 * Get and verify the address.
1da177e4 2376 */
1ce4f28b 2377
66e56cd4 2378 if (likely(saddr == NULL)) {
e40526cb 2379 dev = packet_cached_dev_get(po);
1da177e4
LT		 2380 proto = po->num;
2381 addr = NULL;
2382 } else {
2383 err = -EINVAL;
2384 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2385 goto out;
0fb375fb
EB		 2386 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2387 goto out;
1da177e4
LT		 2388 proto = saddr->sll_protocol;
2389 addr = saddr->sll_addr;
827d9780 2390 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
1da177e4
LT		 2391 }
2392
1da177e4 2393 err = -ENXIO;
e40526cb 2394 if (unlikely(dev == NULL))
1da177e4 2395 goto out_unlock;
d5e76b0a 2396 err = -ENETDOWN;
e40526cb 2397 if (unlikely(!(dev->flags & IFF_UP)))
d5e76b0a
DM		 2398 goto out_unlock;
2399
e40526cb
DB		 2400 if (sock->type == SOCK_RAW)
2401 reserve = dev->hard_header_len;
bfd5f4a3
SS		 2402 if (po->has_vnet_hdr) {
2403 vnet_hdr_len = sizeof(vnet_hdr);
2404
2405 err = -EINVAL;
2406 if (len < vnet_hdr_len)
2407 goto out_unlock;
2408
2409 len -= vnet_hdr_len;
2410
2411 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2412 vnet_hdr_len);
2413 if (err < 0)
2414 goto out_unlock;
2415
2416 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2417 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2418 vnet_hdr.hdr_len))
2419 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2420 vnet_hdr.csum_offset + 2;
2421
2422 err = -EINVAL;
2423 if (vnet_hdr.hdr_len > len)
2424 goto out_unlock;
2425
2426 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2427 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2428 case VIRTIO_NET_HDR_GSO_TCPV4:
2429 gso_type = SKB_GSO_TCPV4;
2430 break;
2431 case VIRTIO_NET_HDR_GSO_TCPV6:
2432 gso_type = SKB_GSO_TCPV6;
2433 break;
2434 case VIRTIO_NET_HDR_GSO_UDP:
2435 gso_type = SKB_GSO_UDP;
2436 break;
2437 default:
2438 goto out_unlock;
2439 }
2440
2441 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2442 gso_type |= SKB_GSO_TCP_ECN;
2443
2444 if (vnet_hdr.gso_size == 0)
2445 goto out_unlock;
2446
2447 }
2448 }
2449
3bdc0eba
BG		 2450 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2451 if (!netif_supports_nofcs(dev)) {
2452 err = -EPROTONOSUPPORT;
2453 goto out_unlock;
2454 }
2455 extra_len = 4; /* We're doing our own CRC */
2456 }
2457
1da177e4 2458 err = -EMSGSIZE;
3bdc0eba 2459 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
1da177e4
LT		 2460 goto out_unlock;
2461
bfd5f4a3 2462 err = -ENOBUFS;
ae641949
HX		 2463 hlen = LL_RESERVED_SPACE(dev);
2464 tlen = dev->needed_tailroom;
2465 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
bfd5f4a3 2466 msg->msg_flags & MSG_DONTWAIT, &err);
40d4e3df 2467 if (skb == NULL)
1da177e4
LT		 2468 goto out_unlock;
2469
bfd5f4a3 2470 skb_set_network_header(skb, reserve);
1da177e4 2471
0c4e8581
SH		 2472 err = -EINVAL;
2473 if (sock->type == SOCK_DGRAM &&
bfd5f4a3 2474 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
0c4e8581 2475 goto out_free;
1da177e4
LT		 2476
2477 /* Returns -EFAULT on error */
bfd5f4a3 2478 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1da177e4
LT		 2479 if (err)
2480 goto out_free;
bf84a010
DB		 2481
2482 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1da177e4 2483
3bdc0eba 2484 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
09effa67
DM		 2485 /* Earlier code assumed this would be a VLAN pkt,
2486 * double-check this now that we have the actual
2487 * packet in hand.
2488 */
2489 struct ethhdr *ehdr;
2490 skb_reset_mac_header(skb);
2491 ehdr = eth_hdr(skb);
2492 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2493 err = -EMSGSIZE;
2494 goto out_free;
2495 }
57f89bfa
BG		 2496 }
2497
09effa67
DM		 2498 skb->protocol = proto;
2499 skb->dev = dev;
1da177e4 2500 skb->priority = sk->sk_priority;
2d37a186 2501 skb->mark = sk->sk_mark;
d346a3fa 2502 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
1da177e4 2503
bfd5f4a3
SS		 2504 if (po->has_vnet_hdr) {
2505 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2506 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2507 vnet_hdr.csum_offset)) {
2508 err = -EINVAL;
2509 goto out_free;
2510 }
2511 }
2512
2513 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2514 skb_shinfo(skb)->gso_type = gso_type;
2515
2516 /* Header must be checked, and gso_segs computed. */
2517 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2518 skb_shinfo(skb)->gso_segs = 0;
2519
2520 len += vnet_hdr_len;
2521 }
2522
d346a3fa
DB		 2523 if (!packet_use_direct_xmit(po))
2524 skb_probe_transport_header(skb, reserve);
3bdc0eba
BG		 2525 if (unlikely(extra_len == 4))
2526 skb->no_fcs = 1;
2527
d346a3fa 2528 err = po->xmit(skb);
1da177e4
LT		 2529 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2530 goto out_unlock;
2531
e40526cb 2532 dev_put(dev);
1da177e4 2533
40d4e3df 2534 return len;
1da177e4
LT		 2535
2536out_free:
2537 kfree_skb(skb);
2538out_unlock:
e40526cb 2539 if (dev)
1da177e4
LT		 2540 dev_put(dev);
2541out:
2542 return err;
2543}
2544
69e3c75f
JB		 2545static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2546 struct msghdr *msg, size_t len)
2547{
69e3c75f
JB		 2548 struct sock *sk = sock->sk;
2549 struct packet_sock *po = pkt_sk(sk);
d346a3fa 2550
69e3c75f
JB		 2551 if (po->tx_ring.pg_vec)
2552 return tpacket_snd(po, msg);
2553 else
69e3c75f
JB		 2554 return packet_snd(sock, msg, len);
2555}
2556
1da177e4
LT		 2557/*
2558 * Close a PACKET socket. This is fairly simple. We immediately go
2559 * to 'closed' state and remove our protocol entry in the device list.
2560 */
2561
2562static int packet_release(struct socket *sock)
2563{
2564 struct sock *sk = sock->sk;
2565 struct packet_sock *po;
d12d01d6 2566 struct net *net;
f6fb8f10 2567 union tpacket_req_u req_u;
1da177e4
LT		 2568
2569 if (!sk)
2570 return 0;
2571
3b1e0a65 2572 net = sock_net(sk);
1da177e4
LT		 2573 po = pkt_sk(sk);
2574
0fa7fa98 2575 mutex_lock(&net->packet.sklist_lock);
808f5114 2576 sk_del_node_init_rcu(sk);
0fa7fa98
PE		 2577 mutex_unlock(&net->packet.sklist_lock);
2578
2579 preempt_disable();
920de804 2580 sock_prot_inuse_add(net, sk->sk_prot, -1);
0fa7fa98 2581 preempt_enable();
1da177e4 2582
808f5114 2583 spin_lock(&po->bind_lock);
ce06b03e 2584 unregister_prot_hook(sk, false);
66e56cd4
DB		 2585 packet_cached_dev_reset(po);
2586
160ff18a
BG		 2587 if (po->prot_hook.dev) {
2588 dev_put(po->prot_hook.dev);
2589 po->prot_hook.dev = NULL;
2590 }
808f5114 2591 spin_unlock(&po->bind_lock);
1da177e4 2592
1da177e4 2593 packet_flush_mclist(sk);
1da177e4 2594
9665d5d6
PS		 2595 if (po->rx_ring.pg_vec) {
2596 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2597 packet_set_ring(sk, &req_u, 1, 0);
9665d5d6 2598 }
69e3c75f 2599
9665d5d6
PS		 2600 if (po->tx_ring.pg_vec) {
2601 memset(&req_u, 0, sizeof(req_u));
f6fb8f10 2602 packet_set_ring(sk, &req_u, 1, 1);
9665d5d6 2603 }
1da177e4 2604
dc99f600
DM		 2605 fanout_release(sk);
2606
808f5114 2607 synchronize_net();
1da177e4
LT		 2608 /*
2609 * Now the socket is dead. No more input will appear.
2610 */
1da177e4
LT		 2611 sock_orphan(sk);
2612 sock->sk = NULL;
2613
2614 /* Purge queues */
2615
2616 skb_queue_purge(&sk->sk_receive_queue);
b0138408 2617 packet_free_pending(po);
17ab56a2 2618 sk_refcnt_debug_release(sk);
1da177e4
LT		 2619
2620 sock_put(sk);
2621 return 0;
2622}
2623
2624/*
2625 * Attach a packet hook.
2626 */
2627
902fefb8 2628static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
1da177e4
LT		 2629{
2630 struct packet_sock *po = pkt_sk(sk);
902fefb8
DB		 2631 const struct net_device *dev_curr;
2632 __be16 proto_curr;
2633 bool need_rehook;
dc99f600 2634
aef950b4
WY		 2635 if (po->fanout) {
2636 if (dev)
2637 dev_put(dev);
2638
dc99f600 2639 return -EINVAL;
aef950b4 2640 }
1da177e4
LT		 2641
2642 lock_sock(sk);
1da177e4 2643 spin_lock(&po->bind_lock);
66e56cd4 2644
902fefb8
DB		 2645 proto_curr = po->prot_hook.type;
2646 dev_curr = po->prot_hook.dev;
2647
2648 need_rehook = proto_curr != proto || dev_curr != dev;
2649
2650 if (need_rehook) {
2651 unregister_prot_hook(sk, true);
1da177e4 2652
902fefb8
DB		 2653 po->num = proto;
2654 po->prot_hook.type = proto;
1da177e4 2655
902fefb8
DB		 2656 if (po->prot_hook.dev)
2657 dev_put(po->prot_hook.dev);
2658
2659 po->prot_hook.dev = dev;
2660
2661 po->ifindex = dev ? dev->ifindex : 0;
2662 packet_cached_dev_assign(po, dev);
2663 }
66e56cd4 2664
902fefb8 2665 if (proto == 0 || !need_rehook)
1da177e4
LT		 2666 goto out_unlock;
2667
be85d4ad 2668 if (!dev || (dev->flags & IFF_UP)) {
ce06b03e 2669 register_prot_hook(sk);
be85d4ad
UT		 2670 } else {
2671 sk->sk_err = ENETDOWN;
2672 if (!sock_flag(sk, SOCK_DEAD))
2673 sk->sk_error_report(sk);
1da177e4
LT		 2674 }
2675
2676out_unlock:
2677 spin_unlock(&po->bind_lock);
2678 release_sock(sk);
2679 return 0;
2680}
2681
2682/*
2683 * Bind a packet socket to a device
2684 */
2685
40d4e3df
ED		 2686static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2687 int addr_len)
1da177e4 2688{
40d4e3df 2689 struct sock *sk = sock->sk;
1da177e4
LT		 2690 char name[15];
2691 struct net_device *dev;
2692 int err = -ENODEV;
1ce4f28b 2693
1da177e4
LT		 2694 /*
2695 * Check legality
2696 */
1ce4f28b 2697
8ae55f04 2698 if (addr_len != sizeof(struct sockaddr))
1da177e4 2699 return -EINVAL;
40d4e3df 2700 strlcpy(name, uaddr->sa_data, sizeof(name));
1da177e4 2701
3b1e0a65 2702 dev = dev_get_by_name(sock_net(sk), name);
160ff18a 2703 if (dev)
1da177e4 2704 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1da177e4
LT		 2705 return err;
2706}
1da177e4
LT		 2707
2708static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2709{
40d4e3df
ED		 2710 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2711 struct sock *sk = sock->sk;
1da177e4
LT		 2712 struct net_device *dev = NULL;
2713 int err;
2714
2715
2716 /*
2717 * Check legality
2718 */
1ce4f28b 2719
1da177e4
LT		 2720 if (addr_len < sizeof(struct sockaddr_ll))
2721 return -EINVAL;
2722 if (sll->sll_family != AF_PACKET)
2723 return -EINVAL;
2724
2725 if (sll->sll_ifindex) {
2726 err = -ENODEV;
3b1e0a65 2727 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1da177e4
LT		 2728 if (dev == NULL)
2729 goto out;
2730 }
2731 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1da177e4
LT		 2732
2733out:
2734 return err;
2735}
2736
2737static struct proto packet_proto = {
2738 .name = "PACKET",
2739 .owner = THIS_MODULE,
2740 .obj_size = sizeof(struct packet_sock),
2741};
2742
2743/*
1ce4f28b 2744 * Create a packet of type SOCK_PACKET.
1da177e4
LT		 2745 */
2746
3f378b68
EP		 2747static int packet_create(struct net *net, struct socket *sock, int protocol,
2748 int kern)
1da177e4
LT		 2749{
2750 struct sock *sk;
2751 struct packet_sock *po;
0e11c91e 2752 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1da177e4
LT		 2753 int err;
2754
df008c91 2755 if (!ns_capable(net->user_ns, CAP_NET_RAW))
1da177e4 2756 return -EPERM;
be02097c
DM		 2757 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2758 sock->type != SOCK_PACKET)
1da177e4
LT		 2759 return -ESOCKTNOSUPPORT;
2760
2761 sock->state = SS_UNCONNECTED;
2762
2763 err = -ENOBUFS;
6257ff21 2764 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1da177e4
LT		 2765 if (sk == NULL)
2766 goto out;
2767
2768 sock->ops = &packet_ops;
1da177e4
LT		 2769 if (sock->type == SOCK_PACKET)
2770 sock->ops = &packet_ops_spkt;
be02097c 2771
1da177e4
LT		 2772 sock_init_data(sock, sk);
2773
2774 po = pkt_sk(sk);
2775 sk->sk_family = PF_PACKET;
0e11c91e 2776 po->num = proto;
d346a3fa 2777 po->xmit = dev_queue_xmit;
66e56cd4 2778
b0138408
DB		 2779 err = packet_alloc_pending(po);
2780 if (err)
2781 goto out2;
2782
66e56cd4 2783 packet_cached_dev_reset(po);
1da177e4
LT		 2784
2785 sk->sk_destruct = packet_sock_destruct;
17ab56a2 2786 sk_refcnt_debug_inc(sk);
1da177e4
LT		 2787
2788 /*
2789 * Attach a protocol block
2790 */
2791
2792 spin_lock_init(&po->bind_lock);
905db440 2793 mutex_init(&po->pg_vec_lock);
1da177e4 2794 po->prot_hook.func = packet_rcv;
be02097c 2795
1da177e4
LT		 2796 if (sock->type == SOCK_PACKET)
2797 po->prot_hook.func = packet_rcv_spkt;
be02097c 2798
1da177e4
LT		 2799 po->prot_hook.af_packet_priv = sk;
2800
0e11c91e
AV		 2801 if (proto) {
2802 po->prot_hook.type = proto;
ce06b03e 2803 register_prot_hook(sk);
1da177e4
LT		 2804 }
2805
0fa7fa98 2806 mutex_lock(&net->packet.sklist_lock);
808f5114 2807 sk_add_node_rcu(sk, &net->packet.sklist);
0fa7fa98
PE		 2808 mutex_unlock(&net->packet.sklist_lock);
2809
2810 preempt_disable();
3680453c 2811 sock_prot_inuse_add(net, &packet_proto, 1);
0fa7fa98 2812 preempt_enable();
808f5114 2813
40d4e3df 2814 return 0;
b0138408
DB		 2815out2:
2816 sk_free(sk);
1da177e4
LT		 2817out:
2818 return err;
2819}
2820
2821/*
2822 * Pull a packet from our receive queue and hand it to the user.
2823 * If necessary we block.
2824 */
2825
2826static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2827 struct msghdr *msg, size_t len, int flags)
2828{
2829 struct sock *sk = sock->sk;
2830 struct sk_buff *skb;
2831 int copied, err;
bfd5f4a3 2832 int vnet_hdr_len = 0;
1da177e4
LT		 2833
2834 err = -EINVAL;
ed85b565 2835 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
1da177e4
LT		 2836 goto out;
2837
2838#if 0
2839 /* What error should we return now? EUNATTACH? */
2840 if (pkt_sk(sk)->ifindex < 0)
2841 return -ENODEV;
2842#endif
2843
ed85b565 2844 if (flags & MSG_ERRQUEUE) {
cb820f8e
RC		 2845 err = sock_recv_errqueue(sk, msg, len,
2846 SOL_PACKET, PACKET_TX_TIMESTAMP);
ed85b565
RC		 2847 goto out;
2848 }
2849
1da177e4
LT		 2850 /*
2851 * Call the generic datagram receiver. This handles all sorts
2852 * of horrible races and re-entrancy so we can forget about it
2853 * in the protocol layers.
2854 *
2855 * Now it will return ENETDOWN, if device have just gone down,
2856 * but then it will block.
2857 */
2858
40d4e3df 2859 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1da177e4
LT		 2860
2861 /*
1ce4f28b 2862 * An error occurred so return it. Because skb_recv_datagram()
1da177e4
LT		 2863 * handles the blocking we don't see and worry about blocking
2864 * retries.
2865 */
2866
8ae55f04 2867 if (skb == NULL)
1da177e4
LT		 2868 goto out;
2869
bfd5f4a3
SS		 2870 if (pkt_sk(sk)->has_vnet_hdr) {
2871 struct virtio_net_hdr vnet_hdr = { 0 };
2872
2873 err = -EINVAL;
2874 vnet_hdr_len = sizeof(vnet_hdr);
1f18b717 2875 if (len < vnet_hdr_len)
bfd5f4a3
SS		 2876 goto out_free;
2877
1f18b717
MK		 2878 len -= vnet_hdr_len;
2879
bfd5f4a3
SS		 2880 if (skb_is_gso(skb)) {
2881 struct skb_shared_info *sinfo = skb_shinfo(skb);
2882
2883 /* This is a hint as to how much should be linear. */
2884 vnet_hdr.hdr_len = skb_headlen(skb);
2885 vnet_hdr.gso_size = sinfo->gso_size;
2886 if (sinfo->gso_type & SKB_GSO_TCPV4)
2887 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2888 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2889 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2890 else if (sinfo->gso_type & SKB_GSO_UDP)
2891 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2892 else if (sinfo->gso_type & SKB_GSO_FCOE)
2893 goto out_free;
2894 else
2895 BUG();
2896 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2897 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2898 } else
2899 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2900
2901 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2902 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
55508d60 2903 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
bfd5f4a3 2904 vnet_hdr.csum_offset = skb->csum_offset;
10a8d94a
JW		 2905 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2906 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
bfd5f4a3
SS		 2907 } /* else everything is zero */
2908
2909 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2910 vnet_hdr_len);
2911 if (err < 0)
2912 goto out_free;
2913 }
2914
f3d33426
HFS		 2915 /* You lose any data beyond the buffer you gave. If it worries
2916 * a user program they can ask the device for its MTU
2917 * anyway.
1da177e4 2918 */
1da177e4 2919 copied = skb->len;
40d4e3df
ED		 2920 if (copied > len) {
2921 copied = len;
2922 msg->msg_flags |= MSG_TRUNC;
1da177e4
LT		 2923 }
2924
2925 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2926 if (err)
2927 goto out_free;
2928
3b885787 2929 sock_recv_ts_and_drops(msg, sk, skb);
1da177e4 2930
f3d33426
HFS		 2931 if (msg->msg_name) {
2932 /* If the address length field is there to be filled
2933 * in, we fill it in now.
2934 */
2935 if (sock->type == SOCK_PACKET) {
342dfc30 2936 __sockaddr_check_size(sizeof(struct sockaddr_pkt));
f3d33426
HFS		 2937 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2938 } else {
2939 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2940 msg->msg_namelen = sll->sll_halen +
2941 offsetof(struct sockaddr_ll, sll_addr);
2942 }
ffbc6111
HX		 2943 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2944 msg->msg_namelen);
f3d33426 2945 }
1da177e4 2946
8dc41944 2947 if (pkt_sk(sk)->auxdata) {
ffbc6111
HX		 2948 struct tpacket_auxdata aux;
2949
2950 aux.tp_status = TP_STATUS_USER;
2951 if (skb->ip_summed == CHECKSUM_PARTIAL)
2952 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2953 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2954 aux.tp_snaplen = skb->len;
2955 aux.tp_mac = 0;
bbe735e4 2956 aux.tp_net = skb_network_offset(skb);
a3bcc23e
BG		 2957 if (vlan_tx_tag_present(skb)) {
2958 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
a0cdfcf3
AW		 2959 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2960 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
a3bcc23e
BG		 2961 } else {
2962 aux.tp_vlan_tci = 0;
a0cdfcf3 2963 aux.tp_vlan_tpid = 0;
a3bcc23e 2964 }
ffbc6111 2965 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
8dc41944
HX		 2966 }
2967
1da177e4
LT		 2968 /*
2969 * Free or return the buffer as appropriate. Again this
2970 * hides all the races and re-entrancy issues from us.
2971 */
bfd5f4a3 2972 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1da177e4
LT		 2973
2974out_free:
2975 skb_free_datagram(sk, skb);
2976out:
2977 return err;
2978}
2979
1da177e4
LT		 2980static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2981 int *uaddr_len, int peer)
2982{
2983 struct net_device *dev;
2984 struct sock *sk = sock->sk;
2985
2986 if (peer)
2987 return -EOPNOTSUPP;
2988
2989 uaddr->sa_family = AF_PACKET;
2dc85bf3 2990 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
654d1f8a
ED		 2991 rcu_read_lock();
2992 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2993 if (dev)
2dc85bf3 2994 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
654d1f8a 2995 rcu_read_unlock();
1da177e4
LT		 2996 *uaddr_len = sizeof(*uaddr);
2997
2998 return 0;
2999}
1da177e4
LT		 3000
3001static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3002 int *uaddr_len, int peer)
3003{
3004 struct net_device *dev;
3005 struct sock *sk = sock->sk;
3006 struct packet_sock *po = pkt_sk(sk);
13cfa97b 3007 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1da177e4
LT		 3008
3009 if (peer)
3010 return -EOPNOTSUPP;
3011
3012 sll->sll_family = AF_PACKET;
3013 sll->sll_ifindex = po->ifindex;
3014 sll->sll_protocol = po->num;
67286640 3015 sll->sll_pkttype = 0;
654d1f8a
ED		 3016 rcu_read_lock();
3017 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1da177e4
LT		 3018 if (dev) {
3019 sll->sll_hatype = dev->type;
3020 sll->sll_halen = dev->addr_len;
3021 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1da177e4
LT		 3022 } else {
3023 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3024 sll->sll_halen = 0;
3025 }
654d1f8a 3026 rcu_read_unlock();
0fb375fb 3027 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1da177e4
LT		 3028
3029 return 0;
3030}
3031
2aeb0b88
WC		 3032static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3033 int what)
1da177e4
LT		 3034{
3035 switch (i->type) {
3036 case PACKET_MR_MULTICAST:
1162563f
JP		 3037 if (i->alen != dev->addr_len)
3038 return -EINVAL;
1da177e4 3039 if (what > 0)
22bedad3 3040 return dev_mc_add(dev, i->addr);
1da177e4 3041 else
22bedad3 3042 return dev_mc_del(dev, i->addr);
1da177e4
LT		 3043 break;
3044 case PACKET_MR_PROMISC:
2aeb0b88 3045 return dev_set_promiscuity(dev, what);
1da177e4
LT		 3046 break;
3047 case PACKET_MR_ALLMULTI:
2aeb0b88 3048 return dev_set_allmulti(dev, what);
1da177e4 3049 break;
d95ed927 3050 case PACKET_MR_UNICAST:
1162563f
JP		 3051 if (i->alen != dev->addr_len)
3052 return -EINVAL;
d95ed927 3053 if (what > 0)
a748ee24 3054 return dev_uc_add(dev, i->addr);
d95ed927 3055 else
a748ee24 3056 return dev_uc_del(dev, i->addr);
d95ed927 3057 break;
40d4e3df
ED		 3058 default:
3059 break;
1da177e4 3060 }
2aeb0b88 3061 return 0;
1da177e4
LT		 3062}
3063
3064static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3065{
40d4e3df 3066 for ( ; i; i = i->next) {
1da177e4
LT		 3067 if (i->ifindex == dev->ifindex)
3068 packet_dev_mc(dev, i, what);
3069 }
3070}
3071
0fb375fb 3072static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3073{
3074 struct packet_sock *po = pkt_sk(sk);
3075 struct packet_mclist *ml, *i;
3076 struct net_device *dev;
3077 int err;
3078
3079 rtnl_lock();
3080
3081 err = -ENODEV;
3b1e0a65 3082 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1da177e4
LT		 3083 if (!dev)
3084 goto done;
3085
3086 err = -EINVAL;
1162563f 3087 if (mreq->mr_alen > dev->addr_len)
1da177e4
LT		 3088 goto done;
3089
3090 err = -ENOBUFS;
8b3a7005 3091 i = kmalloc(sizeof(*i), GFP_KERNEL);
1da177e4
LT		 3092 if (i == NULL)
3093 goto done;
3094
3095 err = 0;
3096 for (ml = po->mclist; ml; ml = ml->next) {
3097 if (ml->ifindex == mreq->mr_ifindex &&
3098 ml->type == mreq->mr_type &&
3099 ml->alen == mreq->mr_alen &&
3100 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3101 ml->count++;
3102 /* Free the new element ... */
3103 kfree(i);
3104 goto done;
3105 }
3106 }
3107
3108 i->type = mreq->mr_type;
3109 i->ifindex = mreq->mr_ifindex;
3110 i->alen = mreq->mr_alen;
3111 memcpy(i->addr, mreq->mr_address, i->alen);
3112 i->count = 1;
3113 i->next = po->mclist;
3114 po->mclist = i;
2aeb0b88
WC		 3115 err = packet_dev_mc(dev, i, 1);
3116 if (err) {
3117 po->mclist = i->next;
3118 kfree(i);
3119 }
1da177e4
LT		 3120
3121done:
3122 rtnl_unlock();
3123 return err;
3124}
3125
0fb375fb 3126static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1da177e4
LT		 3127{
3128 struct packet_mclist *ml, **mlp;
3129
3130 rtnl_lock();
3131
3132 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3133 if (ml->ifindex == mreq->mr_ifindex &&
3134 ml->type == mreq->mr_type &&
3135 ml->alen == mreq->mr_alen &&
3136 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3137 if (--ml->count == 0) {
3138 struct net_device *dev;
3139 *mlp = ml->next;
ad959e76
ED		 3140 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3141 if (dev)
1da177e4 3142 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3143 kfree(ml);
3144 }
3145 rtnl_unlock();
3146 return 0;
3147 }
3148 }
3149 rtnl_unlock();
3150 return -EADDRNOTAVAIL;
3151}
3152
3153static void packet_flush_mclist(struct sock *sk)
3154{
3155 struct packet_sock *po = pkt_sk(sk);
3156 struct packet_mclist *ml;
3157
3158 if (!po->mclist)
3159 return;
3160
3161 rtnl_lock();
3162 while ((ml = po->mclist) != NULL) {
3163 struct net_device *dev;
3164
3165 po->mclist = ml->next;
ad959e76
ED		 3166 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3167 if (dev != NULL)
1da177e4 3168 packet_dev_mc(dev, ml, -1);
1da177e4
LT		 3169 kfree(ml);
3170 }
3171 rtnl_unlock();
3172}
1da177e4
LT		 3173
3174static int
b7058842 3175packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1da177e4
LT		 3176{
3177 struct sock *sk = sock->sk;
8dc41944 3178 struct packet_sock *po = pkt_sk(sk);
1da177e4
LT		 3179 int ret;
3180
3181 if (level != SOL_PACKET)
3182 return -ENOPROTOOPT;
3183
69e3c75f 3184 switch (optname) {
1ce4f28b 3185 case PACKET_ADD_MEMBERSHIP:
1da177e4
LT		 3186 case PACKET_DROP_MEMBERSHIP:
3187 {
0fb375fb
EB		 3188 struct packet_mreq_max mreq;
3189 int len = optlen;
3190 memset(&mreq, 0, sizeof(mreq));
3191 if (len < sizeof(struct packet_mreq))
1da177e4 3192 return -EINVAL;
0fb375fb
EB		 3193 if (len > sizeof(mreq))
3194 len = sizeof(mreq);
40d4e3df 3195 if (copy_from_user(&mreq, optval, len))
1da177e4 3196 return -EFAULT;
0fb375fb
EB		 3197 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3198 return -EINVAL;
1da177e4
LT		 3199 if (optname == PACKET_ADD_MEMBERSHIP)
3200 ret = packet_mc_add(sk, &mreq);
3201 else
3202 ret = packet_mc_drop(sk, &mreq);
3203 return ret;
3204 }
a2efcfa0 3205
1da177e4 3206 case PACKET_RX_RING:
69e3c75f 3207 case PACKET_TX_RING:
1da177e4 3208 {
f6fb8f10 3209 union tpacket_req_u req_u;
3210 int len;
1da177e4 3211
f6fb8f10 3212 switch (po->tp_version) {
3213 case TPACKET_V1:
3214 case TPACKET_V2:
3215 len = sizeof(req_u.req);
3216 break;
3217 case TPACKET_V3:
3218 default:
3219 len = sizeof(req_u.req3);
3220 break;
3221 }
3222 if (optlen < len)
1da177e4 3223 return -EINVAL;
bfd5f4a3
SS		 3224 if (pkt_sk(sk)->has_vnet_hdr)
3225 return -EINVAL;
f6fb8f10 3226 if (copy_from_user(&req_u.req, optval, len))
1da177e4 3227 return -EFAULT;
f6fb8f10 3228 return packet_set_ring(sk, &req_u, 0,
3229 optname == PACKET_TX_RING);
1da177e4
LT		 3230 }
3231 case PACKET_COPY_THRESH:
3232 {
3233 int val;
3234
40d4e3df 3235 if (optlen != sizeof(val))
1da177e4 3236 return -EINVAL;
40d4e3df 3237 if (copy_from_user(&val, optval, sizeof(val)))
1da177e4
LT		 3238 return -EFAULT;
3239
3240 pkt_sk(sk)->copy_thresh = val;
3241 return 0;
3242 }
bbd6ef87
PM		 3243 case PACKET_VERSION:
3244 {
3245 int val;
3246
3247 if (optlen != sizeof(val))
3248 return -EINVAL;
69e3c75f 3249 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
bbd6ef87
PM		 3250 return -EBUSY;
3251 if (copy_from_user(&val, optval, sizeof(val)))
3252 return -EFAULT;
3253 switch (val) {
3254 case TPACKET_V1:
3255 case TPACKET_V2:
f6fb8f10 3256 case TPACKET_V3:
bbd6ef87
PM		 3257 po->tp_version = val;
3258 return 0;
3259 default:
3260 return -EINVAL;
3261 }
3262 }
8913336a
PM		 3263 case PACKET_RESERVE:
3264 {
3265 unsigned int val;
3266
3267 if (optlen != sizeof(val))
3268 return -EINVAL;
69e3c75f 3269 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
8913336a
PM		 3270 return -EBUSY;
3271 if (copy_from_user(&val, optval, sizeof(val)))
3272 return -EFAULT;
3273 po->tp_reserve = val;
3274 return 0;
3275 }
69e3c75f
JB		 3276 case PACKET_LOSS:
3277 {
3278 unsigned int val;
3279
3280 if (optlen != sizeof(val))
3281 return -EINVAL;
3282 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3283 return -EBUSY;
3284 if (copy_from_user(&val, optval, sizeof(val)))
3285 return -EFAULT;
3286 po->tp_loss = !!val;
3287 return 0;
3288 }
8dc41944
HX		 3289 case PACKET_AUXDATA:
3290 {
3291 int val;
3292
3293 if (optlen < sizeof(val))
3294 return -EINVAL;
3295 if (copy_from_user(&val, optval, sizeof(val)))
3296 return -EFAULT;
3297
3298 po->auxdata = !!val;
3299 return 0;
3300 }
80feaacb
PWJ		 3301 case PACKET_ORIGDEV:
3302 {
3303 int val;
3304
3305 if (optlen < sizeof(val))
3306 return -EINVAL;
3307 if (copy_from_user(&val, optval, sizeof(val)))
3308 return -EFAULT;
3309
3310 po->origdev = !!val;
3311 return 0;
3312 }
bfd5f4a3
SS		 3313 case PACKET_VNET_HDR:
3314 {
3315 int val;
3316
3317 if (sock->type != SOCK_RAW)
3318 return -EINVAL;
3319 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3320 return -EBUSY;
3321 if (optlen < sizeof(val))
3322 return -EINVAL;
3323 if (copy_from_user(&val, optval, sizeof(val)))
3324 return -EFAULT;
3325
3326 po->has_vnet_hdr = !!val;
3327 return 0;
3328 }
614f60fa
SM		 3329 case PACKET_TIMESTAMP:
3330 {
3331 int val;
3332
3333 if (optlen != sizeof(val))
3334 return -EINVAL;
3335 if (copy_from_user(&val, optval, sizeof(val)))
3336 return -EFAULT;
3337
3338 po->tp_tstamp = val;
3339 return 0;
3340 }
dc99f600
DM		 3341 case PACKET_FANOUT:
3342 {
3343 int val;
3344
3345 if (optlen != sizeof(val))
3346 return -EINVAL;
3347 if (copy_from_user(&val, optval, sizeof(val)))
3348 return -EFAULT;
3349
3350 return fanout_add(sk, val & 0xffff, val >> 16);
3351 }
5920cd3a
PC		 3352 case PACKET_TX_HAS_OFF:
3353 {
3354 unsigned int val;
3355
3356 if (optlen != sizeof(val))
3357 return -EINVAL;
3358 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3359 return -EBUSY;
3360 if (copy_from_user(&val, optval, sizeof(val)))
3361 return -EFAULT;
3362 po->tp_tx_has_off = !!val;
3363 return 0;
3364 }
d346a3fa
DB		 3365 case PACKET_QDISC_BYPASS:
3366 {
3367 int val;
3368
3369 if (optlen != sizeof(val))
3370 return -EINVAL;
3371 if (copy_from_user(&val, optval, sizeof(val)))
3372 return -EFAULT;
3373
3374 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3375 return 0;
3376 }
1da177e4
LT		 3377 default:
3378 return -ENOPROTOOPT;
3379 }
3380}
3381
3382static int packet_getsockopt(struct socket *sock, int level, int optname,
3383 char __user *optval, int __user *optlen)
3384{
3385 int len;
c06fff6e 3386 int val, lv = sizeof(val);
1da177e4
LT		 3387 struct sock *sk = sock->sk;
3388 struct packet_sock *po = pkt_sk(sk);
c06fff6e 3389 void *data = &val;
ee80fbf3 3390 union tpacket_stats_u st;
1da177e4
LT		 3391
3392 if (level != SOL_PACKET)
3393 return -ENOPROTOOPT;
3394
8ae55f04
KK		 3395 if (get_user(len, optlen))
3396 return -EFAULT;
1da177e4
LT		 3397
3398 if (len < 0)
3399 return -EINVAL;
1ce4f28b 3400
69e3c75f 3401 switch (optname) {
1da177e4 3402 case PACKET_STATISTICS:
1da177e4 3403 spin_lock_bh(&sk->sk_receive_queue.lock);
ee80fbf3
DB		 3404 memcpy(&st, &po->stats, sizeof(st));
3405 memset(&po->stats, 0, sizeof(po->stats));
3406 spin_unlock_bh(&sk->sk_receive_queue.lock);
3407
f6fb8f10 3408 if (po->tp_version == TPACKET_V3) {
c06fff6e 3409 lv = sizeof(struct tpacket_stats_v3);
8bcdeaff 3410 st.stats3.tp_packets += st.stats3.tp_drops;
ee80fbf3 3411 data = &st.stats3;
f6fb8f10 3412 } else {
c06fff6e 3413 lv = sizeof(struct tpacket_stats);
8bcdeaff 3414 st.stats1.tp_packets += st.stats1.tp_drops;
ee80fbf3 3415 data = &st.stats1;
f6fb8f10 3416 }
ee80fbf3 3417
8dc41944
HX		 3418 break;
3419 case PACKET_AUXDATA:
8dc41944 3420 val = po->auxdata;
80feaacb
PWJ		 3421 break;
3422 case PACKET_ORIGDEV:
80feaacb 3423 val = po->origdev;
bfd5f4a3
SS		 3424 break;
3425 case PACKET_VNET_HDR:
bfd5f4a3 3426 val = po->has_vnet_hdr;
1da177e4 3427 break;
bbd6ef87 3428 case PACKET_VERSION:
bbd6ef87 3429 val = po->tp_version;
bbd6ef87
PM		 3430 break;
3431 case PACKET_HDRLEN:
3432 if (len > sizeof(int))
3433 len = sizeof(int);
3434 if (copy_from_user(&val, optval, len))
3435 return -EFAULT;
3436 switch (val) {
3437 case TPACKET_V1:
3438 val = sizeof(struct tpacket_hdr);
3439 break;
3440 case TPACKET_V2:
3441 val = sizeof(struct tpacket2_hdr);
3442 break;
f6fb8f10 3443 case TPACKET_V3:
3444 val = sizeof(struct tpacket3_hdr);
3445 break;
bbd6ef87
PM		 3446 default:
3447 return -EINVAL;
3448 }
bbd6ef87 3449 break;
8913336a 3450 case PACKET_RESERVE:
8913336a 3451 val = po->tp_reserve;
8913336a 3452 break;
69e3c75f 3453 case PACKET_LOSS:
69e3c75f 3454 val = po->tp_loss;
69e3c75f 3455 break;
614f60fa 3456 case PACKET_TIMESTAMP:
614f60fa 3457 val = po->tp_tstamp;
614f60fa 3458 break;
dc99f600 3459 case PACKET_FANOUT:
dc99f600
DM		 3460 val = (po->fanout ?
3461 ((u32)po->fanout->id |
77f65ebd
WB		 3462 ((u32)po->fanout->type << 16) |
3463 ((u32)po->fanout->flags << 24)) :
dc99f600 3464 0);
dc99f600 3465 break;
5920cd3a
PC		 3466 case PACKET_TX_HAS_OFF:
3467 val = po->tp_tx_has_off;
3468 break;
d346a3fa
DB		 3469 case PACKET_QDISC_BYPASS:
3470 val = packet_use_direct_xmit(po);
3471 break;
1da177e4
LT		 3472 default:
3473 return -ENOPROTOOPT;
3474 }
3475
c06fff6e
ED		 3476 if (len > lv)
3477 len = lv;
8ae55f04
KK		 3478 if (put_user(len, optlen))
3479 return -EFAULT;
8dc41944
HX		 3480 if (copy_to_user(optval, data, len))
3481 return -EFAULT;
8ae55f04 3482 return 0;
1da177e4
LT		 3483}
3484
3485
351638e7
JP		 3486static int packet_notifier(struct notifier_block *this,
3487 unsigned long msg, void *ptr)
1da177e4
LT		 3488{
3489 struct sock *sk;
351638e7 3490 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 3491 struct net *net = dev_net(dev);
1da177e4 3492
808f5114 3493 rcu_read_lock();
b67bfe0d 3494 sk_for_each_rcu(sk, &net->packet.sklist) {
1da177e4
LT		 3495 struct packet_sock *po = pkt_sk(sk);
3496
3497 switch (msg) {
3498 case NETDEV_UNREGISTER:
1da177e4
LT		 3499 if (po->mclist)
3500 packet_dev_mclist(dev, po->mclist, -1);
a2efcfa0
DM		 3501 /* fallthrough */
3502
1da177e4
LT		 3503 case NETDEV_DOWN:
3504 if (dev->ifindex == po->ifindex) {
3505 spin_lock(&po->bind_lock);
3506 if (po->running) {
ce06b03e 3507 __unregister_prot_hook(sk, false);
1da177e4
LT		 3508 sk->sk_err = ENETDOWN;
3509 if (!sock_flag(sk, SOCK_DEAD))
3510 sk->sk_error_report(sk);
3511 }
3512 if (msg == NETDEV_UNREGISTER) {
66e56cd4 3513 packet_cached_dev_reset(po);
1da177e4 3514 po->ifindex = -1;
160ff18a
BG		 3515 if (po->prot_hook.dev)
3516 dev_put(po->prot_hook.dev);
1da177e4
LT		 3517 po->prot_hook.dev = NULL;
3518 }
3519 spin_unlock(&po->bind_lock);
3520 }
3521 break;
3522 case NETDEV_UP:
808f5114 3523 if (dev->ifindex == po->ifindex) {
3524 spin_lock(&po->bind_lock);
ce06b03e
DM		 3525 if (po->num)
3526 register_prot_hook(sk);
808f5114 3527 spin_unlock(&po->bind_lock);
1da177e4 3528 }
1da177e4
LT		 3529 break;
3530 }
3531 }
808f5114 3532 rcu_read_unlock();
1da177e4
LT		 3533 return NOTIFY_DONE;
3534}
3535
3536
3537static int packet_ioctl(struct socket *sock, unsigned int cmd,
3538 unsigned long arg)
3539{
3540 struct sock *sk = sock->sk;
3541
69e3c75f 3542 switch (cmd) {
40d4e3df
ED		 3543 case SIOCOUTQ:
3544 {
3545 int amount = sk_wmem_alloc_get(sk);
31e6d363 3546
40d4e3df
ED		 3547 return put_user(amount, (int __user *)arg);
3548 }
3549 case SIOCINQ:
3550 {
3551 struct sk_buff *skb;
3552 int amount = 0;
3553
3554 spin_lock_bh(&sk->sk_receive_queue.lock);
3555 skb = skb_peek(&sk->sk_receive_queue);
3556 if (skb)
3557 amount = skb->len;
3558 spin_unlock_bh(&sk->sk_receive_queue.lock);
3559 return put_user(amount, (int __user *)arg);
3560 }
3561 case SIOCGSTAMP:
3562 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3563 case SIOCGSTAMPNS:
3564 return sock_get_timestampns(sk, (struct timespec __user *)arg);
1ce4f28b 3565
1da177e4 3566#ifdef CONFIG_INET
40d4e3df
ED		 3567 case SIOCADDRT:
3568 case SIOCDELRT:
3569 case SIOCDARP:
3570 case SIOCGARP:
3571 case SIOCSARP:
3572 case SIOCGIFADDR:
3573 case SIOCSIFADDR:
3574 case SIOCGIFBRDADDR:
3575 case SIOCSIFBRDADDR:
3576 case SIOCGIFNETMASK:
3577 case SIOCSIFNETMASK:
3578 case SIOCGIFDSTADDR:
3579 case SIOCSIFDSTADDR:
3580 case SIOCSIFFLAGS:
40d4e3df 3581 return inet_dgram_ops.ioctl(sock, cmd, arg);
1da177e4
LT		 3582#endif
3583
40d4e3df
ED		 3584 default:
3585 return -ENOIOCTLCMD;
1da177e4
LT		 3586 }
3587 return 0;
3588}
3589
40d4e3df 3590static unsigned int packet_poll(struct file *file, struct socket *sock,
1da177e4
LT		 3591 poll_table *wait)
3592{
3593 struct sock *sk = sock->sk;
3594 struct packet_sock *po = pkt_sk(sk);
3595 unsigned int mask = datagram_poll(file, sock, wait);
3596
3597 spin_lock_bh(&sk->sk_receive_queue.lock);
69e3c75f 3598 if (po->rx_ring.pg_vec) {
f6fb8f10 3599 if (!packet_previous_rx_frame(po, &po->rx_ring,
3600 TP_STATUS_KERNEL))
1da177e4
LT		 3601 mask |= POLLIN | POLLRDNORM;
3602 }
3603 spin_unlock_bh(&sk->sk_receive_queue.lock);
69e3c75f
JB		 3604 spin_lock_bh(&sk->sk_write_queue.lock);
3605 if (po->tx_ring.pg_vec) {
3606 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3607 mask |= POLLOUT | POLLWRNORM;
3608 }
3609 spin_unlock_bh(&sk->sk_write_queue.lock);
1da177e4
LT		 3610 return mask;
3611}
3612
3613
3614/* Dirty? Well, I still did not learn better way to account
3615 * for user mmaps.
3616 */
3617
3618static void packet_mm_open(struct vm_area_struct *vma)
3619{
3620 struct file *file = vma->vm_file;
40d4e3df 3621 struct socket *sock = file->private_data;
1da177e4 3622 struct sock *sk = sock->sk;
1ce4f28b 3623
1da177e4
LT		 3624 if (sk)
3625 atomic_inc(&pkt_sk(sk)->mapped);
3626}
3627
3628static void packet_mm_close(struct vm_area_struct *vma)
3629{
3630 struct file *file = vma->vm_file;
40d4e3df 3631 struct socket *sock = file->private_data;
1da177e4 3632 struct sock *sk = sock->sk;
1ce4f28b 3633
1da177e4
LT		 3634 if (sk)
3635 atomic_dec(&pkt_sk(sk)->mapped);
3636}
3637
f0f37e2f 3638static const struct vm_operations_struct packet_mmap_ops = {
40d4e3df
ED		 3639 .open = packet_mm_open,
3640 .close = packet_mm_close,
1da177e4
LT		 3641};
3642
0e3125c7
NH		 3643static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3644 unsigned int len)
1da177e4
LT		 3645{
3646 int i;
3647
4ebf0ae2 3648 for (i = 0; i < len; i++) {
0e3125c7 3649 if (likely(pg_vec[i].buffer)) {
c56b4d90 3650 if (is_vmalloc_addr(pg_vec[i].buffer))
0e3125c7
NH		 3651 vfree(pg_vec[i].buffer);
3652 else
3653 free_pages((unsigned long)pg_vec[i].buffer,
3654 order);
3655 pg_vec[i].buffer = NULL;
3656 }
1da177e4
LT		 3657 }
3658 kfree(pg_vec);
3659}
3660
eea49cc9 3661static char *alloc_one_pg_vec_page(unsigned long order)
4ebf0ae2 3662{
f0d4eb29 3663 char *buffer;
0e3125c7
NH		 3664 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3665 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3666
3667 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3668 if (buffer)
3669 return buffer;
3670
f0d4eb29 3671 /* __get_free_pages failed, fall back to vmalloc */
bbce5a59 3672 buffer = vzalloc((1 << order) * PAGE_SIZE);
0e3125c7
NH		 3673 if (buffer)
3674 return buffer;
3675
f0d4eb29 3676 /* vmalloc failed, lets dig into swap here */
0e3125c7 3677 gfp_flags &= ~__GFP_NORETRY;
f0d4eb29 3678 buffer = (char *) __get_free_pages(gfp_flags, order);
0e3125c7
NH		 3679 if (buffer)
3680 return buffer;
3681
f0d4eb29 3682 /* complete and utter failure */
0e3125c7 3683 return NULL;
4ebf0ae2
DM		 3684}
3685
0e3125c7 3686static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4ebf0ae2
DM		 3687{
3688 unsigned int block_nr = req->tp_block_nr;
0e3125c7 3689 struct pgv *pg_vec;
4ebf0ae2
DM		 3690 int i;
3691
0e3125c7 3692 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
4ebf0ae2
DM		 3693 if (unlikely(!pg_vec))
3694 goto out;
3695
3696 for (i = 0; i < block_nr; i++) {
c56b4d90 3697 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
0e3125c7 3698 if (unlikely(!pg_vec[i].buffer))
4ebf0ae2
DM		 3699 goto out_free_pgvec;
3700 }
3701
3702out:
3703 return pg_vec;
3704
3705out_free_pgvec:
3706 free_pg_vec(pg_vec, order, block_nr);
3707 pg_vec = NULL;
3708 goto out;
3709}
1da177e4 3710
f6fb8f10 3711static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
69e3c75f 3712 int closing, int tx_ring)
1da177e4 3713{
0e3125c7 3714 struct pgv *pg_vec = NULL;
1da177e4 3715 struct packet_sock *po = pkt_sk(sk);
0e11c91e 3716 int was_running, order = 0;
69e3c75f
JB		 3717 struct packet_ring_buffer *rb;
3718 struct sk_buff_head *rb_queue;
0e11c91e 3719 __be16 num;
f6fb8f10 3720 int err = -EINVAL;
3721 /* Added to avoid minimal code churn */
3722 struct tpacket_req *req = &req_u->req;
3723
3724 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3725 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3726 WARN(1, "Tx-ring is not supported.\n");
3727 goto out;
3728 }
1ce4f28b 3729
69e3c75f
JB		 3730 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3731 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
1da177e4 3732
69e3c75f
JB		 3733 err = -EBUSY;
3734 if (!closing) {
3735 if (atomic_read(&po->mapped))
3736 goto out;
b0138408 3737 if (packet_read_pending(rb))
69e3c75f
JB		 3738 goto out;
3739 }
1da177e4 3740
69e3c75f
JB		 3741 if (req->tp_block_nr) {
3742 /* Sanity tests and some calculations */
3743 err = -EBUSY;
3744 if (unlikely(rb->pg_vec))
3745 goto out;
1da177e4 3746
bbd6ef87
PM		 3747 switch (po->tp_version) {
3748 case TPACKET_V1:
3749 po->tp_hdrlen = TPACKET_HDRLEN;
3750 break;
3751 case TPACKET_V2:
3752 po->tp_hdrlen = TPACKET2_HDRLEN;
3753 break;
f6fb8f10 3754 case TPACKET_V3:
3755 po->tp_hdrlen = TPACKET3_HDRLEN;
3756 break;
bbd6ef87
PM		 3757 }
3758
69e3c75f 3759 err = -EINVAL;
4ebf0ae2 3760 if (unlikely((int)req->tp_block_size <= 0))
69e3c75f 3761 goto out;
4ebf0ae2 3762 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
69e3c75f 3763 goto out;
8913336a 3764 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
69e3c75f
JB		 3765 po->tp_reserve))
3766 goto out;
4ebf0ae2 3767 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
69e3c75f 3768 goto out;
1da177e4 3769
69e3c75f
JB		 3770 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3771 if (unlikely(rb->frames_per_block <= 0))
3772 goto out;
3773 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3774 req->tp_frame_nr))
3775 goto out;
1da177e4
LT		 3776
3777 err = -ENOMEM;
4ebf0ae2
DM		 3778 order = get_order(req->tp_block_size);
3779 pg_vec = alloc_pg_vec(req, order);
3780 if (unlikely(!pg_vec))
1da177e4 3781 goto out;
f6fb8f10 3782 switch (po->tp_version) {
3783 case TPACKET_V3:
3784 /* Transmit path is not supported. We checked
3785 * it above but just being paranoid
3786 */
3787 if (!tx_ring)
3788 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3789 break;
3790 default:
3791 break;
3792 }
69e3c75f
JB		 3793 }
3794 /* Done */
3795 else {
3796 err = -EINVAL;
4ebf0ae2 3797 if (unlikely(req->tp_frame_nr))
69e3c75f 3798 goto out;
1da177e4
LT		 3799 }
3800
3801 lock_sock(sk);
3802
3803 /* Detach socket from network */
3804 spin_lock(&po->bind_lock);
3805 was_running = po->running;
3806 num = po->num;
3807 if (was_running) {
1da177e4 3808 po->num = 0;
ce06b03e 3809 __unregister_prot_hook(sk, false);
1da177e4
LT		 3810 }
3811 spin_unlock(&po->bind_lock);
1ce4f28b 3812
1da177e4
LT		 3813 synchronize_net();
3814
3815 err = -EBUSY;
905db440 3816 mutex_lock(&po->pg_vec_lock);
1da177e4
LT		 3817 if (closing || atomic_read(&po->mapped) == 0) {
3818 err = 0;
69e3c75f 3819 spin_lock_bh(&rb_queue->lock);
c053fd96 3820 swap(rb->pg_vec, pg_vec);
69e3c75f
JB		 3821 rb->frame_max = (req->tp_frame_nr - 1);
3822 rb->head = 0;
3823 rb->frame_size = req->tp_frame_size;
3824 spin_unlock_bh(&rb_queue->lock);
3825
c053fd96
CG		 3826 swap(rb->pg_vec_order, order);
3827 swap(rb->pg_vec_len, req->tp_block_nr);
69e3c75f
JB		 3828
3829 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3830 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3831 tpacket_rcv : packet_rcv;
3832 skb_queue_purge(rb_queue);
1da177e4 3833 if (atomic_read(&po->mapped))
40d4e3df
ED		 3834 pr_err("packet_mmap: vma is busy: %d\n",
3835 atomic_read(&po->mapped));
1da177e4 3836 }
905db440 3837 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3838
3839 spin_lock(&po->bind_lock);
ce06b03e 3840 if (was_running) {
1da177e4 3841 po->num = num;
ce06b03e 3842 register_prot_hook(sk);
1da177e4
LT		 3843 }
3844 spin_unlock(&po->bind_lock);
f6fb8f10 3845 if (closing && (po->tp_version > TPACKET_V2)) {
3846 /* Because we don't support block-based V3 on tx-ring */
3847 if (!tx_ring)
3848 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3849 }
1da177e4
LT		 3850 release_sock(sk);
3851
1da177e4
LT		 3852 if (pg_vec)
3853 free_pg_vec(pg_vec, order, req->tp_block_nr);
3854out:
3855 return err;
3856}
3857
69e3c75f
JB		 3858static int packet_mmap(struct file *file, struct socket *sock,
3859 struct vm_area_struct *vma)
1da177e4
LT		 3860{
3861 struct sock *sk = sock->sk;
3862 struct packet_sock *po = pkt_sk(sk);
69e3c75f
JB		 3863 unsigned long size, expected_size;
3864 struct packet_ring_buffer *rb;
1da177e4
LT		 3865 unsigned long start;
3866 int err = -EINVAL;
3867 int i;
3868
3869 if (vma->vm_pgoff)
3870 return -EINVAL;
3871
905db440 3872 mutex_lock(&po->pg_vec_lock);
69e3c75f
JB		 3873
3874 expected_size = 0;
3875 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3876 if (rb->pg_vec) {
3877 expected_size += rb->pg_vec_len
3878 * rb->pg_vec_pages
3879 * PAGE_SIZE;
3880 }
3881 }
3882
3883 if (expected_size == 0)
1da177e4 3884 goto out;
69e3c75f
JB		 3885
3886 size = vma->vm_end - vma->vm_start;
3887 if (size != expected_size)
1da177e4
LT		 3888 goto out;
3889
1da177e4 3890 start = vma->vm_start;
69e3c75f
JB		 3891 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3892 if (rb->pg_vec == NULL)
3893 continue;
3894
3895 for (i = 0; i < rb->pg_vec_len; i++) {
0e3125c7
NH		 3896 struct page *page;
3897 void *kaddr = rb->pg_vec[i].buffer;
69e3c75f
JB		 3898 int pg_num;
3899
c56b4d90
CG		 3900 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3901 page = pgv_to_page(kaddr);
69e3c75f
JB		 3902 err = vm_insert_page(vma, start, page);
3903 if (unlikely(err))
3904 goto out;
3905 start += PAGE_SIZE;
0e3125c7 3906 kaddr += PAGE_SIZE;
69e3c75f 3907 }
4ebf0ae2 3908 }
1da177e4 3909 }
69e3c75f 3910
4ebf0ae2 3911 atomic_inc(&po->mapped);
1da177e4
LT		 3912 vma->vm_ops = &packet_mmap_ops;
3913 err = 0;
3914
3915out:
905db440 3916 mutex_unlock(&po->pg_vec_lock);
1da177e4
LT		 3917 return err;
3918}
1da177e4 3919
90ddc4f0 3920static const struct proto_ops packet_ops_spkt = {
1da177e4
LT		 3921 .family = PF_PACKET,
3922 .owner = THIS_MODULE,
3923 .release = packet_release,
3924 .bind = packet_bind_spkt,
3925 .connect = sock_no_connect,
3926 .socketpair = sock_no_socketpair,
3927 .accept = sock_no_accept,
3928 .getname = packet_getname_spkt,
3929 .poll = datagram_poll,
3930 .ioctl = packet_ioctl,
3931 .listen = sock_no_listen,
3932 .shutdown = sock_no_shutdown,
3933 .setsockopt = sock_no_setsockopt,
3934 .getsockopt = sock_no_getsockopt,
3935 .sendmsg = packet_sendmsg_spkt,
3936 .recvmsg = packet_recvmsg,
3937 .mmap = sock_no_mmap,
3938 .sendpage = sock_no_sendpage,
3939};
1da177e4 3940
90ddc4f0 3941static const struct proto_ops packet_ops = {
1da177e4
LT		 3942 .family = PF_PACKET,
3943 .owner = THIS_MODULE,
3944 .release = packet_release,
3945 .bind = packet_bind,
3946 .connect = sock_no_connect,
3947 .socketpair = sock_no_socketpair,
3948 .accept = sock_no_accept,
1ce4f28b 3949 .getname = packet_getname,
1da177e4
LT		 3950 .poll = packet_poll,
3951 .ioctl = packet_ioctl,
3952 .listen = sock_no_listen,
3953 .shutdown = sock_no_shutdown,
3954 .setsockopt = packet_setsockopt,
3955 .getsockopt = packet_getsockopt,
3956 .sendmsg = packet_sendmsg,
3957 .recvmsg = packet_recvmsg,
3958 .mmap = packet_mmap,
3959 .sendpage = sock_no_sendpage,
3960};
3961
ec1b4cf7 3962static const struct net_proto_family packet_family_ops = {
1da177e4
LT		 3963 .family = PF_PACKET,
3964 .create = packet_create,
3965 .owner = THIS_MODULE,
3966};
3967
3968static struct notifier_block packet_netdev_notifier = {
40d4e3df 3969 .notifier_call = packet_notifier,
1da177e4
LT		 3970};
3971
3972#ifdef CONFIG_PROC_FS
1da177e4
LT		 3973
3974static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
808f5114 3975 __acquires(RCU)
1da177e4 3976{
e372c414 3977 struct net *net = seq_file_net(seq);
808f5114 3978
3979 rcu_read_lock();
3980 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
1da177e4
LT		 3981}
3982
3983static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3984{
1bf40954 3985 struct net *net = seq_file_net(seq);
808f5114 3986 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
1da177e4
LT		 3987}
3988
3989static void packet_seq_stop(struct seq_file *seq, void *v)
808f5114 3990 __releases(RCU)
1da177e4 3991{
808f5114 3992 rcu_read_unlock();
1da177e4
LT		 3993}
3994
1ce4f28b 3995static int packet_seq_show(struct seq_file *seq, void *v)
1da177e4
LT		 3996{
3997 if (v == SEQ_START_TOKEN)
3998 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3999 else {
b7ceabd9 4000 struct sock *s = sk_entry(v);
1da177e4
LT		 4001 const struct packet_sock *po = pkt_sk(s);
4002
4003 seq_printf(seq,
71338aa7 4004 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
1da177e4
LT		 4005 s,
4006 atomic_read(&s->sk_refcnt),
4007 s->sk_type,
4008 ntohs(po->num),
4009 po->ifindex,
4010 po->running,
4011 atomic_read(&s->sk_rmem_alloc),
a7cb5a49 4012 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
40d4e3df 4013 sock_i_ino(s));
1da177e4
LT		 4014 }
4015
4016 return 0;
4017}
4018
56b3d975 4019static const struct seq_operations packet_seq_ops = {
1da177e4
LT		 4020 .start = packet_seq_start,
4021 .next = packet_seq_next,
4022 .stop = packet_seq_stop,
4023 .show = packet_seq_show,
4024};
4025
4026static int packet_seq_open(struct inode *inode, struct file *file)
4027{
e372c414
DL		 4028 return seq_open_net(inode, file, &packet_seq_ops,
4029 sizeof(struct seq_net_private));
1da177e4
LT		 4030}
4031
da7071d7 4032static const struct file_operations packet_seq_fops = {
1da177e4
LT		 4033 .owner = THIS_MODULE,
4034 .open = packet_seq_open,
4035 .read = seq_read,
4036 .llseek = seq_lseek,
e372c414 4037 .release = seq_release_net,
1da177e4
LT		 4038};
4039
4040#endif
4041
2c8c1e72 4042static int __net_init packet_net_init(struct net *net)
d12d01d6 4043{
0fa7fa98 4044 mutex_init(&net->packet.sklist_lock);
2aaef4e4 4045 INIT_HLIST_HEAD(&net->packet.sklist);
d12d01d6 4046
d4beaa66 4047 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
d12d01d6
DL		 4048 return -ENOMEM;
4049
4050 return 0;
4051}
4052
2c8c1e72 4053static void __net_exit packet_net_exit(struct net *net)
d12d01d6 4054{
ece31ffd 4055 remove_proc_entry("packet", net->proc_net);
d12d01d6
DL		 4056}
4057
4058static struct pernet_operations packet_net_ops = {
4059 .init = packet_net_init,
4060 .exit = packet_net_exit,
4061};
4062
4063
1da177e4
LT		 4064static void __exit packet_exit(void)
4065{
1da177e4 4066 unregister_netdevice_notifier(&packet_netdev_notifier);
d12d01d6 4067 unregister_pernet_subsys(&packet_net_ops);
1da177e4
LT		 4068 sock_unregister(PF_PACKET);
4069 proto_unregister(&packet_proto);
4070}
4071
4072static int __init packet_init(void)
4073{
4074 int rc = proto_register(&packet_proto, 0);
4075
4076 if (rc != 0)
4077 goto out;
4078
4079 sock_register(&packet_family_ops);
d12d01d6 4080 register_pernet_subsys(&packet_net_ops);
1da177e4 4081 register_netdevice_notifier(&packet_netdev_notifier);
1da177e4
LT		 4082out:
4083 return rc;
4084}
4085
4086module_init(packet_init);
4087module_exit(packet_exit);
4088MODULE_LICENSE("GPL");
4089MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux 6.x block layer and io_uring tree(s)