2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * PACKET - implements raw packet sockets.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
35 * Ulises Alonso : Frame number limit removal and
36 * packet_set_ring memory leak.
37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
40 * byte arrays at the end of sockaddr_ll
42 * Johann Baudy : Added TX RING.
43 * Chetan Loke : Implemented TPACKET_V3 block abstraction
45 * Copyright (C) 2011, <lokec@ccs.neu.edu>
48 * This program is free software; you can redistribute it and/or
49 * modify it under the terms of the GNU General Public License
50 * as published by the Free Software Foundation; either version
51 * 2 of the License, or (at your option) any later version.
55 #include <linux/types.h>
57 #include <linux/capability.h>
58 #include <linux/fcntl.h>
59 #include <linux/socket.h>
61 #include <linux/inet.h>
62 #include <linux/netdevice.h>
63 #include <linux/if_packet.h>
64 #include <linux/wireless.h>
65 #include <linux/kernel.h>
66 #include <linux/kmod.h>
67 #include <linux/slab.h>
68 #include <linux/vmalloc.h>
69 #include <net/net_namespace.h>
71 #include <net/protocol.h>
72 #include <linux/skbuff.h>
74 #include <linux/errno.h>
75 #include <linux/timer.h>
76 #include <asm/uaccess.h>
77 #include <asm/ioctls.h>
79 #include <asm/cacheflush.h>
81 #include <linux/proc_fs.h>
82 #include <linux/seq_file.h>
83 #include <linux/poll.h>
84 #include <linux/module.h>
85 #include <linux/init.h>
86 #include <linux/mutex.h>
87 #include <linux/if_vlan.h>
88 #include <linux/virtio_net.h>
89 #include <linux/errqueue.h>
90 #include <linux/net_tstamp.h>
91 #include <linux/reciprocal_div.h>
93 #include <net/inet_common.h>
100 - if device has no dev->hard_header routine, it adds and removes ll header
101 inside itself. In this case ll header is invisible outside of device,
102 but higher levels still should reserve dev->hard_header_len.
103 Some devices are enough clever to reallocate skb, when header
104 will not fit to reserved space (tunnel), another ones are silly
106 - packet socket receives packets with pulled ll header,
107 so that SOCK_RAW should push it back.
112 Incoming, dev->hard_header!=NULL
113 mac_header -> ll header
116 Outgoing, dev->hard_header!=NULL
117 mac_header -> ll header
120 Incoming, dev->hard_header==NULL
121 mac_header -> UNKNOWN position. It is very likely, that it points to ll
122 header. PPP makes it, that is wrong, because introduce
123 assymetry between rx and tx paths.
126 Outgoing, dev->hard_header==NULL
127 mac_header -> data. ll header is still not built!
131 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
137 dev->hard_header != NULL
138 mac_header -> ll header
141 dev->hard_header == NULL (ll header is added by device, we cannot control it)
145 We should set nh.raw on output to correct posistion,
146 packet classifier depends on it.
149 /* Private packet socket structures. */
151 /* identical to struct packet_mreq except it has
152 * a longer address field.
154 struct packet_mreq_max {
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
162 struct tpacket_hdr *h1;
163 struct tpacket2_hdr *h2;
164 struct tpacket3_hdr *h3;
168 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
169 int closing, int tx_ring);
171 #define V3_ALIGNMENT (8)
173 #define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
175 #define BLK_PLUS_PRIV(sz_of_priv) \
176 (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
178 #define PGV_FROM_VMALLOC 1
180 #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
181 #define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
182 #define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
183 #define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
184 #define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
185 #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
186 #define BLOCK_PRIV(x) ((void *)((char *)(x) + BLOCK_O2PRIV(x)))
189 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
190 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
191 struct packet_type *pt, struct net_device *orig_dev);
193 static void *packet_previous_frame(struct packet_sock *po,
194 struct packet_ring_buffer *rb,
196 static void packet_increment_head(struct packet_ring_buffer *buff);
197 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *,
198 struct tpacket_block_desc *);
199 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200 struct packet_sock *);
201 static void prb_retire_current_block(struct tpacket_kbdq_core *,
202 struct packet_sock *, unsigned int status);
203 static int prb_queue_frozen(struct tpacket_kbdq_core *);
204 static void prb_open_block(struct tpacket_kbdq_core *,
205 struct tpacket_block_desc *);
206 static void prb_retire_rx_blk_timer_expired(unsigned long);
207 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
208 static void prb_init_blk_timer(struct packet_sock *,
209 struct tpacket_kbdq_core *,
210 void (*func) (unsigned long));
211 static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
212 static void prb_clear_rxhash(struct tpacket_kbdq_core *,
213 struct tpacket3_hdr *);
214 static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
215 struct tpacket3_hdr *);
216 static void packet_flush_mclist(struct sock *sk);
218 struct packet_skb_cb {
219 unsigned int origlen;
221 struct sockaddr_pkt pkt;
222 struct sockaddr_ll ll;
226 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
228 #define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
229 #define GET_PBLOCK_DESC(x, bid) \
230 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
231 #define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
232 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
233 #define GET_NEXT_PRB_BLK_NUM(x) \
234 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
235 ((x)->kactive_blk_num+1) : 0)
237 static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
238 static void __fanout_link(struct sock *sk, struct packet_sock *po);
240 static int packet_direct_xmit(struct sk_buff *skb)
242 struct net_device *dev = skb->dev;
243 const struct net_device_ops *ops = dev->netdev_ops;
244 netdev_features_t features;
245 struct netdev_queue *txq;
249 if (unlikely(!netif_running(dev) ||
250 !netif_carrier_ok(dev))) {
252 return NET_XMIT_DROP;
255 features = netif_skb_features(skb);
256 if (skb_needs_linearize(skb, features) &&
257 __skb_linearize(skb)) {
259 return NET_XMIT_DROP;
262 queue_map = skb_get_queue_mapping(skb);
263 txq = netdev_get_tx_queue(dev, queue_map);
265 __netif_tx_lock_bh(txq);
266 if (unlikely(netif_xmit_frozen_or_stopped(txq))) {
267 ret = NETDEV_TX_BUSY;
272 ret = ops->ndo_start_xmit(skb, dev);
273 if (likely(dev_xmit_complete(ret)))
274 txq_trans_update(txq);
278 __netif_tx_unlock_bh(txq);
282 static struct net_device *packet_cached_dev_get(struct packet_sock *po)
284 struct net_device *dev;
287 dev = rcu_dereference(po->cached_dev);
295 static void packet_cached_dev_assign(struct packet_sock *po,
296 struct net_device *dev)
298 rcu_assign_pointer(po->cached_dev, dev);
301 static void packet_cached_dev_reset(struct packet_sock *po)
303 RCU_INIT_POINTER(po->cached_dev, NULL);
306 static bool packet_use_direct_xmit(const struct packet_sock *po)
308 return po->xmit == packet_direct_xmit;
311 static u16 packet_pick_tx_queue(struct net_device *dev)
313 return (u16) raw_smp_processor_id() % dev->real_num_tx_queues;
316 /* register_prot_hook must be invoked with the po->bind_lock held,
317 * or from a context in which asynchronous accesses to the packet
318 * socket is not possible (packet_create()).
320 static void register_prot_hook(struct sock *sk)
322 struct packet_sock *po = pkt_sk(sk);
326 __fanout_link(sk, po);
328 dev_add_pack(&po->prot_hook);
335 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
336 * held. If the sync parameter is true, we will temporarily drop
337 * the po->bind_lock and do a synchronize_net to make sure no
338 * asynchronous packet processing paths still refer to the elements
339 * of po->prot_hook. If the sync parameter is false, it is the
340 * callers responsibility to take care of this.
342 static void __unregister_prot_hook(struct sock *sk, bool sync)
344 struct packet_sock *po = pkt_sk(sk);
349 __fanout_unlink(sk, po);
351 __dev_remove_pack(&po->prot_hook);
356 spin_unlock(&po->bind_lock);
358 spin_lock(&po->bind_lock);
362 static void unregister_prot_hook(struct sock *sk, bool sync)
364 struct packet_sock *po = pkt_sk(sk);
367 __unregister_prot_hook(sk, sync);
370 static inline __pure struct page *pgv_to_page(void *addr)
372 if (is_vmalloc_addr(addr))
373 return vmalloc_to_page(addr);
374 return virt_to_page(addr);
377 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
379 union tpacket_uhdr h;
382 switch (po->tp_version) {
384 h.h1->tp_status = status;
385 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
388 h.h2->tp_status = status;
389 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
393 WARN(1, "TPACKET version not supported.\n");
400 static int __packet_get_status(struct packet_sock *po, void *frame)
402 union tpacket_uhdr h;
407 switch (po->tp_version) {
409 flush_dcache_page(pgv_to_page(&h.h1->tp_status));
410 return h.h1->tp_status;
412 flush_dcache_page(pgv_to_page(&h.h2->tp_status));
413 return h.h2->tp_status;
416 WARN(1, "TPACKET version not supported.\n");
422 static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec *ts,
425 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
428 if ((flags & SOF_TIMESTAMPING_SYS_HARDWARE) &&
429 ktime_to_timespec_cond(shhwtstamps->syststamp, ts))
430 return TP_STATUS_TS_SYS_HARDWARE;
431 if ((flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
432 ktime_to_timespec_cond(shhwtstamps->hwtstamp, ts))
433 return TP_STATUS_TS_RAW_HARDWARE;
436 if (ktime_to_timespec_cond(skb->tstamp, ts))
437 return TP_STATUS_TS_SOFTWARE;
442 static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
445 union tpacket_uhdr h;
449 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
453 switch (po->tp_version) {
455 h.h1->tp_sec = ts.tv_sec;
456 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
459 h.h2->tp_sec = ts.tv_sec;
460 h.h2->tp_nsec = ts.tv_nsec;
464 WARN(1, "TPACKET version not supported.\n");
468 /* one flush is safe, as both fields always lie on the same cacheline */
469 flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
475 static void *packet_lookup_frame(struct packet_sock *po,
476 struct packet_ring_buffer *rb,
477 unsigned int position,
480 unsigned int pg_vec_pos, frame_offset;
481 union tpacket_uhdr h;
483 pg_vec_pos = position / rb->frames_per_block;
484 frame_offset = position % rb->frames_per_block;
486 h.raw = rb->pg_vec[pg_vec_pos].buffer +
487 (frame_offset * rb->frame_size);
489 if (status != __packet_get_status(po, h.raw))
495 static void *packet_current_frame(struct packet_sock *po,
496 struct packet_ring_buffer *rb,
499 return packet_lookup_frame(po, rb, rb->head, status);
502 static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
504 del_timer_sync(&pkc->retire_blk_timer);
507 static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
509 struct sk_buff_head *rb_queue)
511 struct tpacket_kbdq_core *pkc;
513 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
514 GET_PBDQC_FROM_RB(&po->rx_ring);
516 spin_lock_bh(&rb_queue->lock);
517 pkc->delete_blk_timer = 1;
518 spin_unlock_bh(&rb_queue->lock);
520 prb_del_retire_blk_timer(pkc);
523 static void prb_init_blk_timer(struct packet_sock *po,
524 struct tpacket_kbdq_core *pkc,
525 void (*func) (unsigned long))
527 init_timer(&pkc->retire_blk_timer);
528 pkc->retire_blk_timer.data = (long)po;
529 pkc->retire_blk_timer.function = func;
530 pkc->retire_blk_timer.expires = jiffies;
533 static void prb_setup_retire_blk_timer(struct packet_sock *po, int tx_ring)
535 struct tpacket_kbdq_core *pkc;
540 pkc = tx_ring ? GET_PBDQC_FROM_RB(&po->tx_ring) :
541 GET_PBDQC_FROM_RB(&po->rx_ring);
542 prb_init_blk_timer(po, pkc, prb_retire_rx_blk_timer_expired);
545 static int prb_calc_retire_blk_tmo(struct packet_sock *po,
546 int blk_size_in_bytes)
548 struct net_device *dev;
549 unsigned int mbits = 0, msec = 0, div = 0, tmo = 0;
550 struct ethtool_cmd ecmd;
555 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
556 if (unlikely(!dev)) {
558 return DEFAULT_PRB_RETIRE_TOV;
560 err = __ethtool_get_settings(dev, &ecmd);
561 speed = ethtool_cmd_speed(&ecmd);
565 * If the link speed is so slow you don't really
566 * need to worry about perf anyways
568 if (speed < SPEED_1000 || speed == SPEED_UNKNOWN) {
569 return DEFAULT_PRB_RETIRE_TOV;
576 mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
588 static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
589 union tpacket_req_u *req_u)
591 p1->feature_req_word = req_u->req3.tp_feature_req_word;
594 static void init_prb_bdqc(struct packet_sock *po,
595 struct packet_ring_buffer *rb,
597 union tpacket_req_u *req_u, int tx_ring)
599 struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
600 struct tpacket_block_desc *pbd;
602 memset(p1, 0x0, sizeof(*p1));
604 p1->knxt_seq_num = 1;
606 pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
607 p1->pkblk_start = pg_vec[0].buffer;
608 p1->kblk_size = req_u->req3.tp_block_size;
609 p1->knum_blocks = req_u->req3.tp_block_nr;
610 p1->hdrlen = po->tp_hdrlen;
611 p1->version = po->tp_version;
612 p1->last_kactive_blk_num = 0;
613 po->stats.stats3.tp_freeze_q_cnt = 0;
614 if (req_u->req3.tp_retire_blk_tov)
615 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
617 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
618 req_u->req3.tp_block_size);
619 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
620 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
622 prb_init_ft_ops(p1, req_u);
623 prb_setup_retire_blk_timer(po, tx_ring);
624 prb_open_block(p1, pbd);
627 /* Do NOT update the last_blk_num first.
628 * Assumes sk_buff_head lock is held.
630 static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
632 mod_timer(&pkc->retire_blk_timer,
633 jiffies + pkc->tov_in_jiffies);
634 pkc->last_kactive_blk_num = pkc->kactive_blk_num;
639 * 1) We refresh the timer only when we open a block.
640 * By doing this we don't waste cycles refreshing the timer
641 * on packet-by-packet basis.
643 * With a 1MB block-size, on a 1Gbps line, it will take
644 * i) ~8 ms to fill a block + ii) memcpy etc.
645 * In this cut we are not accounting for the memcpy time.
647 * So, if the user sets the 'tmo' to 10ms then the timer
648 * will never fire while the block is still getting filled
649 * (which is what we want). However, the user could choose
650 * to close a block early and that's fine.
652 * But when the timer does fire, we check whether or not to refresh it.
653 * Since the tmo granularity is in msecs, it is not too expensive
654 * to refresh the timer, lets say every '8' msecs.
655 * Either the user can set the 'tmo' or we can derive it based on
656 * a) line-speed and b) block-size.
657 * prb_calc_retire_blk_tmo() calculates the tmo.
660 static void prb_retire_rx_blk_timer_expired(unsigned long data)
662 struct packet_sock *po = (struct packet_sock *)data;
663 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
665 struct tpacket_block_desc *pbd;
667 spin_lock(&po->sk.sk_receive_queue.lock);
669 frozen = prb_queue_frozen(pkc);
670 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
672 if (unlikely(pkc->delete_blk_timer))
675 /* We only need to plug the race when the block is partially filled.
677 * lock(); increment BLOCK_NUM_PKTS; unlock()
678 * copy_bits() is in progress ...
679 * timer fires on other cpu:
680 * we can't retire the current block because copy_bits
684 if (BLOCK_NUM_PKTS(pbd)) {
685 while (atomic_read(&pkc->blk_fill_in_prog)) {
686 /* Waiting for skb_copy_bits to finish... */
691 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
693 prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
694 if (!prb_dispatch_next_block(pkc, po))
699 /* Case 1. Queue was frozen because user-space was
702 if (prb_curr_blk_in_use(pkc, pbd)) {
704 * Ok, user-space is still behind.
705 * So just refresh the timer.
709 /* Case 2. queue was frozen,user-space caught up,
710 * now the link went idle && the timer fired.
711 * We don't have a block to close.So we open this
712 * block and restart the timer.
713 * opening a block thaws the queue,restarts timer
714 * Thawing/timer-refresh is a side effect.
716 prb_open_block(pkc, pbd);
723 _prb_refresh_rx_retire_blk_timer(pkc);
726 spin_unlock(&po->sk.sk_receive_queue.lock);
729 static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
730 struct tpacket_block_desc *pbd1, __u32 status)
732 /* Flush everything minus the block header */
734 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
739 /* Skip the block header(we know header WILL fit in 4K) */
742 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
743 for (; start < end; start += PAGE_SIZE)
744 flush_dcache_page(pgv_to_page(start));
749 /* Now update the block status. */
751 BLOCK_STATUS(pbd1) = status;
753 /* Flush the block header */
755 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
757 flush_dcache_page(pgv_to_page(start));
767 * 2) Increment active_blk_num
769 * Note:We DONT refresh the timer on purpose.
770 * Because almost always the next block will be opened.
772 static void prb_close_block(struct tpacket_kbdq_core *pkc1,
773 struct tpacket_block_desc *pbd1,
774 struct packet_sock *po, unsigned int stat)
776 __u32 status = TP_STATUS_USER | stat;
778 struct tpacket3_hdr *last_pkt;
779 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
781 if (po->stats.stats3.tp_drops)
782 status |= TP_STATUS_LOSING;
784 last_pkt = (struct tpacket3_hdr *)pkc1->prev;
785 last_pkt->tp_next_offset = 0;
787 /* Get the ts of the last pkt */
788 if (BLOCK_NUM_PKTS(pbd1)) {
789 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
790 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
792 /* Ok, we tmo'd - so get the current time */
795 h1->ts_last_pkt.ts_sec = ts.tv_sec;
796 h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
801 /* Flush the block */
802 prb_flush_block(pkc1, pbd1, status);
804 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
807 static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
809 pkc->reset_pending_on_curr_blk = 0;
813 * Side effect of opening a block:
815 * 1) prb_queue is thawed.
816 * 2) retire_blk_timer is refreshed.
819 static void prb_open_block(struct tpacket_kbdq_core *pkc1,
820 struct tpacket_block_desc *pbd1)
823 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
827 /* We could have just memset this but we will lose the
828 * flexibility of making the priv area sticky
831 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
832 BLOCK_NUM_PKTS(pbd1) = 0;
833 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
837 h1->ts_first_pkt.ts_sec = ts.tv_sec;
838 h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
840 pkc1->pkblk_start = (char *)pbd1;
841 pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
843 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
844 BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
846 pbd1->version = pkc1->version;
847 pkc1->prev = pkc1->nxt_offset;
848 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
850 prb_thaw_queue(pkc1);
851 _prb_refresh_rx_retire_blk_timer(pkc1);
857 * Queue freeze logic:
858 * 1) Assume tp_block_nr = 8 blocks.
859 * 2) At time 't0', user opens Rx ring.
860 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
861 * 4) user-space is either sleeping or processing block '0'.
862 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
863 * it will close block-7,loop around and try to fill block '0'.
865 * __packet_lookup_frame_in_block
866 * prb_retire_current_block()
867 * prb_dispatch_next_block()
868 * |->(BLOCK_STATUS == USER) evaluates to true
869 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
870 * 6) Now there are two cases:
871 * 6.1) Link goes idle right after the queue is frozen.
872 * But remember, the last open_block() refreshed the timer.
873 * When this timer expires,it will refresh itself so that we can
874 * re-open block-0 in near future.
875 * 6.2) Link is busy and keeps on receiving packets. This is a simple
876 * case and __packet_lookup_frame_in_block will check if block-0
877 * is free and can now be re-used.
879 static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
880 struct packet_sock *po)
882 pkc->reset_pending_on_curr_blk = 1;
883 po->stats.stats3.tp_freeze_q_cnt++;
886 #define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
889 * If the next block is free then we will dispatch it
890 * and return a good offset.
891 * Else, we will freeze the queue.
892 * So, caller must check the return value.
894 static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
895 struct packet_sock *po)
897 struct tpacket_block_desc *pbd;
901 /* 1. Get current block num */
902 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
904 /* 2. If this block is currently in_use then freeze the queue */
905 if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
906 prb_freeze_queue(pkc, po);
912 * open this block and return the offset where the first packet
913 * needs to get stored.
915 prb_open_block(pkc, pbd);
916 return (void *)pkc->nxt_offset;
919 static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
920 struct packet_sock *po, unsigned int status)
922 struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
924 /* retire/close the current block */
925 if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
927 * Plug the case where copy_bits() is in progress on
928 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
929 * have space to copy the pkt in the current block and
930 * called prb_retire_current_block()
932 * We don't need to worry about the TMO case because
933 * the timer-handler already handled this case.
935 if (!(status & TP_STATUS_BLK_TMO)) {
936 while (atomic_read(&pkc->blk_fill_in_prog)) {
937 /* Waiting for skb_copy_bits to finish... */
941 prb_close_block(pkc, pbd, po, status);
946 static int prb_curr_blk_in_use(struct tpacket_kbdq_core *pkc,
947 struct tpacket_block_desc *pbd)
949 return TP_STATUS_USER & BLOCK_STATUS(pbd);
952 static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
954 return pkc->reset_pending_on_curr_blk;
957 static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
959 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
960 atomic_dec(&pkc->blk_fill_in_prog);
963 static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
964 struct tpacket3_hdr *ppd)
966 ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
969 static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
970 struct tpacket3_hdr *ppd)
972 ppd->hv1.tp_rxhash = 0;
975 static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
976 struct tpacket3_hdr *ppd)
978 if (vlan_tx_tag_present(pkc->skb)) {
979 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb);
980 ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
981 ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
983 ppd->hv1.tp_vlan_tci = 0;
984 ppd->hv1.tp_vlan_tpid = 0;
985 ppd->tp_status = TP_STATUS_AVAILABLE;
989 static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
990 struct tpacket3_hdr *ppd)
992 ppd->hv1.tp_padding = 0;
993 prb_fill_vlan_info(pkc, ppd);
995 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
996 prb_fill_rxhash(pkc, ppd);
998 prb_clear_rxhash(pkc, ppd);
1001 static void prb_fill_curr_block(char *curr,
1002 struct tpacket_kbdq_core *pkc,
1003 struct tpacket_block_desc *pbd,
1006 struct tpacket3_hdr *ppd;
1008 ppd = (struct tpacket3_hdr *)curr;
1009 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1011 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1012 BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1013 BLOCK_NUM_PKTS(pbd) += 1;
1014 atomic_inc(&pkc->blk_fill_in_prog);
1015 prb_run_all_ft_ops(pkc, ppd);
1018 /* Assumes caller has the sk->rx_queue.lock */
1019 static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1020 struct sk_buff *skb,
1025 struct tpacket_kbdq_core *pkc;
1026 struct tpacket_block_desc *pbd;
1029 pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1030 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1032 /* Queue is frozen when user space is lagging behind */
1033 if (prb_queue_frozen(pkc)) {
1035 * Check if that last block which caused the queue to freeze,
1036 * is still in_use by user-space.
1038 if (prb_curr_blk_in_use(pkc, pbd)) {
1039 /* Can't record this packet */
1043 * Ok, the block was released by user-space.
1044 * Now let's open that block.
1045 * opening a block also thaws the queue.
1046 * Thawing is a side effect.
1048 prb_open_block(pkc, pbd);
1053 curr = pkc->nxt_offset;
1055 end = (char *)pbd + pkc->kblk_size;
1057 /* first try the current block */
1058 if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1059 prb_fill_curr_block(curr, pkc, pbd, len);
1060 return (void *)curr;
1063 /* Ok, close the current block */
1064 prb_retire_current_block(pkc, po, 0);
1066 /* Now, try to dispatch the next block */
1067 curr = (char *)prb_dispatch_next_block(pkc, po);
1069 pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1070 prb_fill_curr_block(curr, pkc, pbd, len);
1071 return (void *)curr;
1075 * No free blocks are available.user_space hasn't caught up yet.
1076 * Queue was just frozen and now this packet will get dropped.
1081 static void *packet_current_rx_frame(struct packet_sock *po,
1082 struct sk_buff *skb,
1083 int status, unsigned int len)
1086 switch (po->tp_version) {
1089 curr = packet_lookup_frame(po, &po->rx_ring,
1090 po->rx_ring.head, status);
1093 return __packet_lookup_frame_in_block(po, skb, status, len);
1095 WARN(1, "TPACKET version not supported\n");
1101 static void *prb_lookup_block(struct packet_sock *po,
1102 struct packet_ring_buffer *rb,
1106 struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1107 struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1109 if (status != BLOCK_STATUS(pbd))
1114 static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1117 if (rb->prb_bdqc.kactive_blk_num)
1118 prev = rb->prb_bdqc.kactive_blk_num-1;
1120 prev = rb->prb_bdqc.knum_blocks-1;
1124 /* Assumes caller has held the rx_queue.lock */
1125 static void *__prb_previous_block(struct packet_sock *po,
1126 struct packet_ring_buffer *rb,
1129 unsigned int previous = prb_previous_blk_num(rb);
1130 return prb_lookup_block(po, rb, previous, status);
1133 static void *packet_previous_rx_frame(struct packet_sock *po,
1134 struct packet_ring_buffer *rb,
1137 if (po->tp_version <= TPACKET_V2)
1138 return packet_previous_frame(po, rb, status);
1140 return __prb_previous_block(po, rb, status);
1143 static void packet_increment_rx_head(struct packet_sock *po,
1144 struct packet_ring_buffer *rb)
1146 switch (po->tp_version) {
1149 return packet_increment_head(rb);
1152 WARN(1, "TPACKET version not supported.\n");
1158 static void *packet_previous_frame(struct packet_sock *po,
1159 struct packet_ring_buffer *rb,
1162 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1163 return packet_lookup_frame(po, rb, previous, status);
1166 static void packet_increment_head(struct packet_ring_buffer *buff)
1168 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1171 static bool packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1173 struct sock *sk = &po->sk;
1176 if (po->prot_hook.func != tpacket_rcv)
1177 return (atomic_read(&sk->sk_rmem_alloc) + skb->truesize)
1180 spin_lock(&sk->sk_receive_queue.lock);
1181 if (po->tp_version == TPACKET_V3)
1182 has_room = prb_lookup_block(po, &po->rx_ring,
1183 po->rx_ring.prb_bdqc.kactive_blk_num,
1186 has_room = packet_lookup_frame(po, &po->rx_ring,
1189 spin_unlock(&sk->sk_receive_queue.lock);
1194 static void packet_sock_destruct(struct sock *sk)
1196 skb_queue_purge(&sk->sk_error_queue);
1198 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1199 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
1201 if (!sock_flag(sk, SOCK_DEAD)) {
1202 pr_err("Attempt to release alive packet socket: %p\n", sk);
1206 sk_refcnt_debug_dec(sk);
1209 static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
1211 int x = atomic_read(&f->rr_cur) + 1;
1219 static unsigned int fanout_demux_hash(struct packet_fanout *f,
1220 struct sk_buff *skb,
1223 return reciprocal_divide(skb->rxhash, num);
1226 static unsigned int fanout_demux_lb(struct packet_fanout *f,
1227 struct sk_buff *skb,
1232 cur = atomic_read(&f->rr_cur);
1233 while ((old = atomic_cmpxchg(&f->rr_cur, cur,
1234 fanout_rr_next(f, num))) != cur)
1239 static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1240 struct sk_buff *skb,
1243 return smp_processor_id() % num;
1246 static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1247 struct sk_buff *skb,
1250 return reciprocal_divide(prandom_u32(), num);
1253 static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1254 struct sk_buff *skb,
1255 unsigned int idx, unsigned int skip,
1260 i = j = min_t(int, f->next[idx], num - 1);
1262 if (i != skip && packet_rcv_has_room(pkt_sk(f->arr[i]), skb)) {
1274 static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1276 return f->flags & (flag >> 8);
1279 static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1280 struct packet_type *pt, struct net_device *orig_dev)
1282 struct packet_fanout *f = pt->af_packet_priv;
1283 unsigned int num = f->num_members;
1284 struct packet_sock *po;
1287 if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
1294 case PACKET_FANOUT_HASH:
1296 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1297 skb = ip_check_defrag(skb, IP_DEFRAG_AF_PACKET);
1302 idx = fanout_demux_hash(f, skb, num);
1304 case PACKET_FANOUT_LB:
1305 idx = fanout_demux_lb(f, skb, num);
1307 case PACKET_FANOUT_CPU:
1308 idx = fanout_demux_cpu(f, skb, num);
1310 case PACKET_FANOUT_RND:
1311 idx = fanout_demux_rnd(f, skb, num);
1313 case PACKET_FANOUT_ROLLOVER:
1314 idx = fanout_demux_rollover(f, skb, 0, (unsigned int) -1, num);
1318 po = pkt_sk(f->arr[idx]);
1319 if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER) &&
1320 unlikely(!packet_rcv_has_room(po, skb))) {
1321 idx = fanout_demux_rollover(f, skb, idx, idx, num);
1322 po = pkt_sk(f->arr[idx]);
1325 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1328 DEFINE_MUTEX(fanout_mutex);
1329 EXPORT_SYMBOL_GPL(fanout_mutex);
1330 static LIST_HEAD(fanout_list);
1332 static void __fanout_link(struct sock *sk, struct packet_sock *po)
1334 struct packet_fanout *f = po->fanout;
1336 spin_lock(&f->lock);
1337 f->arr[f->num_members] = sk;
1340 spin_unlock(&f->lock);
1343 static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1345 struct packet_fanout *f = po->fanout;
1348 spin_lock(&f->lock);
1349 for (i = 0; i < f->num_members; i++) {
1350 if (f->arr[i] == sk)
1353 BUG_ON(i >= f->num_members);
1354 f->arr[i] = f->arr[f->num_members - 1];
1356 spin_unlock(&f->lock);
1359 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1361 if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
1367 static int fanout_add(struct sock *sk, u16 id, u16 type_flags)
1369 struct packet_sock *po = pkt_sk(sk);
1370 struct packet_fanout *f, *match;
1371 u8 type = type_flags & 0xff;
1372 u8 flags = type_flags >> 8;
1376 case PACKET_FANOUT_ROLLOVER:
1377 if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1379 case PACKET_FANOUT_HASH:
1380 case PACKET_FANOUT_LB:
1381 case PACKET_FANOUT_CPU:
1382 case PACKET_FANOUT_RND:
1394 mutex_lock(&fanout_mutex);
1396 list_for_each_entry(f, &fanout_list, list) {
1398 read_pnet(&f->net) == sock_net(sk)) {
1404 if (match && match->flags != flags)
1408 match = kzalloc(sizeof(*match), GFP_KERNEL);
1411 write_pnet(&match->net, sock_net(sk));
1414 match->flags = flags;
1415 atomic_set(&match->rr_cur, 0);
1416 INIT_LIST_HEAD(&match->list);
1417 spin_lock_init(&match->lock);
1418 atomic_set(&match->sk_ref, 0);
1419 match->prot_hook.type = po->prot_hook.type;
1420 match->prot_hook.dev = po->prot_hook.dev;
1421 match->prot_hook.func = packet_rcv_fanout;
1422 match->prot_hook.af_packet_priv = match;
1423 match->prot_hook.id_match = match_fanout_group;
1424 dev_add_pack(&match->prot_hook);
1425 list_add(&match->list, &fanout_list);
1428 if (match->type == type &&
1429 match->prot_hook.type == po->prot_hook.type &&
1430 match->prot_hook.dev == po->prot_hook.dev) {
1432 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) {
1433 __dev_remove_pack(&po->prot_hook);
1435 atomic_inc(&match->sk_ref);
1436 __fanout_link(sk, po);
1441 mutex_unlock(&fanout_mutex);
1445 static void fanout_release(struct sock *sk)
1447 struct packet_sock *po = pkt_sk(sk);
1448 struct packet_fanout *f;
1454 mutex_lock(&fanout_mutex);
1457 if (atomic_dec_and_test(&f->sk_ref)) {
1459 dev_remove_pack(&f->prot_hook);
1462 mutex_unlock(&fanout_mutex);
1465 static const struct proto_ops packet_ops;
1467 static const struct proto_ops packet_ops_spkt;
1469 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1470 struct packet_type *pt, struct net_device *orig_dev)
1473 struct sockaddr_pkt *spkt;
1476 * When we registered the protocol we saved the socket in the data
1477 * field for just this event.
1480 sk = pt->af_packet_priv;
1483 * Yank back the headers [hope the device set this
1484 * right or kerboom...]
1486 * Incoming packets have ll header pulled,
1489 * For outgoing ones skb->data == skb_mac_header(skb)
1490 * so that this procedure is noop.
1493 if (skb->pkt_type == PACKET_LOOPBACK)
1496 if (!net_eq(dev_net(dev), sock_net(sk)))
1499 skb = skb_share_check(skb, GFP_ATOMIC);
1503 /* drop any routing info */
1506 /* drop conntrack reference */
1509 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1511 skb_push(skb, skb->data - skb_mac_header(skb));
1514 * The SOCK_PACKET socket receives _all_ frames.
1517 spkt->spkt_family = dev->type;
1518 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1519 spkt->spkt_protocol = skb->protocol;
1522 * Charge the memory to the socket. This is done specifically
1523 * to prevent sockets using all the memory up.
1526 if (sock_queue_rcv_skb(sk, skb) == 0)
1537 * Output a raw packet to a device layer. This bypasses all the other
1538 * protocol layers and you must therefore supply it with a complete frame
1541 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
1542 struct msghdr *msg, size_t len)
1544 struct sock *sk = sock->sk;
1545 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
1546 struct sk_buff *skb = NULL;
1547 struct net_device *dev;
1553 * Get and verify the address.
1557 if (msg->msg_namelen < sizeof(struct sockaddr))
1559 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1560 proto = saddr->spkt_protocol;
1562 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1565 * Find the device first to size check it
1568 saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1571 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
1577 if (!(dev->flags & IFF_UP))
1581 * You may not queue a frame bigger than the mtu. This is the lowest level
1582 * raw protocol and you must do your own fragmentation at this level.
1585 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1586 if (!netif_supports_nofcs(dev)) {
1587 err = -EPROTONOSUPPORT;
1590 extra_len = 4; /* We're doing our own CRC */
1594 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
1598 size_t reserved = LL_RESERVED_SPACE(dev);
1599 int tlen = dev->needed_tailroom;
1600 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
1603 skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
1606 /* FIXME: Save some space for broken drivers that write a hard
1607 * header at transmission time by themselves. PPP is the notable
1608 * one here. This should really be fixed at the driver level.
1610 skb_reserve(skb, reserved);
1611 skb_reset_network_header(skb);
1613 /* Try to align data part correctly */
1618 skb_reset_network_header(skb);
1620 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
1626 if (len > (dev->mtu + dev->hard_header_len + extra_len)) {
1627 /* Earlier code assumed this would be a VLAN pkt,
1628 * double-check this now that we have the actual
1631 struct ethhdr *ehdr;
1632 skb_reset_mac_header(skb);
1633 ehdr = eth_hdr(skb);
1634 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
1640 skb->protocol = proto;
1642 skb->priority = sk->sk_priority;
1643 skb->mark = sk->sk_mark;
1645 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
1647 if (unlikely(extra_len == 4))
1650 skb_probe_transport_header(skb, 0);
1652 dev_queue_xmit(skb);
1663 static unsigned int run_filter(const struct sk_buff *skb,
1664 const struct sock *sk,
1667 struct sk_filter *filter;
1670 filter = rcu_dereference(sk->sk_filter);
1672 res = SK_RUN_FILTER(filter, skb);
1679 * This function makes lazy skb cloning in hope that most of packets
1680 * are discarded by BPF.
1682 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1683 * and skb->cb are mangled. It works because (and until) packets
1684 * falling here are owned by current CPU. Output packets are cloned
1685 * by dev_queue_xmit_nit(), input packets are processed by net_bh
1686 * sequencially, so that if we return skb to original state on exit,
1687 * we will not harm anyone.
1690 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
1691 struct packet_type *pt, struct net_device *orig_dev)
1694 struct sockaddr_ll *sll;
1695 struct packet_sock *po;
1696 u8 *skb_head = skb->data;
1697 int skb_len = skb->len;
1698 unsigned int snaplen, res;
1700 if (skb->pkt_type == PACKET_LOOPBACK)
1703 sk = pt->af_packet_priv;
1706 if (!net_eq(dev_net(dev), sock_net(sk)))
1711 if (dev->header_ops) {
1712 /* The device has an explicit notion of ll header,
1713 * exported to higher levels.
1715 * Otherwise, the device hides details of its frame
1716 * structure, so that corresponding packet head is
1717 * never delivered to user.
1719 if (sk->sk_type != SOCK_DGRAM)
1720 skb_push(skb, skb->data - skb_mac_header(skb));
1721 else if (skb->pkt_type == PACKET_OUTGOING) {
1722 /* Special case: outgoing packets have ll header at head */
1723 skb_pull(skb, skb_network_offset(skb));
1729 res = run_filter(skb, sk, snaplen);
1731 goto drop_n_restore;
1735 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
1738 if (skb_shared(skb)) {
1739 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
1743 if (skb_head != skb->data) {
1744 skb->data = skb_head;
1751 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
1754 sll = &PACKET_SKB_CB(skb)->sa.ll;
1755 sll->sll_family = AF_PACKET;
1756 sll->sll_hatype = dev->type;
1757 sll->sll_protocol = skb->protocol;
1758 sll->sll_pkttype = skb->pkt_type;
1759 if (unlikely(po->origdev))
1760 sll->sll_ifindex = orig_dev->ifindex;
1762 sll->sll_ifindex = dev->ifindex;
1764 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1766 PACKET_SKB_CB(skb)->origlen = skb->len;
1768 if (pskb_trim(skb, snaplen))
1771 skb_set_owner_r(skb, sk);
1775 /* drop conntrack reference */
1778 spin_lock(&sk->sk_receive_queue.lock);
1779 po->stats.stats1.tp_packets++;
1780 skb->dropcount = atomic_read(&sk->sk_drops);
1781 __skb_queue_tail(&sk->sk_receive_queue, skb);
1782 spin_unlock(&sk->sk_receive_queue.lock);
1783 sk->sk_data_ready(sk, skb->len);
1787 spin_lock(&sk->sk_receive_queue.lock);
1788 po->stats.stats1.tp_drops++;
1789 atomic_inc(&sk->sk_drops);
1790 spin_unlock(&sk->sk_receive_queue.lock);
1793 if (skb_head != skb->data && skb_shared(skb)) {
1794 skb->data = skb_head;
1802 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
1803 struct packet_type *pt, struct net_device *orig_dev)
1806 struct packet_sock *po;
1807 struct sockaddr_ll *sll;
1808 union tpacket_uhdr h;
1809 u8 *skb_head = skb->data;
1810 int skb_len = skb->len;
1811 unsigned int snaplen, res;
1812 unsigned long status = TP_STATUS_USER;
1813 unsigned short macoff, netoff, hdrlen;
1814 struct sk_buff *copy_skb = NULL;
1818 /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
1819 * We may add members to them until current aligned size without forcing
1820 * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
1822 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
1823 BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
1825 if (skb->pkt_type == PACKET_LOOPBACK)
1828 sk = pt->af_packet_priv;
1831 if (!net_eq(dev_net(dev), sock_net(sk)))
1834 if (dev->header_ops) {
1835 if (sk->sk_type != SOCK_DGRAM)
1836 skb_push(skb, skb->data - skb_mac_header(skb));
1837 else if (skb->pkt_type == PACKET_OUTGOING) {
1838 /* Special case: outgoing packets have ll header at head */
1839 skb_pull(skb, skb_network_offset(skb));
1843 if (skb->ip_summed == CHECKSUM_PARTIAL)
1844 status |= TP_STATUS_CSUMNOTREADY;
1848 res = run_filter(skb, sk, snaplen);
1850 goto drop_n_restore;
1854 if (sk->sk_type == SOCK_DGRAM) {
1855 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
1858 unsigned int maclen = skb_network_offset(skb);
1859 netoff = TPACKET_ALIGN(po->tp_hdrlen +
1860 (maclen < 16 ? 16 : maclen)) +
1862 macoff = netoff - maclen;
1864 if (po->tp_version <= TPACKET_V2) {
1865 if (macoff + snaplen > po->rx_ring.frame_size) {
1866 if (po->copy_thresh &&
1867 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
1868 if (skb_shared(skb)) {
1869 copy_skb = skb_clone(skb, GFP_ATOMIC);
1871 copy_skb = skb_get(skb);
1872 skb_head = skb->data;
1875 skb_set_owner_r(copy_skb, sk);
1877 snaplen = po->rx_ring.frame_size - macoff;
1878 if ((int)snaplen < 0)
1882 spin_lock(&sk->sk_receive_queue.lock);
1883 h.raw = packet_current_rx_frame(po, skb,
1884 TP_STATUS_KERNEL, (macoff+snaplen));
1887 if (po->tp_version <= TPACKET_V2) {
1888 packet_increment_rx_head(po, &po->rx_ring);
1890 * LOSING will be reported till you read the stats,
1891 * because it's COR - Clear On Read.
1892 * Anyways, moving it for V1/V2 only as V3 doesn't need this
1895 if (po->stats.stats1.tp_drops)
1896 status |= TP_STATUS_LOSING;
1898 po->stats.stats1.tp_packets++;
1900 status |= TP_STATUS_COPY;
1901 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
1903 spin_unlock(&sk->sk_receive_queue.lock);
1905 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
1907 if (!(ts_status = tpacket_get_timestamp(skb, &ts, po->tp_tstamp)))
1908 getnstimeofday(&ts);
1910 status |= ts_status;
1912 switch (po->tp_version) {
1914 h.h1->tp_len = skb->len;
1915 h.h1->tp_snaplen = snaplen;
1916 h.h1->tp_mac = macoff;
1917 h.h1->tp_net = netoff;
1918 h.h1->tp_sec = ts.tv_sec;
1919 h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
1920 hdrlen = sizeof(*h.h1);
1923 h.h2->tp_len = skb->len;
1924 h.h2->tp_snaplen = snaplen;
1925 h.h2->tp_mac = macoff;
1926 h.h2->tp_net = netoff;
1927 h.h2->tp_sec = ts.tv_sec;
1928 h.h2->tp_nsec = ts.tv_nsec;
1929 if (vlan_tx_tag_present(skb)) {
1930 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
1931 h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
1932 status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1934 h.h2->tp_vlan_tci = 0;
1935 h.h2->tp_vlan_tpid = 0;
1937 memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
1938 hdrlen = sizeof(*h.h2);
1941 /* tp_nxt_offset,vlan are already populated above.
1942 * So DONT clear those fields here
1944 h.h3->tp_status |= status;
1945 h.h3->tp_len = skb->len;
1946 h.h3->tp_snaplen = snaplen;
1947 h.h3->tp_mac = macoff;
1948 h.h3->tp_net = netoff;
1949 h.h3->tp_sec = ts.tv_sec;
1950 h.h3->tp_nsec = ts.tv_nsec;
1951 memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
1952 hdrlen = sizeof(*h.h3);
1958 sll = h.raw + TPACKET_ALIGN(hdrlen);
1959 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
1960 sll->sll_family = AF_PACKET;
1961 sll->sll_hatype = dev->type;
1962 sll->sll_protocol = skb->protocol;
1963 sll->sll_pkttype = skb->pkt_type;
1964 if (unlikely(po->origdev))
1965 sll->sll_ifindex = orig_dev->ifindex;
1967 sll->sll_ifindex = dev->ifindex;
1970 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1974 if (po->tp_version <= TPACKET_V2) {
1975 end = (u8 *)PAGE_ALIGN((unsigned long)h.raw
1976 + macoff + snaplen);
1977 for (start = h.raw; start < end; start += PAGE_SIZE)
1978 flush_dcache_page(pgv_to_page(start));
1983 if (po->tp_version <= TPACKET_V2)
1984 __packet_set_status(po, h.raw, status);
1986 prb_clear_blk_fill_status(&po->rx_ring);
1988 sk->sk_data_ready(sk, 0);
1991 if (skb_head != skb->data && skb_shared(skb)) {
1992 skb->data = skb_head;
2000 po->stats.stats1.tp_drops++;
2001 spin_unlock(&sk->sk_receive_queue.lock);
2003 sk->sk_data_ready(sk, 0);
2004 kfree_skb(copy_skb);
2005 goto drop_n_restore;
2008 static void tpacket_destruct_skb(struct sk_buff *skb)
2010 struct packet_sock *po = pkt_sk(skb->sk);
2013 if (likely(po->tx_ring.pg_vec)) {
2016 ph = skb_shinfo(skb)->destructor_arg;
2017 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
2018 atomic_dec(&po->tx_ring.pending);
2020 ts = __packet_set_timestamp(po, ph, skb);
2021 __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
2027 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2028 void *frame, struct net_device *dev, int size_max,
2029 __be16 proto, unsigned char *addr, int hlen)
2031 union tpacket_uhdr ph;
2032 int to_write, offset, len, tp_len, nr_frags, len_max;
2033 struct socket *sock = po->sk.sk_socket;
2040 skb->protocol = proto;
2042 skb->priority = po->sk.sk_priority;
2043 skb->mark = po->sk.sk_mark;
2044 sock_tx_timestamp(&po->sk, &skb_shinfo(skb)->tx_flags);
2045 skb_shinfo(skb)->destructor_arg = ph.raw;
2047 switch (po->tp_version) {
2049 tp_len = ph.h2->tp_len;
2052 tp_len = ph.h1->tp_len;
2055 if (unlikely(tp_len > size_max)) {
2056 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2060 skb_reserve(skb, hlen);
2061 skb_reset_network_header(skb);
2063 if (!packet_use_direct_xmit(po))
2064 skb_probe_transport_header(skb, 0);
2065 if (unlikely(po->tp_tx_has_off)) {
2066 int off_min, off_max, off;
2067 off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2068 off_max = po->tx_ring.frame_size - tp_len;
2069 if (sock->type == SOCK_DGRAM) {
2070 switch (po->tp_version) {
2072 off = ph.h2->tp_net;
2075 off = ph.h1->tp_net;
2079 switch (po->tp_version) {
2081 off = ph.h2->tp_mac;
2084 off = ph.h1->tp_mac;
2088 if (unlikely((off < off_min) || (off_max < off)))
2090 data = ph.raw + off;
2092 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
2096 if (sock->type == SOCK_DGRAM) {
2097 err = dev_hard_header(skb, dev, ntohs(proto), addr,
2099 if (unlikely(err < 0))
2101 } else if (dev->hard_header_len) {
2102 /* net device doesn't like empty head */
2103 if (unlikely(tp_len <= dev->hard_header_len)) {
2104 pr_err("packet size is too short (%d < %d)\n",
2105 tp_len, dev->hard_header_len);
2109 skb_push(skb, dev->hard_header_len);
2110 err = skb_store_bits(skb, 0, data,
2111 dev->hard_header_len);
2115 data += dev->hard_header_len;
2116 to_write -= dev->hard_header_len;
2119 offset = offset_in_page(data);
2120 len_max = PAGE_SIZE - offset;
2121 len = ((to_write > len_max) ? len_max : to_write);
2123 skb->data_len = to_write;
2124 skb->len += to_write;
2125 skb->truesize += to_write;
2126 atomic_add(to_write, &po->sk.sk_wmem_alloc);
2128 while (likely(to_write)) {
2129 nr_frags = skb_shinfo(skb)->nr_frags;
2131 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2132 pr_err("Packet exceed the number of skb frags(%lu)\n",
2137 page = pgv_to_page(data);
2139 flush_dcache_page(page);
2141 skb_fill_page_desc(skb, nr_frags, page, offset, len);
2144 len_max = PAGE_SIZE;
2145 len = ((to_write > len_max) ? len_max : to_write);
2151 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2153 struct sk_buff *skb;
2154 struct net_device *dev;
2156 int err, reserve = 0;
2158 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2159 bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2160 int tp_len, size_max;
2161 unsigned char *addr;
2163 int status = TP_STATUS_AVAILABLE;
2166 mutex_lock(&po->pg_vec_lock);
2168 if (likely(saddr == NULL)) {
2169 dev = packet_cached_dev_get(po);
2174 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2176 if (msg->msg_namelen < (saddr->sll_halen
2177 + offsetof(struct sockaddr_ll,
2180 proto = saddr->sll_protocol;
2181 addr = saddr->sll_addr;
2182 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
2186 if (unlikely(dev == NULL))
2189 if (unlikely(!(dev->flags & IFF_UP)))
2192 reserve = dev->hard_header_len;
2194 size_max = po->tx_ring.frame_size
2195 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2197 if (size_max > dev->mtu + reserve)
2198 size_max = dev->mtu + reserve;
2201 ph = packet_current_frame(po, &po->tx_ring,
2202 TP_STATUS_SEND_REQUEST);
2203 if (unlikely(ph == NULL)) {
2204 if (need_wait && need_resched())
2209 status = TP_STATUS_SEND_REQUEST;
2210 hlen = LL_RESERVED_SPACE(dev);
2211 tlen = dev->needed_tailroom;
2212 skb = sock_alloc_send_skb(&po->sk,
2213 hlen + tlen + sizeof(struct sockaddr_ll),
2216 if (unlikely(skb == NULL))
2219 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
2222 if (unlikely(tp_len < 0)) {
2224 __packet_set_status(po, ph,
2225 TP_STATUS_AVAILABLE);
2226 packet_increment_head(&po->tx_ring);
2230 status = TP_STATUS_WRONG_FORMAT;
2236 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
2237 skb->destructor = tpacket_destruct_skb;
2238 __packet_set_status(po, ph, TP_STATUS_SENDING);
2239 atomic_inc(&po->tx_ring.pending);
2241 status = TP_STATUS_SEND_REQUEST;
2242 err = po->xmit(skb);
2243 if (unlikely(err > 0)) {
2244 err = net_xmit_errno(err);
2245 if (err && __packet_get_status(po, ph) ==
2246 TP_STATUS_AVAILABLE) {
2247 /* skb was destructed already */
2252 * skb was dropped but not destructed yet;
2253 * let's treat it like congestion or err < 0
2257 packet_increment_head(&po->tx_ring);
2259 } while (likely((ph != NULL) || (need_wait &&
2260 atomic_read(&po->tx_ring.pending))));
2266 __packet_set_status(po, ph, status);
2271 mutex_unlock(&po->pg_vec_lock);
2275 static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2276 size_t reserve, size_t len,
2277 size_t linear, int noblock,
2280 struct sk_buff *skb;
2282 /* Under a page? Don't bother with paged skb. */
2283 if (prepad + len < PAGE_SIZE || !linear)
2286 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
2291 skb_reserve(skb, reserve);
2292 skb_put(skb, linear);
2293 skb->data_len = len - linear;
2294 skb->len += len - linear;
2299 static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2301 struct sock *sk = sock->sk;
2302 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
2303 struct sk_buff *skb;
2304 struct net_device *dev;
2306 unsigned char *addr;
2307 int err, reserve = 0;
2308 struct virtio_net_hdr vnet_hdr = { 0 };
2311 struct packet_sock *po = pkt_sk(sk);
2312 unsigned short gso_type = 0;
2317 * Get and verify the address.
2320 if (likely(saddr == NULL)) {
2321 dev = packet_cached_dev_get(po);
2326 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2328 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2330 proto = saddr->sll_protocol;
2331 addr = saddr->sll_addr;
2332 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
2336 if (unlikely(dev == NULL))
2339 if (unlikely(!(dev->flags & IFF_UP)))
2342 if (sock->type == SOCK_RAW)
2343 reserve = dev->hard_header_len;
2344 if (po->has_vnet_hdr) {
2345 vnet_hdr_len = sizeof(vnet_hdr);
2348 if (len < vnet_hdr_len)
2351 len -= vnet_hdr_len;
2353 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
2358 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2359 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
2361 vnet_hdr.hdr_len = vnet_hdr.csum_start +
2362 vnet_hdr.csum_offset + 2;
2365 if (vnet_hdr.hdr_len > len)
2368 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
2369 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
2370 case VIRTIO_NET_HDR_GSO_TCPV4:
2371 gso_type = SKB_GSO_TCPV4;
2373 case VIRTIO_NET_HDR_GSO_TCPV6:
2374 gso_type = SKB_GSO_TCPV6;
2376 case VIRTIO_NET_HDR_GSO_UDP:
2377 gso_type = SKB_GSO_UDP;
2383 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
2384 gso_type |= SKB_GSO_TCP_ECN;
2386 if (vnet_hdr.gso_size == 0)
2392 if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
2393 if (!netif_supports_nofcs(dev)) {
2394 err = -EPROTONOSUPPORT;
2397 extra_len = 4; /* We're doing our own CRC */
2401 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
2405 hlen = LL_RESERVED_SPACE(dev);
2406 tlen = dev->needed_tailroom;
2407 skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, vnet_hdr.hdr_len,
2408 msg->msg_flags & MSG_DONTWAIT, &err);
2412 skb_set_network_header(skb, reserve);
2415 if (sock->type == SOCK_DGRAM &&
2416 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
2419 /* Returns -EFAULT on error */
2420 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
2424 sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags);
2426 if (!gso_type && (len > dev->mtu + reserve + extra_len)) {
2427 /* Earlier code assumed this would be a VLAN pkt,
2428 * double-check this now that we have the actual
2431 struct ethhdr *ehdr;
2432 skb_reset_mac_header(skb);
2433 ehdr = eth_hdr(skb);
2434 if (ehdr->h_proto != htons(ETH_P_8021Q)) {
2440 skb->protocol = proto;
2442 skb->priority = sk->sk_priority;
2443 skb->mark = sk->sk_mark;
2444 skb_set_queue_mapping(skb, packet_pick_tx_queue(dev));
2446 if (po->has_vnet_hdr) {
2447 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
2448 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
2449 vnet_hdr.csum_offset)) {
2455 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
2456 skb_shinfo(skb)->gso_type = gso_type;
2458 /* Header must be checked, and gso_segs computed. */
2459 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2460 skb_shinfo(skb)->gso_segs = 0;
2462 len += vnet_hdr_len;
2465 if (!packet_use_direct_xmit(po))
2466 skb_probe_transport_header(skb, reserve);
2467 if (unlikely(extra_len == 4))
2470 err = po->xmit(skb);
2471 if (err > 0 && (err = net_xmit_errno(err)) != 0)
2487 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
2488 struct msghdr *msg, size_t len)
2490 struct sock *sk = sock->sk;
2491 struct packet_sock *po = pkt_sk(sk);
2493 if (po->tx_ring.pg_vec)
2494 return tpacket_snd(po, msg);
2496 return packet_snd(sock, msg, len);
2500 * Close a PACKET socket. This is fairly simple. We immediately go
2501 * to 'closed' state and remove our protocol entry in the device list.
2504 static int packet_release(struct socket *sock)
2506 struct sock *sk = sock->sk;
2507 struct packet_sock *po;
2509 union tpacket_req_u req_u;
2517 mutex_lock(&net->packet.sklist_lock);
2518 sk_del_node_init_rcu(sk);
2519 mutex_unlock(&net->packet.sklist_lock);
2522 sock_prot_inuse_add(net, sk->sk_prot, -1);
2525 spin_lock(&po->bind_lock);
2526 unregister_prot_hook(sk, false);
2527 packet_cached_dev_reset(po);
2529 if (po->prot_hook.dev) {
2530 dev_put(po->prot_hook.dev);
2531 po->prot_hook.dev = NULL;
2533 spin_unlock(&po->bind_lock);
2535 packet_flush_mclist(sk);
2537 if (po->rx_ring.pg_vec) {
2538 memset(&req_u, 0, sizeof(req_u));
2539 packet_set_ring(sk, &req_u, 1, 0);
2542 if (po->tx_ring.pg_vec) {
2543 memset(&req_u, 0, sizeof(req_u));
2544 packet_set_ring(sk, &req_u, 1, 1);
2551 * Now the socket is dead. No more input will appear.
2558 skb_queue_purge(&sk->sk_receive_queue);
2559 sk_refcnt_debug_release(sk);
2566 * Attach a packet hook.
2569 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 proto)
2571 struct packet_sock *po = pkt_sk(sk);
2572 const struct net_device *dev_curr;
2584 spin_lock(&po->bind_lock);
2586 proto_curr = po->prot_hook.type;
2587 dev_curr = po->prot_hook.dev;
2589 need_rehook = proto_curr != proto || dev_curr != dev;
2592 unregister_prot_hook(sk, true);
2595 po->prot_hook.type = proto;
2597 if (po->prot_hook.dev)
2598 dev_put(po->prot_hook.dev);
2600 po->prot_hook.dev = dev;
2602 po->ifindex = dev ? dev->ifindex : 0;
2603 packet_cached_dev_assign(po, dev);
2606 if (proto == 0 || !need_rehook)
2609 if (!dev || (dev->flags & IFF_UP)) {
2610 register_prot_hook(sk);
2612 sk->sk_err = ENETDOWN;
2613 if (!sock_flag(sk, SOCK_DEAD))
2614 sk->sk_error_report(sk);
2618 spin_unlock(&po->bind_lock);
2624 * Bind a packet socket to a device
2627 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
2630 struct sock *sk = sock->sk;
2632 struct net_device *dev;
2639 if (addr_len != sizeof(struct sockaddr))
2641 strlcpy(name, uaddr->sa_data, sizeof(name));
2643 dev = dev_get_by_name(sock_net(sk), name);
2645 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
2649 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
2651 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
2652 struct sock *sk = sock->sk;
2653 struct net_device *dev = NULL;
2661 if (addr_len < sizeof(struct sockaddr_ll))
2663 if (sll->sll_family != AF_PACKET)
2666 if (sll->sll_ifindex) {
2668 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
2672 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
2678 static struct proto packet_proto = {
2680 .owner = THIS_MODULE,
2681 .obj_size = sizeof(struct packet_sock),
2685 * Create a packet of type SOCK_PACKET.
2688 static int packet_create(struct net *net, struct socket *sock, int protocol,
2692 struct packet_sock *po;
2693 __be16 proto = (__force __be16)protocol; /* weird, but documented */
2696 if (!ns_capable(net->user_ns, CAP_NET_RAW))
2698 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
2699 sock->type != SOCK_PACKET)
2700 return -ESOCKTNOSUPPORT;
2702 sock->state = SS_UNCONNECTED;
2705 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
2709 sock->ops = &packet_ops;
2710 if (sock->type == SOCK_PACKET)
2711 sock->ops = &packet_ops_spkt;
2713 sock_init_data(sock, sk);
2716 sk->sk_family = PF_PACKET;
2718 po->xmit = dev_queue_xmit;
2720 packet_cached_dev_reset(po);
2722 sk->sk_destruct = packet_sock_destruct;
2723 sk_refcnt_debug_inc(sk);
2726 * Attach a protocol block
2729 spin_lock_init(&po->bind_lock);
2730 mutex_init(&po->pg_vec_lock);
2731 po->prot_hook.func = packet_rcv;
2733 if (sock->type == SOCK_PACKET)
2734 po->prot_hook.func = packet_rcv_spkt;
2736 po->prot_hook.af_packet_priv = sk;
2739 po->prot_hook.type = proto;
2740 register_prot_hook(sk);
2743 mutex_lock(&net->packet.sklist_lock);
2744 sk_add_node_rcu(sk, &net->packet.sklist);
2745 mutex_unlock(&net->packet.sklist_lock);
2748 sock_prot_inuse_add(net, &packet_proto, 1);
2757 * Pull a packet from our receive queue and hand it to the user.
2758 * If necessary we block.
2761 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
2762 struct msghdr *msg, size_t len, int flags)
2764 struct sock *sk = sock->sk;
2765 struct sk_buff *skb;
2767 int vnet_hdr_len = 0;
2770 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
2774 /* What error should we return now? EUNATTACH? */
2775 if (pkt_sk(sk)->ifindex < 0)
2779 if (flags & MSG_ERRQUEUE) {
2780 err = sock_recv_errqueue(sk, msg, len,
2781 SOL_PACKET, PACKET_TX_TIMESTAMP);
2786 * Call the generic datagram receiver. This handles all sorts
2787 * of horrible races and re-entrancy so we can forget about it
2788 * in the protocol layers.
2790 * Now it will return ENETDOWN, if device have just gone down,
2791 * but then it will block.
2794 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
2797 * An error occurred so return it. Because skb_recv_datagram()
2798 * handles the blocking we don't see and worry about blocking
2805 if (pkt_sk(sk)->has_vnet_hdr) {
2806 struct virtio_net_hdr vnet_hdr = { 0 };
2809 vnet_hdr_len = sizeof(vnet_hdr);
2810 if (len < vnet_hdr_len)
2813 len -= vnet_hdr_len;
2815 if (skb_is_gso(skb)) {
2816 struct skb_shared_info *sinfo = skb_shinfo(skb);
2818 /* This is a hint as to how much should be linear. */
2819 vnet_hdr.hdr_len = skb_headlen(skb);
2820 vnet_hdr.gso_size = sinfo->gso_size;
2821 if (sinfo->gso_type & SKB_GSO_TCPV4)
2822 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2823 else if (sinfo->gso_type & SKB_GSO_TCPV6)
2824 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2825 else if (sinfo->gso_type & SKB_GSO_UDP)
2826 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
2827 else if (sinfo->gso_type & SKB_GSO_FCOE)
2831 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
2832 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
2834 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
2836 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2837 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
2838 vnet_hdr.csum_start = skb_checksum_start_offset(skb);
2839 vnet_hdr.csum_offset = skb->csum_offset;
2840 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
2841 vnet_hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID;
2842 } /* else everything is zero */
2844 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
2850 /* You lose any data beyond the buffer you gave. If it worries
2851 * a user program they can ask the device for its MTU
2857 msg->msg_flags |= MSG_TRUNC;
2860 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
2864 sock_recv_ts_and_drops(msg, sk, skb);
2866 if (msg->msg_name) {
2867 /* If the address length field is there to be filled
2868 * in, we fill it in now.
2870 if (sock->type == SOCK_PACKET) {
2871 msg->msg_namelen = sizeof(struct sockaddr_pkt);
2873 struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
2874 msg->msg_namelen = sll->sll_halen +
2875 offsetof(struct sockaddr_ll, sll_addr);
2877 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
2881 if (pkt_sk(sk)->auxdata) {
2882 struct tpacket_auxdata aux;
2884 aux.tp_status = TP_STATUS_USER;
2885 if (skb->ip_summed == CHECKSUM_PARTIAL)
2886 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
2887 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
2888 aux.tp_snaplen = skb->len;
2890 aux.tp_net = skb_network_offset(skb);
2891 if (vlan_tx_tag_present(skb)) {
2892 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
2893 aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
2894 aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2896 aux.tp_vlan_tci = 0;
2897 aux.tp_vlan_tpid = 0;
2899 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
2903 * Free or return the buffer as appropriate. Again this
2904 * hides all the races and re-entrancy issues from us.
2906 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
2909 skb_free_datagram(sk, skb);
2914 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
2915 int *uaddr_len, int peer)
2917 struct net_device *dev;
2918 struct sock *sk = sock->sk;
2923 uaddr->sa_family = AF_PACKET;
2924 memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
2926 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
2928 strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
2930 *uaddr_len = sizeof(*uaddr);
2935 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
2936 int *uaddr_len, int peer)
2938 struct net_device *dev;
2939 struct sock *sk = sock->sk;
2940 struct packet_sock *po = pkt_sk(sk);
2941 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
2946 sll->sll_family = AF_PACKET;
2947 sll->sll_ifindex = po->ifindex;
2948 sll->sll_protocol = po->num;
2949 sll->sll_pkttype = 0;
2951 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
2953 sll->sll_hatype = dev->type;
2954 sll->sll_halen = dev->addr_len;
2955 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
2957 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
2961 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
2966 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
2970 case PACKET_MR_MULTICAST:
2971 if (i->alen != dev->addr_len)
2974 return dev_mc_add(dev, i->addr);
2976 return dev_mc_del(dev, i->addr);
2978 case PACKET_MR_PROMISC:
2979 return dev_set_promiscuity(dev, what);
2981 case PACKET_MR_ALLMULTI:
2982 return dev_set_allmulti(dev, what);
2984 case PACKET_MR_UNICAST:
2985 if (i->alen != dev->addr_len)
2988 return dev_uc_add(dev, i->addr);
2990 return dev_uc_del(dev, i->addr);
2998 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
3000 for ( ; i; i = i->next) {
3001 if (i->ifindex == dev->ifindex)
3002 packet_dev_mc(dev, i, what);
3006 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3008 struct packet_sock *po = pkt_sk(sk);
3009 struct packet_mclist *ml, *i;
3010 struct net_device *dev;
3016 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
3021 if (mreq->mr_alen > dev->addr_len)
3025 i = kmalloc(sizeof(*i), GFP_KERNEL);
3030 for (ml = po->mclist; ml; ml = ml->next) {
3031 if (ml->ifindex == mreq->mr_ifindex &&
3032 ml->type == mreq->mr_type &&
3033 ml->alen == mreq->mr_alen &&
3034 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3036 /* Free the new element ... */
3042 i->type = mreq->mr_type;
3043 i->ifindex = mreq->mr_ifindex;
3044 i->alen = mreq->mr_alen;
3045 memcpy(i->addr, mreq->mr_address, i->alen);
3047 i->next = po->mclist;
3049 err = packet_dev_mc(dev, i, 1);
3051 po->mclist = i->next;
3060 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3062 struct packet_mclist *ml, **mlp;
3066 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3067 if (ml->ifindex == mreq->mr_ifindex &&
3068 ml->type == mreq->mr_type &&
3069 ml->alen == mreq->mr_alen &&
3070 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
3071 if (--ml->count == 0) {
3072 struct net_device *dev;
3074 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3076 packet_dev_mc(dev, ml, -1);
3084 return -EADDRNOTAVAIL;
3087 static void packet_flush_mclist(struct sock *sk)
3089 struct packet_sock *po = pkt_sk(sk);
3090 struct packet_mclist *ml;
3096 while ((ml = po->mclist) != NULL) {
3097 struct net_device *dev;
3099 po->mclist = ml->next;
3100 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
3102 packet_dev_mc(dev, ml, -1);
3109 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
3111 struct sock *sk = sock->sk;
3112 struct packet_sock *po = pkt_sk(sk);
3115 if (level != SOL_PACKET)
3116 return -ENOPROTOOPT;
3119 case PACKET_ADD_MEMBERSHIP:
3120 case PACKET_DROP_MEMBERSHIP:
3122 struct packet_mreq_max mreq;
3124 memset(&mreq, 0, sizeof(mreq));
3125 if (len < sizeof(struct packet_mreq))
3127 if (len > sizeof(mreq))
3129 if (copy_from_user(&mreq, optval, len))
3131 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3133 if (optname == PACKET_ADD_MEMBERSHIP)
3134 ret = packet_mc_add(sk, &mreq);
3136 ret = packet_mc_drop(sk, &mreq);
3140 case PACKET_RX_RING:
3141 case PACKET_TX_RING:
3143 union tpacket_req_u req_u;
3146 switch (po->tp_version) {
3149 len = sizeof(req_u.req);
3153 len = sizeof(req_u.req3);
3158 if (pkt_sk(sk)->has_vnet_hdr)
3160 if (copy_from_user(&req_u.req, optval, len))
3162 return packet_set_ring(sk, &req_u, 0,
3163 optname == PACKET_TX_RING);
3165 case PACKET_COPY_THRESH:
3169 if (optlen != sizeof(val))
3171 if (copy_from_user(&val, optval, sizeof(val)))
3174 pkt_sk(sk)->copy_thresh = val;
3177 case PACKET_VERSION:
3181 if (optlen != sizeof(val))
3183 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3185 if (copy_from_user(&val, optval, sizeof(val)))
3191 po->tp_version = val;
3197 case PACKET_RESERVE:
3201 if (optlen != sizeof(val))
3203 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3205 if (copy_from_user(&val, optval, sizeof(val)))
3207 po->tp_reserve = val;
3214 if (optlen != sizeof(val))
3216 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3218 if (copy_from_user(&val, optval, sizeof(val)))
3220 po->tp_loss = !!val;
3223 case PACKET_AUXDATA:
3227 if (optlen < sizeof(val))
3229 if (copy_from_user(&val, optval, sizeof(val)))
3232 po->auxdata = !!val;
3235 case PACKET_ORIGDEV:
3239 if (optlen < sizeof(val))
3241 if (copy_from_user(&val, optval, sizeof(val)))
3244 po->origdev = !!val;
3247 case PACKET_VNET_HDR:
3251 if (sock->type != SOCK_RAW)
3253 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3255 if (optlen < sizeof(val))
3257 if (copy_from_user(&val, optval, sizeof(val)))
3260 po->has_vnet_hdr = !!val;
3263 case PACKET_TIMESTAMP:
3267 if (optlen != sizeof(val))
3269 if (copy_from_user(&val, optval, sizeof(val)))
3272 po->tp_tstamp = val;
3279 if (optlen != sizeof(val))
3281 if (copy_from_user(&val, optval, sizeof(val)))
3284 return fanout_add(sk, val & 0xffff, val >> 16);
3286 case PACKET_TX_HAS_OFF:
3290 if (optlen != sizeof(val))
3292 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
3294 if (copy_from_user(&val, optval, sizeof(val)))
3296 po->tp_tx_has_off = !!val;
3299 case PACKET_QDISC_BYPASS:
3303 if (optlen != sizeof(val))
3305 if (copy_from_user(&val, optval, sizeof(val)))
3308 po->xmit = val ? packet_direct_xmit : dev_queue_xmit;
3312 return -ENOPROTOOPT;
3316 static int packet_getsockopt(struct socket *sock, int level, int optname,
3317 char __user *optval, int __user *optlen)
3320 int val, lv = sizeof(val);
3321 struct sock *sk = sock->sk;
3322 struct packet_sock *po = pkt_sk(sk);
3324 union tpacket_stats_u st;
3326 if (level != SOL_PACKET)
3327 return -ENOPROTOOPT;
3329 if (get_user(len, optlen))
3336 case PACKET_STATISTICS:
3337 spin_lock_bh(&sk->sk_receive_queue.lock);
3338 memcpy(&st, &po->stats, sizeof(st));
3339 memset(&po->stats, 0, sizeof(po->stats));
3340 spin_unlock_bh(&sk->sk_receive_queue.lock);
3342 if (po->tp_version == TPACKET_V3) {
3343 lv = sizeof(struct tpacket_stats_v3);
3344 st.stats3.tp_packets += st.stats3.tp_drops;
3347 lv = sizeof(struct tpacket_stats);
3348 st.stats1.tp_packets += st.stats1.tp_drops;
3353 case PACKET_AUXDATA:
3356 case PACKET_ORIGDEV:
3359 case PACKET_VNET_HDR:
3360 val = po->has_vnet_hdr;
3362 case PACKET_VERSION:
3363 val = po->tp_version;
3366 if (len > sizeof(int))
3368 if (copy_from_user(&val, optval, len))
3372 val = sizeof(struct tpacket_hdr);
3375 val = sizeof(struct tpacket2_hdr);
3378 val = sizeof(struct tpacket3_hdr);
3384 case PACKET_RESERVE:
3385 val = po->tp_reserve;
3390 case PACKET_TIMESTAMP:
3391 val = po->tp_tstamp;
3395 ((u32)po->fanout->id |
3396 ((u32)po->fanout->type << 16) |
3397 ((u32)po->fanout->flags << 24)) :
3400 case PACKET_TX_HAS_OFF:
3401 val = po->tp_tx_has_off;
3403 case PACKET_QDISC_BYPASS:
3404 val = packet_use_direct_xmit(po);
3407 return -ENOPROTOOPT;
3412 if (put_user(len, optlen))
3414 if (copy_to_user(optval, data, len))
3420 static int packet_notifier(struct notifier_block *this,
3421 unsigned long msg, void *ptr)
3424 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3425 struct net *net = dev_net(dev);
3428 sk_for_each_rcu(sk, &net->packet.sklist) {
3429 struct packet_sock *po = pkt_sk(sk);
3432 case NETDEV_UNREGISTER:
3434 packet_dev_mclist(dev, po->mclist, -1);
3438 if (dev->ifindex == po->ifindex) {
3439 spin_lock(&po->bind_lock);
3441 __unregister_prot_hook(sk, false);
3442 sk->sk_err = ENETDOWN;
3443 if (!sock_flag(sk, SOCK_DEAD))
3444 sk->sk_error_report(sk);
3446 if (msg == NETDEV_UNREGISTER) {
3447 packet_cached_dev_reset(po);
3449 if (po->prot_hook.dev)
3450 dev_put(po->prot_hook.dev);
3451 po->prot_hook.dev = NULL;
3453 spin_unlock(&po->bind_lock);
3457 if (dev->ifindex == po->ifindex) {
3458 spin_lock(&po->bind_lock);
3460 register_prot_hook(sk);
3461 spin_unlock(&po->bind_lock);
3471 static int packet_ioctl(struct socket *sock, unsigned int cmd,
3474 struct sock *sk = sock->sk;
3479 int amount = sk_wmem_alloc_get(sk);
3481 return put_user(amount, (int __user *)arg);
3485 struct sk_buff *skb;
3488 spin_lock_bh(&sk->sk_receive_queue.lock);
3489 skb = skb_peek(&sk->sk_receive_queue);
3492 spin_unlock_bh(&sk->sk_receive_queue.lock);
3493 return put_user(amount, (int __user *)arg);
3496 return sock_get_timestamp(sk, (struct timeval __user *)arg);
3498 return sock_get_timestampns(sk, (struct timespec __user *)arg);
3508 case SIOCGIFBRDADDR:
3509 case SIOCSIFBRDADDR:
3510 case SIOCGIFNETMASK:
3511 case SIOCSIFNETMASK:
3512 case SIOCGIFDSTADDR:
3513 case SIOCSIFDSTADDR:
3515 return inet_dgram_ops.ioctl(sock, cmd, arg);
3519 return -ENOIOCTLCMD;
3524 static unsigned int packet_poll(struct file *file, struct socket *sock,
3527 struct sock *sk = sock->sk;
3528 struct packet_sock *po = pkt_sk(sk);
3529 unsigned int mask = datagram_poll(file, sock, wait);
3531 spin_lock_bh(&sk->sk_receive_queue.lock);
3532 if (po->rx_ring.pg_vec) {
3533 if (!packet_previous_rx_frame(po, &po->rx_ring,
3535 mask |= POLLIN | POLLRDNORM;
3537 spin_unlock_bh(&sk->sk_receive_queue.lock);
3538 spin_lock_bh(&sk->sk_write_queue.lock);
3539 if (po->tx_ring.pg_vec) {
3540 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
3541 mask |= POLLOUT | POLLWRNORM;
3543 spin_unlock_bh(&sk->sk_write_queue.lock);
3548 /* Dirty? Well, I still did not learn better way to account
3552 static void packet_mm_open(struct vm_area_struct *vma)
3554 struct file *file = vma->vm_file;
3555 struct socket *sock = file->private_data;
3556 struct sock *sk = sock->sk;
3559 atomic_inc(&pkt_sk(sk)->mapped);
3562 static void packet_mm_close(struct vm_area_struct *vma)
3564 struct file *file = vma->vm_file;
3565 struct socket *sock = file->private_data;
3566 struct sock *sk = sock->sk;
3569 atomic_dec(&pkt_sk(sk)->mapped);
3572 static const struct vm_operations_struct packet_mmap_ops = {
3573 .open = packet_mm_open,
3574 .close = packet_mm_close,
3577 static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
3582 for (i = 0; i < len; i++) {
3583 if (likely(pg_vec[i].buffer)) {
3584 if (is_vmalloc_addr(pg_vec[i].buffer))
3585 vfree(pg_vec[i].buffer);
3587 free_pages((unsigned long)pg_vec[i].buffer,
3589 pg_vec[i].buffer = NULL;
3595 static char *alloc_one_pg_vec_page(unsigned long order)
3597 char *buffer = NULL;
3598 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
3599 __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
3601 buffer = (char *) __get_free_pages(gfp_flags, order);
3607 * __get_free_pages failed, fall back to vmalloc
3609 buffer = vzalloc((1 << order) * PAGE_SIZE);
3615 * vmalloc failed, lets dig into swap here
3617 gfp_flags &= ~__GFP_NORETRY;
3618 buffer = (char *)__get_free_pages(gfp_flags, order);
3623 * complete and utter failure
3628 static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
3630 unsigned int block_nr = req->tp_block_nr;
3634 pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL);
3635 if (unlikely(!pg_vec))
3638 for (i = 0; i < block_nr; i++) {
3639 pg_vec[i].buffer = alloc_one_pg_vec_page(order);
3640 if (unlikely(!pg_vec[i].buffer))
3641 goto out_free_pgvec;
3648 free_pg_vec(pg_vec, order, block_nr);
3653 static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
3654 int closing, int tx_ring)
3656 struct pgv *pg_vec = NULL;
3657 struct packet_sock *po = pkt_sk(sk);
3658 int was_running, order = 0;
3659 struct packet_ring_buffer *rb;
3660 struct sk_buff_head *rb_queue;
3663 /* Added to avoid minimal code churn */
3664 struct tpacket_req *req = &req_u->req;
3666 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */
3667 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) {
3668 WARN(1, "Tx-ring is not supported.\n");
3672 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
3673 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
3677 if (atomic_read(&po->mapped))
3679 if (atomic_read(&rb->pending))
3683 if (req->tp_block_nr) {
3684 /* Sanity tests and some calculations */
3686 if (unlikely(rb->pg_vec))
3689 switch (po->tp_version) {
3691 po->tp_hdrlen = TPACKET_HDRLEN;
3694 po->tp_hdrlen = TPACKET2_HDRLEN;
3697 po->tp_hdrlen = TPACKET3_HDRLEN;
3702 if (unlikely((int)req->tp_block_size <= 0))
3704 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
3706 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
3709 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
3712 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
3713 if (unlikely(rb->frames_per_block <= 0))
3715 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
3720 order = get_order(req->tp_block_size);
3721 pg_vec = alloc_pg_vec(req, order);
3722 if (unlikely(!pg_vec))
3724 switch (po->tp_version) {
3726 /* Transmit path is not supported. We checked
3727 * it above but just being paranoid
3730 init_prb_bdqc(po, rb, pg_vec, req_u, tx_ring);
3739 if (unlikely(req->tp_frame_nr))
3745 /* Detach socket from network */
3746 spin_lock(&po->bind_lock);
3747 was_running = po->running;
3751 __unregister_prot_hook(sk, false);
3753 spin_unlock(&po->bind_lock);
3758 mutex_lock(&po->pg_vec_lock);
3759 if (closing || atomic_read(&po->mapped) == 0) {
3761 spin_lock_bh(&rb_queue->lock);
3762 swap(rb->pg_vec, pg_vec);
3763 rb->frame_max = (req->tp_frame_nr - 1);
3765 rb->frame_size = req->tp_frame_size;
3766 spin_unlock_bh(&rb_queue->lock);
3768 swap(rb->pg_vec_order, order);
3769 swap(rb->pg_vec_len, req->tp_block_nr);
3771 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
3772 po->prot_hook.func = (po->rx_ring.pg_vec) ?
3773 tpacket_rcv : packet_rcv;
3774 skb_queue_purge(rb_queue);
3775 if (atomic_read(&po->mapped))
3776 pr_err("packet_mmap: vma is busy: %d\n",
3777 atomic_read(&po->mapped));
3779 mutex_unlock(&po->pg_vec_lock);
3781 spin_lock(&po->bind_lock);
3784 register_prot_hook(sk);
3786 spin_unlock(&po->bind_lock);
3787 if (closing && (po->tp_version > TPACKET_V2)) {
3788 /* Because we don't support block-based V3 on tx-ring */
3790 prb_shutdown_retire_blk_timer(po, tx_ring, rb_queue);
3795 free_pg_vec(pg_vec, order, req->tp_block_nr);
3800 static int packet_mmap(struct file *file, struct socket *sock,
3801 struct vm_area_struct *vma)
3803 struct sock *sk = sock->sk;
3804 struct packet_sock *po = pkt_sk(sk);
3805 unsigned long size, expected_size;
3806 struct packet_ring_buffer *rb;
3807 unsigned long start;
3814 mutex_lock(&po->pg_vec_lock);
3817 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3819 expected_size += rb->pg_vec_len
3825 if (expected_size == 0)
3828 size = vma->vm_end - vma->vm_start;
3829 if (size != expected_size)
3832 start = vma->vm_start;
3833 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
3834 if (rb->pg_vec == NULL)
3837 for (i = 0; i < rb->pg_vec_len; i++) {
3839 void *kaddr = rb->pg_vec[i].buffer;
3842 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
3843 page = pgv_to_page(kaddr);
3844 err = vm_insert_page(vma, start, page);
3853 atomic_inc(&po->mapped);
3854 vma->vm_ops = &packet_mmap_ops;
3858 mutex_unlock(&po->pg_vec_lock);
3862 static const struct proto_ops packet_ops_spkt = {
3863 .family = PF_PACKET,
3864 .owner = THIS_MODULE,
3865 .release = packet_release,
3866 .bind = packet_bind_spkt,
3867 .connect = sock_no_connect,
3868 .socketpair = sock_no_socketpair,
3869 .accept = sock_no_accept,
3870 .getname = packet_getname_spkt,
3871 .poll = datagram_poll,
3872 .ioctl = packet_ioctl,
3873 .listen = sock_no_listen,
3874 .shutdown = sock_no_shutdown,
3875 .setsockopt = sock_no_setsockopt,
3876 .getsockopt = sock_no_getsockopt,
3877 .sendmsg = packet_sendmsg_spkt,
3878 .recvmsg = packet_recvmsg,
3879 .mmap = sock_no_mmap,
3880 .sendpage = sock_no_sendpage,
3883 static const struct proto_ops packet_ops = {
3884 .family = PF_PACKET,
3885 .owner = THIS_MODULE,
3886 .release = packet_release,
3887 .bind = packet_bind,
3888 .connect = sock_no_connect,
3889 .socketpair = sock_no_socketpair,
3890 .accept = sock_no_accept,
3891 .getname = packet_getname,
3892 .poll = packet_poll,
3893 .ioctl = packet_ioctl,
3894 .listen = sock_no_listen,
3895 .shutdown = sock_no_shutdown,
3896 .setsockopt = packet_setsockopt,
3897 .getsockopt = packet_getsockopt,
3898 .sendmsg = packet_sendmsg,
3899 .recvmsg = packet_recvmsg,
3900 .mmap = packet_mmap,
3901 .sendpage = sock_no_sendpage,
3904 static const struct net_proto_family packet_family_ops = {
3905 .family = PF_PACKET,
3906 .create = packet_create,
3907 .owner = THIS_MODULE,
3910 static struct notifier_block packet_netdev_notifier = {
3911 .notifier_call = packet_notifier,
3914 #ifdef CONFIG_PROC_FS
3916 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
3919 struct net *net = seq_file_net(seq);
3922 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
3925 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3927 struct net *net = seq_file_net(seq);
3928 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
3931 static void packet_seq_stop(struct seq_file *seq, void *v)
3937 static int packet_seq_show(struct seq_file *seq, void *v)
3939 if (v == SEQ_START_TOKEN)
3940 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3942 struct sock *s = sk_entry(v);
3943 const struct packet_sock *po = pkt_sk(s);
3946 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
3948 atomic_read(&s->sk_refcnt),
3953 atomic_read(&s->sk_rmem_alloc),
3954 from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
3961 static const struct seq_operations packet_seq_ops = {
3962 .start = packet_seq_start,
3963 .next = packet_seq_next,
3964 .stop = packet_seq_stop,
3965 .show = packet_seq_show,
3968 static int packet_seq_open(struct inode *inode, struct file *file)
3970 return seq_open_net(inode, file, &packet_seq_ops,
3971 sizeof(struct seq_net_private));
3974 static const struct file_operations packet_seq_fops = {
3975 .owner = THIS_MODULE,
3976 .open = packet_seq_open,
3978 .llseek = seq_lseek,
3979 .release = seq_release_net,
3984 static int __net_init packet_net_init(struct net *net)
3986 mutex_init(&net->packet.sklist_lock);
3987 INIT_HLIST_HEAD(&net->packet.sklist);
3989 if (!proc_create("packet", 0, net->proc_net, &packet_seq_fops))
3995 static void __net_exit packet_net_exit(struct net *net)
3997 remove_proc_entry("packet", net->proc_net);
4000 static struct pernet_operations packet_net_ops = {
4001 .init = packet_net_init,
4002 .exit = packet_net_exit,
4006 static void __exit packet_exit(void)
4008 unregister_netdevice_notifier(&packet_netdev_notifier);
4009 unregister_pernet_subsys(&packet_net_ops);
4010 sock_unregister(PF_PACKET);
4011 proto_unregister(&packet_proto);
4014 static int __init packet_init(void)
4016 int rc = proto_register(&packet_proto, 0);
4021 sock_register(&packet_family_ops);
4022 register_pernet_subsys(&packet_net_ops);
4023 register_netdevice_notifier(&packet_netdev_notifier);
4028 module_init(packet_init);
4029 module_exit(packet_exit);
4030 MODULE_LICENSE("GPL");
4031 MODULE_ALIAS_NETPROTO(PF_PACKET);