dev: introduce skb_scrub_packet()

[~andy/linux] / net / core / dev.c

diff --git a/net/core/dev.c b/net/core/dev.c
index 9c18557f93c6e6aee110a9160f4fe02fe240c4f9..370354a9c5f6926e977ce4374541bc4b6e2e5dac 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -129,6 +129,8 @@
 #include <linux/inetdevice.h>
 #include <linux/cpu_rmap.h>
 #include <linux/static_key.h>
+#include <linux/hashtable.h>
+#include <linux/vmalloc.h>
 
 #include "net-sysfs.h"
 
@@ -166,6 +168,12 @@ static struct list_head offload_base __read_mostly;
 DEFINE_RWLOCK(dev_base_lock);
 EXPORT_SYMBOL(dev_base_lock);
 
+/* protects napi_hash addition/deletion and napi_gen_id */
+static DEFINE_SPINLOCK(napi_hash_lock);
+
+static unsigned int napi_gen_id;
+static DEFINE_HASHTABLE(napi_hash, 8);
+
 seqcount_t devnet_rename_seq;
 
 static inline void dev_base_seq_inc(struct net *net)
@@ -1644,22 +1652,13 @@ int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
                }
        }
 
-       skb_orphan(skb);
-
        if (unlikely(!is_skb_forwardable(dev, skb))) {
                atomic_long_inc(&dev->rx_dropped);
                kfree_skb(skb);
                return NET_RX_DROP;
        }
-       skb->skb_iif = 0;
-       skb_dst_drop(skb);
-       skb->tstamp.tv64 = 0;
-       skb->pkt_type = PACKET_HOST;
+       skb_scrub_packet(skb);
        skb->protocol = eth_type_trans(skb, dev);
-       skb->mark = 0;
-       secpath_reset(skb);
-       nf_reset(skb);
-       nf_reset_trace(skb);
        return netif_rx(skb);
 }
 EXPORT_SYMBOL_GPL(dev_forward_skb);
@@ -4136,6 +4135,58 @@ void napi_complete(struct napi_struct *n)
 }
 EXPORT_SYMBOL(napi_complete);
 
+/* must be called under rcu_read_lock(), as we dont take a reference */
+struct napi_struct *napi_by_id(unsigned int napi_id)
+{
+       unsigned int hash = napi_id % HASH_SIZE(napi_hash);
+       struct napi_struct *napi;
+
+       hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
+               if (napi->napi_id == napi_id)
+                       return napi;
+
+       return NULL;
+}
+EXPORT_SYMBOL_GPL(napi_by_id);
+
+void napi_hash_add(struct napi_struct *napi)
+{
+       if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) {
+
+               spin_lock(&napi_hash_lock);
+
+               /* 0 is not a valid id, we also skip an id that is taken
+                * we expect both events to be extremely rare
+                */
+               napi->napi_id = 0;
+               while (!napi->napi_id) {
+                       napi->napi_id = ++napi_gen_id;
+                       if (napi_by_id(napi->napi_id))
+                               napi->napi_id = 0;
+               }
+
+               hlist_add_head_rcu(&napi->napi_hash_node,
+                       &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
+
+               spin_unlock(&napi_hash_lock);
+       }
+}
+EXPORT_SYMBOL_GPL(napi_hash_add);
+
+/* Warning : caller is responsible to make sure rcu grace period
+ * is respected before freeing memory containing @napi
+ */
+void napi_hash_del(struct napi_struct *napi)
+{
+       spin_lock(&napi_hash_lock);
+
+       if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state))
+               hlist_del_rcu(&napi->napi_hash_node);
+
+       spin_unlock(&napi_hash_lock);
+}
+EXPORT_SYMBOL_GPL(napi_hash_del);
+
 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
                    int (*poll)(struct napi_struct *, int), int weight)
 {
@@ -5194,17 +5245,28 @@ static void netdev_init_one_queue(struct net_device *dev,
 #endif
 }
 
+static void netif_free_tx_queues(struct net_device *dev)
+{
+       if (is_vmalloc_addr(dev->_tx))
+               vfree(dev->_tx);
+       else
+               kfree(dev->_tx);
+}
+
 static int netif_alloc_netdev_queues(struct net_device *dev)
 {
        unsigned int count = dev->num_tx_queues;
        struct netdev_queue *tx;
+       size_t sz = count * sizeof(*tx);
 
-       BUG_ON(count < 1);
-
-       tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
-       if (!tx)
-               return -ENOMEM;
+       BUG_ON(count < 1 || count > 0xffff);
 
+       tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
+       if (!tx) {
+               tx = vzalloc(sz);
+               if (!tx)
+                       return -ENOMEM;
+       }
        dev->_tx = tx;
 
        netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
@@ -5752,7 +5814,7 @@ free_all:
 
 free_pcpu:
        free_percpu(dev->pcpu_refcnt);
-       kfree(dev->_tx);
+       netif_free_tx_queues(dev);
 #ifdef CONFIG_RPS
        kfree(dev->_rx);
 #endif
@@ -5777,7 +5839,7 @@ void free_netdev(struct net_device *dev)
 
        release_net(dev_net(dev));
 
-       kfree(dev->_tx);
+       netif_free_tx_queues(dev);
 #ifdef CONFIG_RPS
        kfree(dev->_rx);
 #endif