Pileus Git - ~andy/linux/blob - net/openvswitch/datapath.c

   1 /*
   2  * Copyright (c) 2007-2013 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/init.h>
  22 #include <linux/module.h>
  23 #include <linux/if_arp.h>
  24 #include <linux/if_vlan.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/jhash.h>
  28 #include <linux/delay.h>
  29 #include <linux/time.h>
  30 #include <linux/etherdevice.h>
  31 #include <linux/genetlink.h>
  32 #include <linux/kernel.h>
  33 #include <linux/kthread.h>
  34 #include <linux/mutex.h>
  35 #include <linux/percpu.h>
  36 #include <linux/rcupdate.h>
  37 #include <linux/tcp.h>
  38 #include <linux/udp.h>
  39 #include <linux/ethtool.h>
  40 #include <linux/wait.h>
  41 #include <asm/div64.h>
  42 #include <linux/highmem.h>
  43 #include <linux/netfilter_bridge.h>
  44 #include <linux/netfilter_ipv4.h>
  45 #include <linux/inetdevice.h>
  46 #include <linux/list.h>
  47 #include <linux/lockdep.h>
  48 #include <linux/openvswitch.h>
  49 #include <linux/rculist.h>
  50 #include <linux/dmi.h>
  51 #include <linux/workqueue.h>
  52 #include <net/genetlink.h>
  53 #include <net/net_namespace.h>
  54 #include <net/netns/generic.h>
  55
  56 #include "datapath.h"
  57 #include "flow.h"
  58 #include "flow_netlink.h"
  59 #include "vport-internal_dev.h"
  60 #include "vport-netdev.h"
  61
  62 int ovs_net_id __read_mostly;
  63
  64 static void ovs_notify(struct genl_family *family,
  65                        struct sk_buff *skb, struct genl_info *info)
  66 {
  67         genl_notify(family, skb, genl_info_net(info), info->snd_portid,
  68                     0, info->nlhdr, GFP_KERNEL);
  69 }
  70
  71 /**
  72  * DOC: Locking:
  73  *
  74  * All writes e.g. Writes to device state (add/remove datapath, port, set
  75  * operations on vports, etc.), Writes to other state (flow table
  76  * modifications, set miscellaneous datapath parameters, etc.) are protected
  77  * by ovs_lock.
  78  *
  79  * Reads are protected by RCU.
  80  *
  81  * There are a few special cases (mostly stats) that have their own
  82  * synchronization but they nest under all of above and don't interact with
  83  * each other.
  84  *
  85  * The RTNL lock nests inside ovs_mutex.
  86  */
  87
  88 static DEFINE_MUTEX(ovs_mutex);
  89
  90 void ovs_lock(void)
  91 {
  92         mutex_lock(&ovs_mutex);
  93 }
  94
  95 void ovs_unlock(void)
  96 {
  97         mutex_unlock(&ovs_mutex);
  98 }
  99
 100 #ifdef CONFIG_LOCKDEP
 101 int lockdep_ovsl_is_held(void)
 102 {
 103         if (debug_locks)
 104                 return lockdep_is_held(&ovs_mutex);
 105         else
 106                 return 1;
 107 }
 108 #endif
 109
 110 static struct vport *new_vport(const struct vport_parms *);
 111 static int queue_gso_packets(struct net *, int dp_ifindex, struct sk_buff *,
 112                              const struct dp_upcall_info *);
 113 static int queue_userspace_packet(struct net *, int dp_ifindex,
 114                                   struct sk_buff *,
 115                                   const struct dp_upcall_info *);
 116
 117 /* Must be called with rcu_read_lock or ovs_mutex. */
 118 static struct datapath *get_dp(struct net *net, int dp_ifindex)
 119 {
 120         struct datapath *dp = NULL;
 121         struct net_device *dev;
 122
 123         rcu_read_lock();
 124         dev = dev_get_by_index_rcu(net, dp_ifindex);
 125         if (dev) {
 126                 struct vport *vport = ovs_internal_dev_get_vport(dev);
 127                 if (vport)
 128                         dp = vport->dp;
 129         }
 130         rcu_read_unlock();
 131
 132         return dp;
 133 }
 134
 135 /* Must be called with rcu_read_lock or ovs_mutex. */
 136 const char *ovs_dp_name(const struct datapath *dp)
 137 {
 138         struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
 139         return vport->ops->get_name(vport);
 140 }
 141
 142 static int get_dpifindex(struct datapath *dp)
 143 {
 144         struct vport *local;
 145         int ifindex;
 146
 147         rcu_read_lock();
 148
 149         local = ovs_vport_rcu(dp, OVSP_LOCAL);
 150         if (local)
 151                 ifindex = netdev_vport_priv(local)->dev->ifindex;
 152         else
 153                 ifindex = 0;
 154
 155         rcu_read_unlock();
 156
 157         return ifindex;
 158 }
 159
 160 static void destroy_dp_rcu(struct rcu_head *rcu)
 161 {
 162         struct datapath *dp = container_of(rcu, struct datapath, rcu);
 163
 164         ovs_flow_tbl_destroy(&dp->table);
 165         free_percpu(dp->stats_percpu);
 166         release_net(ovs_dp_get_net(dp));
 167         kfree(dp->ports);
 168         kfree(dp);
 169 }
 170
 171 static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
 172                                             u16 port_no)
 173 {
 174         return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
 175 }
 176
 177 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
 178 {
 179         struct vport *vport;
 180         struct hlist_head *head;
 181
 182         head = vport_hash_bucket(dp, port_no);
 183         hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
 184                 if (vport->port_no == port_no)
 185                         return vport;
 186         }
 187         return NULL;
 188 }
 189
 190 /* Called with ovs_mutex. */
 191 static struct vport *new_vport(const struct vport_parms *parms)
 192 {
 193         struct vport *vport;
 194
 195         vport = ovs_vport_add(parms);
 196         if (!IS_ERR(vport)) {
 197                 struct datapath *dp = parms->dp;
 198                 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
 199
 200                 hlist_add_head_rcu(&vport->dp_hash_node, head);
 201         }
 202         return vport;
 203 }
 204
 205 void ovs_dp_detach_port(struct vport *p)
 206 {
 207         ASSERT_OVSL();
 208
 209         /* First drop references to device. */
 210         hlist_del_rcu(&p->dp_hash_node);
 211
 212         /* Then destroy it. */
 213         ovs_vport_del(p);
 214 }
 215
 216 /* Must be called with rcu_read_lock. */
 217 void ovs_dp_process_received_packet(struct vport *p, struct sk_buff *skb)
 218 {
 219         struct datapath *dp = p->dp;
 220         struct sw_flow *flow;
 221         struct dp_stats_percpu *stats;
 222         struct sw_flow_key key;
 223         u64 *stats_counter;
 224         u32 n_mask_hit;
 225         int error;
 226
 227         stats = this_cpu_ptr(dp->stats_percpu);
 228
 229         /* Extract flow from 'skb' into 'key'. */
 230         error = ovs_flow_extract(skb, p->port_no, &key);
 231         if (unlikely(error)) {
 232                 kfree_skb(skb);
 233                 return;
 234         }
 235
 236         /* Look up flow. */
 237         flow = ovs_flow_tbl_lookup_stats(&dp->table, &key, &n_mask_hit);
 238         if (unlikely(!flow)) {
 239                 struct dp_upcall_info upcall;
 240
 241                 upcall.cmd = OVS_PACKET_CMD_MISS;
 242                 upcall.key = &key;
 243                 upcall.userdata = NULL;
 244                 upcall.portid = p->upcall_portid;
 245                 ovs_dp_upcall(dp, skb, &upcall);
 246                 consume_skb(skb);
 247                 stats_counter = &stats->n_missed;
 248                 goto out;
 249         }
 250
 251         OVS_CB(skb)->flow = flow;
 252         OVS_CB(skb)->pkt_key = &key;
 253
 254         stats_counter = &stats->n_hit;
 255         ovs_flow_used(OVS_CB(skb)->flow, skb);
 256         ovs_execute_actions(dp, skb);
 257
 258 out:
 259         /* Update datapath statistics. */
 260         u64_stats_update_begin(&stats->sync);
 261         (*stats_counter)++;
 262         stats->n_mask_hit += n_mask_hit;
 263         u64_stats_update_end(&stats->sync);
 264 }
 265
 266 static struct genl_family dp_packet_genl_family = {
 267         .id = GENL_ID_GENERATE,
 268         .hdrsize = sizeof(struct ovs_header),
 269         .name = OVS_PACKET_FAMILY,
 270         .version = OVS_PACKET_VERSION,
 271         .maxattr = OVS_PACKET_ATTR_MAX,
 272         .netnsok = true,
 273         .parallel_ops = true,
 274 };
 275
 276 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
 277                   const struct dp_upcall_info *upcall_info)
 278 {
 279         struct dp_stats_percpu *stats;
 280         int dp_ifindex;
 281         int err;
 282
 283         if (upcall_info->portid == 0) {
 284                 err = -ENOTCONN;
 285                 goto err;
 286         }
 287
 288         dp_ifindex = get_dpifindex(dp);
 289         if (!dp_ifindex) {
 290                 err = -ENODEV;
 291                 goto err;
 292         }
 293
 294         if (!skb_is_gso(skb))
 295                 err = queue_userspace_packet(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
 296         else
 297                 err = queue_gso_packets(ovs_dp_get_net(dp), dp_ifindex, skb, upcall_info);
 298         if (err)
 299                 goto err;
 300
 301         return 0;
 302
 303 err:
 304         stats = this_cpu_ptr(dp->stats_percpu);
 305
 306         u64_stats_update_begin(&stats->sync);
 307         stats->n_lost++;
 308         u64_stats_update_end(&stats->sync);
 309
 310         return err;
 311 }
 312
 313 static int queue_gso_packets(struct net *net, int dp_ifindex,
 314                              struct sk_buff *skb,
 315                              const struct dp_upcall_info *upcall_info)
 316 {
 317         unsigned short gso_type = skb_shinfo(skb)->gso_type;
 318         struct dp_upcall_info later_info;
 319         struct sw_flow_key later_key;
 320         struct sk_buff *segs, *nskb;
 321         int err;
 322
 323         segs = __skb_gso_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM, false);
 324         if (IS_ERR(segs))
 325                 return PTR_ERR(segs);
 326
 327         /* Queue all of the segments. */
 328         skb = segs;
 329         do {
 330                 err = queue_userspace_packet(net, dp_ifindex, skb, upcall_info);
 331                 if (err)
 332                         break;
 333
 334                 if (skb == segs && gso_type & SKB_GSO_UDP) {
 335                         /* The initial flow key extracted by ovs_flow_extract()
 336                          * in this case is for a first fragment, so we need to
 337                          * properly mark later fragments.
 338                          */
 339                         later_key = *upcall_info->key;
 340                         later_key.ip.frag = OVS_FRAG_TYPE_LATER;
 341
 342                         later_info = *upcall_info;
 343                         later_info.key = &later_key;
 344                         upcall_info = &later_info;
 345                 }
 346         } while ((skb = skb->next));
 347
 348         /* Free all of the segments. */
 349         skb = segs;
 350         do {
 351                 nskb = skb->next;
 352                 if (err)
 353                         kfree_skb(skb);
 354                 else
 355                         consume_skb(skb);
 356         } while ((skb = nskb));
 357         return err;
 358 }
 359
 360 static size_t key_attr_size(void)
 361 {
 362         return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
 363                 + nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
 364                   + nla_total_size(8)   /* OVS_TUNNEL_KEY_ATTR_ID */
 365                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_SRC */
 366                   + nla_total_size(4)   /* OVS_TUNNEL_KEY_ATTR_IPV4_DST */
 367                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TOS */
 368                   + nla_total_size(1)   /* OVS_TUNNEL_KEY_ATTR_TTL */
 369                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
 370                   + nla_total_size(0)   /* OVS_TUNNEL_KEY_ATTR_CSUM */
 371                 + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
 372                 + nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
 373                 + nla_total_size(12)  /* OVS_KEY_ATTR_ETHERNET */
 374                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 375                 + nla_total_size(4)   /* OVS_KEY_ATTR_8021Q */
 376                 + nla_total_size(0)   /* OVS_KEY_ATTR_ENCAP */
 377                 + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
 378                 + nla_total_size(40)  /* OVS_KEY_ATTR_IPV6 */
 379                 + nla_total_size(2)   /* OVS_KEY_ATTR_ICMPV6 */
 380                 + nla_total_size(28); /* OVS_KEY_ATTR_ND */
 381 }
 382
 383 static size_t upcall_msg_size(const struct sk_buff *skb,
 384                               const struct nlattr *userdata)
 385 {
 386         size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
 387                 + nla_total_size(skb->len) /* OVS_PACKET_ATTR_PACKET */
 388                 + nla_total_size(key_attr_size()); /* OVS_PACKET_ATTR_KEY */
 389
 390         /* OVS_PACKET_ATTR_USERDATA */
 391         if (userdata)
 392                 size += NLA_ALIGN(userdata->nla_len);
 393
 394         return size;
 395 }
 396
 397 static int queue_userspace_packet(struct net *net, int dp_ifindex,
 398                                   struct sk_buff *skb,
 399                                   const struct dp_upcall_info *upcall_info)
 400 {
 401         struct ovs_header *upcall;
 402         struct sk_buff *nskb = NULL;
 403         struct sk_buff *user_skb; /* to be queued to userspace */
 404         struct nlattr *nla;
 405         struct genl_info info = {
 406                 .dst_sk = net->genl_sock,
 407                 .snd_portid = upcall_info->portid,
 408         };
 409         size_t len;
 410         int err;
 411
 412         if (vlan_tx_tag_present(skb)) {
 413                 nskb = skb_clone(skb, GFP_ATOMIC);
 414                 if (!nskb)
 415                         return -ENOMEM;
 416
 417                 nskb = __vlan_put_tag(nskb, nskb->vlan_proto, vlan_tx_tag_get(nskb));
 418                 if (!nskb)
 419                         return -ENOMEM;
 420
 421                 nskb->vlan_tci = 0;
 422                 skb = nskb;
 423         }
 424
 425         if (nla_attr_size(skb->len) > USHRT_MAX) {
 426                 err = -EFBIG;
 427                 goto out;
 428         }
 429
 430         len = upcall_msg_size(skb, upcall_info->userdata);
 431         user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
 432         if (!user_skb) {
 433                 err = -ENOMEM;
 434                 goto out;
 435         }
 436
 437         upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
 438                              0, upcall_info->cmd);
 439         upcall->dp_ifindex = dp_ifindex;
 440
 441         nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_KEY);
 442         ovs_nla_put_flow(upcall_info->key, upcall_info->key, user_skb);
 443         nla_nest_end(user_skb, nla);
 444
 445         if (upcall_info->userdata)
 446                 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
 447                           nla_len(upcall_info->userdata),
 448                           nla_data(upcall_info->userdata));
 449
 450         nla = __nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, skb->len);
 451
 452         skb_copy_and_csum_dev(skb, nla_data(nla));
 453
 454         genlmsg_end(user_skb, upcall);
 455         err = genlmsg_unicast(net, user_skb, upcall_info->portid);
 456
 457 out:
 458         kfree_skb(nskb);
 459         return err;
 460 }
 461
 462 static void clear_stats(struct sw_flow *flow)
 463 {
 464         flow->used = 0;
 465         flow->tcp_flags = 0;
 466         flow->packet_count = 0;
 467         flow->byte_count = 0;
 468 }
 469
 470 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
 471 {
 472         struct ovs_header *ovs_header = info->userhdr;
 473         struct nlattr **a = info->attrs;
 474         struct sw_flow_actions *acts;
 475         struct sk_buff *packet;
 476         struct sw_flow *flow;
 477         struct datapath *dp;
 478         struct ethhdr *eth;
 479         int len;
 480         int err;
 481
 482         err = -EINVAL;
 483         if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
 484             !a[OVS_PACKET_ATTR_ACTIONS])
 485                 goto err;
 486
 487         len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
 488         packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
 489         err = -ENOMEM;
 490         if (!packet)
 491                 goto err;
 492         skb_reserve(packet, NET_IP_ALIGN);
 493
 494         nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
 495
 496         skb_reset_mac_header(packet);
 497         eth = eth_hdr(packet);
 498
 499         /* Normally, setting the skb 'protocol' field would be handled by a
 500          * call to eth_type_trans(), but it assumes there's a sending
 501          * device, which we may not have. */
 502         if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
 503                 packet->protocol = eth->h_proto;
 504         else
 505                 packet->protocol = htons(ETH_P_802_2);
 506
 507         /* Build an sw_flow for sending this packet. */
 508         flow = ovs_flow_alloc();
 509         err = PTR_ERR(flow);
 510         if (IS_ERR(flow))
 511                 goto err_kfree_skb;
 512
 513         err = ovs_flow_extract(packet, -1, &flow->key);
 514         if (err)
 515                 goto err_flow_free;
 516
 517         err = ovs_nla_get_flow_metadata(flow, a[OVS_PACKET_ATTR_KEY]);
 518         if (err)
 519                 goto err_flow_free;
 520         acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_PACKET_ATTR_ACTIONS]));
 521         err = PTR_ERR(acts);
 522         if (IS_ERR(acts))
 523                 goto err_flow_free;
 524
 525         err = ovs_nla_copy_actions(a[OVS_PACKET_ATTR_ACTIONS],
 526                                    &flow->key, 0, &acts);
 527         rcu_assign_pointer(flow->sf_acts, acts);
 528         if (err)
 529                 goto err_flow_free;
 530
 531         OVS_CB(packet)->flow = flow;
 532         OVS_CB(packet)->pkt_key = &flow->key;
 533         packet->priority = flow->key.phy.priority;
 534         packet->mark = flow->key.phy.skb_mark;
 535
 536         rcu_read_lock();
 537         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 538         err = -ENODEV;
 539         if (!dp)
 540                 goto err_unlock;
 541
 542         local_bh_disable();
 543         err = ovs_execute_actions(dp, packet);
 544         local_bh_enable();
 545         rcu_read_unlock();
 546
 547         ovs_flow_free(flow, false);
 548         return err;
 549
 550 err_unlock:
 551         rcu_read_unlock();
 552 err_flow_free:
 553         ovs_flow_free(flow, false);
 554 err_kfree_skb:
 555         kfree_skb(packet);
 556 err:
 557         return err;
 558 }
 559
 560 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
 561         [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
 562         [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
 563         [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
 564 };
 565
 566 static const struct genl_ops dp_packet_genl_ops[] = {
 567         { .cmd = OVS_PACKET_CMD_EXECUTE,
 568           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
 569           .policy = packet_policy,
 570           .doit = ovs_packet_cmd_execute
 571         }
 572 };
 573
 574 static void get_dp_stats(struct datapath *dp, struct ovs_dp_stats *stats,
 575                          struct ovs_dp_megaflow_stats *mega_stats)
 576 {
 577         int i;
 578
 579         memset(mega_stats, 0, sizeof(*mega_stats));
 580
 581         stats->n_flows = ovs_flow_tbl_count(&dp->table);
 582         mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
 583
 584         stats->n_hit = stats->n_missed = stats->n_lost = 0;
 585
 586         for_each_possible_cpu(i) {
 587                 const struct dp_stats_percpu *percpu_stats;
 588                 struct dp_stats_percpu local_stats;
 589                 unsigned int start;
 590
 591                 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
 592
 593                 do {
 594                         start = u64_stats_fetch_begin_bh(&percpu_stats->sync);
 595                         local_stats = *percpu_stats;
 596                 } while (u64_stats_fetch_retry_bh(&percpu_stats->sync, start));
 597
 598                 stats->n_hit += local_stats.n_hit;
 599                 stats->n_missed += local_stats.n_missed;
 600                 stats->n_lost += local_stats.n_lost;
 601                 mega_stats->n_mask_hit += local_stats.n_mask_hit;
 602         }
 603 }
 604
 605 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
 606         [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
 607         [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
 608         [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
 609 };
 610
 611 static struct genl_family dp_flow_genl_family = {
 612         .id = GENL_ID_GENERATE,
 613         .hdrsize = sizeof(struct ovs_header),
 614         .name = OVS_FLOW_FAMILY,
 615         .version = OVS_FLOW_VERSION,
 616         .maxattr = OVS_FLOW_ATTR_MAX,
 617         .netnsok = true,
 618         .parallel_ops = true,
 619 };
 620
 621 static struct genl_multicast_group ovs_dp_flow_multicast_group = {
 622         .name = OVS_FLOW_MCGROUP
 623 };
 624
 625 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts)
 626 {
 627         return NLMSG_ALIGN(sizeof(struct ovs_header))
 628                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_KEY */
 629                 + nla_total_size(key_attr_size()) /* OVS_FLOW_ATTR_MASK */
 630                 + nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
 631                 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
 632                 + nla_total_size(8) /* OVS_FLOW_ATTR_USED */
 633                 + nla_total_size(acts->actions_len); /* OVS_FLOW_ATTR_ACTIONS */
 634 }
 635
 636 /* Called with ovs_mutex. */
 637 static int ovs_flow_cmd_fill_info(struct sw_flow *flow, struct datapath *dp,
 638                                   struct sk_buff *skb, u32 portid,
 639                                   u32 seq, u32 flags, u8 cmd)
 640 {
 641         const int skb_orig_len = skb->len;
 642         struct nlattr *start;
 643         struct ovs_flow_stats stats;
 644         struct ovs_header *ovs_header;
 645         struct nlattr *nla;
 646         unsigned long used;
 647         u8 tcp_flags;
 648         int err;
 649
 650         ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, flags, cmd);
 651         if (!ovs_header)
 652                 return -EMSGSIZE;
 653
 654         ovs_header->dp_ifindex = get_dpifindex(dp);
 655
 656         /* Fill flow key. */
 657         nla = nla_nest_start(skb, OVS_FLOW_ATTR_KEY);
 658         if (!nla)
 659                 goto nla_put_failure;
 660
 661         err = ovs_nla_put_flow(&flow->unmasked_key, &flow->unmasked_key, skb);
 662         if (err)
 663                 goto error;
 664         nla_nest_end(skb, nla);
 665
 666         nla = nla_nest_start(skb, OVS_FLOW_ATTR_MASK);
 667         if (!nla)
 668                 goto nla_put_failure;
 669
 670         err = ovs_nla_put_flow(&flow->key, &flow->mask->key, skb);
 671         if (err)
 672                 goto error;
 673
 674         nla_nest_end(skb, nla);
 675
 676         spin_lock_bh(&flow->lock);
 677         used = flow->used;
 678         stats.n_packets = flow->packet_count;
 679         stats.n_bytes = flow->byte_count;
 680         tcp_flags = (u8)ntohs(flow->tcp_flags);
 681         spin_unlock_bh(&flow->lock);
 682
 683         if (used &&
 684             nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
 685                 goto nla_put_failure;
 686
 687         if (stats.n_packets &&
 688             nla_put(skb, OVS_FLOW_ATTR_STATS,
 689                     sizeof(struct ovs_flow_stats), &stats))
 690                 goto nla_put_failure;
 691
 692         if (tcp_flags &&
 693             nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, tcp_flags))
 694                 goto nla_put_failure;
 695
 696         /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
 697          * this is the first flow to be dumped into 'skb'.  This is unusual for
 698          * Netlink but individual action lists can be longer than
 699          * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
 700          * The userspace caller can always fetch the actions separately if it
 701          * really wants them.  (Most userspace callers in fact don't care.)
 702          *
 703          * This can only fail for dump operations because the skb is always
 704          * properly sized for single flows.
 705          */
 706         start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
 707         if (start) {
 708                 const struct sw_flow_actions *sf_acts;
 709
 710                 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
 711
 712                 err = ovs_nla_put_actions(sf_acts->actions,
 713                                           sf_acts->actions_len, skb);
 714                 if (!err)
 715                         nla_nest_end(skb, start);
 716                 else {
 717                         if (skb_orig_len)
 718                                 goto error;
 719
 720                         nla_nest_cancel(skb, start);
 721                 }
 722         } else if (skb_orig_len)
 723                 goto nla_put_failure;
 724
 725         return genlmsg_end(skb, ovs_header);
 726
 727 nla_put_failure:
 728         err = -EMSGSIZE;
 729 error:
 730         genlmsg_cancel(skb, ovs_header);
 731         return err;
 732 }
 733
 734 static struct sk_buff *ovs_flow_cmd_alloc_info(struct sw_flow *flow,
 735                                                struct genl_info *info)
 736 {
 737         size_t len;
 738
 739         len = ovs_flow_cmd_msg_size(ovsl_dereference(flow->sf_acts));
 740
 741         return genlmsg_new_unicast(len, info, GFP_KERNEL);
 742 }
 743
 744 static struct sk_buff *ovs_flow_cmd_build_info(struct sw_flow *flow,
 745                                                struct datapath *dp,
 746                                                struct genl_info *info,
 747                                                u8 cmd)
 748 {
 749         struct sk_buff *skb;
 750         int retval;
 751
 752         skb = ovs_flow_cmd_alloc_info(flow, info);
 753         if (!skb)
 754                 return ERR_PTR(-ENOMEM);
 755
 756         retval = ovs_flow_cmd_fill_info(flow, dp, skb, info->snd_portid,
 757                                         info->snd_seq, 0, cmd);
 758         BUG_ON(retval < 0);
 759         return skb;
 760 }
 761
 762 static int ovs_flow_cmd_new_or_set(struct sk_buff *skb, struct genl_info *info)
 763 {
 764         struct nlattr **a = info->attrs;
 765         struct ovs_header *ovs_header = info->userhdr;
 766         struct sw_flow_key key, masked_key;
 767         struct sw_flow *flow = NULL;
 768         struct sw_flow_mask mask;
 769         struct sk_buff *reply;
 770         struct datapath *dp;
 771         struct sw_flow_actions *acts = NULL;
 772         struct sw_flow_match match;
 773         int error;
 774
 775         /* Extract key. */
 776         error = -EINVAL;
 777         if (!a[OVS_FLOW_ATTR_KEY])
 778                 goto error;
 779
 780         ovs_match_init(&match, &key, &mask);
 781         error = ovs_nla_get_match(&match,
 782                                   a[OVS_FLOW_ATTR_KEY], a[OVS_FLOW_ATTR_MASK]);
 783         if (error)
 784                 goto error;
 785
 786         /* Validate actions. */
 787         if (a[OVS_FLOW_ATTR_ACTIONS]) {
 788                 acts = ovs_nla_alloc_flow_actions(nla_len(a[OVS_FLOW_ATTR_ACTIONS]));
 789                 error = PTR_ERR(acts);
 790                 if (IS_ERR(acts))
 791                         goto error;
 792
 793                 ovs_flow_mask_key(&masked_key, &key, &mask);
 794                 error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS],
 795                                              &masked_key, 0, &acts);
 796                 if (error) {
 797                         OVS_NLERR("Flow actions may not be safe on all matching packets.\n");
 798                         goto err_kfree;
 799                 }
 800         } else if (info->genlhdr->cmd == OVS_FLOW_CMD_NEW) {
 801                 error = -EINVAL;
 802                 goto error;
 803         }
 804
 805         ovs_lock();
 806         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 807         error = -ENODEV;
 808         if (!dp)
 809                 goto err_unlock_ovs;
 810
 811         /* Check if this is a duplicate flow */
 812         flow = ovs_flow_tbl_lookup(&dp->table, &key);
 813         if (!flow) {
 814                 /* Bail out if we're not allowed to create a new flow. */
 815                 error = -ENOENT;
 816                 if (info->genlhdr->cmd == OVS_FLOW_CMD_SET)
 817                         goto err_unlock_ovs;
 818
 819                 /* Allocate flow. */
 820                 flow = ovs_flow_alloc();
 821                 if (IS_ERR(flow)) {
 822                         error = PTR_ERR(flow);
 823                         goto err_unlock_ovs;
 824                 }
 825                 clear_stats(flow);
 826
 827                 flow->key = masked_key;
 828                 flow->unmasked_key = key;
 829                 rcu_assign_pointer(flow->sf_acts, acts);
 830
 831                 /* Put flow in bucket. */
 832                 error = ovs_flow_tbl_insert(&dp->table, flow, &mask);
 833                 if (error) {
 834                         acts = NULL;
 835                         goto err_flow_free;
 836                 }
 837
 838                 reply = ovs_flow_cmd_build_info(flow, dp, info, OVS_FLOW_CMD_NEW);
 839         } else {
 840                 /* We found a matching flow. */
 841                 struct sw_flow_actions *old_acts;
 842
 843                 /* Bail out if we're not allowed to modify an existing flow.
 844                  * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
 845                  * because Generic Netlink treats the latter as a dump
 846                  * request.  We also accept NLM_F_EXCL in case that bug ever
 847                  * gets fixed.
 848                  */
 849                 error = -EEXIST;
 850                 if (info->genlhdr->cmd == OVS_FLOW_CMD_NEW &&
 851                     info->nlhdr->nlmsg_flags & (NLM_F_CREATE | NLM_F_EXCL))
 852                         goto err_unlock_ovs;
 853
 854                 /* The unmasked key has to be the same for flow updates. */
 855                 error = -EINVAL;
 856                 if (!ovs_flow_cmp_unmasked_key(flow, &match)) {
 857                         OVS_NLERR("Flow modification message rejected, unmasked key does not match.\n");
 858                         goto err_unlock_ovs;
 859                 }
 860
 861                 /* Update actions. */
 862                 old_acts = ovsl_dereference(flow->sf_acts);
 863                 rcu_assign_pointer(flow->sf_acts, acts);
 864                 ovs_nla_free_flow_actions(old_acts);
 865
 866                 reply = ovs_flow_cmd_build_info(flow, dp, info, OVS_FLOW_CMD_NEW);
 867
 868                 /* Clear stats. */
 869                 if (a[OVS_FLOW_ATTR_CLEAR]) {
 870                         spin_lock_bh(&flow->lock);
 871                         clear_stats(flow);
 872                         spin_unlock_bh(&flow->lock);
 873                 }
 874         }
 875         ovs_unlock();
 876
 877         if (!IS_ERR(reply))
 878                 ovs_notify(&dp_flow_genl_family, reply, info);
 879         else
 880                 genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
 881                              0, PTR_ERR(reply));
 882         return 0;
 883
 884 err_flow_free:
 885         ovs_flow_free(flow, false);
 886 err_unlock_ovs:
 887         ovs_unlock();
 888 err_kfree:
 889         kfree(acts);
 890 error:
 891         return error;
 892 }
 893
 894 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
 895 {
 896         struct nlattr **a = info->attrs;
 897         struct ovs_header *ovs_header = info->userhdr;
 898         struct sw_flow_key key;
 899         struct sk_buff *reply;
 900         struct sw_flow *flow;
 901         struct datapath *dp;
 902         struct sw_flow_match match;
 903         int err;
 904
 905         if (!a[OVS_FLOW_ATTR_KEY]) {
 906                 OVS_NLERR("Flow get message rejected, Key attribute missing.\n");
 907                 return -EINVAL;
 908         }
 909
 910         ovs_match_init(&match, &key, NULL);
 911         err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
 912         if (err)
 913                 return err;
 914
 915         ovs_lock();
 916         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 917         if (!dp) {
 918                 err = -ENODEV;
 919                 goto unlock;
 920         }
 921
 922         flow = ovs_flow_tbl_lookup(&dp->table, &key);
 923         if (!flow || !ovs_flow_cmp_unmasked_key(flow, &match)) {
 924                 err = -ENOENT;
 925                 goto unlock;
 926         }
 927
 928         reply = ovs_flow_cmd_build_info(flow, dp, info, OVS_FLOW_CMD_NEW);
 929         if (IS_ERR(reply)) {
 930                 err = PTR_ERR(reply);
 931                 goto unlock;
 932         }
 933
 934         ovs_unlock();
 935         return genlmsg_reply(reply, info);
 936 unlock:
 937         ovs_unlock();
 938         return err;
 939 }
 940
 941 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
 942 {
 943         struct nlattr **a = info->attrs;
 944         struct ovs_header *ovs_header = info->userhdr;
 945         struct sw_flow_key key;
 946         struct sk_buff *reply;
 947         struct sw_flow *flow;
 948         struct datapath *dp;
 949         struct sw_flow_match match;
 950         int err;
 951
 952         ovs_lock();
 953         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
 954         if (!dp) {
 955                 err = -ENODEV;
 956                 goto unlock;
 957         }
 958
 959         if (!a[OVS_FLOW_ATTR_KEY]) {
 960                 err = ovs_flow_tbl_flush(&dp->table);
 961                 goto unlock;
 962         }
 963
 964         ovs_match_init(&match, &key, NULL);
 965         err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL);
 966         if (err)
 967                 goto unlock;
 968
 969         flow = ovs_flow_tbl_lookup(&dp->table, &key);
 970         if (!flow || !ovs_flow_cmp_unmasked_key(flow, &match)) {
 971                 err = -ENOENT;
 972                 goto unlock;
 973         }
 974
 975         reply = ovs_flow_cmd_alloc_info(flow, info);
 976         if (!reply) {
 977                 err = -ENOMEM;
 978                 goto unlock;
 979         }
 980
 981         ovs_flow_tbl_remove(&dp->table, flow);
 982
 983         err = ovs_flow_cmd_fill_info(flow, dp, reply, info->snd_portid,
 984                                      info->snd_seq, 0, OVS_FLOW_CMD_DEL);
 985         BUG_ON(err < 0);
 986
 987         ovs_flow_free(flow, true);
 988         ovs_unlock();
 989
 990         ovs_notify(&dp_flow_genl_family, reply, info);
 991         return 0;
 992 unlock:
 993         ovs_unlock();
 994         return err;
 995 }
 996
 997 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
 998 {
 999         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1000         struct table_instance *ti;
1001         struct datapath *dp;
1002
1003         rcu_read_lock();
1004         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1005         if (!dp) {
1006                 rcu_read_unlock();
1007                 return -ENODEV;
1008         }
1009
1010         ti = rcu_dereference(dp->table.ti);
1011         for (;;) {
1012                 struct sw_flow *flow;
1013                 u32 bucket, obj;
1014
1015                 bucket = cb->args[0];
1016                 obj = cb->args[1];
1017                 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1018                 if (!flow)
1019                         break;
1020
1021                 if (ovs_flow_cmd_fill_info(flow, dp, skb,
1022                                            NETLINK_CB(cb->skb).portid,
1023                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
1024                                            OVS_FLOW_CMD_NEW) < 0)
1025                         break;
1026
1027                 cb->args[0] = bucket;
1028                 cb->args[1] = obj;
1029         }
1030         rcu_read_unlock();
1031         return skb->len;
1032 }
1033
1034 static const struct genl_ops dp_flow_genl_ops[] = {
1035         { .cmd = OVS_FLOW_CMD_NEW,
1036           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1037           .policy = flow_policy,
1038           .doit = ovs_flow_cmd_new_or_set
1039         },
1040         { .cmd = OVS_FLOW_CMD_DEL,
1041           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1042           .policy = flow_policy,
1043           .doit = ovs_flow_cmd_del
1044         },
1045         { .cmd = OVS_FLOW_CMD_GET,
1046           .flags = 0,               /* OK for unprivileged users. */
1047           .policy = flow_policy,
1048           .doit = ovs_flow_cmd_get,
1049           .dumpit = ovs_flow_cmd_dump
1050         },
1051         { .cmd = OVS_FLOW_CMD_SET,
1052           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1053           .policy = flow_policy,
1054           .doit = ovs_flow_cmd_new_or_set,
1055         },
1056 };
1057
1058 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1059         [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1060         [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1061 };
1062
1063 static struct genl_family dp_datapath_genl_family = {
1064         .id = GENL_ID_GENERATE,
1065         .hdrsize = sizeof(struct ovs_header),
1066         .name = OVS_DATAPATH_FAMILY,
1067         .version = OVS_DATAPATH_VERSION,
1068         .maxattr = OVS_DP_ATTR_MAX,
1069         .netnsok = true,
1070         .parallel_ops = true,
1071 };
1072
1073 static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
1074         .name = OVS_DATAPATH_MCGROUP
1075 };
1076
1077 static size_t ovs_dp_cmd_msg_size(void)
1078 {
1079         size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1080
1081         msgsize += nla_total_size(IFNAMSIZ);
1082         msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1083         msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
1084
1085         return msgsize;
1086 }
1087
1088 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1089                                 u32 portid, u32 seq, u32 flags, u8 cmd)
1090 {
1091         struct ovs_header *ovs_header;
1092         struct ovs_dp_stats dp_stats;
1093         struct ovs_dp_megaflow_stats dp_megaflow_stats;
1094         int err;
1095
1096         ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1097                                    flags, cmd);
1098         if (!ovs_header)
1099                 goto error;
1100
1101         ovs_header->dp_ifindex = get_dpifindex(dp);
1102
1103         rcu_read_lock();
1104         err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
1105         rcu_read_unlock();
1106         if (err)
1107                 goto nla_put_failure;
1108
1109         get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
1110         if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1111                         &dp_stats))
1112                 goto nla_put_failure;
1113
1114         if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1115                         sizeof(struct ovs_dp_megaflow_stats),
1116                         &dp_megaflow_stats))
1117                 goto nla_put_failure;
1118
1119         return genlmsg_end(skb, ovs_header);
1120
1121 nla_put_failure:
1122         genlmsg_cancel(skb, ovs_header);
1123 error:
1124         return -EMSGSIZE;
1125 }
1126
1127 static struct sk_buff *ovs_dp_cmd_build_info(struct datapath *dp,
1128                                              struct genl_info *info, u8 cmd)
1129 {
1130         struct sk_buff *skb;
1131         int retval;
1132
1133         skb = genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
1134         if (!skb)
1135                 return ERR_PTR(-ENOMEM);
1136
1137         retval = ovs_dp_cmd_fill_info(dp, skb, info->snd_portid, info->snd_seq, 0, cmd);
1138         if (retval < 0) {
1139                 kfree_skb(skb);
1140                 return ERR_PTR(retval);
1141         }
1142         return skb;
1143 }
1144
1145 /* Called with ovs_mutex. */
1146 static struct datapath *lookup_datapath(struct net *net,
1147                                         struct ovs_header *ovs_header,
1148                                         struct nlattr *a[OVS_DP_ATTR_MAX + 1])
1149 {
1150         struct datapath *dp;
1151
1152         if (!a[OVS_DP_ATTR_NAME])
1153                 dp = get_dp(net, ovs_header->dp_ifindex);
1154         else {
1155                 struct vport *vport;
1156
1157                 rcu_read_lock();
1158                 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1159                 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
1160                 rcu_read_unlock();
1161         }
1162         return dp ? dp : ERR_PTR(-ENODEV);
1163 }
1164
1165 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1166 {
1167         struct nlattr **a = info->attrs;
1168         struct vport_parms parms;
1169         struct sk_buff *reply;
1170         struct datapath *dp;
1171         struct vport *vport;
1172         struct ovs_net *ovs_net;
1173         int err, i;
1174
1175         err = -EINVAL;
1176         if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
1177                 goto err;
1178
1179         ovs_lock();
1180
1181         err = -ENOMEM;
1182         dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1183         if (dp == NULL)
1184                 goto err_unlock_ovs;
1185
1186         ovs_dp_set_net(dp, hold_net(sock_net(skb->sk)));
1187
1188         /* Allocate table. */
1189         err = ovs_flow_tbl_init(&dp->table);
1190         if (err)
1191                 goto err_free_dp;
1192
1193         dp->stats_percpu = alloc_percpu(struct dp_stats_percpu);
1194         if (!dp->stats_percpu) {
1195                 err = -ENOMEM;
1196                 goto err_destroy_table;
1197         }
1198
1199         for_each_possible_cpu(i) {
1200                 struct dp_stats_percpu *dpath_stats;
1201                 dpath_stats = per_cpu_ptr(dp->stats_percpu, i);
1202                 u64_stats_init(&dpath_stats->sync);
1203         }
1204
1205         dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1206                             GFP_KERNEL);
1207         if (!dp->ports) {
1208                 err = -ENOMEM;
1209                 goto err_destroy_percpu;
1210         }
1211
1212         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1213                 INIT_HLIST_HEAD(&dp->ports[i]);
1214
1215         /* Set up our datapath device. */
1216         parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1217         parms.type = OVS_VPORT_TYPE_INTERNAL;
1218         parms.options = NULL;
1219         parms.dp = dp;
1220         parms.port_no = OVSP_LOCAL;
1221         parms.upcall_portid = nla_get_u32(a[OVS_DP_ATTR_UPCALL_PID]);
1222
1223         vport = new_vport(&parms);
1224         if (IS_ERR(vport)) {
1225                 err = PTR_ERR(vport);
1226                 if (err == -EBUSY)
1227                         err = -EEXIST;
1228
1229                 goto err_destroy_ports_array;
1230         }
1231
1232         reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_NEW);
1233         err = PTR_ERR(reply);
1234         if (IS_ERR(reply))
1235                 goto err_destroy_local_port;
1236
1237         ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1238         list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1239
1240         ovs_unlock();
1241
1242         ovs_notify(&dp_datapath_genl_family, reply, info);
1243         return 0;
1244
1245 err_destroy_local_port:
1246         ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1247 err_destroy_ports_array:
1248         kfree(dp->ports);
1249 err_destroy_percpu:
1250         free_percpu(dp->stats_percpu);
1251 err_destroy_table:
1252         ovs_flow_tbl_destroy(&dp->table);
1253 err_free_dp:
1254         release_net(ovs_dp_get_net(dp));
1255         kfree(dp);
1256 err_unlock_ovs:
1257         ovs_unlock();
1258 err:
1259         return err;
1260 }
1261
1262 /* Called with ovs_mutex. */
1263 static void __dp_destroy(struct datapath *dp)
1264 {
1265         int i;
1266
1267         for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1268                 struct vport *vport;
1269                 struct hlist_node *n;
1270
1271                 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1272                         if (vport->port_no != OVSP_LOCAL)
1273                                 ovs_dp_detach_port(vport);
1274         }
1275
1276         list_del_rcu(&dp->list_node);
1277
1278         /* OVSP_LOCAL is datapath internal port. We need to make sure that
1279          * all port in datapath are destroyed first before freeing datapath.
1280          */
1281         ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1282
1283         call_rcu(&dp->rcu, destroy_dp_rcu);
1284 }
1285
1286 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1287 {
1288         struct sk_buff *reply;
1289         struct datapath *dp;
1290         int err;
1291
1292         ovs_lock();
1293         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1294         err = PTR_ERR(dp);
1295         if (IS_ERR(dp))
1296                 goto unlock;
1297
1298         reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_DEL);
1299         err = PTR_ERR(reply);
1300         if (IS_ERR(reply))
1301                 goto unlock;
1302
1303         __dp_destroy(dp);
1304         ovs_unlock();
1305
1306         ovs_notify(&dp_datapath_genl_family, reply, info);
1307
1308         return 0;
1309 unlock:
1310         ovs_unlock();
1311         return err;
1312 }
1313
1314 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
1315 {
1316         struct sk_buff *reply;
1317         struct datapath *dp;
1318         int err;
1319
1320         ovs_lock();
1321         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1322         err = PTR_ERR(dp);
1323         if (IS_ERR(dp))
1324                 goto unlock;
1325
1326         reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_NEW);
1327         if (IS_ERR(reply)) {
1328                 err = PTR_ERR(reply);
1329                 genl_set_err(&dp_datapath_genl_family, sock_net(skb->sk), 0,
1330                              0, err);
1331                 err = 0;
1332                 goto unlock;
1333         }
1334
1335         ovs_unlock();
1336         ovs_notify(&dp_datapath_genl_family, reply, info);
1337
1338         return 0;
1339 unlock:
1340         ovs_unlock();
1341         return err;
1342 }
1343
1344 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1345 {
1346         struct sk_buff *reply;
1347         struct datapath *dp;
1348         int err;
1349
1350         ovs_lock();
1351         dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1352         if (IS_ERR(dp)) {
1353                 err = PTR_ERR(dp);
1354                 goto unlock;
1355         }
1356
1357         reply = ovs_dp_cmd_build_info(dp, info, OVS_DP_CMD_NEW);
1358         if (IS_ERR(reply)) {
1359                 err = PTR_ERR(reply);
1360                 goto unlock;
1361         }
1362
1363         ovs_unlock();
1364         return genlmsg_reply(reply, info);
1365
1366 unlock:
1367         ovs_unlock();
1368         return err;
1369 }
1370
1371 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1372 {
1373         struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1374         struct datapath *dp;
1375         int skip = cb->args[0];
1376         int i = 0;
1377
1378         rcu_read_lock();
1379         list_for_each_entry_rcu(dp, &ovs_net->dps, list_node) {
1380                 if (i >= skip &&
1381                     ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1382                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
1383                                          OVS_DP_CMD_NEW) < 0)
1384                         break;
1385                 i++;
1386         }
1387         rcu_read_unlock();
1388
1389         cb->args[0] = i;
1390
1391         return skb->len;
1392 }
1393
1394 static const struct genl_ops dp_datapath_genl_ops[] = {
1395         { .cmd = OVS_DP_CMD_NEW,
1396           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1397           .policy = datapath_policy,
1398           .doit = ovs_dp_cmd_new
1399         },
1400         { .cmd = OVS_DP_CMD_DEL,
1401           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1402           .policy = datapath_policy,
1403           .doit = ovs_dp_cmd_del
1404         },
1405         { .cmd = OVS_DP_CMD_GET,
1406           .flags = 0,               /* OK for unprivileged users. */
1407           .policy = datapath_policy,
1408           .doit = ovs_dp_cmd_get,
1409           .dumpit = ovs_dp_cmd_dump
1410         },
1411         { .cmd = OVS_DP_CMD_SET,
1412           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1413           .policy = datapath_policy,
1414           .doit = ovs_dp_cmd_set,
1415         },
1416 };
1417
1418 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
1419         [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1420         [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
1421         [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
1422         [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
1423         [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1424         [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
1425 };
1426
1427 struct genl_family dp_vport_genl_family = {
1428         .id = GENL_ID_GENERATE,
1429         .hdrsize = sizeof(struct ovs_header),
1430         .name = OVS_VPORT_FAMILY,
1431         .version = OVS_VPORT_VERSION,
1432         .maxattr = OVS_VPORT_ATTR_MAX,
1433         .netnsok = true,
1434         .parallel_ops = true,
1435 };
1436
1437 struct genl_multicast_group ovs_dp_vport_multicast_group = {
1438         .name = OVS_VPORT_MCGROUP
1439 };
1440
1441 /* Called with ovs_mutex or RCU read lock. */
1442 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1443                                    u32 portid, u32 seq, u32 flags, u8 cmd)
1444 {
1445         struct ovs_header *ovs_header;
1446         struct ovs_vport_stats vport_stats;
1447         int err;
1448
1449         ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1450                                  flags, cmd);
1451         if (!ovs_header)
1452                 return -EMSGSIZE;
1453
1454         ovs_header->dp_ifindex = get_dpifindex(vport->dp);
1455
1456         if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1457             nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1458             nla_put_string(skb, OVS_VPORT_ATTR_NAME, vport->ops->get_name(vport)) ||
1459             nla_put_u32(skb, OVS_VPORT_ATTR_UPCALL_PID, vport->upcall_portid))
1460                 goto nla_put_failure;
1461
1462         ovs_vport_get_stats(vport, &vport_stats);
1463         if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
1464                     &vport_stats))
1465                 goto nla_put_failure;
1466
1467         err = ovs_vport_get_options(vport, skb);
1468         if (err == -EMSGSIZE)
1469                 goto error;
1470
1471         return genlmsg_end(skb, ovs_header);
1472
1473 nla_put_failure:
1474         err = -EMSGSIZE;
1475 error:
1476         genlmsg_cancel(skb, ovs_header);
1477         return err;
1478 }
1479
1480 /* Called with ovs_mutex or RCU read lock. */
1481 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1482                                          u32 seq, u8 cmd)
1483 {
1484         struct sk_buff *skb;
1485         int retval;
1486
1487         skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
1488         if (!skb)
1489                 return ERR_PTR(-ENOMEM);
1490
1491         retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1492         BUG_ON(retval < 0);
1493
1494         return skb;
1495 }
1496
1497 /* Called with ovs_mutex or RCU read lock. */
1498 static struct vport *lookup_vport(struct net *net,
1499                                   struct ovs_header *ovs_header,
1500                                   struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
1501 {
1502         struct datapath *dp;
1503         struct vport *vport;
1504
1505         if (a[OVS_VPORT_ATTR_NAME]) {
1506                 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1507                 if (!vport)
1508                         return ERR_PTR(-ENODEV);
1509                 if (ovs_header->dp_ifindex &&
1510                     ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1511                         return ERR_PTR(-ENODEV);
1512                 return vport;
1513         } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1514                 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1515
1516                 if (port_no >= DP_MAX_PORTS)
1517                         return ERR_PTR(-EFBIG);
1518
1519                 dp = get_dp(net, ovs_header->dp_ifindex);
1520                 if (!dp)
1521                         return ERR_PTR(-ENODEV);
1522
1523                 vport = ovs_vport_ovsl_rcu(dp, port_no);
1524                 if (!vport)
1525                         return ERR_PTR(-ENODEV);
1526                 return vport;
1527         } else
1528                 return ERR_PTR(-EINVAL);
1529 }
1530
1531 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
1532 {
1533         struct nlattr **a = info->attrs;
1534         struct ovs_header *ovs_header = info->userhdr;
1535         struct vport_parms parms;
1536         struct sk_buff *reply;
1537         struct vport *vport;
1538         struct datapath *dp;
1539         u32 port_no;
1540         int err;
1541
1542         err = -EINVAL;
1543         if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
1544             !a[OVS_VPORT_ATTR_UPCALL_PID])
1545                 goto exit;
1546
1547         ovs_lock();
1548         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1549         err = -ENODEV;
1550         if (!dp)
1551                 goto exit_unlock;
1552
1553         if (a[OVS_VPORT_ATTR_PORT_NO]) {
1554                 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
1555
1556                 err = -EFBIG;
1557                 if (port_no >= DP_MAX_PORTS)
1558                         goto exit_unlock;
1559
1560                 vport = ovs_vport_ovsl(dp, port_no);
1561                 err = -EBUSY;
1562                 if (vport)
1563                         goto exit_unlock;
1564         } else {
1565                 for (port_no = 1; ; port_no++) {
1566                         if (port_no >= DP_MAX_PORTS) {
1567                                 err = -EFBIG;
1568                                 goto exit_unlock;
1569                         }
1570                         vport = ovs_vport_ovsl(dp, port_no);
1571                         if (!vport)
1572                                 break;
1573                 }
1574         }
1575
1576         parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
1577         parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
1578         parms.options = a[OVS_VPORT_ATTR_OPTIONS];
1579         parms.dp = dp;
1580         parms.port_no = port_no;
1581         parms.upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);
1582
1583         vport = new_vport(&parms);
1584         err = PTR_ERR(vport);
1585         if (IS_ERR(vport))
1586                 goto exit_unlock;
1587
1588         err = 0;
1589         reply = ovs_vport_cmd_build_info(vport, info->snd_portid, info->snd_seq,
1590                                          OVS_VPORT_CMD_NEW);
1591         if (IS_ERR(reply)) {
1592                 err = PTR_ERR(reply);
1593                 ovs_dp_detach_port(vport);
1594                 goto exit_unlock;
1595         }
1596
1597         ovs_notify(&dp_vport_genl_family, reply, info);
1598
1599 exit_unlock:
1600         ovs_unlock();
1601 exit:
1602         return err;
1603 }
1604
1605 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
1606 {
1607         struct nlattr **a = info->attrs;
1608         struct sk_buff *reply;
1609         struct vport *vport;
1610         int err;
1611
1612         ovs_lock();
1613         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1614         err = PTR_ERR(vport);
1615         if (IS_ERR(vport))
1616                 goto exit_unlock;
1617
1618         if (a[OVS_VPORT_ATTR_TYPE] &&
1619             nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
1620                 err = -EINVAL;
1621                 goto exit_unlock;
1622         }
1623
1624         reply = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1625         if (!reply) {
1626                 err = -ENOMEM;
1627                 goto exit_unlock;
1628         }
1629
1630         if (a[OVS_VPORT_ATTR_OPTIONS]) {
1631                 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
1632                 if (err)
1633                         goto exit_free;
1634         }
1635
1636         if (a[OVS_VPORT_ATTR_UPCALL_PID])
1637                 vport->upcall_portid = nla_get_u32(a[OVS_VPORT_ATTR_UPCALL_PID]);
1638
1639         err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
1640                                       info->snd_seq, 0, OVS_VPORT_CMD_NEW);
1641         BUG_ON(err < 0);
1642
1643         ovs_unlock();
1644         ovs_notify(&dp_vport_genl_family, reply, info);
1645         return 0;
1646
1647 exit_free:
1648         kfree_skb(reply);
1649 exit_unlock:
1650         ovs_unlock();
1651         return err;
1652 }
1653
1654 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
1655 {
1656         struct nlattr **a = info->attrs;
1657         struct sk_buff *reply;
1658         struct vport *vport;
1659         int err;
1660
1661         ovs_lock();
1662         vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1663         err = PTR_ERR(vport);
1664         if (IS_ERR(vport))
1665                 goto exit_unlock;
1666
1667         if (vport->port_no == OVSP_LOCAL) {
1668                 err = -EINVAL;
1669                 goto exit_unlock;
1670         }
1671
1672         reply = ovs_vport_cmd_build_info(vport, info->snd_portid,
1673                                          info->snd_seq, OVS_VPORT_CMD_DEL);
1674         err = PTR_ERR(reply);
1675         if (IS_ERR(reply))
1676                 goto exit_unlock;
1677
1678         err = 0;
1679         ovs_dp_detach_port(vport);
1680
1681         ovs_notify(&dp_vport_genl_family, reply, info);
1682
1683 exit_unlock:
1684         ovs_unlock();
1685         return err;
1686 }
1687
1688 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
1689 {
1690         struct nlattr **a = info->attrs;
1691         struct ovs_header *ovs_header = info->userhdr;
1692         struct sk_buff *reply;
1693         struct vport *vport;
1694         int err;
1695
1696         rcu_read_lock();
1697         vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
1698         err = PTR_ERR(vport);
1699         if (IS_ERR(vport))
1700                 goto exit_unlock;
1701
1702         reply = ovs_vport_cmd_build_info(vport, info->snd_portid,
1703                                          info->snd_seq, OVS_VPORT_CMD_NEW);
1704         err = PTR_ERR(reply);
1705         if (IS_ERR(reply))
1706                 goto exit_unlock;
1707
1708         rcu_read_unlock();
1709
1710         return genlmsg_reply(reply, info);
1711
1712 exit_unlock:
1713         rcu_read_unlock();
1714         return err;
1715 }
1716
1717 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
1718 {
1719         struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1720         struct datapath *dp;
1721         int bucket = cb->args[0], skip = cb->args[1];
1722         int i, j = 0;
1723
1724         dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1725         if (!dp)
1726                 return -ENODEV;
1727
1728         rcu_read_lock();
1729         for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
1730                 struct vport *vport;
1731
1732                 j = 0;
1733                 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
1734                         if (j >= skip &&
1735                             ovs_vport_cmd_fill_info(vport, skb,
1736                                                     NETLINK_CB(cb->skb).portid,
1737                                                     cb->nlh->nlmsg_seq,
1738                                                     NLM_F_MULTI,
1739                                                     OVS_VPORT_CMD_NEW) < 0)
1740                                 goto out;
1741
1742                         j++;
1743                 }
1744                 skip = 0;
1745         }
1746 out:
1747         rcu_read_unlock();
1748
1749         cb->args[0] = i;
1750         cb->args[1] = j;
1751
1752         return skb->len;
1753 }
1754
1755 static const struct genl_ops dp_vport_genl_ops[] = {
1756         { .cmd = OVS_VPORT_CMD_NEW,
1757           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1758           .policy = vport_policy,
1759           .doit = ovs_vport_cmd_new
1760         },
1761         { .cmd = OVS_VPORT_CMD_DEL,
1762           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1763           .policy = vport_policy,
1764           .doit = ovs_vport_cmd_del
1765         },
1766         { .cmd = OVS_VPORT_CMD_GET,
1767           .flags = 0,               /* OK for unprivileged users. */
1768           .policy = vport_policy,
1769           .doit = ovs_vport_cmd_get,
1770           .dumpit = ovs_vport_cmd_dump
1771         },
1772         { .cmd = OVS_VPORT_CMD_SET,
1773           .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
1774           .policy = vport_policy,
1775           .doit = ovs_vport_cmd_set,
1776         },
1777 };
1778
1779 struct genl_family_and_ops {
1780         struct genl_family *family;
1781         const struct genl_ops *ops;
1782         int n_ops;
1783         const struct genl_multicast_group *group;
1784 };
1785
1786 static const struct genl_family_and_ops dp_genl_families[] = {
1787         { &dp_datapath_genl_family,
1788           dp_datapath_genl_ops, ARRAY_SIZE(dp_datapath_genl_ops),
1789           &ovs_dp_datapath_multicast_group },
1790         { &dp_vport_genl_family,
1791           dp_vport_genl_ops, ARRAY_SIZE(dp_vport_genl_ops),
1792           &ovs_dp_vport_multicast_group },
1793         { &dp_flow_genl_family,
1794           dp_flow_genl_ops, ARRAY_SIZE(dp_flow_genl_ops),
1795           &ovs_dp_flow_multicast_group },
1796         { &dp_packet_genl_family,
1797           dp_packet_genl_ops, ARRAY_SIZE(dp_packet_genl_ops),
1798           NULL },
1799 };
1800
1801 static void dp_unregister_genl(int n_families)
1802 {
1803         int i;
1804
1805         for (i = 0; i < n_families; i++)
1806                 genl_unregister_family(dp_genl_families[i].family);
1807 }
1808
1809 static int dp_register_genl(void)
1810 {
1811         int n_registered;
1812         int err;
1813         int i;
1814
1815         n_registered = 0;
1816         for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
1817                 const struct genl_family_and_ops *f = &dp_genl_families[i];
1818
1819                 f->family->ops = f->ops;
1820                 f->family->n_ops = f->n_ops;
1821                 f->family->mcgrps = f->group;
1822                 f->family->n_mcgrps = f->group ? 1 : 0;
1823                 err = genl_register_family(f->family);
1824                 if (err)
1825                         goto error;
1826                 n_registered++;
1827         }
1828
1829         return 0;
1830
1831 error:
1832         dp_unregister_genl(n_registered);
1833         return err;
1834 }
1835
1836 static int __net_init ovs_init_net(struct net *net)
1837 {
1838         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
1839
1840         INIT_LIST_HEAD(&ovs_net->dps);
1841         INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
1842         return 0;
1843 }
1844
1845 static void __net_exit ovs_exit_net(struct net *net)
1846 {
1847         struct datapath *dp, *dp_next;
1848         struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
1849
1850         ovs_lock();
1851         list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
1852                 __dp_destroy(dp);
1853         ovs_unlock();
1854
1855         cancel_work_sync(&ovs_net->dp_notify_work);
1856 }
1857
1858 static struct pernet_operations ovs_net_ops = {
1859         .init = ovs_init_net,
1860         .exit = ovs_exit_net,
1861         .id   = &ovs_net_id,
1862         .size = sizeof(struct ovs_net),
1863 };
1864
1865 static int __init dp_init(void)
1866 {
1867         int err;
1868
1869         BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
1870
1871         pr_info("Open vSwitch switching datapath\n");
1872
1873         err = ovs_flow_init();
1874         if (err)
1875                 goto error;
1876
1877         err = ovs_vport_init();
1878         if (err)
1879                 goto error_flow_exit;
1880
1881         err = register_pernet_device(&ovs_net_ops);
1882         if (err)
1883                 goto error_vport_exit;
1884
1885         err = register_netdevice_notifier(&ovs_dp_device_notifier);
1886         if (err)
1887                 goto error_netns_exit;
1888
1889         err = dp_register_genl();
1890         if (err < 0)
1891                 goto error_unreg_notifier;
1892
1893         return 0;
1894
1895 error_unreg_notifier:
1896         unregister_netdevice_notifier(&ovs_dp_device_notifier);
1897 error_netns_exit:
1898         unregister_pernet_device(&ovs_net_ops);
1899 error_vport_exit:
1900         ovs_vport_exit();
1901 error_flow_exit:
1902         ovs_flow_exit();
1903 error:
1904         return err;
1905 }
1906
1907 static void dp_cleanup(void)
1908 {
1909         dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
1910         unregister_netdevice_notifier(&ovs_dp_device_notifier);
1911         unregister_pernet_device(&ovs_net_ops);
1912         rcu_barrier();
1913         ovs_vport_exit();
1914         ovs_flow_exit();
1915 }
1916
1917 module_init(dp_init);
1918 module_exit(dp_cleanup);
1919
1920 MODULE_DESCRIPTION("Open vSwitch switching datapath");
1921 MODULE_LICENSE("GPL");