Pileus Git - ~andy/linux/blob - drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/sctp.h>
  46 #include <linux/ipv6.h>
  47 #include <linux/slab.h>
  48 #include <net/checksum.h>
  49 #include <net/ip6_checksum.h>
  50 #include <linux/ethtool.h>
  51 #include <linux/if.h>
  52 #include <linux/if_vlan.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "ixgbevf.h"
  56
  57 const char ixgbevf_driver_name[] = "ixgbevf";
  58 static const char ixgbevf_driver_string[] =
  59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  60
  61 #define DRV_VERSION "2.11.3-k"
  62 const char ixgbevf_driver_version[] = DRV_VERSION;
  63 static char ixgbevf_copyright[] =
  64         "Copyright (c) 2009 - 2012 Intel Corporation.";
  65
  66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  67         [board_82599_vf] = &ixgbevf_82599_vf_info,
  68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  69 };
  70
  71 /* ixgbevf_pci_tbl - PCI Device ID Table
  72  *
  73  * Wildcard entries (PCI_ANY_ID) should come last
  74  * Last entry must be all 0s
  75  *
  76  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  77  *   Class, Class Mask, private data (not used) }
  78  */
  79 static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = {
  80         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
  81         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
  82         /* required last entry */
  83         {0, }
  84 };
  85 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  86
  87 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  88 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
  89 MODULE_LICENSE("GPL");
  90 MODULE_VERSION(DRV_VERSION);
  91
  92 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  93 static int debug = -1;
  94 module_param(debug, int, 0);
  95 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  96
  97 /* forward decls */
  98 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
  99 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
 100
 101 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
 102                                            struct ixgbevf_ring *rx_ring,
 103                                            u32 val)
 104 {
 105         /*
 106          * Force memory writes to complete before letting h/w
 107          * know there are new descriptors to fetch.  (Only
 108          * applicable for weak-ordered memory model archs,
 109          * such as IA-64).
 110          */
 111         wmb();
 112         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
 113 }
 114
 115 /**
 116  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 117  * @adapter: pointer to adapter struct
 118  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 119  * @queue: queue to map the corresponding interrupt to
 120  * @msix_vector: the vector to map to the corresponding queue
 121  */
 122 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 123                              u8 queue, u8 msix_vector)
 124 {
 125         u32 ivar, index;
 126         struct ixgbe_hw *hw = &adapter->hw;
 127         if (direction == -1) {
 128                 /* other causes */
 129                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 130                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 131                 ivar &= ~0xFF;
 132                 ivar |= msix_vector;
 133                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 134         } else {
 135                 /* tx or rx causes */
 136                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 137                 index = ((16 * (queue & 1)) + (8 * direction));
 138                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 139                 ivar &= ~(0xFF << index);
 140                 ivar |= (msix_vector << index);
 141                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 142         }
 143 }
 144
 145 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
 146                                                struct ixgbevf_tx_buffer
 147                                                *tx_buffer_info)
 148 {
 149         if (tx_buffer_info->dma) {
 150                 if (tx_buffer_info->mapped_as_page)
 151                         dma_unmap_page(tx_ring->dev,
 152                                        tx_buffer_info->dma,
 153                                        tx_buffer_info->length,
 154                                        DMA_TO_DEVICE);
 155                 else
 156                         dma_unmap_single(tx_ring->dev,
 157                                          tx_buffer_info->dma,
 158                                          tx_buffer_info->length,
 159                                          DMA_TO_DEVICE);
 160                 tx_buffer_info->dma = 0;
 161         }
 162         if (tx_buffer_info->skb) {
 163                 dev_kfree_skb_any(tx_buffer_info->skb);
 164                 tx_buffer_info->skb = NULL;
 165         }
 166         tx_buffer_info->time_stamp = 0;
 167         /* tx_buffer_info must be completely set up in the transmit path */
 168 }
 169
 170 #define IXGBE_MAX_TXD_PWR       14
 171 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 172
 173 /* Tx Descriptors needed, worst case */
 174 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 175 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 176
 177 static void ixgbevf_tx_timeout(struct net_device *netdev);
 178
 179 /**
 180  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 181  * @q_vector: board private structure
 182  * @tx_ring: tx ring to clean
 183  **/
 184 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 185                                  struct ixgbevf_ring *tx_ring)
 186 {
 187         struct ixgbevf_adapter *adapter = q_vector->adapter;
 188         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
 189         struct ixgbevf_tx_buffer *tx_buffer_info;
 190         unsigned int i, count = 0;
 191         unsigned int total_bytes = 0, total_packets = 0;
 192
 193         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 194                 return true;
 195
 196         i = tx_ring->next_to_clean;
 197         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 198         eop_desc = tx_buffer_info->next_to_watch;
 199
 200         do {
 201                 bool cleaned = false;
 202
 203                 /* if next_to_watch is not set then there is no work pending */
 204                 if (!eop_desc)
 205                         break;
 206
 207                 /* prevent any other reads prior to eop_desc */
 208                 read_barrier_depends();
 209
 210                 /* if DD is not set pending work has not been completed */
 211                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
 212                         break;
 213
 214                 /* clear next_to_watch to prevent false hangs */
 215                 tx_buffer_info->next_to_watch = NULL;
 216
 217                 for ( ; !cleaned; count++) {
 218                         struct sk_buff *skb;
 219                         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 220                         cleaned = (tx_desc == eop_desc);
 221                         skb = tx_buffer_info->skb;
 222
 223                         if (cleaned && skb) {
 224                                 unsigned int segs, bytecount;
 225
 226                                 /* gso_segs is currently only valid for tcp */
 227                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
 228                                 /* multiply data chunks by size of headers */
 229                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
 230                                             skb->len;
 231                                 total_packets += segs;
 232                                 total_bytes += bytecount;
 233                         }
 234
 235                         ixgbevf_unmap_and_free_tx_resource(tx_ring,
 236                                                            tx_buffer_info);
 237
 238                         tx_desc->wb.status = 0;
 239
 240                         i++;
 241                         if (i == tx_ring->count)
 242                                 i = 0;
 243
 244                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 245                 }
 246
 247                 eop_desc = tx_buffer_info->next_to_watch;
 248         } while (count < tx_ring->count);
 249
 250         tx_ring->next_to_clean = i;
 251
 252 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 253         if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
 254                      (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 255                 /* Make sure that anybody stopping the queue after this
 256                  * sees the new next_to_clean.
 257                  */
 258                 smp_mb();
 259                 if (__netif_subqueue_stopped(tx_ring->netdev,
 260                                              tx_ring->queue_index) &&
 261                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 262                         netif_wake_subqueue(tx_ring->netdev,
 263                                             tx_ring->queue_index);
 264                         ++adapter->restart_queue;
 265                 }
 266         }
 267
 268         u64_stats_update_begin(&tx_ring->syncp);
 269         tx_ring->total_bytes += total_bytes;
 270         tx_ring->total_packets += total_packets;
 271         u64_stats_update_end(&tx_ring->syncp);
 272         q_vector->tx.total_bytes += total_bytes;
 273         q_vector->tx.total_packets += total_packets;
 274
 275         return count < tx_ring->count;
 276 }
 277
 278 /**
 279  * ixgbevf_receive_skb - Send a completed packet up the stack
 280  * @q_vector: structure containing interrupt and ring information
 281  * @skb: packet to send up
 282  * @status: hardware indication of status of receive
 283  * @rx_desc: rx descriptor
 284  **/
 285 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
 286                                 struct sk_buff *skb, u8 status,
 287                                 union ixgbe_adv_rx_desc *rx_desc)
 288 {
 289         struct ixgbevf_adapter *adapter = q_vector->adapter;
 290         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
 291         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
 292
 293         if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
 294                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
 295
 296         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
 297                 napi_gro_receive(&q_vector->napi, skb);
 298         else
 299                 netif_rx(skb);
 300 }
 301
 302 /**
 303  * ixgbevf_rx_skb - Helper function to determine proper Rx method
 304  * @q_vector: structure containing interrupt and ring information
 305  * @skb: packet to send up
 306  * @status: hardware indication of status of receive
 307  * @rx_desc: rx descriptor
 308  **/
 309 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
 310                            struct sk_buff *skb, u8 status,
 311                            union ixgbe_adv_rx_desc *rx_desc)
 312 {
 313 #ifdef CONFIG_NET_RX_BUSY_POLL
 314         skb_mark_napi_id(skb, &q_vector->napi);
 315
 316         if (ixgbevf_qv_busy_polling(q_vector)) {
 317                 netif_receive_skb(skb);
 318                 /* exit early if we busy polled */
 319                 return;
 320         }
 321 #endif /* CONFIG_NET_RX_BUSY_POLL */
 322
 323         ixgbevf_receive_skb(q_vector, skb, status, rx_desc);
 324 }
 325
 326 /**
 327  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 328  * @ring: pointer to Rx descriptor ring structure
 329  * @status_err: hardware indication of status of receive
 330  * @skb: skb currently being received and modified
 331  **/
 332 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
 333                                        u32 status_err, struct sk_buff *skb)
 334 {
 335         skb_checksum_none_assert(skb);
 336
 337         /* Rx csum disabled */
 338         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 339                 return;
 340
 341         /* if IP and error */
 342         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
 343             (status_err & IXGBE_RXDADV_ERR_IPE)) {
 344                 ring->hw_csum_rx_error++;
 345                 return;
 346         }
 347
 348         if (!(status_err & IXGBE_RXD_STAT_L4CS))
 349                 return;
 350
 351         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
 352                 ring->hw_csum_rx_error++;
 353                 return;
 354         }
 355
 356         /* It must be a TCP or UDP packet with a valid checksum */
 357         skb->ip_summed = CHECKSUM_UNNECESSARY;
 358         ring->hw_csum_rx_good++;
 359 }
 360
 361 /**
 362  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 363  * @adapter: address of board private structure
 364  **/
 365 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
 366                                      struct ixgbevf_ring *rx_ring,
 367                                      int cleaned_count)
 368 {
 369         struct pci_dev *pdev = adapter->pdev;
 370         union ixgbe_adv_rx_desc *rx_desc;
 371         struct ixgbevf_rx_buffer *bi;
 372         unsigned int i = rx_ring->next_to_use;
 373
 374         bi = &rx_ring->rx_buffer_info[i];
 375
 376         while (cleaned_count--) {
 377                 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 378
 379                 if (!bi->skb) {
 380                         struct sk_buff *skb;
 381
 382                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
 383                                                         rx_ring->rx_buf_len);
 384                         if (!skb) {
 385                                 adapter->alloc_rx_buff_failed++;
 386                                 goto no_buffers;
 387                         }
 388                         bi->skb = skb;
 389
 390                         bi->dma = dma_map_single(&pdev->dev, skb->data,
 391                                                  rx_ring->rx_buf_len,
 392                                                  DMA_FROM_DEVICE);
 393                         if (dma_mapping_error(&pdev->dev, bi->dma)) {
 394                                 dev_kfree_skb(skb);
 395                                 bi->skb = NULL;
 396                                 dev_err(&pdev->dev, "RX DMA map failed\n");
 397                                 break;
 398                         }
 399                 }
 400                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
 401
 402                 i++;
 403                 if (i == rx_ring->count)
 404                         i = 0;
 405                 bi = &rx_ring->rx_buffer_info[i];
 406         }
 407
 408 no_buffers:
 409         if (rx_ring->next_to_use != i) {
 410                 rx_ring->next_to_use = i;
 411                 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
 412         }
 413 }
 414
 415 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 416                                              u32 qmask)
 417 {
 418         struct ixgbe_hw *hw = &adapter->hw;
 419
 420         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 421 }
 422
 423 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 424                                 struct ixgbevf_ring *rx_ring,
 425                                 int budget)
 426 {
 427         struct ixgbevf_adapter *adapter = q_vector->adapter;
 428         struct pci_dev *pdev = adapter->pdev;
 429         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
 430         struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
 431         struct sk_buff *skb;
 432         unsigned int i;
 433         u32 len, staterr;
 434         int cleaned_count = 0;
 435         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 436
 437         i = rx_ring->next_to_clean;
 438         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 439         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 440         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 441
 442         while (staterr & IXGBE_RXD_STAT_DD) {
 443                 if (!budget)
 444                         break;
 445                 budget--;
 446
 447                 rmb(); /* read descriptor and rx_buffer_info after status DD */
 448                 len = le16_to_cpu(rx_desc->wb.upper.length);
 449                 skb = rx_buffer_info->skb;
 450                 prefetch(skb->data - NET_IP_ALIGN);
 451                 rx_buffer_info->skb = NULL;
 452
 453                 if (rx_buffer_info->dma) {
 454                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
 455                                          rx_ring->rx_buf_len,
 456                                          DMA_FROM_DEVICE);
 457                         rx_buffer_info->dma = 0;
 458                         skb_put(skb, len);
 459                 }
 460
 461                 i++;
 462                 if (i == rx_ring->count)
 463                         i = 0;
 464
 465                 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
 466                 prefetch(next_rxd);
 467                 cleaned_count++;
 468
 469                 next_buffer = &rx_ring->rx_buffer_info[i];
 470
 471                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
 472                         skb->next = next_buffer->skb;
 473                         IXGBE_CB(skb->next)->prev = skb;
 474                         adapter->non_eop_descs++;
 475                         goto next_desc;
 476                 }
 477
 478                 /* we should not be chaining buffers, if we did drop the skb */
 479                 if (IXGBE_CB(skb)->prev) {
 480                         do {
 481                                 struct sk_buff *this = skb;
 482                                 skb = IXGBE_CB(skb)->prev;
 483                                 dev_kfree_skb(this);
 484                         } while (skb);
 485                         goto next_desc;
 486                 }
 487
 488                 /* ERR_MASK will only have valid bits if EOP set */
 489                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
 490                         dev_kfree_skb_irq(skb);
 491                         goto next_desc;
 492                 }
 493
 494                 ixgbevf_rx_checksum(rx_ring, staterr, skb);
 495
 496                 /* probably a little skewed due to removing CRC */
 497                 total_rx_bytes += skb->len;
 498                 total_rx_packets++;
 499
 500                 /*
 501                  * Work around issue of some types of VM to VM loop back
 502                  * packets not getting split correctly
 503                  */
 504                 if (staterr & IXGBE_RXD_STAT_LB) {
 505                         u32 header_fixup_len = skb_headlen(skb);
 506                         if (header_fixup_len < 14)
 507                                 skb_push(skb, header_fixup_len);
 508                 }
 509                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 510
 511                 /* Workaround hardware that can't do proper VEPA multicast
 512                  * source pruning.
 513                  */
 514                 if ((skb->pkt_type & (PACKET_BROADCAST | PACKET_MULTICAST)) &&
 515                     ether_addr_equal(adapter->netdev->dev_addr,
 516                                      eth_hdr(skb)->h_source)) {
 517                         dev_kfree_skb_irq(skb);
 518                         goto next_desc;
 519                 }
 520
 521                 ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc);
 522
 523 next_desc:
 524                 rx_desc->wb.upper.status_error = 0;
 525
 526                 /* return some buffers to hardware, one at a time is too slow */
 527                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 528                         ixgbevf_alloc_rx_buffers(adapter, rx_ring,
 529                                                  cleaned_count);
 530                         cleaned_count = 0;
 531                 }
 532
 533                 /* use prefetched values */
 534                 rx_desc = next_rxd;
 535                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
 536
 537                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 538         }
 539
 540         rx_ring->next_to_clean = i;
 541         cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
 542
 543         if (cleaned_count)
 544                 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
 545
 546         u64_stats_update_begin(&rx_ring->syncp);
 547         rx_ring->total_packets += total_rx_packets;
 548         rx_ring->total_bytes += total_rx_bytes;
 549         u64_stats_update_end(&rx_ring->syncp);
 550         q_vector->rx.total_packets += total_rx_packets;
 551         q_vector->rx.total_bytes += total_rx_bytes;
 552
 553         return total_rx_packets;
 554 }
 555
 556 /**
 557  * ixgbevf_poll - NAPI polling calback
 558  * @napi: napi struct with our devices info in it
 559  * @budget: amount of work driver is allowed to do this pass, in packets
 560  *
 561  * This function will clean more than one or more rings associated with a
 562  * q_vector.
 563  **/
 564 static int ixgbevf_poll(struct napi_struct *napi, int budget)
 565 {
 566         struct ixgbevf_q_vector *q_vector =
 567                 container_of(napi, struct ixgbevf_q_vector, napi);
 568         struct ixgbevf_adapter *adapter = q_vector->adapter;
 569         struct ixgbevf_ring *ring;
 570         int per_ring_budget;
 571         bool clean_complete = true;
 572
 573         ixgbevf_for_each_ring(ring, q_vector->tx)
 574                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
 575
 576 #ifdef CONFIG_NET_RX_BUSY_POLL
 577         if (!ixgbevf_qv_lock_napi(q_vector))
 578                 return budget;
 579 #endif
 580
 581         /* attempt to distribute budget to each queue fairly, but don't allow
 582          * the budget to go below 1 because we'll exit polling */
 583         if (q_vector->rx.count > 1)
 584                 per_ring_budget = max(budget/q_vector->rx.count, 1);
 585         else
 586                 per_ring_budget = budget;
 587
 588         adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
 589         ixgbevf_for_each_ring(ring, q_vector->rx)
 590                 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
 591                                                         per_ring_budget)
 592                                    < per_ring_budget);
 593         adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
 594
 595 #ifdef CONFIG_NET_RX_BUSY_POLL
 596         ixgbevf_qv_unlock_napi(q_vector);
 597 #endif
 598
 599         /* If all work not completed, return budget and keep polling */
 600         if (!clean_complete)
 601                 return budget;
 602         /* all work done, exit the polling mode */
 603         napi_complete(napi);
 604         if (adapter->rx_itr_setting & 1)
 605                 ixgbevf_set_itr(q_vector);
 606         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 607                 ixgbevf_irq_enable_queues(adapter,
 608                                           1 << q_vector->v_idx);
 609
 610         return 0;
 611 }
 612
 613 /**
 614  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 615  * @q_vector: structure containing interrupt and ring information
 616  */
 617 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
 618 {
 619         struct ixgbevf_adapter *adapter = q_vector->adapter;
 620         struct ixgbe_hw *hw = &adapter->hw;
 621         int v_idx = q_vector->v_idx;
 622         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
 623
 624         /*
 625          * set the WDIS bit to not clear the timer bits and cause an
 626          * immediate assertion of the interrupt
 627          */
 628         itr_reg |= IXGBE_EITR_CNT_WDIS;
 629
 630         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 631 }
 632
 633 #ifdef CONFIG_NET_RX_BUSY_POLL
 634 /* must be called with local_bh_disable()d */
 635 static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
 636 {
 637         struct ixgbevf_q_vector *q_vector =
 638                         container_of(napi, struct ixgbevf_q_vector, napi);
 639         struct ixgbevf_adapter *adapter = q_vector->adapter;
 640         struct ixgbevf_ring  *ring;
 641         int found = 0;
 642
 643         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 644                 return LL_FLUSH_FAILED;
 645
 646         if (!ixgbevf_qv_lock_poll(q_vector))
 647                 return LL_FLUSH_BUSY;
 648
 649         ixgbevf_for_each_ring(ring, q_vector->rx) {
 650                 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
 651                 if (found)
 652                         break;
 653         }
 654
 655         ixgbevf_qv_unlock_poll(q_vector);
 656
 657         return found;
 658 }
 659 #endif /* CONFIG_NET_RX_BUSY_POLL */
 660
 661 /**
 662  * ixgbevf_configure_msix - Configure MSI-X hardware
 663  * @adapter: board private structure
 664  *
 665  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
 666  * interrupts.
 667  **/
 668 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
 669 {
 670         struct ixgbevf_q_vector *q_vector;
 671         int q_vectors, v_idx;
 672
 673         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 674         adapter->eims_enable_mask = 0;
 675
 676         /*
 677          * Populate the IVAR table and set the ITR values to the
 678          * corresponding register.
 679          */
 680         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
 681                 struct ixgbevf_ring *ring;
 682                 q_vector = adapter->q_vector[v_idx];
 683
 684                 ixgbevf_for_each_ring(ring, q_vector->rx)
 685                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
 686
 687                 ixgbevf_for_each_ring(ring, q_vector->tx)
 688                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
 689
 690                 if (q_vector->tx.ring && !q_vector->rx.ring) {
 691                         /* tx only vector */
 692                         if (adapter->tx_itr_setting == 1)
 693                                 q_vector->itr = IXGBE_10K_ITR;
 694                         else
 695                                 q_vector->itr = adapter->tx_itr_setting;
 696                 } else {
 697                         /* rx or rx/tx vector */
 698                         if (adapter->rx_itr_setting == 1)
 699                                 q_vector->itr = IXGBE_20K_ITR;
 700                         else
 701                                 q_vector->itr = adapter->rx_itr_setting;
 702                 }
 703
 704                 /* add q_vector eims value to global eims_enable_mask */
 705                 adapter->eims_enable_mask |= 1 << v_idx;
 706
 707                 ixgbevf_write_eitr(q_vector);
 708         }
 709
 710         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
 711         /* setup eims_other and add value to global eims_enable_mask */
 712         adapter->eims_other = 1 << v_idx;
 713         adapter->eims_enable_mask |= adapter->eims_other;
 714 }
 715
 716 enum latency_range {
 717         lowest_latency = 0,
 718         low_latency = 1,
 719         bulk_latency = 2,
 720         latency_invalid = 255
 721 };
 722
 723 /**
 724  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
 725  * @q_vector: structure containing interrupt and ring information
 726  * @ring_container: structure containing ring performance data
 727  *
 728  *      Stores a new ITR value based on packets and byte
 729  *      counts during the last interrupt.  The advantage of per interrupt
 730  *      computation is faster updates and more accurate ITR for the current
 731  *      traffic pattern.  Constants in this function were computed
 732  *      based on theoretical maximum wire speed and thresholds were set based
 733  *      on testing data as well as attempting to minimize response time
 734  *      while increasing bulk throughput.
 735  **/
 736 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
 737                                struct ixgbevf_ring_container *ring_container)
 738 {
 739         int bytes = ring_container->total_bytes;
 740         int packets = ring_container->total_packets;
 741         u32 timepassed_us;
 742         u64 bytes_perint;
 743         u8 itr_setting = ring_container->itr;
 744
 745         if (packets == 0)
 746                 return;
 747
 748         /* simple throttlerate management
 749          *    0-20MB/s lowest (100000 ints/s)
 750          *   20-100MB/s low   (20000 ints/s)
 751          *  100-1249MB/s bulk (8000 ints/s)
 752          */
 753         /* what was last interrupt timeslice? */
 754         timepassed_us = q_vector->itr >> 2;
 755         bytes_perint = bytes / timepassed_us; /* bytes/usec */
 756
 757         switch (itr_setting) {
 758         case lowest_latency:
 759                 if (bytes_perint > 10)
 760                         itr_setting = low_latency;
 761                 break;
 762         case low_latency:
 763                 if (bytes_perint > 20)
 764                         itr_setting = bulk_latency;
 765                 else if (bytes_perint <= 10)
 766                         itr_setting = lowest_latency;
 767                 break;
 768         case bulk_latency:
 769                 if (bytes_perint <= 20)
 770                         itr_setting = low_latency;
 771                 break;
 772         }
 773
 774         /* clear work counters since we have the values we need */
 775         ring_container->total_bytes = 0;
 776         ring_container->total_packets = 0;
 777
 778         /* write updated itr to ring container */
 779         ring_container->itr = itr_setting;
 780 }
 781
 782 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
 783 {
 784         u32 new_itr = q_vector->itr;
 785         u8 current_itr;
 786
 787         ixgbevf_update_itr(q_vector, &q_vector->tx);
 788         ixgbevf_update_itr(q_vector, &q_vector->rx);
 789
 790         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
 791
 792         switch (current_itr) {
 793         /* counts and packets in update_itr are dependent on these numbers */
 794         case lowest_latency:
 795                 new_itr = IXGBE_100K_ITR;
 796                 break;
 797         case low_latency:
 798                 new_itr = IXGBE_20K_ITR;
 799                 break;
 800         case bulk_latency:
 801         default:
 802                 new_itr = IXGBE_8K_ITR;
 803                 break;
 804         }
 805
 806         if (new_itr != q_vector->itr) {
 807                 /* do an exponential smoothing */
 808                 new_itr = (10 * new_itr * q_vector->itr) /
 809                           ((9 * new_itr) + q_vector->itr);
 810
 811                 /* save the algorithm value here */
 812                 q_vector->itr = new_itr;
 813
 814                 ixgbevf_write_eitr(q_vector);
 815         }
 816 }
 817
 818 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
 819 {
 820         struct ixgbevf_adapter *adapter = data;
 821         struct ixgbe_hw *hw = &adapter->hw;
 822
 823         hw->mac.get_link_status = 1;
 824
 825         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 826                 mod_timer(&adapter->watchdog_timer, jiffies);
 827
 828         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
 829
 830         return IRQ_HANDLED;
 831 }
 832
 833 /**
 834  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
 835  * @irq: unused
 836  * @data: pointer to our q_vector struct for this interrupt vector
 837  **/
 838 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
 839 {
 840         struct ixgbevf_q_vector *q_vector = data;
 841
 842         /* EIAM disabled interrupts (on this vector) for us */
 843         if (q_vector->rx.ring || q_vector->tx.ring)
 844                 napi_schedule(&q_vector->napi);
 845
 846         return IRQ_HANDLED;
 847 }
 848
 849 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
 850                                      int r_idx)
 851 {
 852         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 853
 854         a->rx_ring[r_idx].next = q_vector->rx.ring;
 855         q_vector->rx.ring = &a->rx_ring[r_idx];
 856         q_vector->rx.count++;
 857 }
 858
 859 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
 860                                      int t_idx)
 861 {
 862         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 863
 864         a->tx_ring[t_idx].next = q_vector->tx.ring;
 865         q_vector->tx.ring = &a->tx_ring[t_idx];
 866         q_vector->tx.count++;
 867 }
 868
 869 /**
 870  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
 871  * @adapter: board private structure to initialize
 872  *
 873  * This function maps descriptor rings to the queue-specific vectors
 874  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 875  * one vector per ring/queue, but on a constrained vector budget, we
 876  * group the rings as "efficiently" as possible.  You would add new
 877  * mapping configurations in here.
 878  **/
 879 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
 880 {
 881         int q_vectors;
 882         int v_start = 0;
 883         int rxr_idx = 0, txr_idx = 0;
 884         int rxr_remaining = adapter->num_rx_queues;
 885         int txr_remaining = adapter->num_tx_queues;
 886         int i, j;
 887         int rqpv, tqpv;
 888         int err = 0;
 889
 890         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 891
 892         /*
 893          * The ideal configuration...
 894          * We have enough vectors to map one per queue.
 895          */
 896         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
 897                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
 898                         map_vector_to_rxq(adapter, v_start, rxr_idx);
 899
 900                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
 901                         map_vector_to_txq(adapter, v_start, txr_idx);
 902                 goto out;
 903         }
 904
 905         /*
 906          * If we don't have enough vectors for a 1-to-1
 907          * mapping, we'll have to group them so there are
 908          * multiple queues per vector.
 909          */
 910         /* Re-adjusting *qpv takes care of the remainder. */
 911         for (i = v_start; i < q_vectors; i++) {
 912                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
 913                 for (j = 0; j < rqpv; j++) {
 914                         map_vector_to_rxq(adapter, i, rxr_idx);
 915                         rxr_idx++;
 916                         rxr_remaining--;
 917                 }
 918         }
 919         for (i = v_start; i < q_vectors; i++) {
 920                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
 921                 for (j = 0; j < tqpv; j++) {
 922                         map_vector_to_txq(adapter, i, txr_idx);
 923                         txr_idx++;
 924                         txr_remaining--;
 925                 }
 926         }
 927
 928 out:
 929         return err;
 930 }
 931
 932 /**
 933  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
 934  * @adapter: board private structure
 935  *
 936  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
 937  * interrupts from the kernel.
 938  **/
 939 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
 940 {
 941         struct net_device *netdev = adapter->netdev;
 942         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 943         int vector, err;
 944         int ri = 0, ti = 0;
 945
 946         for (vector = 0; vector < q_vectors; vector++) {
 947                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
 948                 struct msix_entry *entry = &adapter->msix_entries[vector];
 949
 950                 if (q_vector->tx.ring && q_vector->rx.ring) {
 951                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 952                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
 953                         ti++;
 954                 } else if (q_vector->rx.ring) {
 955                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 956                                  "%s-%s-%d", netdev->name, "rx", ri++);
 957                 } else if (q_vector->tx.ring) {
 958                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 959                                  "%s-%s-%d", netdev->name, "tx", ti++);
 960                 } else {
 961                         /* skip this unused q_vector */
 962                         continue;
 963                 }
 964                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
 965                                   q_vector->name, q_vector);
 966                 if (err) {
 967                         hw_dbg(&adapter->hw,
 968                                "request_irq failed for MSIX interrupt "
 969                                "Error: %d\n", err);
 970                         goto free_queue_irqs;
 971                 }
 972         }
 973
 974         err = request_irq(adapter->msix_entries[vector].vector,
 975                           &ixgbevf_msix_other, 0, netdev->name, adapter);
 976         if (err) {
 977                 hw_dbg(&adapter->hw,
 978                        "request_irq for msix_other failed: %d\n", err);
 979                 goto free_queue_irqs;
 980         }
 981
 982         return 0;
 983
 984 free_queue_irqs:
 985         while (vector) {
 986                 vector--;
 987                 free_irq(adapter->msix_entries[vector].vector,
 988                          adapter->q_vector[vector]);
 989         }
 990         /* This failure is non-recoverable - it indicates the system is
 991          * out of MSIX vector resources and the VF driver cannot run
 992          * without them.  Set the number of msix vectors to zero
 993          * indicating that not enough can be allocated.  The error
 994          * will be returned to the user indicating device open failed.
 995          * Any further attempts to force the driver to open will also
 996          * fail.  The only way to recover is to unload the driver and
 997          * reload it again.  If the system has recovered some MSIX
 998          * vectors then it may succeed.
 999          */
1000         adapter->num_msix_vectors = 0;
1001         return err;
1002 }
1003
1004 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1005 {
1006         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1007
1008         for (i = 0; i < q_vectors; i++) {
1009                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
1010                 q_vector->rx.ring = NULL;
1011                 q_vector->tx.ring = NULL;
1012                 q_vector->rx.count = 0;
1013                 q_vector->tx.count = 0;
1014         }
1015 }
1016
1017 /**
1018  * ixgbevf_request_irq - initialize interrupts
1019  * @adapter: board private structure
1020  *
1021  * Attempts to configure interrupts using the best available
1022  * capabilities of the hardware and kernel.
1023  **/
1024 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1025 {
1026         int err = 0;
1027
1028         err = ixgbevf_request_msix_irqs(adapter);
1029
1030         if (err)
1031                 hw_dbg(&adapter->hw,
1032                        "request_irq failed, Error %d\n", err);
1033
1034         return err;
1035 }
1036
1037 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1038 {
1039         int i, q_vectors;
1040
1041         q_vectors = adapter->num_msix_vectors;
1042         i = q_vectors - 1;
1043
1044         free_irq(adapter->msix_entries[i].vector, adapter);
1045         i--;
1046
1047         for (; i >= 0; i--) {
1048                 /* free only the irqs that were actually requested */
1049                 if (!adapter->q_vector[i]->rx.ring &&
1050                     !adapter->q_vector[i]->tx.ring)
1051                         continue;
1052
1053                 free_irq(adapter->msix_entries[i].vector,
1054                          adapter->q_vector[i]);
1055         }
1056
1057         ixgbevf_reset_q_vectors(adapter);
1058 }
1059
1060 /**
1061  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1062  * @adapter: board private structure
1063  **/
1064 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1065 {
1066         struct ixgbe_hw *hw = &adapter->hw;
1067         int i;
1068
1069         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1070         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1071         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1072
1073         IXGBE_WRITE_FLUSH(hw);
1074
1075         for (i = 0; i < adapter->num_msix_vectors; i++)
1076                 synchronize_irq(adapter->msix_entries[i].vector);
1077 }
1078
1079 /**
1080  * ixgbevf_irq_enable - Enable default interrupt generation settings
1081  * @adapter: board private structure
1082  **/
1083 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1084 {
1085         struct ixgbe_hw *hw = &adapter->hw;
1086
1087         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1088         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1089         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1090 }
1091
1092 /**
1093  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1094  * @adapter: board private structure
1095  *
1096  * Configure the Tx unit of the MAC after a reset.
1097  **/
1098 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1099 {
1100         u64 tdba;
1101         struct ixgbe_hw *hw = &adapter->hw;
1102         u32 i, j, tdlen, txctrl;
1103
1104         /* Setup the HW Tx Head and Tail descriptor pointers */
1105         for (i = 0; i < adapter->num_tx_queues; i++) {
1106                 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1107                 j = ring->reg_idx;
1108                 tdba = ring->dma;
1109                 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1110                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1111                                 (tdba & DMA_BIT_MASK(32)));
1112                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1113                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1114                 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1115                 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1116                 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1117                 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1118                 /* Disable Tx Head Writeback RO bit, since this hoses
1119                  * bookkeeping if things aren't delivered in order.
1120                  */
1121                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1122                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1123                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1124         }
1125 }
1126
1127 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1128
1129 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1130 {
1131         struct ixgbevf_ring *rx_ring;
1132         struct ixgbe_hw *hw = &adapter->hw;
1133         u32 srrctl;
1134
1135         rx_ring = &adapter->rx_ring[index];
1136
1137         srrctl = IXGBE_SRRCTL_DROP_EN;
1138
1139         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1140
1141         srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1142                   IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1143
1144         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1145 }
1146
1147 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1148 {
1149         struct ixgbe_hw *hw = &adapter->hw;
1150
1151         /* PSRTYPE must be initialized in 82599 */
1152         u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1153                       IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1154                       IXGBE_PSRTYPE_L2HDR;
1155
1156         if (adapter->num_rx_queues > 1)
1157                 psrtype |= 1 << 29;
1158
1159         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1160 }
1161
1162 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1163 {
1164         struct ixgbe_hw *hw = &adapter->hw;
1165         struct net_device *netdev = adapter->netdev;
1166         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1167         int i;
1168         u16 rx_buf_len;
1169
1170         /* notify the PF of our intent to use this size of frame */
1171         ixgbevf_rlpml_set_vf(hw, max_frame);
1172
1173         /* PF will allow an extra 4 bytes past for vlan tagged frames */
1174         max_frame += VLAN_HLEN;
1175
1176         /*
1177          * Allocate buffer sizes that fit well into 32K and
1178          * take into account max frame size of 9.5K
1179          */
1180         if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1181             (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1182                 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1183         else if (max_frame <= IXGBEVF_RXBUFFER_2K)
1184                 rx_buf_len = IXGBEVF_RXBUFFER_2K;
1185         else if (max_frame <= IXGBEVF_RXBUFFER_4K)
1186                 rx_buf_len = IXGBEVF_RXBUFFER_4K;
1187         else if (max_frame <= IXGBEVF_RXBUFFER_8K)
1188                 rx_buf_len = IXGBEVF_RXBUFFER_8K;
1189         else
1190                 rx_buf_len = IXGBEVF_RXBUFFER_10K;
1191
1192         for (i = 0; i < adapter->num_rx_queues; i++)
1193                 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1194 }
1195
1196 /**
1197  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1198  * @adapter: board private structure
1199  *
1200  * Configure the Rx unit of the MAC after a reset.
1201  **/
1202 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1203 {
1204         u64 rdba;
1205         struct ixgbe_hw *hw = &adapter->hw;
1206         int i, j;
1207         u32 rdlen;
1208
1209         ixgbevf_setup_psrtype(adapter);
1210
1211         /* set_rx_buffer_len must be called before ring initialization */
1212         ixgbevf_set_rx_buffer_len(adapter);
1213
1214         rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1215         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1216          * the Base and Length of the Rx Descriptor Ring */
1217         for (i = 0; i < adapter->num_rx_queues; i++) {
1218                 rdba = adapter->rx_ring[i].dma;
1219                 j = adapter->rx_ring[i].reg_idx;
1220                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1221                                 (rdba & DMA_BIT_MASK(32)));
1222                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1223                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1224                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1225                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1226                 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1227                 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1228
1229                 ixgbevf_configure_srrctl(adapter, j);
1230         }
1231 }
1232
1233 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1234                                    __be16 proto, u16 vid)
1235 {
1236         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1237         struct ixgbe_hw *hw = &adapter->hw;
1238         int err;
1239
1240         spin_lock_bh(&adapter->mbx_lock);
1241
1242         /* add VID to filter table */
1243         err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1244
1245         spin_unlock_bh(&adapter->mbx_lock);
1246
1247         /* translate error return types so error makes sense */
1248         if (err == IXGBE_ERR_MBX)
1249                 return -EIO;
1250
1251         if (err == IXGBE_ERR_INVALID_ARGUMENT)
1252                 return -EACCES;
1253
1254         set_bit(vid, adapter->active_vlans);
1255
1256         return err;
1257 }
1258
1259 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1260                                     __be16 proto, u16 vid)
1261 {
1262         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1263         struct ixgbe_hw *hw = &adapter->hw;
1264         int err = -EOPNOTSUPP;
1265
1266         spin_lock_bh(&adapter->mbx_lock);
1267
1268         /* remove VID from filter table */
1269         err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1270
1271         spin_unlock_bh(&adapter->mbx_lock);
1272
1273         clear_bit(vid, adapter->active_vlans);
1274
1275         return err;
1276 }
1277
1278 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1279 {
1280         u16 vid;
1281
1282         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1283                 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1284                                         htons(ETH_P_8021Q), vid);
1285 }
1286
1287 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1288 {
1289         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1290         struct ixgbe_hw *hw = &adapter->hw;
1291         int count = 0;
1292
1293         if ((netdev_uc_count(netdev)) > 10) {
1294                 pr_err("Too many unicast filters - No Space\n");
1295                 return -ENOSPC;
1296         }
1297
1298         if (!netdev_uc_empty(netdev)) {
1299                 struct netdev_hw_addr *ha;
1300                 netdev_for_each_uc_addr(ha, netdev) {
1301                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1302                         udelay(200);
1303                 }
1304         } else {
1305                 /*
1306                  * If the list is empty then send message to PF driver to
1307                  * clear all macvlans on this VF.
1308                  */
1309                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1310         }
1311
1312         return count;
1313 }
1314
1315 /**
1316  * ixgbevf_set_rx_mode - Multicast and unicast set
1317  * @netdev: network interface device structure
1318  *
1319  * The set_rx_method entry point is called whenever the multicast address
1320  * list, unicast address list or the network interface flags are updated.
1321  * This routine is responsible for configuring the hardware for proper
1322  * multicast mode and configuring requested unicast filters.
1323  **/
1324 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1325 {
1326         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1327         struct ixgbe_hw *hw = &adapter->hw;
1328
1329         spin_lock_bh(&adapter->mbx_lock);
1330
1331         /* reprogram multicast list */
1332         hw->mac.ops.update_mc_addr_list(hw, netdev);
1333
1334         ixgbevf_write_uc_addr_list(netdev);
1335
1336         spin_unlock_bh(&adapter->mbx_lock);
1337 }
1338
1339 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1340 {
1341         int q_idx;
1342         struct ixgbevf_q_vector *q_vector;
1343         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1344
1345         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1346                 q_vector = adapter->q_vector[q_idx];
1347 #ifdef CONFIG_NET_RX_BUSY_POLL
1348                 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1349 #endif
1350                 napi_enable(&q_vector->napi);
1351         }
1352 }
1353
1354 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1355 {
1356         int q_idx;
1357         struct ixgbevf_q_vector *q_vector;
1358         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1359
1360         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1361                 q_vector = adapter->q_vector[q_idx];
1362                 napi_disable(&q_vector->napi);
1363 #ifdef CONFIG_NET_RX_BUSY_POLL
1364                 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1365                         pr_info("QV %d locked\n", q_idx);
1366                         usleep_range(1000, 20000);
1367                 }
1368 #endif /* CONFIG_NET_RX_BUSY_POLL */
1369         }
1370 }
1371
1372 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1373 {
1374         struct net_device *netdev = adapter->netdev;
1375         int i;
1376
1377         ixgbevf_set_rx_mode(netdev);
1378
1379         ixgbevf_restore_vlan(adapter);
1380
1381         ixgbevf_configure_tx(adapter);
1382         ixgbevf_configure_rx(adapter);
1383         for (i = 0; i < adapter->num_rx_queues; i++) {
1384                 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1385                 ixgbevf_alloc_rx_buffers(adapter, ring,
1386                                          IXGBE_DESC_UNUSED(ring));
1387         }
1388 }
1389
1390 #define IXGBEVF_MAX_RX_DESC_POLL 10
1391 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1392                                          int rxr)
1393 {
1394         struct ixgbe_hw *hw = &adapter->hw;
1395         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1396         u32 rxdctl;
1397         int j = adapter->rx_ring[rxr].reg_idx;
1398
1399         do {
1400                 usleep_range(1000, 2000);
1401                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1402         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1403
1404         if (!wait_loop)
1405                 hw_dbg(hw, "RXDCTL.ENABLE queue %d not set while polling\n",
1406                        rxr);
1407
1408         ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1409                                 (adapter->rx_ring[rxr].count - 1));
1410 }
1411
1412 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1413                                      struct ixgbevf_ring *ring)
1414 {
1415         struct ixgbe_hw *hw = &adapter->hw;
1416         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1417         u32 rxdctl;
1418         u8 reg_idx = ring->reg_idx;
1419
1420         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1421         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1422
1423         /* write value back with RXDCTL.ENABLE bit cleared */
1424         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1425
1426         /* the hardware may take up to 100us to really disable the rx queue */
1427         do {
1428                 udelay(10);
1429                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1430         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1431
1432         if (!wait_loop)
1433                 hw_dbg(hw, "RXDCTL.ENABLE queue %d not cleared while polling\n",
1434                        reg_idx);
1435 }
1436
1437 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1438 {
1439         /* Only save pre-reset stats if there are some */
1440         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1441                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1442                         adapter->stats.base_vfgprc;
1443                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1444                         adapter->stats.base_vfgptc;
1445                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1446                         adapter->stats.base_vfgorc;
1447                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1448                         adapter->stats.base_vfgotc;
1449                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1450                         adapter->stats.base_vfmprc;
1451         }
1452 }
1453
1454 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1455 {
1456         struct ixgbe_hw *hw = &adapter->hw;
1457
1458         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1459         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1460         adapter->stats.last_vfgorc |=
1461                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1462         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1463         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1464         adapter->stats.last_vfgotc |=
1465                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1466         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1467
1468         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1469         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1470         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1471         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1472         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1473 }
1474
1475 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1476 {
1477         struct ixgbe_hw *hw = &adapter->hw;
1478         int api[] = { ixgbe_mbox_api_11,
1479                       ixgbe_mbox_api_10,
1480                       ixgbe_mbox_api_unknown };
1481         int err = 0, idx = 0;
1482
1483         spin_lock_bh(&adapter->mbx_lock);
1484
1485         while (api[idx] != ixgbe_mbox_api_unknown) {
1486                 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1487                 if (!err)
1488                         break;
1489                 idx++;
1490         }
1491
1492         spin_unlock_bh(&adapter->mbx_lock);
1493 }
1494
1495 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1496 {
1497         struct net_device *netdev = adapter->netdev;
1498         struct ixgbe_hw *hw = &adapter->hw;
1499         int i, j = 0;
1500         int num_rx_rings = adapter->num_rx_queues;
1501         u32 txdctl, rxdctl;
1502
1503         for (i = 0; i < adapter->num_tx_queues; i++) {
1504                 j = adapter->tx_ring[i].reg_idx;
1505                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1506                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1507                 txdctl |= (8 << 16);
1508                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1509         }
1510
1511         for (i = 0; i < adapter->num_tx_queues; i++) {
1512                 j = adapter->tx_ring[i].reg_idx;
1513                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1514                 txdctl |= IXGBE_TXDCTL_ENABLE;
1515                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1516         }
1517
1518         for (i = 0; i < num_rx_rings; i++) {
1519                 j = adapter->rx_ring[i].reg_idx;
1520                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1521                 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1522                 if (hw->mac.type == ixgbe_mac_X540_vf) {
1523                         rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1524                         rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1525                                    IXGBE_RXDCTL_RLPML_EN);
1526                 }
1527                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1528                 ixgbevf_rx_desc_queue_enable(adapter, i);
1529         }
1530
1531         ixgbevf_configure_msix(adapter);
1532
1533         spin_lock_bh(&adapter->mbx_lock);
1534
1535         if (is_valid_ether_addr(hw->mac.addr))
1536                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1537         else
1538                 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1539
1540         spin_unlock_bh(&adapter->mbx_lock);
1541
1542         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1543         ixgbevf_napi_enable_all(adapter);
1544
1545         /* enable transmits */
1546         netif_tx_start_all_queues(netdev);
1547
1548         ixgbevf_save_reset_stats(adapter);
1549         ixgbevf_init_last_counter_stats(adapter);
1550
1551         hw->mac.get_link_status = 1;
1552         mod_timer(&adapter->watchdog_timer, jiffies);
1553 }
1554
1555 static int ixgbevf_reset_queues(struct ixgbevf_adapter *adapter)
1556 {
1557         struct ixgbe_hw *hw = &adapter->hw;
1558         struct ixgbevf_ring *rx_ring;
1559         unsigned int def_q = 0;
1560         unsigned int num_tcs = 0;
1561         unsigned int num_rx_queues = 1;
1562         int err, i;
1563
1564         spin_lock_bh(&adapter->mbx_lock);
1565
1566         /* fetch queue configuration from the PF */
1567         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1568
1569         spin_unlock_bh(&adapter->mbx_lock);
1570
1571         if (err)
1572                 return err;
1573
1574         if (num_tcs > 1) {
1575                 /* update default Tx ring register index */
1576                 adapter->tx_ring[0].reg_idx = def_q;
1577
1578                 /* we need as many queues as traffic classes */
1579                 num_rx_queues = num_tcs;
1580         }
1581
1582         /* nothing to do if we have the correct number of queues */
1583         if (adapter->num_rx_queues == num_rx_queues)
1584                 return 0;
1585
1586         /* allocate new rings */
1587         rx_ring = kcalloc(num_rx_queues,
1588                           sizeof(struct ixgbevf_ring), GFP_KERNEL);
1589         if (!rx_ring)
1590                 return -ENOMEM;
1591
1592         /* setup ring fields */
1593         for (i = 0; i < num_rx_queues; i++) {
1594                 rx_ring[i].count = adapter->rx_ring_count;
1595                 rx_ring[i].queue_index = i;
1596                 rx_ring[i].reg_idx = i;
1597                 rx_ring[i].dev = &adapter->pdev->dev;
1598                 rx_ring[i].netdev = adapter->netdev;
1599
1600                 /* allocate resources on the ring */
1601                 err = ixgbevf_setup_rx_resources(adapter, &rx_ring[i]);
1602                 if (err) {
1603                         while (i) {
1604                                 i--;
1605                                 ixgbevf_free_rx_resources(adapter, &rx_ring[i]);
1606                         }
1607                         kfree(rx_ring);
1608                         return err;
1609                 }
1610         }
1611
1612         /* free the existing rings and queues */
1613         ixgbevf_free_all_rx_resources(adapter);
1614         adapter->num_rx_queues = 0;
1615         kfree(adapter->rx_ring);
1616
1617         /* move new rings into position on the adapter struct */
1618         adapter->rx_ring = rx_ring;
1619         adapter->num_rx_queues = num_rx_queues;
1620
1621         /* reset ring to vector mapping */
1622         ixgbevf_reset_q_vectors(adapter);
1623         ixgbevf_map_rings_to_vectors(adapter);
1624
1625         return 0;
1626 }
1627
1628 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1629 {
1630         struct ixgbe_hw *hw = &adapter->hw;
1631
1632         ixgbevf_reset_queues(adapter);
1633
1634         ixgbevf_configure(adapter);
1635
1636         ixgbevf_up_complete(adapter);
1637
1638         /* clear any pending interrupts, may auto mask */
1639         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1640
1641         ixgbevf_irq_enable(adapter);
1642 }
1643
1644 /**
1645  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1646  * @adapter: board private structure
1647  * @rx_ring: ring to free buffers from
1648  **/
1649 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1650                                   struct ixgbevf_ring *rx_ring)
1651 {
1652         struct pci_dev *pdev = adapter->pdev;
1653         unsigned long size;
1654         unsigned int i;
1655
1656         if (!rx_ring->rx_buffer_info)
1657                 return;
1658
1659         /* Free all the Rx ring sk_buffs */
1660         for (i = 0; i < rx_ring->count; i++) {
1661                 struct ixgbevf_rx_buffer *rx_buffer_info;
1662
1663                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1664                 if (rx_buffer_info->dma) {
1665                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
1666                                          rx_ring->rx_buf_len,
1667                                          DMA_FROM_DEVICE);
1668                         rx_buffer_info->dma = 0;
1669                 }
1670                 if (rx_buffer_info->skb) {
1671                         struct sk_buff *skb = rx_buffer_info->skb;
1672                         rx_buffer_info->skb = NULL;
1673                         do {
1674                                 struct sk_buff *this = skb;
1675                                 skb = IXGBE_CB(skb)->prev;
1676                                 dev_kfree_skb(this);
1677                         } while (skb);
1678                 }
1679         }
1680
1681         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1682         memset(rx_ring->rx_buffer_info, 0, size);
1683
1684         /* Zero out the descriptor ring */
1685         memset(rx_ring->desc, 0, rx_ring->size);
1686
1687         rx_ring->next_to_clean = 0;
1688         rx_ring->next_to_use = 0;
1689
1690         if (rx_ring->head)
1691                 writel(0, adapter->hw.hw_addr + rx_ring->head);
1692         if (rx_ring->tail)
1693                 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1694 }
1695
1696 /**
1697  * ixgbevf_clean_tx_ring - Free Tx Buffers
1698  * @adapter: board private structure
1699  * @tx_ring: ring to be cleaned
1700  **/
1701 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1702                                   struct ixgbevf_ring *tx_ring)
1703 {
1704         struct ixgbevf_tx_buffer *tx_buffer_info;
1705         unsigned long size;
1706         unsigned int i;
1707
1708         if (!tx_ring->tx_buffer_info)
1709                 return;
1710
1711         /* Free all the Tx ring sk_buffs */
1712         for (i = 0; i < tx_ring->count; i++) {
1713                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1714                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
1715         }
1716
1717         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1718         memset(tx_ring->tx_buffer_info, 0, size);
1719
1720         memset(tx_ring->desc, 0, tx_ring->size);
1721
1722         tx_ring->next_to_use = 0;
1723         tx_ring->next_to_clean = 0;
1724
1725         if (tx_ring->head)
1726                 writel(0, adapter->hw.hw_addr + tx_ring->head);
1727         if (tx_ring->tail)
1728                 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1729 }
1730
1731 /**
1732  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1733  * @adapter: board private structure
1734  **/
1735 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1736 {
1737         int i;
1738
1739         for (i = 0; i < adapter->num_rx_queues; i++)
1740                 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1741 }
1742
1743 /**
1744  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1745  * @adapter: board private structure
1746  **/
1747 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1748 {
1749         int i;
1750
1751         for (i = 0; i < adapter->num_tx_queues; i++)
1752                 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1753 }
1754
1755 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1756 {
1757         struct net_device *netdev = adapter->netdev;
1758         struct ixgbe_hw *hw = &adapter->hw;
1759         u32 txdctl;
1760         int i, j;
1761
1762         /* signal that we are down to the interrupt handler */
1763         set_bit(__IXGBEVF_DOWN, &adapter->state);
1764
1765         /* disable all enabled rx queues */
1766         for (i = 0; i < adapter->num_rx_queues; i++)
1767                 ixgbevf_disable_rx_queue(adapter, &adapter->rx_ring[i]);
1768
1769         netif_tx_disable(netdev);
1770
1771         msleep(10);
1772
1773         netif_tx_stop_all_queues(netdev);
1774
1775         ixgbevf_irq_disable(adapter);
1776
1777         ixgbevf_napi_disable_all(adapter);
1778
1779         del_timer_sync(&adapter->watchdog_timer);
1780         /* can't call flush scheduled work here because it can deadlock
1781          * if linkwatch_event tries to acquire the rtnl_lock which we are
1782          * holding */
1783         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1784                 msleep(1);
1785
1786         /* disable transmits in the hardware now that interrupts are off */
1787         for (i = 0; i < adapter->num_tx_queues; i++) {
1788                 j = adapter->tx_ring[i].reg_idx;
1789                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1790                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1791                                 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1792         }
1793
1794         netif_carrier_off(netdev);
1795
1796         if (!pci_channel_offline(adapter->pdev))
1797                 ixgbevf_reset(adapter);
1798
1799         ixgbevf_clean_all_tx_rings(adapter);
1800         ixgbevf_clean_all_rx_rings(adapter);
1801 }
1802
1803 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1804 {
1805         WARN_ON(in_interrupt());
1806
1807         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1808                 msleep(1);
1809
1810         ixgbevf_down(adapter);
1811         ixgbevf_up(adapter);
1812
1813         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1814 }
1815
1816 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1817 {
1818         struct ixgbe_hw *hw = &adapter->hw;
1819         struct net_device *netdev = adapter->netdev;
1820
1821         if (hw->mac.ops.reset_hw(hw)) {
1822                 hw_dbg(hw, "PF still resetting\n");
1823         } else {
1824                 hw->mac.ops.init_hw(hw);
1825                 ixgbevf_negotiate_api(adapter);
1826         }
1827
1828         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1829                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1830                        netdev->addr_len);
1831                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1832                        netdev->addr_len);
1833         }
1834 }
1835
1836 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1837                                         int vectors)
1838 {
1839         int err = 0;
1840         int vector_threshold;
1841
1842         /* We'll want at least 2 (vector_threshold):
1843          * 1) TxQ[0] + RxQ[0] handler
1844          * 2) Other (Link Status Change, etc.)
1845          */
1846         vector_threshold = MIN_MSIX_COUNT;
1847
1848         /* The more we get, the more we will assign to Tx/Rx Cleanup
1849          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1850          * Right now, we simply care about how many we'll get; we'll
1851          * set them up later while requesting irq's.
1852          */
1853         while (vectors >= vector_threshold) {
1854                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1855                                       vectors);
1856                 if (!err || err < 0) /* Success or a nasty failure. */
1857                         break;
1858                 else /* err == number of vectors we should try again with */
1859                         vectors = err;
1860         }
1861
1862         if (vectors < vector_threshold)
1863                 err = -ENOMEM;
1864
1865         if (err) {
1866                 dev_err(&adapter->pdev->dev,
1867                         "Unable to allocate MSI-X interrupts\n");
1868                 kfree(adapter->msix_entries);
1869                 adapter->msix_entries = NULL;
1870         } else {
1871                 /*
1872                  * Adjust for only the vectors we'll use, which is minimum
1873                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1874                  * vectors we were allocated.
1875                  */
1876                 adapter->num_msix_vectors = vectors;
1877         }
1878
1879         return err;
1880 }
1881
1882 /**
1883  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
1884  * @adapter: board private structure to initialize
1885  *
1886  * This is the top level queue allocation routine.  The order here is very
1887  * important, starting with the "most" number of features turned on at once,
1888  * and ending with the smallest set of features.  This way large combinations
1889  * can be allocated if they're turned on, and smaller combinations are the
1890  * fallthrough conditions.
1891  *
1892  **/
1893 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1894 {
1895         /* Start with base case */
1896         adapter->num_rx_queues = 1;
1897         adapter->num_tx_queues = 1;
1898 }
1899
1900 /**
1901  * ixgbevf_alloc_queues - Allocate memory for all rings
1902  * @adapter: board private structure to initialize
1903  *
1904  * We allocate one ring per queue at run-time since we don't know the
1905  * number of queues at compile-time.  The polling_netdev array is
1906  * intended for Multiqueue, but should work fine with a single queue.
1907  **/
1908 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1909 {
1910         int i;
1911
1912         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1913                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1914         if (!adapter->tx_ring)
1915                 goto err_tx_ring_allocation;
1916
1917         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1918                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1919         if (!adapter->rx_ring)
1920                 goto err_rx_ring_allocation;
1921
1922         for (i = 0; i < adapter->num_tx_queues; i++) {
1923                 adapter->tx_ring[i].count = adapter->tx_ring_count;
1924                 adapter->tx_ring[i].queue_index = i;
1925                 /* reg_idx may be remapped later by DCB config */
1926                 adapter->tx_ring[i].reg_idx = i;
1927                 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1928                 adapter->tx_ring[i].netdev = adapter->netdev;
1929         }
1930
1931         for (i = 0; i < adapter->num_rx_queues; i++) {
1932                 adapter->rx_ring[i].count = adapter->rx_ring_count;
1933                 adapter->rx_ring[i].queue_index = i;
1934                 adapter->rx_ring[i].reg_idx = i;
1935                 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1936                 adapter->rx_ring[i].netdev = adapter->netdev;
1937         }
1938
1939         return 0;
1940
1941 err_rx_ring_allocation:
1942         kfree(adapter->tx_ring);
1943 err_tx_ring_allocation:
1944         return -ENOMEM;
1945 }
1946
1947 /**
1948  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1949  * @adapter: board private structure to initialize
1950  *
1951  * Attempt to configure the interrupts using the best available
1952  * capabilities of the hardware and the kernel.
1953  **/
1954 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1955 {
1956         struct net_device *netdev = adapter->netdev;
1957         int err = 0;
1958         int vector, v_budget;
1959
1960         /*
1961          * It's easy to be greedy for MSI-X vectors, but it really
1962          * doesn't do us much good if we have a lot more vectors
1963          * than CPU's.  So let's be conservative and only ask for
1964          * (roughly) the same number of vectors as there are CPU's.
1965          * The default is to use pairs of vectors.
1966          */
1967         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1968         v_budget = min_t(int, v_budget, num_online_cpus());
1969         v_budget += NON_Q_VECTORS;
1970
1971         /* A failure in MSI-X entry allocation isn't fatal, but it does
1972          * mean we disable MSI-X capabilities of the adapter. */
1973         adapter->msix_entries = kcalloc(v_budget,
1974                                         sizeof(struct msix_entry), GFP_KERNEL);
1975         if (!adapter->msix_entries) {
1976                 err = -ENOMEM;
1977                 goto out;
1978         }
1979
1980         for (vector = 0; vector < v_budget; vector++)
1981                 adapter->msix_entries[vector].entry = vector;
1982
1983         err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
1984         if (err)
1985                 goto out;
1986
1987         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
1988         if (err)
1989                 goto out;
1990
1991         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
1992
1993 out:
1994         return err;
1995 }
1996
1997 /**
1998  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1999  * @adapter: board private structure to initialize
2000  *
2001  * We allocate one q_vector per queue interrupt.  If allocation fails we
2002  * return -ENOMEM.
2003  **/
2004 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2005 {
2006         int q_idx, num_q_vectors;
2007         struct ixgbevf_q_vector *q_vector;
2008
2009         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2010
2011         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2012                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2013                 if (!q_vector)
2014                         goto err_out;
2015                 q_vector->adapter = adapter;
2016                 q_vector->v_idx = q_idx;
2017                 netif_napi_add(adapter->netdev, &q_vector->napi,
2018                                ixgbevf_poll, 64);
2019 #ifdef CONFIG_NET_RX_BUSY_POLL
2020                 napi_hash_add(&q_vector->napi);
2021 #endif
2022                 adapter->q_vector[q_idx] = q_vector;
2023         }
2024
2025         return 0;
2026
2027 err_out:
2028         while (q_idx) {
2029                 q_idx--;
2030                 q_vector = adapter->q_vector[q_idx];
2031 #ifdef CONFIG_NET_RX_BUSY_POLL
2032                 napi_hash_del(&q_vector->napi);
2033 #endif
2034                 netif_napi_del(&q_vector->napi);
2035                 kfree(q_vector);
2036                 adapter->q_vector[q_idx] = NULL;
2037         }
2038         return -ENOMEM;
2039 }
2040
2041 /**
2042  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2043  * @adapter: board private structure to initialize
2044  *
2045  * This function frees the memory allocated to the q_vectors.  In addition if
2046  * NAPI is enabled it will delete any references to the NAPI struct prior
2047  * to freeing the q_vector.
2048  **/
2049 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2050 {
2051         int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2052
2053         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2054                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2055
2056                 adapter->q_vector[q_idx] = NULL;
2057 #ifdef CONFIG_NET_RX_BUSY_POLL
2058                 napi_hash_del(&q_vector->napi);
2059 #endif
2060                 netif_napi_del(&q_vector->napi);
2061                 kfree(q_vector);
2062         }
2063 }
2064
2065 /**
2066  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2067  * @adapter: board private structure
2068  *
2069  **/
2070 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2071 {
2072         pci_disable_msix(adapter->pdev);
2073         kfree(adapter->msix_entries);
2074         adapter->msix_entries = NULL;
2075 }
2076
2077 /**
2078  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2079  * @adapter: board private structure to initialize
2080  *
2081  **/
2082 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2083 {
2084         int err;
2085
2086         /* Number of supported queues */
2087         ixgbevf_set_num_queues(adapter);
2088
2089         err = ixgbevf_set_interrupt_capability(adapter);
2090         if (err) {
2091                 hw_dbg(&adapter->hw,
2092                        "Unable to setup interrupt capabilities\n");
2093                 goto err_set_interrupt;
2094         }
2095
2096         err = ixgbevf_alloc_q_vectors(adapter);
2097         if (err) {
2098                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2099                        "vectors\n");
2100                 goto err_alloc_q_vectors;
2101         }
2102
2103         err = ixgbevf_alloc_queues(adapter);
2104         if (err) {
2105                 pr_err("Unable to allocate memory for queues\n");
2106                 goto err_alloc_queues;
2107         }
2108
2109         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2110                "Tx Queue count = %u\n",
2111                (adapter->num_rx_queues > 1) ? "Enabled" :
2112                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2113
2114         set_bit(__IXGBEVF_DOWN, &adapter->state);
2115
2116         return 0;
2117 err_alloc_queues:
2118         ixgbevf_free_q_vectors(adapter);
2119 err_alloc_q_vectors:
2120         ixgbevf_reset_interrupt_capability(adapter);
2121 err_set_interrupt:
2122         return err;
2123 }
2124
2125 /**
2126  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2127  * @adapter: board private structure to clear interrupt scheme on
2128  *
2129  * We go through and clear interrupt specific resources and reset the structure
2130  * to pre-load conditions
2131  **/
2132 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2133 {
2134         adapter->num_tx_queues = 0;
2135         adapter->num_rx_queues = 0;
2136
2137         ixgbevf_free_q_vectors(adapter);
2138         ixgbevf_reset_interrupt_capability(adapter);
2139 }
2140
2141 /**
2142  * ixgbevf_sw_init - Initialize general software structures
2143  * (struct ixgbevf_adapter)
2144  * @adapter: board private structure to initialize
2145  *
2146  * ixgbevf_sw_init initializes the Adapter private data structure.
2147  * Fields are initialized based on PCI device information and
2148  * OS network device settings (MTU size).
2149  **/
2150 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2151 {
2152         struct ixgbe_hw *hw = &adapter->hw;
2153         struct pci_dev *pdev = adapter->pdev;
2154         struct net_device *netdev = adapter->netdev;
2155         int err;
2156
2157         /* PCI config space info */
2158
2159         hw->vendor_id = pdev->vendor;
2160         hw->device_id = pdev->device;
2161         hw->revision_id = pdev->revision;
2162         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2163         hw->subsystem_device_id = pdev->subsystem_device;
2164
2165         hw->mbx.ops.init_params(hw);
2166
2167         /* assume legacy case in which PF would only give VF 2 queues */
2168         hw->mac.max_tx_queues = 2;
2169         hw->mac.max_rx_queues = 2;
2170
2171         /* lock to protect mailbox accesses */
2172         spin_lock_init(&adapter->mbx_lock);
2173
2174         err = hw->mac.ops.reset_hw(hw);
2175         if (err) {
2176                 dev_info(&pdev->dev,
2177                          "PF still in reset state.  Is the PF interface up?\n");
2178         } else {
2179                 err = hw->mac.ops.init_hw(hw);
2180                 if (err) {
2181                         pr_err("init_shared_code failed: %d\n", err);
2182                         goto out;
2183                 }
2184                 ixgbevf_negotiate_api(adapter);
2185                 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2186                 if (err)
2187                         dev_info(&pdev->dev, "Error reading MAC address\n");
2188                 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2189                         dev_info(&pdev->dev,
2190                                  "MAC address not assigned by administrator.\n");
2191                 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2192         }
2193
2194         if (!is_valid_ether_addr(netdev->dev_addr)) {
2195                 dev_info(&pdev->dev, "Assigning random MAC address\n");
2196                 eth_hw_addr_random(netdev);
2197                 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
2198         }
2199
2200         /* Enable dynamic interrupt throttling rates */
2201         adapter->rx_itr_setting = 1;
2202         adapter->tx_itr_setting = 1;
2203
2204         /* set default ring sizes */
2205         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2206         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2207
2208         set_bit(__IXGBEVF_DOWN, &adapter->state);
2209         return 0;
2210
2211 out:
2212         return err;
2213 }
2214
2215 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
2216         {                                                       \
2217                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
2218                 if (current_counter < last_counter)             \
2219                         counter += 0x100000000LL;               \
2220                 last_counter = current_counter;                 \
2221                 counter &= 0xFFFFFFFF00000000LL;                \
2222                 counter |= current_counter;                     \
2223         }
2224
2225 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2226         {                                                                \
2227                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
2228                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
2229                 u64 current_counter = (current_counter_msb << 32) |      \
2230                         current_counter_lsb;                             \
2231                 if (current_counter < last_counter)                      \
2232                         counter += 0x1000000000LL;                       \
2233                 last_counter = current_counter;                          \
2234                 counter &= 0xFFFFFFF000000000LL;                         \
2235                 counter |= current_counter;                              \
2236         }
2237 /**
2238  * ixgbevf_update_stats - Update the board statistics counters.
2239  * @adapter: board private structure
2240  **/
2241 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2242 {
2243         struct ixgbe_hw *hw = &adapter->hw;
2244         int i;
2245
2246         if (!adapter->link_up)
2247                 return;
2248
2249         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2250                                 adapter->stats.vfgprc);
2251         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2252                                 adapter->stats.vfgptc);
2253         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2254                                 adapter->stats.last_vfgorc,
2255                                 adapter->stats.vfgorc);
2256         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2257                                 adapter->stats.last_vfgotc,
2258                                 adapter->stats.vfgotc);
2259         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2260                                 adapter->stats.vfmprc);
2261
2262         for (i = 0;  i  < adapter->num_rx_queues;  i++) {
2263                 adapter->hw_csum_rx_error +=
2264                         adapter->rx_ring[i].hw_csum_rx_error;
2265                 adapter->hw_csum_rx_good +=
2266                         adapter->rx_ring[i].hw_csum_rx_good;
2267                 adapter->rx_ring[i].hw_csum_rx_error = 0;
2268                 adapter->rx_ring[i].hw_csum_rx_good = 0;
2269         }
2270 }
2271
2272 /**
2273  * ixgbevf_watchdog - Timer Call-back
2274  * @data: pointer to adapter cast into an unsigned long
2275  **/
2276 static void ixgbevf_watchdog(unsigned long data)
2277 {
2278         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2279         struct ixgbe_hw *hw = &adapter->hw;
2280         u32 eics = 0;
2281         int i;
2282
2283         /*
2284          * Do the watchdog outside of interrupt context due to the lovely
2285          * delays that some of the newer hardware requires
2286          */
2287
2288         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2289                 goto watchdog_short_circuit;
2290
2291         /* get one bit for every active tx/rx interrupt vector */
2292         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2293                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2294                 if (qv->rx.ring || qv->tx.ring)
2295                         eics |= 1 << i;
2296         }
2297
2298         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
2299
2300 watchdog_short_circuit:
2301         schedule_work(&adapter->watchdog_task);
2302 }
2303
2304 /**
2305  * ixgbevf_tx_timeout - Respond to a Tx Hang
2306  * @netdev: network interface device structure
2307  **/
2308 static void ixgbevf_tx_timeout(struct net_device *netdev)
2309 {
2310         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2311
2312         /* Do the reset outside of interrupt context */
2313         schedule_work(&adapter->reset_task);
2314 }
2315
2316 static void ixgbevf_reset_task(struct work_struct *work)
2317 {
2318         struct ixgbevf_adapter *adapter;
2319         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2320
2321         /* If we're already down or resetting, just bail */
2322         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2323             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2324                 return;
2325
2326         adapter->tx_timeout_count++;
2327
2328         ixgbevf_reinit_locked(adapter);
2329 }
2330
2331 /**
2332  * ixgbevf_watchdog_task - worker thread to bring link up
2333  * @work: pointer to work_struct containing our data
2334  **/
2335 static void ixgbevf_watchdog_task(struct work_struct *work)
2336 {
2337         struct ixgbevf_adapter *adapter = container_of(work,
2338                                                        struct ixgbevf_adapter,
2339                                                        watchdog_task);
2340         struct net_device *netdev = adapter->netdev;
2341         struct ixgbe_hw *hw = &adapter->hw;
2342         u32 link_speed = adapter->link_speed;
2343         bool link_up = adapter->link_up;
2344         s32 need_reset;
2345
2346         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2347
2348         /*
2349          * Always check the link on the watchdog because we have
2350          * no LSC interrupt
2351          */
2352         spin_lock_bh(&adapter->mbx_lock);
2353
2354         need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
2355
2356         spin_unlock_bh(&adapter->mbx_lock);
2357
2358         if (need_reset) {
2359                 adapter->link_up = link_up;
2360                 adapter->link_speed = link_speed;
2361                 netif_carrier_off(netdev);
2362                 netif_tx_stop_all_queues(netdev);
2363                 schedule_work(&adapter->reset_task);
2364                 goto pf_has_reset;
2365         }
2366         adapter->link_up = link_up;
2367         adapter->link_speed = link_speed;
2368
2369         if (link_up) {
2370                 if (!netif_carrier_ok(netdev)) {
2371                         char *link_speed_string;
2372                         switch (link_speed) {
2373                         case IXGBE_LINK_SPEED_10GB_FULL:
2374                                 link_speed_string = "10 Gbps";
2375                                 break;
2376                         case IXGBE_LINK_SPEED_1GB_FULL:
2377                                 link_speed_string = "1 Gbps";
2378                                 break;
2379                         case IXGBE_LINK_SPEED_100_FULL:
2380                                 link_speed_string = "100 Mbps";
2381                                 break;
2382                         default:
2383                                 link_speed_string = "unknown speed";
2384                                 break;
2385                         }
2386                         dev_info(&adapter->pdev->dev,
2387                                 "NIC Link is Up, %s\n", link_speed_string);
2388                         netif_carrier_on(netdev);
2389                         netif_tx_wake_all_queues(netdev);
2390                 }
2391         } else {
2392                 adapter->link_up = false;
2393                 adapter->link_speed = 0;
2394                 if (netif_carrier_ok(netdev)) {
2395                         dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
2396                         netif_carrier_off(netdev);
2397                         netif_tx_stop_all_queues(netdev);
2398                 }
2399         }
2400
2401         ixgbevf_update_stats(adapter);
2402
2403 pf_has_reset:
2404         /* Reset the timer */
2405         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2406                 mod_timer(&adapter->watchdog_timer,
2407                           round_jiffies(jiffies + (2 * HZ)));
2408
2409         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2410 }
2411
2412 /**
2413  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2414  * @adapter: board private structure
2415  * @tx_ring: Tx descriptor ring for a specific queue
2416  *
2417  * Free all transmit software resources
2418  **/
2419 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2420                                struct ixgbevf_ring *tx_ring)
2421 {
2422         struct pci_dev *pdev = adapter->pdev;
2423
2424         ixgbevf_clean_tx_ring(adapter, tx_ring);
2425
2426         vfree(tx_ring->tx_buffer_info);
2427         tx_ring->tx_buffer_info = NULL;
2428
2429         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2430                           tx_ring->dma);
2431
2432         tx_ring->desc = NULL;
2433 }
2434
2435 /**
2436  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2437  * @adapter: board private structure
2438  *
2439  * Free all transmit software resources
2440  **/
2441 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2442 {
2443         int i;
2444
2445         for (i = 0; i < adapter->num_tx_queues; i++)
2446                 if (adapter->tx_ring[i].desc)
2447                         ixgbevf_free_tx_resources(adapter,
2448                                                   &adapter->tx_ring[i]);
2449
2450 }
2451
2452 /**
2453  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2454  * @adapter: board private structure
2455  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2456  *
2457  * Return 0 on success, negative on failure
2458  **/
2459 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2460                                struct ixgbevf_ring *tx_ring)
2461 {
2462         struct pci_dev *pdev = adapter->pdev;
2463         int size;
2464
2465         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2466         tx_ring->tx_buffer_info = vzalloc(size);
2467         if (!tx_ring->tx_buffer_info)
2468                 goto err;
2469
2470         /* round up to nearest 4K */
2471         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2472         tx_ring->size = ALIGN(tx_ring->size, 4096);
2473
2474         tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2475                                            &tx_ring->dma, GFP_KERNEL);
2476         if (!tx_ring->desc)
2477                 goto err;
2478
2479         tx_ring->next_to_use = 0;
2480         tx_ring->next_to_clean = 0;
2481         return 0;
2482
2483 err:
2484         vfree(tx_ring->tx_buffer_info);
2485         tx_ring->tx_buffer_info = NULL;
2486         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2487                "descriptor ring\n");
2488         return -ENOMEM;
2489 }
2490
2491 /**
2492  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2493  * @adapter: board private structure
2494  *
2495  * If this function returns with an error, then it's possible one or
2496  * more of the rings is populated (while the rest are not).  It is the
2497  * callers duty to clean those orphaned rings.
2498  *
2499  * Return 0 on success, negative on failure
2500  **/
2501 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2502 {
2503         int i, err = 0;
2504
2505         for (i = 0; i < adapter->num_tx_queues; i++) {
2506                 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2507                 if (!err)
2508                         continue;
2509                 hw_dbg(&adapter->hw,
2510                        "Allocation for Tx Queue %u failed\n", i);
2511                 break;
2512         }
2513
2514         return err;
2515 }
2516
2517 /**
2518  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2519  * @adapter: board private structure
2520  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2521  *
2522  * Returns 0 on success, negative on failure
2523  **/
2524 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2525                                struct ixgbevf_ring *rx_ring)
2526 {
2527         struct pci_dev *pdev = adapter->pdev;
2528         int size;
2529
2530         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2531         rx_ring->rx_buffer_info = vzalloc(size);
2532         if (!rx_ring->rx_buffer_info)
2533                 goto alloc_failed;
2534
2535         /* Round up to nearest 4K */
2536         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2537         rx_ring->size = ALIGN(rx_ring->size, 4096);
2538
2539         rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2540                                            &rx_ring->dma, GFP_KERNEL);
2541
2542         if (!rx_ring->desc) {
2543                 vfree(rx_ring->rx_buffer_info);
2544                 rx_ring->rx_buffer_info = NULL;
2545                 goto alloc_failed;
2546         }
2547
2548         rx_ring->next_to_clean = 0;
2549         rx_ring->next_to_use = 0;
2550
2551         return 0;
2552 alloc_failed:
2553         return -ENOMEM;
2554 }
2555
2556 /**
2557  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2558  * @adapter: board private structure
2559  *
2560  * If this function returns with an error, then it's possible one or
2561  * more of the rings is populated (while the rest are not).  It is the
2562  * callers duty to clean those orphaned rings.
2563  *
2564  * Return 0 on success, negative on failure
2565  **/
2566 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2567 {
2568         int i, err = 0;
2569
2570         for (i = 0; i < adapter->num_rx_queues; i++) {
2571                 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2572                 if (!err)
2573                         continue;
2574                 hw_dbg(&adapter->hw,
2575                        "Allocation for Rx Queue %u failed\n", i);
2576                 break;
2577         }
2578         return err;
2579 }
2580
2581 /**
2582  * ixgbevf_free_rx_resources - Free Rx Resources
2583  * @adapter: board private structure
2584  * @rx_ring: ring to clean the resources from
2585  *
2586  * Free all receive software resources
2587  **/
2588 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2589                                struct ixgbevf_ring *rx_ring)
2590 {
2591         struct pci_dev *pdev = adapter->pdev;
2592
2593         ixgbevf_clean_rx_ring(adapter, rx_ring);
2594
2595         vfree(rx_ring->rx_buffer_info);
2596         rx_ring->rx_buffer_info = NULL;
2597
2598         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2599                           rx_ring->dma);
2600
2601         rx_ring->desc = NULL;
2602 }
2603
2604 /**
2605  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2606  * @adapter: board private structure
2607  *
2608  * Free all receive software resources
2609  **/
2610 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2611 {
2612         int i;
2613
2614         for (i = 0; i < adapter->num_rx_queues; i++)
2615                 if (adapter->rx_ring[i].desc)
2616                         ixgbevf_free_rx_resources(adapter,
2617                                                   &adapter->rx_ring[i]);
2618 }
2619
2620 static int ixgbevf_setup_queues(struct ixgbevf_adapter *adapter)
2621 {
2622         struct ixgbe_hw *hw = &adapter->hw;
2623         struct ixgbevf_ring *rx_ring;
2624         unsigned int def_q = 0;
2625         unsigned int num_tcs = 0;
2626         unsigned int num_rx_queues = 1;
2627         int err, i;
2628
2629         spin_lock_bh(&adapter->mbx_lock);
2630
2631         /* fetch queue configuration from the PF */
2632         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2633
2634         spin_unlock_bh(&adapter->mbx_lock);
2635
2636         if (err)
2637                 return err;
2638
2639         if (num_tcs > 1) {
2640                 /* update default Tx ring register index */
2641                 adapter->tx_ring[0].reg_idx = def_q;
2642
2643                 /* we need as many queues as traffic classes */
2644                 num_rx_queues = num_tcs;
2645         }
2646
2647         /* nothing to do if we have the correct number of queues */
2648         if (adapter->num_rx_queues == num_rx_queues)
2649                 return 0;
2650
2651         /* allocate new rings */
2652         rx_ring = kcalloc(num_rx_queues,
2653                           sizeof(struct ixgbevf_ring), GFP_KERNEL);
2654         if (!rx_ring)
2655                 return -ENOMEM;
2656
2657         /* setup ring fields */
2658         for (i = 0; i < num_rx_queues; i++) {
2659                 rx_ring[i].count = adapter->rx_ring_count;
2660                 rx_ring[i].queue_index = i;
2661                 rx_ring[i].reg_idx = i;
2662                 rx_ring[i].dev = &adapter->pdev->dev;
2663                 rx_ring[i].netdev = adapter->netdev;
2664         }
2665
2666         /* free the existing ring and queues */
2667         adapter->num_rx_queues = 0;
2668         kfree(adapter->rx_ring);
2669
2670         /* move new rings into position on the adapter struct */
2671         adapter->rx_ring = rx_ring;
2672         adapter->num_rx_queues = num_rx_queues;
2673
2674         return 0;
2675 }
2676
2677 /**
2678  * ixgbevf_open - Called when a network interface is made active
2679  * @netdev: network interface device structure
2680  *
2681  * Returns 0 on success, negative value on failure
2682  *
2683  * The open entry point is called when a network interface is made
2684  * active by the system (IFF_UP).  At this point all resources needed
2685  * for transmit and receive operations are allocated, the interrupt
2686  * handler is registered with the OS, the watchdog timer is started,
2687  * and the stack is notified that the interface is ready.
2688  **/
2689 static int ixgbevf_open(struct net_device *netdev)
2690 {
2691         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2692         struct ixgbe_hw *hw = &adapter->hw;
2693         int err;
2694
2695         /* A previous failure to open the device because of a lack of
2696          * available MSIX vector resources may have reset the number
2697          * of msix vectors variable to zero.  The only way to recover
2698          * is to unload/reload the driver and hope that the system has
2699          * been able to recover some MSIX vector resources.
2700          */
2701         if (!adapter->num_msix_vectors)
2702                 return -ENOMEM;
2703
2704         /* disallow open during test */
2705         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2706                 return -EBUSY;
2707
2708         if (hw->adapter_stopped) {
2709                 ixgbevf_reset(adapter);
2710                 /* if adapter is still stopped then PF isn't up and
2711                  * the vf can't start. */
2712                 if (hw->adapter_stopped) {
2713                         err = IXGBE_ERR_MBX;
2714                         pr_err("Unable to start - perhaps the PF Driver isn't "
2715                                "up yet\n");
2716                         goto err_setup_reset;
2717                 }
2718         }
2719
2720         /* setup queue reg_idx and Rx queue count */
2721         err = ixgbevf_setup_queues(adapter);
2722         if (err)
2723                 goto err_setup_queues;
2724
2725         /* allocate transmit descriptors */
2726         err = ixgbevf_setup_all_tx_resources(adapter);
2727         if (err)
2728                 goto err_setup_tx;
2729
2730         /* allocate receive descriptors */
2731         err = ixgbevf_setup_all_rx_resources(adapter);
2732         if (err)
2733                 goto err_setup_rx;
2734
2735         ixgbevf_configure(adapter);
2736
2737         /*
2738          * Map the Tx/Rx rings to the vectors we were allotted.
2739          * if request_irq will be called in this function map_rings
2740          * must be called *before* up_complete
2741          */
2742         ixgbevf_map_rings_to_vectors(adapter);
2743
2744         ixgbevf_up_complete(adapter);
2745
2746         /* clear any pending interrupts, may auto mask */
2747         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2748         err = ixgbevf_request_irq(adapter);
2749         if (err)
2750                 goto err_req_irq;
2751
2752         ixgbevf_irq_enable(adapter);
2753
2754         return 0;
2755
2756 err_req_irq:
2757         ixgbevf_down(adapter);
2758 err_setup_rx:
2759         ixgbevf_free_all_rx_resources(adapter);
2760 err_setup_tx:
2761         ixgbevf_free_all_tx_resources(adapter);
2762 err_setup_queues:
2763         ixgbevf_reset(adapter);
2764
2765 err_setup_reset:
2766
2767         return err;
2768 }
2769
2770 /**
2771  * ixgbevf_close - Disables a network interface
2772  * @netdev: network interface device structure
2773  *
2774  * Returns 0, this is not allowed to fail
2775  *
2776  * The close entry point is called when an interface is de-activated
2777  * by the OS.  The hardware is still under the drivers control, but
2778  * needs to be disabled.  A global MAC reset is issued to stop the
2779  * hardware, and all transmit and receive resources are freed.
2780  **/
2781 static int ixgbevf_close(struct net_device *netdev)
2782 {
2783         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2784
2785         ixgbevf_down(adapter);
2786         ixgbevf_free_irq(adapter);
2787
2788         ixgbevf_free_all_tx_resources(adapter);
2789         ixgbevf_free_all_rx_resources(adapter);
2790
2791         return 0;
2792 }
2793
2794 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2795                                 u32 vlan_macip_lens, u32 type_tucmd,
2796                                 u32 mss_l4len_idx)
2797 {
2798         struct ixgbe_adv_tx_context_desc *context_desc;
2799         u16 i = tx_ring->next_to_use;
2800
2801         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
2802
2803         i++;
2804         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2805
2806         /* set bits to identify this as an advanced context descriptor */
2807         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2808
2809         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
2810         context_desc->seqnum_seed       = 0;
2811         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
2812         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
2813 }
2814
2815 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2816                        struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2817 {
2818         u32 vlan_macip_lens, type_tucmd;
2819         u32 mss_l4len_idx, l4len;
2820
2821         if (!skb_is_gso(skb))
2822                 return 0;
2823
2824         if (skb_header_cloned(skb)) {
2825                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2826                 if (err)
2827                         return err;
2828         }
2829
2830         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2831         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2832
2833         if (skb->protocol == htons(ETH_P_IP)) {
2834                 struct iphdr *iph = ip_hdr(skb);
2835                 iph->tot_len = 0;
2836                 iph->check = 0;
2837                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2838                                                          iph->daddr, 0,
2839                                                          IPPROTO_TCP,
2840                                                          0);
2841                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2842         } else if (skb_is_gso_v6(skb)) {
2843                 ipv6_hdr(skb)->payload_len = 0;
2844                 tcp_hdr(skb)->check =
2845                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2846                                      &ipv6_hdr(skb)->daddr,
2847                                      0, IPPROTO_TCP, 0);
2848         }
2849
2850         /* compute header lengths */
2851         l4len = tcp_hdrlen(skb);
2852         *hdr_len += l4len;
2853         *hdr_len = skb_transport_offset(skb) + l4len;
2854
2855         /* mss_l4len_id: use 1 as index for TSO */
2856         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2857         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2858         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2859
2860         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2861         vlan_macip_lens = skb_network_header_len(skb);
2862         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2863         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2864
2865         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2866                             type_tucmd, mss_l4len_idx);
2867
2868         return 1;
2869 }
2870
2871 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2872                             struct sk_buff *skb, u32 tx_flags)
2873 {
2874         u32 vlan_macip_lens = 0;
2875         u32 mss_l4len_idx = 0;
2876         u32 type_tucmd = 0;
2877
2878         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2879                 u8 l4_hdr = 0;
2880                 switch (skb->protocol) {
2881                 case __constant_htons(ETH_P_IP):
2882                         vlan_macip_lens |= skb_network_header_len(skb);
2883                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2884                         l4_hdr = ip_hdr(skb)->protocol;
2885                         break;
2886                 case __constant_htons(ETH_P_IPV6):
2887                         vlan_macip_lens |= skb_network_header_len(skb);
2888                         l4_hdr = ipv6_hdr(skb)->nexthdr;
2889                         break;
2890                 default:
2891                         if (unlikely(net_ratelimit())) {
2892                                 dev_warn(tx_ring->dev,
2893                                  "partial checksum but proto=%x!\n",
2894                                  skb->protocol);
2895                         }
2896                         break;
2897                 }
2898
2899                 switch (l4_hdr) {
2900                 case IPPROTO_TCP:
2901                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2902                         mss_l4len_idx = tcp_hdrlen(skb) <<
2903                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2904                         break;
2905                 case IPPROTO_SCTP:
2906                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2907                         mss_l4len_idx = sizeof(struct sctphdr) <<
2908                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2909                         break;
2910                 case IPPROTO_UDP:
2911                         mss_l4len_idx = sizeof(struct udphdr) <<
2912                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2913                         break;
2914                 default:
2915                         if (unlikely(net_ratelimit())) {
2916                                 dev_warn(tx_ring->dev,
2917                                  "partial checksum but l4 proto=%x!\n",
2918                                  l4_hdr);
2919                         }
2920                         break;
2921                 }
2922         }
2923
2924         /* vlan_macip_lens: MACLEN, VLAN tag */
2925         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2926         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2927
2928         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2929                             type_tucmd, mss_l4len_idx);
2930
2931         return (skb->ip_summed == CHECKSUM_PARTIAL);
2932 }
2933
2934 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2935                           struct sk_buff *skb, u32 tx_flags)
2936 {
2937         struct ixgbevf_tx_buffer *tx_buffer_info;
2938         unsigned int len;
2939         unsigned int total = skb->len;
2940         unsigned int offset = 0, size;
2941         int count = 0;
2942         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2943         unsigned int f;
2944         int i;
2945
2946         i = tx_ring->next_to_use;
2947
2948         len = min(skb_headlen(skb), total);
2949         while (len) {
2950                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2951                 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2952
2953                 tx_buffer_info->length = size;
2954                 tx_buffer_info->mapped_as_page = false;
2955                 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
2956                                                      skb->data + offset,
2957                                                      size, DMA_TO_DEVICE);
2958                 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
2959                         goto dma_error;
2960
2961                 len -= size;
2962                 total -= size;
2963                 offset += size;
2964                 count++;
2965                 i++;
2966                 if (i == tx_ring->count)
2967                         i = 0;
2968         }
2969
2970         for (f = 0; f < nr_frags; f++) {
2971                 const struct skb_frag_struct *frag;
2972
2973                 frag = &skb_shinfo(skb)->frags[f];
2974                 len = min((unsigned int)skb_frag_size(frag), total);
2975                 offset = 0;
2976
2977                 while (len) {
2978                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
2979                         size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2980
2981                         tx_buffer_info->length = size;
2982                         tx_buffer_info->dma =
2983                                 skb_frag_dma_map(tx_ring->dev, frag,
2984                                                  offset, size, DMA_TO_DEVICE);
2985                         if (dma_mapping_error(tx_ring->dev,
2986                                               tx_buffer_info->dma))
2987                                 goto dma_error;
2988                         tx_buffer_info->mapped_as_page = true;
2989
2990                         len -= size;
2991                         total -= size;
2992                         offset += size;
2993                         count++;
2994                         i++;
2995                         if (i == tx_ring->count)
2996                                 i = 0;
2997                 }
2998                 if (total == 0)
2999                         break;
3000         }
3001
3002         if (i == 0)
3003                 i = tx_ring->count - 1;
3004         else
3005                 i = i - 1;
3006         tx_ring->tx_buffer_info[i].skb = skb;
3007
3008         return count;
3009
3010 dma_error:
3011         dev_err(tx_ring->dev, "TX DMA map failed\n");
3012
3013         /* clear timestamp and dma mappings for failed tx_buffer_info map */
3014         tx_buffer_info->dma = 0;
3015         count--;
3016
3017         /* clear timestamp and dma mappings for remaining portion of packet */
3018         while (count >= 0) {
3019                 count--;
3020                 i--;
3021                 if (i < 0)
3022                         i += tx_ring->count;
3023                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
3024                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
3025         }
3026
3027         return count;
3028 }
3029
3030 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
3031                              int count, unsigned int first, u32 paylen,
3032                              u8 hdr_len)
3033 {
3034         union ixgbe_adv_tx_desc *tx_desc = NULL;
3035         struct ixgbevf_tx_buffer *tx_buffer_info;
3036         u32 olinfo_status = 0, cmd_type_len = 0;
3037         unsigned int i;
3038
3039         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
3040
3041         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
3042
3043         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
3044
3045         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
3046                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
3047
3048         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3049                 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
3050
3051         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
3052                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
3053
3054                 /* use index 1 context for tso */
3055                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
3056                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3057                         olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
3058         }
3059
3060         /*
3061          * Check Context must be set if Tx switch is enabled, which it
3062          * always is for case where virtual functions are running
3063          */
3064         olinfo_status |= IXGBE_ADVTXD_CC;
3065
3066         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
3067
3068         i = tx_ring->next_to_use;
3069         while (count--) {
3070                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
3071                 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
3072                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
3073                 tx_desc->read.cmd_type_len =
3074                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
3075                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3076                 i++;
3077                 if (i == tx_ring->count)
3078                         i = 0;
3079         }
3080
3081         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
3082
3083         tx_ring->tx_buffer_info[first].time_stamp = jiffies;
3084
3085         /* Force memory writes to complete before letting h/w
3086          * know there are new descriptors to fetch.  (Only
3087          * applicable for weak-ordered memory model archs,
3088          * such as IA-64).
3089          */
3090         wmb();
3091
3092         tx_ring->tx_buffer_info[first].next_to_watch = tx_desc;
3093         tx_ring->next_to_use = i;
3094 }
3095
3096 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3097 {
3098         struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
3099
3100         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
3101         /* Herbert's original patch had:
3102          *  smp_mb__after_netif_stop_queue();
3103          * but since that doesn't exist yet, just open code it. */
3104         smp_mb();
3105
3106         /* We need to check again in a case another CPU has just
3107          * made room available. */
3108         if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
3109                 return -EBUSY;
3110
3111         /* A reprieve! - use start_queue because it doesn't call schedule */
3112         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
3113         ++adapter->restart_queue;
3114         return 0;
3115 }
3116
3117 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3118 {
3119         if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
3120                 return 0;
3121         return __ixgbevf_maybe_stop_tx(tx_ring, size);
3122 }
3123
3124 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3125 {
3126         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3127         struct ixgbevf_ring *tx_ring;
3128         unsigned int first;
3129         unsigned int tx_flags = 0;
3130         u8 hdr_len = 0;
3131         int r_idx = 0, tso;
3132         u16 count = TXD_USE_COUNT(skb_headlen(skb));
3133 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3134         unsigned short f;
3135 #endif
3136         u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
3137         if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
3138                 dev_kfree_skb(skb);
3139                 return NETDEV_TX_OK;
3140         }
3141
3142         tx_ring = &adapter->tx_ring[r_idx];
3143
3144         /*
3145          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3146          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3147          *       + 2 desc gap to keep tail from touching head,
3148          *       + 1 desc for context descriptor,
3149          * otherwise try next time
3150          */
3151 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3152         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3153                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3154 #else
3155         count += skb_shinfo(skb)->nr_frags;
3156 #endif
3157         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
3158                 adapter->tx_busy++;
3159                 return NETDEV_TX_BUSY;
3160         }
3161
3162         if (vlan_tx_tag_present(skb)) {
3163                 tx_flags |= vlan_tx_tag_get(skb);
3164                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3165                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3166         }
3167
3168         first = tx_ring->next_to_use;
3169
3170         if (skb->protocol == htons(ETH_P_IP))
3171                 tx_flags |= IXGBE_TX_FLAGS_IPV4;
3172         tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
3173         if (tso < 0) {
3174                 dev_kfree_skb_any(skb);
3175                 return NETDEV_TX_OK;
3176         }
3177
3178         if (tso)
3179                 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
3180         else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
3181                 tx_flags |= IXGBE_TX_FLAGS_CSUM;
3182
3183         ixgbevf_tx_queue(tx_ring, tx_flags,
3184                          ixgbevf_tx_map(tx_ring, skb, tx_flags),
3185                          first, skb->len, hdr_len);
3186
3187         writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
3188
3189         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
3190
3191         return NETDEV_TX_OK;
3192 }
3193
3194 /**
3195  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3196  * @netdev: network interface device structure
3197  * @p: pointer to an address structure
3198  *
3199  * Returns 0 on success, negative on failure
3200  **/
3201 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3202 {
3203         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3204         struct ixgbe_hw *hw = &adapter->hw;
3205         struct sockaddr *addr = p;
3206
3207         if (!is_valid_ether_addr(addr->sa_data))
3208                 return -EADDRNOTAVAIL;
3209
3210         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3211         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3212
3213         spin_lock_bh(&adapter->mbx_lock);
3214
3215         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
3216
3217         spin_unlock_bh(&adapter->mbx_lock);
3218
3219         return 0;
3220 }
3221
3222 /**
3223  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3224  * @netdev: network interface device structure
3225  * @new_mtu: new value for maximum frame size
3226  *
3227  * Returns 0 on success, negative on failure
3228  **/
3229 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3230 {
3231         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3232         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3233         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
3234
3235         switch (adapter->hw.api_version) {
3236         case ixgbe_mbox_api_11:
3237                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3238                 break;
3239         default:
3240                 if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
3241                         max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3242                 break;
3243         }
3244
3245         /* MTU < 68 is an error and causes problems on some kernels */
3246         if ((new_mtu < 68) || (max_frame > max_possible_frame))
3247                 return -EINVAL;
3248
3249         hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
3250                netdev->mtu, new_mtu);
3251         /* must set new MTU before calling down or up */
3252         netdev->mtu = new_mtu;
3253
3254         if (netif_running(netdev))
3255                 ixgbevf_reinit_locked(adapter);
3256
3257         return 0;
3258 }
3259
3260 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
3261 {
3262         struct net_device *netdev = pci_get_drvdata(pdev);
3263         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3264 #ifdef CONFIG_PM
3265         int retval = 0;
3266 #endif
3267
3268         netif_device_detach(netdev);
3269
3270         if (netif_running(netdev)) {
3271                 rtnl_lock();
3272                 ixgbevf_down(adapter);
3273                 ixgbevf_free_irq(adapter);
3274                 ixgbevf_free_all_tx_resources(adapter);
3275                 ixgbevf_free_all_rx_resources(adapter);
3276                 rtnl_unlock();
3277         }
3278
3279         ixgbevf_clear_interrupt_scheme(adapter);
3280
3281 #ifdef CONFIG_PM
3282         retval = pci_save_state(pdev);
3283         if (retval)
3284                 return retval;
3285
3286 #endif
3287         pci_disable_device(pdev);
3288
3289         return 0;
3290 }
3291
3292 #ifdef CONFIG_PM
3293 static int ixgbevf_resume(struct pci_dev *pdev)
3294 {
3295         struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev);
3296         struct net_device *netdev = adapter->netdev;
3297         u32 err;
3298
3299         pci_set_power_state(pdev, PCI_D0);
3300         pci_restore_state(pdev);
3301         /*
3302          * pci_restore_state clears dev->state_saved so call
3303          * pci_save_state to restore it.
3304          */
3305         pci_save_state(pdev);
3306
3307         err = pci_enable_device_mem(pdev);
3308         if (err) {
3309                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3310                 return err;
3311         }
3312         pci_set_master(pdev);
3313
3314         ixgbevf_reset(adapter);
3315
3316         rtnl_lock();
3317         err = ixgbevf_init_interrupt_scheme(adapter);
3318         rtnl_unlock();
3319         if (err) {
3320                 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3321                 return err;
3322         }
3323
3324         if (netif_running(netdev)) {
3325                 err = ixgbevf_open(netdev);
3326                 if (err)
3327                         return err;
3328         }
3329
3330         netif_device_attach(netdev);
3331
3332         return err;
3333 }
3334
3335 #endif /* CONFIG_PM */
3336 static void ixgbevf_shutdown(struct pci_dev *pdev)
3337 {
3338         ixgbevf_suspend(pdev, PMSG_SUSPEND);
3339 }
3340
3341 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3342                                                 struct rtnl_link_stats64 *stats)
3343 {
3344         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3345         unsigned int start;
3346         u64 bytes, packets;
3347         const struct ixgbevf_ring *ring;
3348         int i;
3349
3350         ixgbevf_update_stats(adapter);
3351
3352         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3353
3354         for (i = 0; i < adapter->num_rx_queues; i++) {
3355                 ring = &adapter->rx_ring[i];
3356                 do {
3357                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3358                         bytes = ring->total_bytes;
3359                         packets = ring->total_packets;
3360                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3361                 stats->rx_bytes += bytes;
3362                 stats->rx_packets += packets;
3363         }
3364
3365         for (i = 0; i < adapter->num_tx_queues; i++) {
3366                 ring = &adapter->tx_ring[i];
3367                 do {
3368                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3369                         bytes = ring->total_bytes;
3370                         packets = ring->total_packets;
3371                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3372                 stats->tx_bytes += bytes;
3373                 stats->tx_packets += packets;
3374         }
3375
3376         return stats;
3377 }
3378
3379 static const struct net_device_ops ixgbevf_netdev_ops = {
3380         .ndo_open               = ixgbevf_open,
3381         .ndo_stop               = ixgbevf_close,
3382         .ndo_start_xmit         = ixgbevf_xmit_frame,
3383         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
3384         .ndo_get_stats64        = ixgbevf_get_stats,
3385         .ndo_validate_addr      = eth_validate_addr,
3386         .ndo_set_mac_address    = ixgbevf_set_mac,
3387         .ndo_change_mtu         = ixgbevf_change_mtu,
3388         .ndo_tx_timeout         = ixgbevf_tx_timeout,
3389         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
3390         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
3391 #ifdef CONFIG_NET_RX_BUSY_POLL
3392         .ndo_busy_poll          = ixgbevf_busy_poll_recv,
3393 #endif
3394 };
3395
3396 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3397 {
3398         dev->netdev_ops = &ixgbevf_netdev_ops;
3399         ixgbevf_set_ethtool_ops(dev);
3400         dev->watchdog_timeo = 5 * HZ;
3401 }
3402
3403 /**
3404  * ixgbevf_probe - Device Initialization Routine
3405  * @pdev: PCI device information struct
3406  * @ent: entry in ixgbevf_pci_tbl
3407  *
3408  * Returns 0 on success, negative on failure
3409  *
3410  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3411  * The OS initialization, configuring of the adapter private structure,
3412  * and a hardware reset occur.
3413  **/
3414 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3415 {
3416         struct net_device *netdev;
3417         struct ixgbevf_adapter *adapter = NULL;
3418         struct ixgbe_hw *hw = NULL;
3419         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3420         static int cards_found;
3421         int err, pci_using_dac;
3422
3423         err = pci_enable_device(pdev);
3424         if (err)
3425                 return err;
3426
3427         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3428             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
3429                 pci_using_dac = 1;
3430         } else {
3431                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
3432                 if (err) {
3433                         err = dma_set_coherent_mask(&pdev->dev,
3434                                                     DMA_BIT_MASK(32));
3435                         if (err) {
3436                                 dev_err(&pdev->dev, "No usable DMA "
3437                                         "configuration, aborting\n");
3438                                 goto err_dma;
3439                         }
3440                 }
3441                 pci_using_dac = 0;
3442         }
3443
3444         err = pci_request_regions(pdev, ixgbevf_driver_name);
3445         if (err) {
3446                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3447                 goto err_pci_reg;
3448         }
3449
3450         pci_set_master(pdev);
3451
3452         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3453                                    MAX_TX_QUEUES);
3454         if (!netdev) {
3455                 err = -ENOMEM;
3456                 goto err_alloc_etherdev;
3457         }
3458
3459         SET_NETDEV_DEV(netdev, &pdev->dev);
3460
3461         pci_set_drvdata(pdev, netdev);
3462         adapter = netdev_priv(netdev);
3463
3464         adapter->netdev = netdev;
3465         adapter->pdev = pdev;
3466         hw = &adapter->hw;
3467         hw->back = adapter;
3468         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3469
3470         /*
3471          * call save state here in standalone driver because it relies on
3472          * adapter struct to exist, and needs to call netdev_priv
3473          */
3474         pci_save_state(pdev);
3475
3476         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3477                               pci_resource_len(pdev, 0));
3478         if (!hw->hw_addr) {
3479                 err = -EIO;
3480                 goto err_ioremap;
3481         }
3482
3483         ixgbevf_assign_netdev_ops(netdev);
3484
3485         adapter->bd_number = cards_found;
3486
3487         /* Setup hw api */
3488         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3489         hw->mac.type  = ii->mac;
3490
3491         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3492                sizeof(struct ixgbe_mbx_operations));
3493
3494         /* setup the private structure */
3495         err = ixgbevf_sw_init(adapter);
3496         if (err)
3497                 goto err_sw_init;
3498
3499         /* The HW MAC address was set and/or determined in sw_init */
3500         if (!is_valid_ether_addr(netdev->dev_addr)) {
3501                 pr_err("invalid MAC address\n");
3502                 err = -EIO;
3503                 goto err_sw_init;
3504         }
3505
3506         netdev->hw_features = NETIF_F_SG |
3507                            NETIF_F_IP_CSUM |
3508                            NETIF_F_IPV6_CSUM |
3509                            NETIF_F_TSO |
3510                            NETIF_F_TSO6 |
3511                            NETIF_F_RXCSUM;
3512
3513         netdev->features = netdev->hw_features |
3514                            NETIF_F_HW_VLAN_CTAG_TX |
3515                            NETIF_F_HW_VLAN_CTAG_RX |
3516                            NETIF_F_HW_VLAN_CTAG_FILTER;
3517
3518         netdev->vlan_features |= NETIF_F_TSO;
3519         netdev->vlan_features |= NETIF_F_TSO6;
3520         netdev->vlan_features |= NETIF_F_IP_CSUM;
3521         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3522         netdev->vlan_features |= NETIF_F_SG;
3523
3524         if (pci_using_dac)
3525                 netdev->features |= NETIF_F_HIGHDMA;
3526
3527         netdev->priv_flags |= IFF_UNICAST_FLT;
3528
3529         init_timer(&adapter->watchdog_timer);
3530         adapter->watchdog_timer.function = ixgbevf_watchdog;
3531         adapter->watchdog_timer.data = (unsigned long)adapter;
3532
3533         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3534         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3535
3536         err = ixgbevf_init_interrupt_scheme(adapter);
3537         if (err)
3538                 goto err_sw_init;
3539
3540         strcpy(netdev->name, "eth%d");
3541
3542         err = register_netdev(netdev);
3543         if (err)
3544                 goto err_register;
3545
3546         netif_carrier_off(netdev);
3547
3548         ixgbevf_init_last_counter_stats(adapter);
3549
3550         /* print the MAC address */
3551         hw_dbg(hw, "%pM\n", netdev->dev_addr);
3552
3553         hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3554
3555         hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3556         cards_found++;
3557         return 0;
3558
3559 err_register:
3560         ixgbevf_clear_interrupt_scheme(adapter);
3561 err_sw_init:
3562         ixgbevf_reset_interrupt_capability(adapter);
3563         iounmap(hw->hw_addr);
3564 err_ioremap:
3565         free_netdev(netdev);
3566 err_alloc_etherdev:
3567         pci_release_regions(pdev);
3568 err_pci_reg:
3569 err_dma:
3570         pci_disable_device(pdev);
3571         return err;
3572 }
3573
3574 /**
3575  * ixgbevf_remove - Device Removal Routine
3576  * @pdev: PCI device information struct
3577  *
3578  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3579  * that it should release a PCI device.  The could be caused by a
3580  * Hot-Plug event, or because the driver is going to be removed from
3581  * memory.
3582  **/
3583 static void ixgbevf_remove(struct pci_dev *pdev)
3584 {
3585         struct net_device *netdev = pci_get_drvdata(pdev);
3586         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3587
3588         set_bit(__IXGBEVF_DOWN, &adapter->state);
3589
3590         del_timer_sync(&adapter->watchdog_timer);
3591
3592         cancel_work_sync(&adapter->reset_task);
3593         cancel_work_sync(&adapter->watchdog_task);
3594
3595         if (netdev->reg_state == NETREG_REGISTERED)
3596                 unregister_netdev(netdev);
3597
3598         ixgbevf_clear_interrupt_scheme(adapter);
3599         ixgbevf_reset_interrupt_capability(adapter);
3600
3601         iounmap(adapter->hw.hw_addr);
3602         pci_release_regions(pdev);
3603
3604         hw_dbg(&adapter->hw, "Remove complete\n");
3605
3606         kfree(adapter->tx_ring);
3607         kfree(adapter->rx_ring);
3608
3609         free_netdev(netdev);
3610
3611         pci_disable_device(pdev);
3612 }
3613
3614 /**
3615  * ixgbevf_io_error_detected - called when PCI error is detected
3616  * @pdev: Pointer to PCI device
3617  * @state: The current pci connection state
3618  *
3619  * This function is called after a PCI bus error affecting
3620  * this device has been detected.
3621  */
3622 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3623                                                   pci_channel_state_t state)
3624 {
3625         struct net_device *netdev = pci_get_drvdata(pdev);
3626         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3627
3628         netif_device_detach(netdev);
3629
3630         if (state == pci_channel_io_perm_failure)
3631                 return PCI_ERS_RESULT_DISCONNECT;
3632
3633         if (netif_running(netdev))
3634                 ixgbevf_down(adapter);
3635
3636         pci_disable_device(pdev);
3637
3638         /* Request a slot slot reset. */
3639         return PCI_ERS_RESULT_NEED_RESET;
3640 }
3641
3642 /**
3643  * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3644  * @pdev: Pointer to PCI device
3645  *
3646  * Restart the card from scratch, as if from a cold-boot. Implementation
3647  * resembles the first-half of the ixgbevf_resume routine.
3648  */
3649 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3650 {
3651         struct net_device *netdev = pci_get_drvdata(pdev);
3652         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3653
3654         if (pci_enable_device_mem(pdev)) {
3655                 dev_err(&pdev->dev,
3656                         "Cannot re-enable PCI device after reset.\n");
3657                 return PCI_ERS_RESULT_DISCONNECT;
3658         }
3659
3660         pci_set_master(pdev);
3661
3662         ixgbevf_reset(adapter);
3663
3664         return PCI_ERS_RESULT_RECOVERED;
3665 }
3666
3667 /**
3668  * ixgbevf_io_resume - called when traffic can start flowing again.
3669  * @pdev: Pointer to PCI device
3670  *
3671  * This callback is called when the error recovery driver tells us that
3672  * its OK to resume normal operation. Implementation resembles the
3673  * second-half of the ixgbevf_resume routine.
3674  */
3675 static void ixgbevf_io_resume(struct pci_dev *pdev)
3676 {
3677         struct net_device *netdev = pci_get_drvdata(pdev);
3678         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3679
3680         if (netif_running(netdev))
3681                 ixgbevf_up(adapter);
3682
3683         netif_device_attach(netdev);
3684 }
3685
3686 /* PCI Error Recovery (ERS) */
3687 static const struct pci_error_handlers ixgbevf_err_handler = {
3688         .error_detected = ixgbevf_io_error_detected,
3689         .slot_reset = ixgbevf_io_slot_reset,
3690         .resume = ixgbevf_io_resume,
3691 };
3692
3693 static struct pci_driver ixgbevf_driver = {
3694         .name     = ixgbevf_driver_name,
3695         .id_table = ixgbevf_pci_tbl,
3696         .probe    = ixgbevf_probe,
3697         .remove   = ixgbevf_remove,
3698 #ifdef CONFIG_PM
3699         /* Power Management Hooks */
3700         .suspend  = ixgbevf_suspend,
3701         .resume   = ixgbevf_resume,
3702 #endif
3703         .shutdown = ixgbevf_shutdown,
3704         .err_handler = &ixgbevf_err_handler
3705 };
3706
3707 /**
3708  * ixgbevf_init_module - Driver Registration Routine
3709  *
3710  * ixgbevf_init_module is the first routine called when the driver is
3711  * loaded. All it does is register with the PCI subsystem.
3712  **/
3713 static int __init ixgbevf_init_module(void)
3714 {
3715         int ret;
3716         pr_info("%s - version %s\n", ixgbevf_driver_string,
3717                 ixgbevf_driver_version);
3718
3719         pr_info("%s\n", ixgbevf_copyright);
3720
3721         ret = pci_register_driver(&ixgbevf_driver);
3722         return ret;
3723 }
3724
3725 module_init(ixgbevf_init_module);
3726
3727 /**
3728  * ixgbevf_exit_module - Driver Exit Cleanup Routine
3729  *
3730  * ixgbevf_exit_module is called just before the driver is removed
3731  * from memory.
3732  **/
3733 static void __exit ixgbevf_exit_module(void)
3734 {
3735         pci_unregister_driver(&ixgbevf_driver);
3736 }
3737
3738 #ifdef DEBUG
3739 /**
3740  * ixgbevf_get_hw_dev_name - return device name string
3741  * used by hardware layer to print debugging information
3742  **/
3743 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3744 {
3745         struct ixgbevf_adapter *adapter = hw->back;
3746         return adapter->netdev->name;
3747 }
3748
3749 #endif
3750 module_exit(ixgbevf_exit_module);
3751
3752 /* ixgbevf_main.c */