Pileus Git - ~andy/linux/blob - drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 /* Local includes */
  28 #include "i40e.h"
  29 #ifdef CONFIG_I40E_VXLAN
  30 #include <net/vxlan.h>
  31 #endif
  32
  33 const char i40e_driver_name[] = "i40e";
  34 static const char i40e_driver_string[] =
  35                         "Intel(R) Ethernet Connection XL710 Network Driver";
  36
  37 #define DRV_KERN "-k"
  38
  39 #define DRV_VERSION_MAJOR 0
  40 #define DRV_VERSION_MINOR 3
  41 #define DRV_VERSION_BUILD 28
  42 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  43              __stringify(DRV_VERSION_MINOR) "." \
  44              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  45 const char i40e_driver_version_str[] = DRV_VERSION;
  46 static const char i40e_copyright[] = "Copyright (c) 2013 Intel Corporation.";
  47
  48 /* a bit of forward declarations */
  49 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  50 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  51 static int i40e_add_vsi(struct i40e_vsi *vsi);
  52 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  53 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  54 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  55 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  56 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  57
  58 /* i40e_pci_tbl - PCI Device ID Table
  59  *
  60  * Last entry must be all 0s
  61  *
  62  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  63  *   Class, Class Mask, private data (not used) }
  64  */
  65 static DEFINE_PCI_DEVICE_TABLE(i40e_pci_tbl) = {
  66         {PCI_VDEVICE(INTEL, I40E_SFP_XL710_DEVICE_ID), 0},
  67         {PCI_VDEVICE(INTEL, I40E_SFP_X710_DEVICE_ID), 0},
  68         {PCI_VDEVICE(INTEL, I40E_QEMU_DEVICE_ID), 0},
  69         {PCI_VDEVICE(INTEL, I40E_KX_A_DEVICE_ID), 0},
  70         {PCI_VDEVICE(INTEL, I40E_KX_B_DEVICE_ID), 0},
  71         {PCI_VDEVICE(INTEL, I40E_KX_C_DEVICE_ID), 0},
  72         {PCI_VDEVICE(INTEL, I40E_KX_D_DEVICE_ID), 0},
  73         {PCI_VDEVICE(INTEL, I40E_QSFP_A_DEVICE_ID), 0},
  74         {PCI_VDEVICE(INTEL, I40E_QSFP_B_DEVICE_ID), 0},
  75         {PCI_VDEVICE(INTEL, I40E_QSFP_C_DEVICE_ID), 0},
  76         /* required last entry */
  77         {0, }
  78 };
  79 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  80
  81 #define I40E_MAX_VF_COUNT 128
  82 static int debug = -1;
  83 module_param(debug, int, 0);
  84 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  85
  86 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  87 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  88 MODULE_LICENSE("GPL");
  89 MODULE_VERSION(DRV_VERSION);
  90
  91 /**
  92  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
  93  * @hw:   pointer to the HW structure
  94  * @mem:  ptr to mem struct to fill out
  95  * @size: size of memory requested
  96  * @alignment: what to align the allocation to
  97  **/
  98 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
  99                             u64 size, u32 alignment)
 100 {
 101         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 102
 103         mem->size = ALIGN(size, alignment);
 104         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 105                                       &mem->pa, GFP_KERNEL);
 106         if (!mem->va)
 107                 return -ENOMEM;
 108
 109         return 0;
 110 }
 111
 112 /**
 113  * i40e_free_dma_mem_d - OS specific memory free for shared code
 114  * @hw:   pointer to the HW structure
 115  * @mem:  ptr to mem struct to free
 116  **/
 117 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 118 {
 119         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 120
 121         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 122         mem->va = NULL;
 123         mem->pa = 0;
 124         mem->size = 0;
 125
 126         return 0;
 127 }
 128
 129 /**
 130  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 131  * @hw:   pointer to the HW structure
 132  * @mem:  ptr to mem struct to fill out
 133  * @size: size of memory requested
 134  **/
 135 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 136                              u32 size)
 137 {
 138         mem->size = size;
 139         mem->va = kzalloc(size, GFP_KERNEL);
 140
 141         if (!mem->va)
 142                 return -ENOMEM;
 143
 144         return 0;
 145 }
 146
 147 /**
 148  * i40e_free_virt_mem_d - OS specific memory free for shared code
 149  * @hw:   pointer to the HW structure
 150  * @mem:  ptr to mem struct to free
 151  **/
 152 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 153 {
 154         /* it's ok to kfree a NULL pointer */
 155         kfree(mem->va);
 156         mem->va = NULL;
 157         mem->size = 0;
 158
 159         return 0;
 160 }
 161
 162 /**
 163  * i40e_get_lump - find a lump of free generic resource
 164  * @pf: board private structure
 165  * @pile: the pile of resource to search
 166  * @needed: the number of items needed
 167  * @id: an owner id to stick on the items assigned
 168  *
 169  * Returns the base item index of the lump, or negative for error
 170  *
 171  * The search_hint trick and lack of advanced fit-finding only work
 172  * because we're highly likely to have all the same size lump requests.
 173  * Linear search time and any fragmentation should be minimal.
 174  **/
 175 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 176                          u16 needed, u16 id)
 177 {
 178         int ret = -ENOMEM;
 179         int i, j;
 180
 181         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 182                 dev_info(&pf->pdev->dev,
 183                          "param err: pile=%p needed=%d id=0x%04x\n",
 184                          pile, needed, id);
 185                 return -EINVAL;
 186         }
 187
 188         /* start the linear search with an imperfect hint */
 189         i = pile->search_hint;
 190         while (i < pile->num_entries) {
 191                 /* skip already allocated entries */
 192                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 193                         i++;
 194                         continue;
 195                 }
 196
 197                 /* do we have enough in this lump? */
 198                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 199                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 200                                 break;
 201                 }
 202
 203                 if (j == needed) {
 204                         /* there was enough, so assign it to the requestor */
 205                         for (j = 0; j < needed; j++)
 206                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 207                         ret = i;
 208                         pile->search_hint = i + j;
 209                         break;
 210                 } else {
 211                         /* not enough, so skip over it and continue looking */
 212                         i += j;
 213                 }
 214         }
 215
 216         return ret;
 217 }
 218
 219 /**
 220  * i40e_put_lump - return a lump of generic resource
 221  * @pile: the pile of resource to search
 222  * @index: the base item index
 223  * @id: the owner id of the items assigned
 224  *
 225  * Returns the count of items in the lump
 226  **/
 227 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 228 {
 229         int valid_id = (id | I40E_PILE_VALID_BIT);
 230         int count = 0;
 231         int i;
 232
 233         if (!pile || index >= pile->num_entries)
 234                 return -EINVAL;
 235
 236         for (i = index;
 237              i < pile->num_entries && pile->list[i] == valid_id;
 238              i++) {
 239                 pile->list[i] = 0;
 240                 count++;
 241         }
 242
 243         if (count && index < pile->search_hint)
 244                 pile->search_hint = index;
 245
 246         return count;
 247 }
 248
 249 /**
 250  * i40e_service_event_schedule - Schedule the service task to wake up
 251  * @pf: board private structure
 252  *
 253  * If not already scheduled, this puts the task into the work queue
 254  **/
 255 static void i40e_service_event_schedule(struct i40e_pf *pf)
 256 {
 257         if (!test_bit(__I40E_DOWN, &pf->state) &&
 258             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
 259             !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
 260                 schedule_work(&pf->service_task);
 261 }
 262
 263 /**
 264  * i40e_tx_timeout - Respond to a Tx Hang
 265  * @netdev: network interface device structure
 266  *
 267  * If any port has noticed a Tx timeout, it is likely that the whole
 268  * device is munged, not just the one netdev port, so go for the full
 269  * reset.
 270  **/
 271 static void i40e_tx_timeout(struct net_device *netdev)
 272 {
 273         struct i40e_netdev_priv *np = netdev_priv(netdev);
 274         struct i40e_vsi *vsi = np->vsi;
 275         struct i40e_pf *pf = vsi->back;
 276
 277         pf->tx_timeout_count++;
 278
 279         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 280                 pf->tx_timeout_recovery_level = 0;
 281         pf->tx_timeout_last_recovery = jiffies;
 282         netdev_info(netdev, "tx_timeout recovery level %d\n",
 283                     pf->tx_timeout_recovery_level);
 284
 285         switch (pf->tx_timeout_recovery_level) {
 286         case 0:
 287                 /* disable and re-enable queues for the VSI */
 288                 if (in_interrupt()) {
 289                         set_bit(__I40E_REINIT_REQUESTED, &pf->state);
 290                         set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
 291                 } else {
 292                         i40e_vsi_reinit_locked(vsi);
 293                 }
 294                 break;
 295         case 1:
 296                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 297                 break;
 298         case 2:
 299                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 300                 break;
 301         case 3:
 302                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 303                 break;
 304         default:
 305                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 306                 i40e_down(vsi);
 307                 break;
 308         }
 309         i40e_service_event_schedule(pf);
 310         pf->tx_timeout_recovery_level++;
 311 }
 312
 313 /**
 314  * i40e_release_rx_desc - Store the new tail and head values
 315  * @rx_ring: ring to bump
 316  * @val: new head index
 317  **/
 318 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 319 {
 320         rx_ring->next_to_use = val;
 321
 322         /* Force memory writes to complete before letting h/w
 323          * know there are new descriptors to fetch.  (Only
 324          * applicable for weak-ordered memory model archs,
 325          * such as IA-64).
 326          */
 327         wmb();
 328         writel(val, rx_ring->tail);
 329 }
 330
 331 /**
 332  * i40e_get_vsi_stats_struct - Get System Network Statistics
 333  * @vsi: the VSI we care about
 334  *
 335  * Returns the address of the device statistics structure.
 336  * The statistics are actually updated from the service task.
 337  **/
 338 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 339 {
 340         return &vsi->net_stats;
 341 }
 342
 343 /**
 344  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 345  * @netdev: network interface device structure
 346  *
 347  * Returns the address of the device statistics structure.
 348  * The statistics are actually updated from the service task.
 349  **/
 350 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 351                                              struct net_device *netdev,
 352                                              struct rtnl_link_stats64 *stats)
 353 {
 354         struct i40e_netdev_priv *np = netdev_priv(netdev);
 355         struct i40e_vsi *vsi = np->vsi;
 356         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 357         int i;
 358
 359
 360         if (test_bit(__I40E_DOWN, &vsi->state))
 361                 return stats;
 362
 363         if (!vsi->tx_rings)
 364                 return stats;
 365
 366         rcu_read_lock();
 367         for (i = 0; i < vsi->num_queue_pairs; i++) {
 368                 struct i40e_ring *tx_ring, *rx_ring;
 369                 u64 bytes, packets;
 370                 unsigned int start;
 371
 372                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 373                 if (!tx_ring)
 374                         continue;
 375
 376                 do {
 377                         start = u64_stats_fetch_begin_bh(&tx_ring->syncp);
 378                         packets = tx_ring->stats.packets;
 379                         bytes   = tx_ring->stats.bytes;
 380                 } while (u64_stats_fetch_retry_bh(&tx_ring->syncp, start));
 381
 382                 stats->tx_packets += packets;
 383                 stats->tx_bytes   += bytes;
 384                 rx_ring = &tx_ring[1];
 385
 386                 do {
 387                         start = u64_stats_fetch_begin_bh(&rx_ring->syncp);
 388                         packets = rx_ring->stats.packets;
 389                         bytes   = rx_ring->stats.bytes;
 390                 } while (u64_stats_fetch_retry_bh(&rx_ring->syncp, start));
 391
 392                 stats->rx_packets += packets;
 393                 stats->rx_bytes   += bytes;
 394         }
 395         rcu_read_unlock();
 396
 397         /* following stats updated by ixgbe_watchdog_task() */
 398         stats->multicast        = vsi_stats->multicast;
 399         stats->tx_errors        = vsi_stats->tx_errors;
 400         stats->tx_dropped       = vsi_stats->tx_dropped;
 401         stats->rx_errors        = vsi_stats->rx_errors;
 402         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 403         stats->rx_length_errors = vsi_stats->rx_length_errors;
 404
 405         return stats;
 406 }
 407
 408 /**
 409  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 410  * @vsi: the VSI to have its stats reset
 411  **/
 412 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 413 {
 414         struct rtnl_link_stats64 *ns;
 415         int i;
 416
 417         if (!vsi)
 418                 return;
 419
 420         ns = i40e_get_vsi_stats_struct(vsi);
 421         memset(ns, 0, sizeof(*ns));
 422         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 423         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 424         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 425         if (vsi->rx_rings && vsi->rx_rings[0]) {
 426                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 427                         memset(&vsi->rx_rings[i]->stats, 0 ,
 428                                sizeof(vsi->rx_rings[i]->stats));
 429                         memset(&vsi->rx_rings[i]->rx_stats, 0 ,
 430                                sizeof(vsi->rx_rings[i]->rx_stats));
 431                         memset(&vsi->tx_rings[i]->stats, 0 ,
 432                                sizeof(vsi->tx_rings[i]->stats));
 433                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 434                                sizeof(vsi->tx_rings[i]->tx_stats));
 435                 }
 436         }
 437         vsi->stat_offsets_loaded = false;
 438 }
 439
 440 /**
 441  * i40e_pf_reset_stats - Reset all of the stats for the given pf
 442  * @pf: the PF to be reset
 443  **/
 444 void i40e_pf_reset_stats(struct i40e_pf *pf)
 445 {
 446         memset(&pf->stats, 0, sizeof(pf->stats));
 447         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 448         pf->stat_offsets_loaded = false;
 449 }
 450
 451 /**
 452  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 453  * @hw: ptr to the hardware info
 454  * @hireg: the high 32 bit reg to read
 455  * @loreg: the low 32 bit reg to read
 456  * @offset_loaded: has the initial offset been loaded yet
 457  * @offset: ptr to current offset value
 458  * @stat: ptr to the stat
 459  *
 460  * Since the device stats are not reset at PFReset, they likely will not
 461  * be zeroed when the driver starts.  We'll save the first values read
 462  * and use them as offsets to be subtracted from the raw values in order
 463  * to report stats that count from zero.  In the process, we also manage
 464  * the potential roll-over.
 465  **/
 466 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 467                                bool offset_loaded, u64 *offset, u64 *stat)
 468 {
 469         u64 new_data;
 470
 471         if (hw->device_id == I40E_QEMU_DEVICE_ID) {
 472                 new_data = rd32(hw, loreg);
 473                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 474         } else {
 475                 new_data = rd64(hw, loreg);
 476         }
 477         if (!offset_loaded)
 478                 *offset = new_data;
 479         if (likely(new_data >= *offset))
 480                 *stat = new_data - *offset;
 481         else
 482                 *stat = (new_data + ((u64)1 << 48)) - *offset;
 483         *stat &= 0xFFFFFFFFFFFFULL;
 484 }
 485
 486 /**
 487  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 488  * @hw: ptr to the hardware info
 489  * @reg: the hw reg to read
 490  * @offset_loaded: has the initial offset been loaded yet
 491  * @offset: ptr to current offset value
 492  * @stat: ptr to the stat
 493  **/
 494 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 495                                bool offset_loaded, u64 *offset, u64 *stat)
 496 {
 497         u32 new_data;
 498
 499         new_data = rd32(hw, reg);
 500         if (!offset_loaded)
 501                 *offset = new_data;
 502         if (likely(new_data >= *offset))
 503                 *stat = (u32)(new_data - *offset);
 504         else
 505                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
 506 }
 507
 508 /**
 509  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 510  * @vsi: the VSI to be updated
 511  **/
 512 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 513 {
 514         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 515         struct i40e_pf *pf = vsi->back;
 516         struct i40e_hw *hw = &pf->hw;
 517         struct i40e_eth_stats *oes;
 518         struct i40e_eth_stats *es;     /* device's eth stats */
 519
 520         es = &vsi->eth_stats;
 521         oes = &vsi->eth_stats_offsets;
 522
 523         /* Gather up the stats that the hw collects */
 524         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 525                            vsi->stat_offsets_loaded,
 526                            &oes->tx_errors, &es->tx_errors);
 527         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 528                            vsi->stat_offsets_loaded,
 529                            &oes->rx_discards, &es->rx_discards);
 530
 531         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 532                            I40E_GLV_GORCL(stat_idx),
 533                            vsi->stat_offsets_loaded,
 534                            &oes->rx_bytes, &es->rx_bytes);
 535         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 536                            I40E_GLV_UPRCL(stat_idx),
 537                            vsi->stat_offsets_loaded,
 538                            &oes->rx_unicast, &es->rx_unicast);
 539         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 540                            I40E_GLV_MPRCL(stat_idx),
 541                            vsi->stat_offsets_loaded,
 542                            &oes->rx_multicast, &es->rx_multicast);
 543         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 544                            I40E_GLV_BPRCL(stat_idx),
 545                            vsi->stat_offsets_loaded,
 546                            &oes->rx_broadcast, &es->rx_broadcast);
 547
 548         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 549                            I40E_GLV_GOTCL(stat_idx),
 550                            vsi->stat_offsets_loaded,
 551                            &oes->tx_bytes, &es->tx_bytes);
 552         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 553                            I40E_GLV_UPTCL(stat_idx),
 554                            vsi->stat_offsets_loaded,
 555                            &oes->tx_unicast, &es->tx_unicast);
 556         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 557                            I40E_GLV_MPTCL(stat_idx),
 558                            vsi->stat_offsets_loaded,
 559                            &oes->tx_multicast, &es->tx_multicast);
 560         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 561                            I40E_GLV_BPTCL(stat_idx),
 562                            vsi->stat_offsets_loaded,
 563                            &oes->tx_broadcast, &es->tx_broadcast);
 564         vsi->stat_offsets_loaded = true;
 565 }
 566
 567 /**
 568  * i40e_update_veb_stats - Update Switch component statistics
 569  * @veb: the VEB being updated
 570  **/
 571 static void i40e_update_veb_stats(struct i40e_veb *veb)
 572 {
 573         struct i40e_pf *pf = veb->pf;
 574         struct i40e_hw *hw = &pf->hw;
 575         struct i40e_eth_stats *oes;
 576         struct i40e_eth_stats *es;     /* device's eth stats */
 577         int idx = 0;
 578
 579         idx = veb->stats_idx;
 580         es = &veb->stats;
 581         oes = &veb->stats_offsets;
 582
 583         /* Gather up the stats that the hw collects */
 584         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 585                            veb->stat_offsets_loaded,
 586                            &oes->tx_discards, &es->tx_discards);
 587         if (hw->revision_id > 0)
 588                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 589                                    veb->stat_offsets_loaded,
 590                                    &oes->rx_unknown_protocol,
 591                                    &es->rx_unknown_protocol);
 592         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 593                            veb->stat_offsets_loaded,
 594                            &oes->rx_bytes, &es->rx_bytes);
 595         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 596                            veb->stat_offsets_loaded,
 597                            &oes->rx_unicast, &es->rx_unicast);
 598         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 599                            veb->stat_offsets_loaded,
 600                            &oes->rx_multicast, &es->rx_multicast);
 601         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 602                            veb->stat_offsets_loaded,
 603                            &oes->rx_broadcast, &es->rx_broadcast);
 604
 605         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 606                            veb->stat_offsets_loaded,
 607                            &oes->tx_bytes, &es->tx_bytes);
 608         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 609                            veb->stat_offsets_loaded,
 610                            &oes->tx_unicast, &es->tx_unicast);
 611         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 612                            veb->stat_offsets_loaded,
 613                            &oes->tx_multicast, &es->tx_multicast);
 614         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 615                            veb->stat_offsets_loaded,
 616                            &oes->tx_broadcast, &es->tx_broadcast);
 617         veb->stat_offsets_loaded = true;
 618 }
 619
 620 /**
 621  * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
 622  * @pf: the corresponding PF
 623  *
 624  * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
 625  **/
 626 static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
 627 {
 628         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 629         struct i40e_hw_port_stats *nsd = &pf->stats;
 630         struct i40e_hw *hw = &pf->hw;
 631         u64 xoff = 0;
 632         u16 i, v;
 633
 634         if ((hw->fc.current_mode != I40E_FC_FULL) &&
 635             (hw->fc.current_mode != I40E_FC_RX_PAUSE))
 636                 return;
 637
 638         xoff = nsd->link_xoff_rx;
 639         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 640                            pf->stat_offsets_loaded,
 641                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 642
 643         /* No new LFC xoff rx */
 644         if (!(nsd->link_xoff_rx - xoff))
 645                 return;
 646
 647         /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
 648         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 649                 struct i40e_vsi *vsi = pf->vsi[v];
 650
 651                 if (!vsi)
 652                         continue;
 653
 654                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 655                         struct i40e_ring *ring = vsi->tx_rings[i];
 656                         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
 657                 }
 658         }
 659 }
 660
 661 /**
 662  * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
 663  * @pf: the corresponding PF
 664  *
 665  * Update the Rx XOFF counter (PAUSE frames) in PFC mode
 666  **/
 667 static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
 668 {
 669         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 670         struct i40e_hw_port_stats *nsd = &pf->stats;
 671         bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
 672         struct i40e_dcbx_config *dcb_cfg;
 673         struct i40e_hw *hw = &pf->hw;
 674         u16 i, v;
 675         u8 tc;
 676
 677         dcb_cfg = &hw->local_dcbx_config;
 678
 679         /* See if DCB enabled with PFC TC */
 680         if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
 681             !(dcb_cfg->pfc.pfcenable)) {
 682                 i40e_update_link_xoff_rx(pf);
 683                 return;
 684         }
 685
 686         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 687                 u64 prio_xoff = nsd->priority_xoff_rx[i];
 688                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 689                                    pf->stat_offsets_loaded,
 690                                    &osd->priority_xoff_rx[i],
 691                                    &nsd->priority_xoff_rx[i]);
 692
 693                 /* No new PFC xoff rx */
 694                 if (!(nsd->priority_xoff_rx[i] - prio_xoff))
 695                         continue;
 696                 /* Get the TC for given priority */
 697                 tc = dcb_cfg->etscfg.prioritytable[i];
 698                 xoff[tc] = true;
 699         }
 700
 701         /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
 702         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 703                 struct i40e_vsi *vsi = pf->vsi[v];
 704
 705                 if (!vsi)
 706                         continue;
 707
 708                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 709                         struct i40e_ring *ring = vsi->tx_rings[i];
 710
 711                         tc = ring->dcb_tc;
 712                         if (xoff[tc])
 713                                 clear_bit(__I40E_HANG_CHECK_ARMED,
 714                                           &ring->state);
 715                 }
 716         }
 717 }
 718
 719 /**
 720  * i40e_update_stats - Update the board statistics counters.
 721  * @vsi: the VSI to be updated
 722  *
 723  * There are a few instances where we store the same stat in a
 724  * couple of different structs.  This is partly because we have
 725  * the netdev stats that need to be filled out, which is slightly
 726  * different from the "eth_stats" defined by the chip and used in
 727  * VF communications.  We sort it all out here in a central place.
 728  **/
 729 void i40e_update_stats(struct i40e_vsi *vsi)
 730 {
 731         struct i40e_pf *pf = vsi->back;
 732         struct i40e_hw *hw = &pf->hw;
 733         struct rtnl_link_stats64 *ons;
 734         struct rtnl_link_stats64 *ns;   /* netdev stats */
 735         struct i40e_eth_stats *oes;
 736         struct i40e_eth_stats *es;     /* device's eth stats */
 737         u32 tx_restart, tx_busy;
 738         u32 rx_page, rx_buf;
 739         u64 rx_p, rx_b;
 740         u64 tx_p, tx_b;
 741         int i;
 742         u16 q;
 743
 744         if (test_bit(__I40E_DOWN, &vsi->state) ||
 745             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 746                 return;
 747
 748         ns = i40e_get_vsi_stats_struct(vsi);
 749         ons = &vsi->net_stats_offsets;
 750         es = &vsi->eth_stats;
 751         oes = &vsi->eth_stats_offsets;
 752
 753         /* Gather up the netdev and vsi stats that the driver collects
 754          * on the fly during packet processing
 755          */
 756         rx_b = rx_p = 0;
 757         tx_b = tx_p = 0;
 758         tx_restart = tx_busy = 0;
 759         rx_page = 0;
 760         rx_buf = 0;
 761         rcu_read_lock();
 762         for (q = 0; q < vsi->num_queue_pairs; q++) {
 763                 struct i40e_ring *p;
 764                 u64 bytes, packets;
 765                 unsigned int start;
 766
 767                 /* locate Tx ring */
 768                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 769
 770                 do {
 771                         start = u64_stats_fetch_begin_bh(&p->syncp);
 772                         packets = p->stats.packets;
 773                         bytes = p->stats.bytes;
 774                 } while (u64_stats_fetch_retry_bh(&p->syncp, start));
 775                 tx_b += bytes;
 776                 tx_p += packets;
 777                 tx_restart += p->tx_stats.restart_queue;
 778                 tx_busy += p->tx_stats.tx_busy;
 779
 780                 /* Rx queue is part of the same block as Tx queue */
 781                 p = &p[1];
 782                 do {
 783                         start = u64_stats_fetch_begin_bh(&p->syncp);
 784                         packets = p->stats.packets;
 785                         bytes = p->stats.bytes;
 786                 } while (u64_stats_fetch_retry_bh(&p->syncp, start));
 787                 rx_b += bytes;
 788                 rx_p += packets;
 789                 rx_buf += p->rx_stats.alloc_buff_failed;
 790                 rx_page += p->rx_stats.alloc_page_failed;
 791         }
 792         rcu_read_unlock();
 793         vsi->tx_restart = tx_restart;
 794         vsi->tx_busy = tx_busy;
 795         vsi->rx_page_failed = rx_page;
 796         vsi->rx_buf_failed = rx_buf;
 797
 798         ns->rx_packets = rx_p;
 799         ns->rx_bytes = rx_b;
 800         ns->tx_packets = tx_p;
 801         ns->tx_bytes = tx_b;
 802
 803         i40e_update_eth_stats(vsi);
 804         /* update netdev stats from eth stats */
 805         ons->rx_errors = oes->rx_errors;
 806         ns->rx_errors = es->rx_errors;
 807         ons->tx_errors = oes->tx_errors;
 808         ns->tx_errors = es->tx_errors;
 809         ons->multicast = oes->rx_multicast;
 810         ns->multicast = es->rx_multicast;
 811         ons->tx_dropped = oes->tx_discards;
 812         ns->tx_dropped = es->tx_discards;
 813
 814         /* Get the port data only if this is the main PF VSI */
 815         if (vsi == pf->vsi[pf->lan_vsi]) {
 816                 struct i40e_hw_port_stats *nsd = &pf->stats;
 817                 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 818
 819                 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 820                                    I40E_GLPRT_GORCL(hw->port),
 821                                    pf->stat_offsets_loaded,
 822                                    &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 823                 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 824                                    I40E_GLPRT_GOTCL(hw->port),
 825                                    pf->stat_offsets_loaded,
 826                                    &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 827                 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 828                                    pf->stat_offsets_loaded,
 829                                    &osd->eth.rx_discards,
 830                                    &nsd->eth.rx_discards);
 831                 i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
 832                                    pf->stat_offsets_loaded,
 833                                    &osd->eth.tx_discards,
 834                                    &nsd->eth.tx_discards);
 835                 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 836                                    I40E_GLPRT_MPRCL(hw->port),
 837                                    pf->stat_offsets_loaded,
 838                                    &osd->eth.rx_multicast,
 839                                    &nsd->eth.rx_multicast);
 840
 841                 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 842                                    pf->stat_offsets_loaded,
 843                                    &osd->tx_dropped_link_down,
 844                                    &nsd->tx_dropped_link_down);
 845
 846                 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 847                                    pf->stat_offsets_loaded,
 848                                    &osd->crc_errors, &nsd->crc_errors);
 849                 ns->rx_crc_errors = nsd->crc_errors;
 850
 851                 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 852                                    pf->stat_offsets_loaded,
 853                                    &osd->illegal_bytes, &nsd->illegal_bytes);
 854                 ns->rx_errors = nsd->crc_errors
 855                                 + nsd->illegal_bytes;
 856
 857                 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 858                                    pf->stat_offsets_loaded,
 859                                    &osd->mac_local_faults,
 860                                    &nsd->mac_local_faults);
 861                 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 862                                    pf->stat_offsets_loaded,
 863                                    &osd->mac_remote_faults,
 864                                    &nsd->mac_remote_faults);
 865
 866                 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 867                                    pf->stat_offsets_loaded,
 868                                    &osd->rx_length_errors,
 869                                    &nsd->rx_length_errors);
 870                 ns->rx_length_errors = nsd->rx_length_errors;
 871
 872                 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 873                                    pf->stat_offsets_loaded,
 874                                    &osd->link_xon_rx, &nsd->link_xon_rx);
 875                 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 876                                    pf->stat_offsets_loaded,
 877                                    &osd->link_xon_tx, &nsd->link_xon_tx);
 878                 i40e_update_prio_xoff_rx(pf);  /* handles I40E_GLPRT_LXOFFRXC */
 879                 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 880                                    pf->stat_offsets_loaded,
 881                                    &osd->link_xoff_tx, &nsd->link_xoff_tx);
 882
 883                 for (i = 0; i < 8; i++) {
 884                         i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 885                                            pf->stat_offsets_loaded,
 886                                            &osd->priority_xon_rx[i],
 887                                            &nsd->priority_xon_rx[i]);
 888                         i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 889                                            pf->stat_offsets_loaded,
 890                                            &osd->priority_xon_tx[i],
 891                                            &nsd->priority_xon_tx[i]);
 892                         i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 893                                            pf->stat_offsets_loaded,
 894                                            &osd->priority_xoff_tx[i],
 895                                            &nsd->priority_xoff_tx[i]);
 896                         i40e_stat_update32(hw,
 897                                            I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 898                                            pf->stat_offsets_loaded,
 899                                            &osd->priority_xon_2_xoff[i],
 900                                            &nsd->priority_xon_2_xoff[i]);
 901                 }
 902
 903                 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 904                                    I40E_GLPRT_PRC64L(hw->port),
 905                                    pf->stat_offsets_loaded,
 906                                    &osd->rx_size_64, &nsd->rx_size_64);
 907                 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 908                                    I40E_GLPRT_PRC127L(hw->port),
 909                                    pf->stat_offsets_loaded,
 910                                    &osd->rx_size_127, &nsd->rx_size_127);
 911                 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 912                                    I40E_GLPRT_PRC255L(hw->port),
 913                                    pf->stat_offsets_loaded,
 914                                    &osd->rx_size_255, &nsd->rx_size_255);
 915                 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 916                                    I40E_GLPRT_PRC511L(hw->port),
 917                                    pf->stat_offsets_loaded,
 918                                    &osd->rx_size_511, &nsd->rx_size_511);
 919                 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 920                                    I40E_GLPRT_PRC1023L(hw->port),
 921                                    pf->stat_offsets_loaded,
 922                                    &osd->rx_size_1023, &nsd->rx_size_1023);
 923                 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 924                                    I40E_GLPRT_PRC1522L(hw->port),
 925                                    pf->stat_offsets_loaded,
 926                                    &osd->rx_size_1522, &nsd->rx_size_1522);
 927                 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 928                                    I40E_GLPRT_PRC9522L(hw->port),
 929                                    pf->stat_offsets_loaded,
 930                                    &osd->rx_size_big, &nsd->rx_size_big);
 931
 932                 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
 933                                    I40E_GLPRT_PTC64L(hw->port),
 934                                    pf->stat_offsets_loaded,
 935                                    &osd->tx_size_64, &nsd->tx_size_64);
 936                 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
 937                                    I40E_GLPRT_PTC127L(hw->port),
 938                                    pf->stat_offsets_loaded,
 939                                    &osd->tx_size_127, &nsd->tx_size_127);
 940                 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
 941                                    I40E_GLPRT_PTC255L(hw->port),
 942                                    pf->stat_offsets_loaded,
 943                                    &osd->tx_size_255, &nsd->tx_size_255);
 944                 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
 945                                    I40E_GLPRT_PTC511L(hw->port),
 946                                    pf->stat_offsets_loaded,
 947                                    &osd->tx_size_511, &nsd->tx_size_511);
 948                 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
 949                                    I40E_GLPRT_PTC1023L(hw->port),
 950                                    pf->stat_offsets_loaded,
 951                                    &osd->tx_size_1023, &nsd->tx_size_1023);
 952                 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
 953                                    I40E_GLPRT_PTC1522L(hw->port),
 954                                    pf->stat_offsets_loaded,
 955                                    &osd->tx_size_1522, &nsd->tx_size_1522);
 956                 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
 957                                    I40E_GLPRT_PTC9522L(hw->port),
 958                                    pf->stat_offsets_loaded,
 959                                    &osd->tx_size_big, &nsd->tx_size_big);
 960
 961                 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
 962                                    pf->stat_offsets_loaded,
 963                                    &osd->rx_undersize, &nsd->rx_undersize);
 964                 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
 965                                    pf->stat_offsets_loaded,
 966                                    &osd->rx_fragments, &nsd->rx_fragments);
 967                 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
 968                                    pf->stat_offsets_loaded,
 969                                    &osd->rx_oversize, &nsd->rx_oversize);
 970                 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
 971                                    pf->stat_offsets_loaded,
 972                                    &osd->rx_jabber, &nsd->rx_jabber);
 973         }
 974
 975         pf->stat_offsets_loaded = true;
 976 }
 977
 978 /**
 979  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
 980  * @vsi: the VSI to be searched
 981  * @macaddr: the MAC address
 982  * @vlan: the vlan
 983  * @is_vf: make sure its a vf filter, else doesn't matter
 984  * @is_netdev: make sure its a netdev filter, else doesn't matter
 985  *
 986  * Returns ptr to the filter object or NULL
 987  **/
 988 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
 989                                                 u8 *macaddr, s16 vlan,
 990                                                 bool is_vf, bool is_netdev)
 991 {
 992         struct i40e_mac_filter *f;
 993
 994         if (!vsi || !macaddr)
 995                 return NULL;
 996
 997         list_for_each_entry(f, &vsi->mac_filter_list, list) {
 998                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
 999                     (vlan == f->vlan)    &&
1000                     (!is_vf || f->is_vf) &&
1001                     (!is_netdev || f->is_netdev))
1002                         return f;
1003         }
1004         return NULL;
1005 }
1006
1007 /**
1008  * i40e_find_mac - Find a mac addr in the macvlan filters list
1009  * @vsi: the VSI to be searched
1010  * @macaddr: the MAC address we are searching for
1011  * @is_vf: make sure its a vf filter, else doesn't matter
1012  * @is_netdev: make sure its a netdev filter, else doesn't matter
1013  *
1014  * Returns the first filter with the provided MAC address or NULL if
1015  * MAC address was not found
1016  **/
1017 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
1018                                       bool is_vf, bool is_netdev)
1019 {
1020         struct i40e_mac_filter *f;
1021
1022         if (!vsi || !macaddr)
1023                 return NULL;
1024
1025         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1026                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1027                     (!is_vf || f->is_vf) &&
1028                     (!is_netdev || f->is_netdev))
1029                         return f;
1030         }
1031         return NULL;
1032 }
1033
1034 /**
1035  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1036  * @vsi: the VSI to be searched
1037  *
1038  * Returns true if VSI is in vlan mode or false otherwise
1039  **/
1040 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1041 {
1042         struct i40e_mac_filter *f;
1043
1044         /* Only -1 for all the filters denotes not in vlan mode
1045          * so we have to go through all the list in order to make sure
1046          */
1047         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1048                 if (f->vlan >= 0)
1049                         return true;
1050         }
1051
1052         return false;
1053 }
1054
1055 /**
1056  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1057  * @vsi: the VSI to be searched
1058  * @macaddr: the mac address to be filtered
1059  * @is_vf: true if it is a vf
1060  * @is_netdev: true if it is a netdev
1061  *
1062  * Goes through all the macvlan filters and adds a
1063  * macvlan filter for each unique vlan that already exists
1064  *
1065  * Returns first filter found on success, else NULL
1066  **/
1067 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1068                                              bool is_vf, bool is_netdev)
1069 {
1070         struct i40e_mac_filter *f;
1071
1072         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1073                 if (!i40e_find_filter(vsi, macaddr, f->vlan,
1074                                       is_vf, is_netdev)) {
1075                         if (!i40e_add_filter(vsi, macaddr, f->vlan,
1076                                                 is_vf, is_netdev))
1077                                 return NULL;
1078                 }
1079         }
1080
1081         return list_first_entry_or_null(&vsi->mac_filter_list,
1082                                         struct i40e_mac_filter, list);
1083 }
1084
1085 /**
1086  * i40e_add_filter - Add a mac/vlan filter to the VSI
1087  * @vsi: the VSI to be searched
1088  * @macaddr: the MAC address
1089  * @vlan: the vlan
1090  * @is_vf: make sure its a vf filter, else doesn't matter
1091  * @is_netdev: make sure its a netdev filter, else doesn't matter
1092  *
1093  * Returns ptr to the filter object or NULL when no memory available.
1094  **/
1095 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1096                                         u8 *macaddr, s16 vlan,
1097                                         bool is_vf, bool is_netdev)
1098 {
1099         struct i40e_mac_filter *f;
1100
1101         if (!vsi || !macaddr)
1102                 return NULL;
1103
1104         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1105         if (!f) {
1106                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1107                 if (!f)
1108                         goto add_filter_out;
1109
1110                 memcpy(f->macaddr, macaddr, ETH_ALEN);
1111                 f->vlan = vlan;
1112                 f->changed = true;
1113
1114                 INIT_LIST_HEAD(&f->list);
1115                 list_add(&f->list, &vsi->mac_filter_list);
1116         }
1117
1118         /* increment counter and add a new flag if needed */
1119         if (is_vf) {
1120                 if (!f->is_vf) {
1121                         f->is_vf = true;
1122                         f->counter++;
1123                 }
1124         } else if (is_netdev) {
1125                 if (!f->is_netdev) {
1126                         f->is_netdev = true;
1127                         f->counter++;
1128                 }
1129         } else {
1130                 f->counter++;
1131         }
1132
1133         /* changed tells sync_filters_subtask to
1134          * push the filter down to the firmware
1135          */
1136         if (f->changed) {
1137                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1138                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1139         }
1140
1141 add_filter_out:
1142         return f;
1143 }
1144
1145 /**
1146  * i40e_del_filter - Remove a mac/vlan filter from the VSI
1147  * @vsi: the VSI to be searched
1148  * @macaddr: the MAC address
1149  * @vlan: the vlan
1150  * @is_vf: make sure it's a vf filter, else doesn't matter
1151  * @is_netdev: make sure it's a netdev filter, else doesn't matter
1152  **/
1153 void i40e_del_filter(struct i40e_vsi *vsi,
1154                      u8 *macaddr, s16 vlan,
1155                      bool is_vf, bool is_netdev)
1156 {
1157         struct i40e_mac_filter *f;
1158
1159         if (!vsi || !macaddr)
1160                 return;
1161
1162         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1163         if (!f || f->counter == 0)
1164                 return;
1165
1166         if (is_vf) {
1167                 if (f->is_vf) {
1168                         f->is_vf = false;
1169                         f->counter--;
1170                 }
1171         } else if (is_netdev) {
1172                 if (f->is_netdev) {
1173                         f->is_netdev = false;
1174                         f->counter--;
1175                 }
1176         } else {
1177                 /* make sure we don't remove a filter in use by vf or netdev */
1178                 int min_f = 0;
1179                 min_f += (f->is_vf ? 1 : 0);
1180                 min_f += (f->is_netdev ? 1 : 0);
1181
1182                 if (f->counter > min_f)
1183                         f->counter--;
1184         }
1185
1186         /* counter == 0 tells sync_filters_subtask to
1187          * remove the filter from the firmware's list
1188          */
1189         if (f->counter == 0) {
1190                 f->changed = true;
1191                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1192                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1193         }
1194 }
1195
1196 /**
1197  * i40e_set_mac - NDO callback to set mac address
1198  * @netdev: network interface device structure
1199  * @p: pointer to an address structure
1200  *
1201  * Returns 0 on success, negative on failure
1202  **/
1203 static int i40e_set_mac(struct net_device *netdev, void *p)
1204 {
1205         struct i40e_netdev_priv *np = netdev_priv(netdev);
1206         struct i40e_vsi *vsi = np->vsi;
1207         struct sockaddr *addr = p;
1208         struct i40e_mac_filter *f;
1209
1210         if (!is_valid_ether_addr(addr->sa_data))
1211                 return -EADDRNOTAVAIL;
1212
1213         netdev_info(netdev, "set mac address=%pM\n", addr->sa_data);
1214
1215         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
1216                 return 0;
1217
1218         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1219             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1220                 return -EADDRNOTAVAIL;
1221
1222         if (vsi->type == I40E_VSI_MAIN) {
1223                 i40e_status ret;
1224                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1225                                                 I40E_AQC_WRITE_TYPE_LAA_ONLY,
1226                                                 addr->sa_data, NULL);
1227                 if (ret) {
1228                         netdev_info(netdev,
1229                                     "Addr change for Main VSI failed: %d\n",
1230                                     ret);
1231                         return -EADDRNOTAVAIL;
1232                 }
1233
1234                 memcpy(vsi->back->hw.mac.addr, addr->sa_data, netdev->addr_len);
1235         }
1236
1237         /* In order to be sure to not drop any packets, add the new address
1238          * then delete the old one.
1239          */
1240         f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY, false, false);
1241         if (!f)
1242                 return -ENOMEM;
1243
1244         i40e_sync_vsi_filters(vsi);
1245         i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, false, false);
1246         i40e_sync_vsi_filters(vsi);
1247
1248         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1249
1250         return 0;
1251 }
1252
1253 /**
1254  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1255  * @vsi: the VSI being setup
1256  * @ctxt: VSI context structure
1257  * @enabled_tc: Enabled TCs bitmap
1258  * @is_add: True if called before Add VSI
1259  *
1260  * Setup VSI queue mapping for enabled traffic classes.
1261  **/
1262 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1263                                      struct i40e_vsi_context *ctxt,
1264                                      u8 enabled_tc,
1265                                      bool is_add)
1266 {
1267         struct i40e_pf *pf = vsi->back;
1268         u16 sections = 0;
1269         u8 netdev_tc = 0;
1270         u16 numtc = 0;
1271         u16 qcount;
1272         u8 offset;
1273         u16 qmap;
1274         int i;
1275
1276         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1277         offset = 0;
1278
1279         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1280                 /* Find numtc from enabled TC bitmap */
1281                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1282                         if (enabled_tc & (1 << i)) /* TC is enabled */
1283                                 numtc++;
1284                 }
1285                 if (!numtc) {
1286                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1287                         numtc = 1;
1288                 }
1289         } else {
1290                 /* At least TC0 is enabled in case of non-DCB case */
1291                 numtc = 1;
1292         }
1293
1294         vsi->tc_config.numtc = numtc;
1295         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1296
1297         /* Setup queue offset/count for all TCs for given VSI */
1298         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1299                 /* See if the given TC is enabled for the given VSI */
1300                 if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
1301                         int pow, num_qps;
1302
1303                         vsi->tc_config.tc_info[i].qoffset = offset;
1304                         switch (vsi->type) {
1305                         case I40E_VSI_MAIN:
1306                                 if (i == 0)
1307                                         qcount = pf->rss_size;
1308                                 else
1309                                         qcount = pf->num_tc_qps;
1310                                 vsi->tc_config.tc_info[i].qcount = qcount;
1311                                 break;
1312                         case I40E_VSI_FDIR:
1313                         case I40E_VSI_SRIOV:
1314                         case I40E_VSI_VMDQ2:
1315                         default:
1316                                 qcount = vsi->alloc_queue_pairs;
1317                                 vsi->tc_config.tc_info[i].qcount = qcount;
1318                                 WARN_ON(i != 0);
1319                                 break;
1320                         }
1321
1322                         /* find the power-of-2 of the number of queue pairs */
1323                         num_qps = vsi->tc_config.tc_info[i].qcount;
1324                         pow = 0;
1325                         while (num_qps &&
1326                               ((1 << pow) < vsi->tc_config.tc_info[i].qcount)) {
1327                                 pow++;
1328                                 num_qps >>= 1;
1329                         }
1330
1331                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1332                         qmap =
1333                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1334                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1335
1336                         offset += vsi->tc_config.tc_info[i].qcount;
1337                 } else {
1338                         /* TC is not enabled so set the offset to
1339                          * default queue and allocate one queue
1340                          * for the given TC.
1341                          */
1342                         vsi->tc_config.tc_info[i].qoffset = 0;
1343                         vsi->tc_config.tc_info[i].qcount = 1;
1344                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1345
1346                         qmap = 0;
1347                 }
1348                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1349         }
1350
1351         /* Set actual Tx/Rx queue pairs */
1352         vsi->num_queue_pairs = offset;
1353
1354         /* Scheduler section valid can only be set for ADD VSI */
1355         if (is_add) {
1356                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1357
1358                 ctxt->info.up_enable_bits = enabled_tc;
1359         }
1360         if (vsi->type == I40E_VSI_SRIOV) {
1361                 ctxt->info.mapping_flags |=
1362                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1363                 for (i = 0; i < vsi->num_queue_pairs; i++)
1364                         ctxt->info.queue_mapping[i] =
1365                                                cpu_to_le16(vsi->base_queue + i);
1366         } else {
1367                 ctxt->info.mapping_flags |=
1368                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1369                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1370         }
1371         ctxt->info.valid_sections |= cpu_to_le16(sections);
1372 }
1373
1374 /**
1375  * i40e_set_rx_mode - NDO callback to set the netdev filters
1376  * @netdev: network interface device structure
1377  **/
1378 static void i40e_set_rx_mode(struct net_device *netdev)
1379 {
1380         struct i40e_netdev_priv *np = netdev_priv(netdev);
1381         struct i40e_mac_filter *f, *ftmp;
1382         struct i40e_vsi *vsi = np->vsi;
1383         struct netdev_hw_addr *uca;
1384         struct netdev_hw_addr *mca;
1385         struct netdev_hw_addr *ha;
1386
1387         /* add addr if not already in the filter list */
1388         netdev_for_each_uc_addr(uca, netdev) {
1389                 if (!i40e_find_mac(vsi, uca->addr, false, true)) {
1390                         if (i40e_is_vsi_in_vlan(vsi))
1391                                 i40e_put_mac_in_vlan(vsi, uca->addr,
1392                                                      false, true);
1393                         else
1394                                 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
1395                                                 false, true);
1396                 }
1397         }
1398
1399         netdev_for_each_mc_addr(mca, netdev) {
1400                 if (!i40e_find_mac(vsi, mca->addr, false, true)) {
1401                         if (i40e_is_vsi_in_vlan(vsi))
1402                                 i40e_put_mac_in_vlan(vsi, mca->addr,
1403                                                      false, true);
1404                         else
1405                                 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
1406                                                 false, true);
1407                 }
1408         }
1409
1410         /* remove filter if not in netdev list */
1411         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1412                 bool found = false;
1413
1414                 if (!f->is_netdev)
1415                         continue;
1416
1417                 if (is_multicast_ether_addr(f->macaddr)) {
1418                         netdev_for_each_mc_addr(mca, netdev) {
1419                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
1420                                         found = true;
1421                                         break;
1422                                 }
1423                         }
1424                 } else {
1425                         netdev_for_each_uc_addr(uca, netdev) {
1426                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
1427                                         found = true;
1428                                         break;
1429                                 }
1430                         }
1431
1432                         for_each_dev_addr(netdev, ha) {
1433                                 if (ether_addr_equal(ha->addr, f->macaddr)) {
1434                                         found = true;
1435                                         break;
1436                                 }
1437                         }
1438                 }
1439                 if (!found)
1440                         i40e_del_filter(
1441                            vsi, f->macaddr, I40E_VLAN_ANY, false, true);
1442         }
1443
1444         /* check for other flag changes */
1445         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1446                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1447                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1448         }
1449 }
1450
1451 /**
1452  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1453  * @vsi: ptr to the VSI
1454  *
1455  * Push any outstanding VSI filter changes through the AdminQ.
1456  *
1457  * Returns 0 or error value
1458  **/
1459 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1460 {
1461         struct i40e_mac_filter *f, *ftmp;
1462         bool promisc_forced_on = false;
1463         bool add_happened = false;
1464         int filter_list_len = 0;
1465         u32 changed_flags = 0;
1466         i40e_status aq_ret = 0;
1467         struct i40e_pf *pf;
1468         int num_add = 0;
1469         int num_del = 0;
1470         u16 cmd_flags;
1471
1472         /* empty array typed pointers, kcalloc later */
1473         struct i40e_aqc_add_macvlan_element_data *add_list;
1474         struct i40e_aqc_remove_macvlan_element_data *del_list;
1475
1476         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
1477                 usleep_range(1000, 2000);
1478         pf = vsi->back;
1479
1480         if (vsi->netdev) {
1481                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
1482                 vsi->current_netdev_flags = vsi->netdev->flags;
1483         }
1484
1485         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
1486                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
1487
1488                 filter_list_len = pf->hw.aq.asq_buf_size /
1489                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1490                 del_list = kcalloc(filter_list_len,
1491                             sizeof(struct i40e_aqc_remove_macvlan_element_data),
1492                             GFP_KERNEL);
1493                 if (!del_list)
1494                         return -ENOMEM;
1495
1496                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1497                         if (!f->changed)
1498                                 continue;
1499
1500                         if (f->counter != 0)
1501                                 continue;
1502                         f->changed = false;
1503                         cmd_flags = 0;
1504
1505                         /* add to delete list */
1506                         memcpy(del_list[num_del].mac_addr,
1507                                f->macaddr, ETH_ALEN);
1508                         del_list[num_del].vlan_tag =
1509                                 cpu_to_le16((u16)(f->vlan ==
1510                                             I40E_VLAN_ANY ? 0 : f->vlan));
1511
1512                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1513                         del_list[num_del].flags = cmd_flags;
1514                         num_del++;
1515
1516                         /* unlink from filter list */
1517                         list_del(&f->list);
1518                         kfree(f);
1519
1520                         /* flush a full buffer */
1521                         if (num_del == filter_list_len) {
1522                                 aq_ret = i40e_aq_remove_macvlan(&pf->hw,
1523                                             vsi->seid, del_list, num_del,
1524                                             NULL);
1525                                 num_del = 0;
1526                                 memset(del_list, 0, sizeof(*del_list));
1527
1528                                 if (aq_ret)
1529                                         dev_info(&pf->pdev->dev,
1530                                                  "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
1531                                                  aq_ret,
1532                                                  pf->hw.aq.asq_last_status);
1533                         }
1534                 }
1535                 if (num_del) {
1536                         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
1537                                                      del_list, num_del, NULL);
1538                         num_del = 0;
1539
1540                         if (aq_ret)
1541                                 dev_info(&pf->pdev->dev,
1542                                          "ignoring delete macvlan error, err %d, aq_err %d\n",
1543                                          aq_ret, pf->hw.aq.asq_last_status);
1544                 }
1545
1546                 kfree(del_list);
1547                 del_list = NULL;
1548
1549                 /* do all the adds now */
1550                 filter_list_len = pf->hw.aq.asq_buf_size /
1551                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1552                 add_list = kcalloc(filter_list_len,
1553                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1554                                GFP_KERNEL);
1555                 if (!add_list)
1556                         return -ENOMEM;
1557
1558                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1559                         if (!f->changed)
1560                                 continue;
1561
1562                         if (f->counter == 0)
1563                                 continue;
1564                         f->changed = false;
1565                         add_happened = true;
1566                         cmd_flags = 0;
1567
1568                         /* add to add array */
1569                         memcpy(add_list[num_add].mac_addr,
1570                                f->macaddr, ETH_ALEN);
1571                         add_list[num_add].vlan_tag =
1572                                 cpu_to_le16(
1573                                  (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
1574                         add_list[num_add].queue_number = 0;
1575
1576                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
1577                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
1578                         num_add++;
1579
1580                         /* flush a full buffer */
1581                         if (num_add == filter_list_len) {
1582                                 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1583                                                              add_list, num_add,
1584                                                              NULL);
1585                                 num_add = 0;
1586
1587                                 if (aq_ret)
1588                                         break;
1589                                 memset(add_list, 0, sizeof(*add_list));
1590                         }
1591                 }
1592                 if (num_add) {
1593                         aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1594                                                      add_list, num_add, NULL);
1595                         num_add = 0;
1596                 }
1597                 kfree(add_list);
1598                 add_list = NULL;
1599
1600                 if (add_happened && (!aq_ret)) {
1601                         /* do nothing */;
1602                 } else if (add_happened && (aq_ret)) {
1603                         dev_info(&pf->pdev->dev,
1604                                  "add filter failed, err %d, aq_err %d\n",
1605                                  aq_ret, pf->hw.aq.asq_last_status);
1606                         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
1607                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1608                                       &vsi->state)) {
1609                                 promisc_forced_on = true;
1610                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1611                                         &vsi->state);
1612                                 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
1613                         }
1614                 }
1615         }
1616
1617         /* check for changes in promiscuous modes */
1618         if (changed_flags & IFF_ALLMULTI) {
1619                 bool cur_multipromisc;
1620                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
1621                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
1622                                                                vsi->seid,
1623                                                                cur_multipromisc,
1624                                                                NULL);
1625                 if (aq_ret)
1626                         dev_info(&pf->pdev->dev,
1627                                  "set multi promisc failed, err %d, aq_err %d\n",
1628                                  aq_ret, pf->hw.aq.asq_last_status);
1629         }
1630         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
1631                 bool cur_promisc;
1632                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
1633                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1634                                         &vsi->state));
1635                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
1636                                                              vsi->seid,
1637                                                              cur_promisc, NULL);
1638                 if (aq_ret)
1639                         dev_info(&pf->pdev->dev,
1640                                  "set uni promisc failed, err %d, aq_err %d\n",
1641                                  aq_ret, pf->hw.aq.asq_last_status);
1642                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
1643                                                    vsi->seid,
1644                                                    cur_promisc, NULL);
1645                 if (aq_ret)
1646                         dev_info(&pf->pdev->dev,
1647                                  "set brdcast promisc failed, err %d, aq_err %d\n",
1648                                  aq_ret, pf->hw.aq.asq_last_status);
1649         }
1650
1651         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
1652         return 0;
1653 }
1654
1655 /**
1656  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
1657  * @pf: board private structure
1658  **/
1659 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
1660 {
1661         int v;
1662
1663         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
1664                 return;
1665         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
1666
1667         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
1668                 if (pf->vsi[v] &&
1669                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
1670                         i40e_sync_vsi_filters(pf->vsi[v]);
1671         }
1672 }
1673
1674 /**
1675  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
1676  * @netdev: network interface device structure
1677  * @new_mtu: new value for maximum frame size
1678  *
1679  * Returns 0 on success, negative on failure
1680  **/
1681 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
1682 {
1683         struct i40e_netdev_priv *np = netdev_priv(netdev);
1684         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
1685         struct i40e_vsi *vsi = np->vsi;
1686
1687         /* MTU < 68 is an error and causes problems on some kernels */
1688         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1689                 return -EINVAL;
1690
1691         netdev_info(netdev, "changing MTU from %d to %d\n",
1692                     netdev->mtu, new_mtu);
1693         netdev->mtu = new_mtu;
1694         if (netif_running(netdev))
1695                 i40e_vsi_reinit_locked(vsi);
1696
1697         return 0;
1698 }
1699
1700 /**
1701  * i40e_ioctl - Access the hwtstamp interface
1702  * @netdev: network interface device structure
1703  * @ifr: interface request data
1704  * @cmd: ioctl command
1705  **/
1706 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1707 {
1708         struct i40e_netdev_priv *np = netdev_priv(netdev);
1709         struct i40e_pf *pf = np->vsi->back;
1710
1711         switch (cmd) {
1712         case SIOCGHWTSTAMP:
1713                 return i40e_ptp_get_ts_config(pf, ifr);
1714         case SIOCSHWTSTAMP:
1715                 return i40e_ptp_set_ts_config(pf, ifr);
1716         default:
1717                 return -EOPNOTSUPP;
1718         }
1719 }
1720
1721 /**
1722  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
1723  * @vsi: the vsi being adjusted
1724  **/
1725 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
1726 {
1727         struct i40e_vsi_context ctxt;
1728         i40e_status ret;
1729
1730         if ((vsi->info.valid_sections &
1731              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1732             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
1733                 return;  /* already enabled */
1734
1735         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1736         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1737                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
1738
1739         ctxt.seid = vsi->seid;
1740         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1741         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1742         if (ret) {
1743                 dev_info(&vsi->back->pdev->dev,
1744                          "%s: update vsi failed, aq_err=%d\n",
1745                          __func__, vsi->back->hw.aq.asq_last_status);
1746         }
1747 }
1748
1749 /**
1750  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
1751  * @vsi: the vsi being adjusted
1752  **/
1753 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
1754 {
1755         struct i40e_vsi_context ctxt;
1756         i40e_status ret;
1757
1758         if ((vsi->info.valid_sections &
1759              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1760             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
1761              I40E_AQ_VSI_PVLAN_EMOD_MASK))
1762                 return;  /* already disabled */
1763
1764         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1765         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1766                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
1767
1768         ctxt.seid = vsi->seid;
1769         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1770         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1771         if (ret) {
1772                 dev_info(&vsi->back->pdev->dev,
1773                          "%s: update vsi failed, aq_err=%d\n",
1774                          __func__, vsi->back->hw.aq.asq_last_status);
1775         }
1776 }
1777
1778 /**
1779  * i40e_vlan_rx_register - Setup or shutdown vlan offload
1780  * @netdev: network interface to be adjusted
1781  * @features: netdev features to test if VLAN offload is enabled or not
1782  **/
1783 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
1784 {
1785         struct i40e_netdev_priv *np = netdev_priv(netdev);
1786         struct i40e_vsi *vsi = np->vsi;
1787
1788         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1789                 i40e_vlan_stripping_enable(vsi);
1790         else
1791                 i40e_vlan_stripping_disable(vsi);
1792 }
1793
1794 /**
1795  * i40e_vsi_add_vlan - Add vsi membership for given vlan
1796  * @vsi: the vsi being configured
1797  * @vid: vlan id to be added (0 = untagged only , -1 = any)
1798  **/
1799 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
1800 {
1801         struct i40e_mac_filter *f, *add_f;
1802         bool is_netdev, is_vf;
1803
1804         is_vf = (vsi->type == I40E_VSI_SRIOV);
1805         is_netdev = !!(vsi->netdev);
1806
1807         if (is_netdev) {
1808                 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
1809                                         is_vf, is_netdev);
1810                 if (!add_f) {
1811                         dev_info(&vsi->back->pdev->dev,
1812                                  "Could not add vlan filter %d for %pM\n",
1813                                  vid, vsi->netdev->dev_addr);
1814                         return -ENOMEM;
1815                 }
1816         }
1817
1818         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1819                 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1820                 if (!add_f) {
1821                         dev_info(&vsi->back->pdev->dev,
1822                                  "Could not add vlan filter %d for %pM\n",
1823                                  vid, f->macaddr);
1824                         return -ENOMEM;
1825                 }
1826         }
1827
1828         /* Now if we add a vlan tag, make sure to check if it is the first
1829          * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
1830          * with 0, so we now accept untagged and specified tagged traffic
1831          * (and not any taged and untagged)
1832          */
1833         if (vid > 0) {
1834                 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
1835                                                   I40E_VLAN_ANY,
1836                                                   is_vf, is_netdev)) {
1837                         i40e_del_filter(vsi, vsi->netdev->dev_addr,
1838                                         I40E_VLAN_ANY, is_vf, is_netdev);
1839                         add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
1840                                                 is_vf, is_netdev);
1841                         if (!add_f) {
1842                                 dev_info(&vsi->back->pdev->dev,
1843                                          "Could not add filter 0 for %pM\n",
1844                                          vsi->netdev->dev_addr);
1845                                 return -ENOMEM;
1846                         }
1847                 }
1848
1849                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1850                         if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1851                                              is_vf, is_netdev)) {
1852                                 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1853                                                 is_vf, is_netdev);
1854                                 add_f = i40e_add_filter(vsi, f->macaddr,
1855                                                         0, is_vf, is_netdev);
1856                                 if (!add_f) {
1857                                         dev_info(&vsi->back->pdev->dev,
1858                                                  "Could not add filter 0 for %pM\n",
1859                                                  f->macaddr);
1860                                         return -ENOMEM;
1861                                 }
1862                         }
1863                 }
1864         }
1865
1866         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1867             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1868                 return 0;
1869
1870         return i40e_sync_vsi_filters(vsi);
1871 }
1872
1873 /**
1874  * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
1875  * @vsi: the vsi being configured
1876  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
1877  *
1878  * Return: 0 on success or negative otherwise
1879  **/
1880 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
1881 {
1882         struct net_device *netdev = vsi->netdev;
1883         struct i40e_mac_filter *f, *add_f;
1884         bool is_vf, is_netdev;
1885         int filter_count = 0;
1886
1887         is_vf = (vsi->type == I40E_VSI_SRIOV);
1888         is_netdev = !!(netdev);
1889
1890         if (is_netdev)
1891                 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
1892
1893         list_for_each_entry(f, &vsi->mac_filter_list, list)
1894                 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1895
1896         /* go through all the filters for this VSI and if there is only
1897          * vid == 0 it means there are no other filters, so vid 0 must
1898          * be replaced with -1. This signifies that we should from now
1899          * on accept any traffic (with any tag present, or untagged)
1900          */
1901         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1902                 if (is_netdev) {
1903                         if (f->vlan &&
1904                             ether_addr_equal(netdev->dev_addr, f->macaddr))
1905                                 filter_count++;
1906                 }
1907
1908                 if (f->vlan)
1909                         filter_count++;
1910         }
1911
1912         if (!filter_count && is_netdev) {
1913                 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
1914                 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
1915                                     is_vf, is_netdev);
1916                 if (!f) {
1917                         dev_info(&vsi->back->pdev->dev,
1918                                  "Could not add filter %d for %pM\n",
1919                                  I40E_VLAN_ANY, netdev->dev_addr);
1920                         return -ENOMEM;
1921                 }
1922         }
1923
1924         if (!filter_count) {
1925                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1926                         i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
1927                         add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1928                                             is_vf, is_netdev);
1929                         if (!add_f) {
1930                                 dev_info(&vsi->back->pdev->dev,
1931                                          "Could not add filter %d for %pM\n",
1932                                          I40E_VLAN_ANY, f->macaddr);
1933                                 return -ENOMEM;
1934                         }
1935                 }
1936         }
1937
1938         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1939             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1940                 return 0;
1941
1942         return i40e_sync_vsi_filters(vsi);
1943 }
1944
1945 /**
1946  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
1947  * @netdev: network interface to be adjusted
1948  * @vid: vlan id to be added
1949  *
1950  * net_device_ops implementation for adding vlan ids
1951  **/
1952 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
1953                                 __always_unused __be16 proto, u16 vid)
1954 {
1955         struct i40e_netdev_priv *np = netdev_priv(netdev);
1956         struct i40e_vsi *vsi = np->vsi;
1957         int ret = 0;
1958
1959         if (vid > 4095)
1960                 return -EINVAL;
1961
1962         netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
1963
1964         /* If the network stack called us with vid = 0, we should
1965          * indicate to i40e_vsi_add_vlan() that we want to receive
1966          * any traffic (i.e. with any vlan tag, or untagged)
1967          */
1968         ret = i40e_vsi_add_vlan(vsi, vid ? vid : I40E_VLAN_ANY);
1969
1970         if (!ret && (vid < VLAN_N_VID))
1971                 set_bit(vid, vsi->active_vlans);
1972
1973         return ret;
1974 }
1975
1976 /**
1977  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
1978  * @netdev: network interface to be adjusted
1979  * @vid: vlan id to be removed
1980  *
1981  * net_device_ops implementation for adding vlan ids
1982  **/
1983 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
1984                                  __always_unused __be16 proto, u16 vid)
1985 {
1986         struct i40e_netdev_priv *np = netdev_priv(netdev);
1987         struct i40e_vsi *vsi = np->vsi;
1988
1989         netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
1990
1991         /* return code is ignored as there is nothing a user
1992          * can do about failure to remove and a log message was
1993          * already printed from the other function
1994          */
1995         i40e_vsi_kill_vlan(vsi, vid);
1996
1997         clear_bit(vid, vsi->active_vlans);
1998
1999         return 0;
2000 }
2001
2002 /**
2003  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2004  * @vsi: the vsi being brought back up
2005  **/
2006 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2007 {
2008         u16 vid;
2009
2010         if (!vsi->netdev)
2011                 return;
2012
2013         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2014
2015         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2016                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2017                                      vid);
2018 }
2019
2020 /**
2021  * i40e_vsi_add_pvid - Add pvid for the VSI
2022  * @vsi: the vsi being adjusted
2023  * @vid: the vlan id to set as a PVID
2024  **/
2025 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2026 {
2027         struct i40e_vsi_context ctxt;
2028         i40e_status aq_ret;
2029
2030         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2031         vsi->info.pvid = cpu_to_le16(vid);
2032         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2033                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2034                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2035
2036         ctxt.seid = vsi->seid;
2037         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2038         aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2039         if (aq_ret) {
2040                 dev_info(&vsi->back->pdev->dev,
2041                          "%s: update vsi failed, aq_err=%d\n",
2042                          __func__, vsi->back->hw.aq.asq_last_status);
2043                 return -ENOENT;
2044         }
2045
2046         return 0;
2047 }
2048
2049 /**
2050  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2051  * @vsi: the vsi being adjusted
2052  *
2053  * Just use the vlan_rx_register() service to put it back to normal
2054  **/
2055 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2056 {
2057         i40e_vlan_stripping_disable(vsi);
2058
2059         vsi->info.pvid = 0;
2060 }
2061
2062 /**
2063  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2064  * @vsi: ptr to the VSI
2065  *
2066  * If this function returns with an error, then it's possible one or
2067  * more of the rings is populated (while the rest are not).  It is the
2068  * callers duty to clean those orphaned rings.
2069  *
2070  * Return 0 on success, negative on failure
2071  **/
2072 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2073 {
2074         int i, err = 0;
2075
2076         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2077                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2078
2079         return err;
2080 }
2081
2082 /**
2083  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2084  * @vsi: ptr to the VSI
2085  *
2086  * Free VSI's transmit software resources
2087  **/
2088 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2089 {
2090         int i;
2091
2092         if (!vsi->tx_rings)
2093                 return;
2094
2095         for (i = 0; i < vsi->num_queue_pairs; i++)
2096                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2097                         i40e_free_tx_resources(vsi->tx_rings[i]);
2098 }
2099
2100 /**
2101  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2102  * @vsi: ptr to the VSI
2103  *
2104  * If this function returns with an error, then it's possible one or
2105  * more of the rings is populated (while the rest are not).  It is the
2106  * callers duty to clean those orphaned rings.
2107  *
2108  * Return 0 on success, negative on failure
2109  **/
2110 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2111 {
2112         int i, err = 0;
2113
2114         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2115                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2116         return err;
2117 }
2118
2119 /**
2120  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2121  * @vsi: ptr to the VSI
2122  *
2123  * Free all receive software resources
2124  **/
2125 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2126 {
2127         int i;
2128
2129         if (!vsi->rx_rings)
2130                 return;
2131
2132         for (i = 0; i < vsi->num_queue_pairs; i++)
2133                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2134                         i40e_free_rx_resources(vsi->rx_rings[i]);
2135 }
2136
2137 /**
2138  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2139  * @ring: The Tx ring to configure
2140  *
2141  * Configure the Tx descriptor ring in the HMC context.
2142  **/
2143 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2144 {
2145         struct i40e_vsi *vsi = ring->vsi;
2146         u16 pf_q = vsi->base_queue + ring->queue_index;
2147         struct i40e_hw *hw = &vsi->back->hw;
2148         struct i40e_hmc_obj_txq tx_ctx;
2149         i40e_status err = 0;
2150         u32 qtx_ctl = 0;
2151
2152         /* some ATR related tx ring init */
2153         if (vsi->back->flags & I40E_FLAG_FDIR_ATR_ENABLED) {
2154                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2155                 ring->atr_count = 0;
2156         } else {
2157                 ring->atr_sample_rate = 0;
2158         }
2159
2160         /* initialize XPS */
2161         if (ring->q_vector && ring->netdev &&
2162             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2163                 netif_set_xps_queue(ring->netdev,
2164                                     &ring->q_vector->affinity_mask,
2165                                     ring->queue_index);
2166
2167         /* clear the context structure first */
2168         memset(&tx_ctx, 0, sizeof(tx_ctx));
2169
2170         tx_ctx.new_context = 1;
2171         tx_ctx.base = (ring->dma / 128);
2172         tx_ctx.qlen = ring->count;
2173         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FDIR_ENABLED |
2174                                                I40E_FLAG_FDIR_ATR_ENABLED));
2175         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2176
2177         /* As part of VSI creation/update, FW allocates certain
2178          * Tx arbitration queue sets for each TC enabled for
2179          * the VSI. The FW returns the handles to these queue
2180          * sets as part of the response buffer to Add VSI,
2181          * Update VSI, etc. AQ commands. It is expected that
2182          * these queue set handles be associated with the Tx
2183          * queues by the driver as part of the TX queue context
2184          * initialization. This has to be done regardless of
2185          * DCB as by default everything is mapped to TC0.
2186          */
2187         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2188         tx_ctx.rdylist_act = 0;
2189
2190         /* clear the context in the HMC */
2191         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2192         if (err) {
2193                 dev_info(&vsi->back->pdev->dev,
2194                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2195                          ring->queue_index, pf_q, err);
2196                 return -ENOMEM;
2197         }
2198
2199         /* set the context in the HMC */
2200         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2201         if (err) {
2202                 dev_info(&vsi->back->pdev->dev,
2203                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2204                          ring->queue_index, pf_q, err);
2205                 return -ENOMEM;
2206         }
2207
2208         /* Now associate this queue with this PCI function */
2209         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2210         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2211                     I40E_QTX_CTL_PF_INDX_MASK);
2212         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2213         i40e_flush(hw);
2214
2215         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
2216
2217         /* cache tail off for easier writes later */
2218         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2219
2220         return 0;
2221 }
2222
2223 /**
2224  * i40e_configure_rx_ring - Configure a receive ring context
2225  * @ring: The Rx ring to configure
2226  *
2227  * Configure the Rx descriptor ring in the HMC context.
2228  **/
2229 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2230 {
2231         struct i40e_vsi *vsi = ring->vsi;
2232         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2233         u16 pf_q = vsi->base_queue + ring->queue_index;
2234         struct i40e_hw *hw = &vsi->back->hw;
2235         struct i40e_hmc_obj_rxq rx_ctx;
2236         i40e_status err = 0;
2237
2238         ring->state = 0;
2239
2240         /* clear the context structure first */
2241         memset(&rx_ctx, 0, sizeof(rx_ctx));
2242
2243         ring->rx_buf_len = vsi->rx_buf_len;
2244         ring->rx_hdr_len = vsi->rx_hdr_len;
2245
2246         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2247         rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2248
2249         rx_ctx.base = (ring->dma / 128);
2250         rx_ctx.qlen = ring->count;
2251
2252         if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
2253                 set_ring_16byte_desc_enabled(ring);
2254                 rx_ctx.dsize = 0;
2255         } else {
2256                 rx_ctx.dsize = 1;
2257         }
2258
2259         rx_ctx.dtype = vsi->dtype;
2260         if (vsi->dtype) {
2261                 set_ring_ps_enabled(ring);
2262                 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
2263                                   I40E_RX_SPLIT_IP      |
2264                                   I40E_RX_SPLIT_TCP_UDP |
2265                                   I40E_RX_SPLIT_SCTP;
2266         } else {
2267                 rx_ctx.hsplit_0 = 0;
2268         }
2269
2270         rx_ctx.rxmax = min_t(u16, vsi->max_frame,
2271                                   (chain_len * ring->rx_buf_len));
2272         rx_ctx.tphrdesc_ena = 1;
2273         rx_ctx.tphwdesc_ena = 1;
2274         rx_ctx.tphdata_ena = 1;
2275         rx_ctx.tphhead_ena = 1;
2276         if (hw->revision_id == 0)
2277                 rx_ctx.lrxqthresh = 0;
2278         else
2279                 rx_ctx.lrxqthresh = 2;
2280         rx_ctx.crcstrip = 1;
2281         rx_ctx.l2tsel = 1;
2282         rx_ctx.showiv = 1;
2283
2284         /* clear the context in the HMC */
2285         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2286         if (err) {
2287                 dev_info(&vsi->back->pdev->dev,
2288                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2289                          ring->queue_index, pf_q, err);
2290                 return -ENOMEM;
2291         }
2292
2293         /* set the context in the HMC */
2294         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2295         if (err) {
2296                 dev_info(&vsi->back->pdev->dev,
2297                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2298                          ring->queue_index, pf_q, err);
2299                 return -ENOMEM;
2300         }
2301
2302         /* cache tail for quicker writes, and clear the reg before use */
2303         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2304         writel(0, ring->tail);
2305
2306         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
2307
2308         return 0;
2309 }
2310
2311 /**
2312  * i40e_vsi_configure_tx - Configure the VSI for Tx
2313  * @vsi: VSI structure describing this set of rings and resources
2314  *
2315  * Configure the Tx VSI for operation.
2316  **/
2317 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
2318 {
2319         int err = 0;
2320         u16 i;
2321
2322         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
2323                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
2324
2325         return err;
2326 }
2327
2328 /**
2329  * i40e_vsi_configure_rx - Configure the VSI for Rx
2330  * @vsi: the VSI being configured
2331  *
2332  * Configure the Rx VSI for operation.
2333  **/
2334 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
2335 {
2336         int err = 0;
2337         u16 i;
2338
2339         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
2340                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
2341                                + ETH_FCS_LEN + VLAN_HLEN;
2342         else
2343                 vsi->max_frame = I40E_RXBUFFER_2048;
2344
2345         /* figure out correct receive buffer length */
2346         switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
2347                                     I40E_FLAG_RX_PS_ENABLED)) {
2348         case I40E_FLAG_RX_1BUF_ENABLED:
2349                 vsi->rx_hdr_len = 0;
2350                 vsi->rx_buf_len = vsi->max_frame;
2351                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2352                 break;
2353         case I40E_FLAG_RX_PS_ENABLED:
2354                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2355                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2356                 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
2357                 break;
2358         default:
2359                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2360                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2361                 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
2362                 break;
2363         }
2364
2365         /* round up for the chip's needs */
2366         vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
2367                                 (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2368         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
2369                                 (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2370
2371         /* set up individual rings */
2372         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2373                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
2374
2375         return err;
2376 }
2377
2378 /**
2379  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
2380  * @vsi: ptr to the VSI
2381  **/
2382 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
2383 {
2384         u16 qoffset, qcount;
2385         int i, n;
2386
2387         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
2388                 return;
2389
2390         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
2391                 if (!(vsi->tc_config.enabled_tc & (1 << n)))
2392                         continue;
2393
2394                 qoffset = vsi->tc_config.tc_info[n].qoffset;
2395                 qcount = vsi->tc_config.tc_info[n].qcount;
2396                 for (i = qoffset; i < (qoffset + qcount); i++) {
2397                         struct i40e_ring *rx_ring = vsi->rx_rings[i];
2398                         struct i40e_ring *tx_ring = vsi->tx_rings[i];
2399                         rx_ring->dcb_tc = n;
2400                         tx_ring->dcb_tc = n;
2401                 }
2402         }
2403 }
2404
2405 /**
2406  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
2407  * @vsi: ptr to the VSI
2408  **/
2409 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
2410 {
2411         if (vsi->netdev)
2412                 i40e_set_rx_mode(vsi->netdev);
2413 }
2414
2415 /**
2416  * i40e_vsi_configure - Set up the VSI for action
2417  * @vsi: the VSI being configured
2418  **/
2419 static int i40e_vsi_configure(struct i40e_vsi *vsi)
2420 {
2421         int err;
2422
2423         i40e_set_vsi_rx_mode(vsi);
2424         i40e_restore_vlan(vsi);
2425         i40e_vsi_config_dcb_rings(vsi);
2426         err = i40e_vsi_configure_tx(vsi);
2427         if (!err)
2428                 err = i40e_vsi_configure_rx(vsi);
2429
2430         return err;
2431 }
2432
2433 /**
2434  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
2435  * @vsi: the VSI being configured
2436  **/
2437 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
2438 {
2439         struct i40e_pf *pf = vsi->back;
2440         struct i40e_q_vector *q_vector;
2441         struct i40e_hw *hw = &pf->hw;
2442         u16 vector;
2443         int i, q;
2444         u32 val;
2445         u32 qp;
2446
2447         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
2448          * and PFINT_LNKLSTn registers, e.g.:
2449          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
2450          */
2451         qp = vsi->base_queue;
2452         vector = vsi->base_vector;
2453         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
2454                 q_vector = vsi->q_vectors[i];
2455                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2456                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
2457                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
2458                      q_vector->rx.itr);
2459                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2460                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
2461                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
2462                      q_vector->tx.itr);
2463
2464                 /* Linked list for the queuepairs assigned to this vector */
2465                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
2466                 for (q = 0; q < q_vector->num_ringpairs; q++) {
2467                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2468                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
2469                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2470                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
2471                               (I40E_QUEUE_TYPE_TX
2472                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2473
2474                         wr32(hw, I40E_QINT_RQCTL(qp), val);
2475
2476                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2477                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
2478                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2479                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
2480                               (I40E_QUEUE_TYPE_RX
2481                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2482
2483                         /* Terminate the linked list */
2484                         if (q == (q_vector->num_ringpairs - 1))
2485                                 val |= (I40E_QUEUE_END_OF_LIST
2486                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2487
2488                         wr32(hw, I40E_QINT_TQCTL(qp), val);
2489                         qp++;
2490                 }
2491         }
2492
2493         i40e_flush(hw);
2494 }
2495
2496 /**
2497  * i40e_enable_misc_int_causes - enable the non-queue interrupts
2498  * @hw: ptr to the hardware info
2499  **/
2500 static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
2501 {
2502         u32 val;
2503
2504         /* clear things first */
2505         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2506         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2507
2508         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
2509               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
2510               I40E_PFINT_ICR0_ENA_GRST_MASK          |
2511               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
2512               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
2513               I40E_PFINT_ICR0_ENA_TIMESYNC_MASK      |
2514               I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK  |
2515               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
2516               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
2517               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2518
2519         wr32(hw, I40E_PFINT_ICR0_ENA, val);
2520
2521         /* SW_ITR_IDX = 0, but don't change INTENA */
2522         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2523                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2524
2525         /* OTHER_ITR_IDX = 0 */
2526         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2527 }
2528
2529 /**
2530  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
2531  * @vsi: the VSI being configured
2532  **/
2533 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
2534 {
2535         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
2536         struct i40e_pf *pf = vsi->back;
2537         struct i40e_hw *hw = &pf->hw;
2538         u32 val;
2539
2540         /* set the ITR configuration */
2541         q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2542         q_vector->rx.latency_range = I40E_LOW_LATENCY;
2543         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
2544         q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2545         q_vector->tx.latency_range = I40E_LOW_LATENCY;
2546         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
2547
2548         i40e_enable_misc_int_causes(hw);
2549
2550         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2551         wr32(hw, I40E_PFINT_LNKLST0, 0);
2552
2553         /* Associate the queue pair to the vector and enable the q int */
2554         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
2555               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2556               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2557
2558         wr32(hw, I40E_QINT_RQCTL(0), val);
2559
2560         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
2561               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2562               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2563
2564         wr32(hw, I40E_QINT_TQCTL(0), val);
2565         i40e_flush(hw);
2566 }
2567
2568 /**
2569  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
2570  * @pf: board private structure
2571  **/
2572 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
2573 {
2574         struct i40e_hw *hw = &pf->hw;
2575
2576         wr32(hw, I40E_PFINT_DYN_CTL0,
2577              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2578         i40e_flush(hw);
2579 }
2580
2581 /**
2582  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
2583  * @pf: board private structure
2584  **/
2585 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
2586 {
2587         struct i40e_hw *hw = &pf->hw;
2588         u32 val;
2589
2590         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
2591               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2592               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
2593
2594         wr32(hw, I40E_PFINT_DYN_CTL0, val);
2595         i40e_flush(hw);
2596 }
2597
2598 /**
2599  * i40e_irq_dynamic_enable - Enable default interrupt generation settings
2600  * @vsi: pointer to a vsi
2601  * @vector: enable a particular Hw Interrupt vector
2602  **/
2603 void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
2604 {
2605         struct i40e_pf *pf = vsi->back;
2606         struct i40e_hw *hw = &pf->hw;
2607         u32 val;
2608
2609         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2610               I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2611               (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2612         wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
2613         /* skip the flush */
2614 }
2615
2616 /**
2617  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
2618  * @irq: interrupt number
2619  * @data: pointer to a q_vector
2620  **/
2621 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
2622 {
2623         struct i40e_q_vector *q_vector = data;
2624
2625         if (!q_vector->tx.ring && !q_vector->rx.ring)
2626                 return IRQ_HANDLED;
2627
2628         napi_schedule(&q_vector->napi);
2629
2630         return IRQ_HANDLED;
2631 }
2632
2633 /**
2634  * i40e_fdir_clean_rings - Interrupt Handler for FDIR rings
2635  * @irq: interrupt number
2636  * @data: pointer to a q_vector
2637  **/
2638 static irqreturn_t i40e_fdir_clean_rings(int irq, void *data)
2639 {
2640         struct i40e_q_vector *q_vector = data;
2641
2642         if (!q_vector->tx.ring && !q_vector->rx.ring)
2643                 return IRQ_HANDLED;
2644
2645         pr_info("fdir ring cleaning needed\n");
2646
2647         return IRQ_HANDLED;
2648 }
2649
2650 /**
2651  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
2652  * @vsi: the VSI being configured
2653  * @basename: name for the vector
2654  *
2655  * Allocates MSI-X vectors and requests interrupts from the kernel.
2656  **/
2657 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
2658 {
2659         int q_vectors = vsi->num_q_vectors;
2660         struct i40e_pf *pf = vsi->back;
2661         int base = vsi->base_vector;
2662         int rx_int_idx = 0;
2663         int tx_int_idx = 0;
2664         int vector, err;
2665
2666         for (vector = 0; vector < q_vectors; vector++) {
2667                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
2668
2669                 if (q_vector->tx.ring && q_vector->rx.ring) {
2670                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2671                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2672                         tx_int_idx++;
2673                 } else if (q_vector->rx.ring) {
2674                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2675                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
2676                 } else if (q_vector->tx.ring) {
2677                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2678                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
2679                 } else {
2680                         /* skip this unused q_vector */
2681                         continue;
2682                 }
2683                 err = request_irq(pf->msix_entries[base + vector].vector,
2684                                   vsi->irq_handler,
2685                                   0,
2686                                   q_vector->name,
2687                                   q_vector);
2688                 if (err) {
2689                         dev_info(&pf->pdev->dev,
2690                                  "%s: request_irq failed, error: %d\n",
2691                                  __func__, err);
2692                         goto free_queue_irqs;
2693                 }
2694                 /* assign the mask for this irq */
2695                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2696                                       &q_vector->affinity_mask);
2697         }
2698
2699         return 0;
2700
2701 free_queue_irqs:
2702         while (vector) {
2703                 vector--;
2704                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2705                                       NULL);
2706                 free_irq(pf->msix_entries[base + vector].vector,
2707                          &(vsi->q_vectors[vector]));
2708         }
2709         return err;
2710 }
2711
2712 /**
2713  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
2714  * @vsi: the VSI being un-configured
2715  **/
2716 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
2717 {
2718         struct i40e_pf *pf = vsi->back;
2719         struct i40e_hw *hw = &pf->hw;
2720         int base = vsi->base_vector;
2721         int i;
2722
2723         for (i = 0; i < vsi->num_queue_pairs; i++) {
2724                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
2725                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
2726         }
2727
2728         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2729                 for (i = vsi->base_vector;
2730                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2731                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
2732
2733                 i40e_flush(hw);
2734                 for (i = 0; i < vsi->num_q_vectors; i++)
2735                         synchronize_irq(pf->msix_entries[i + base].vector);
2736         } else {
2737                 /* Legacy and MSI mode - this stops all interrupt handling */
2738                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
2739                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
2740                 i40e_flush(hw);
2741                 synchronize_irq(pf->pdev->irq);
2742         }
2743 }
2744
2745 /**
2746  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
2747  * @vsi: the VSI being configured
2748  **/
2749 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
2750 {
2751         struct i40e_pf *pf = vsi->back;
2752         int i;
2753
2754         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2755                 for (i = vsi->base_vector;
2756                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2757                         i40e_irq_dynamic_enable(vsi, i);
2758         } else {
2759                 i40e_irq_dynamic_enable_icr0(pf);
2760         }
2761
2762         i40e_flush(&pf->hw);
2763         return 0;
2764 }
2765
2766 /**
2767  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
2768  * @pf: board private structure
2769  **/
2770 static void i40e_stop_misc_vector(struct i40e_pf *pf)
2771 {
2772         /* Disable ICR 0 */
2773         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
2774         i40e_flush(&pf->hw);
2775 }
2776
2777 /**
2778  * i40e_intr - MSI/Legacy and non-queue interrupt handler
2779  * @irq: interrupt number
2780  * @data: pointer to a q_vector
2781  *
2782  * This is the handler used for all MSI/Legacy interrupts, and deals
2783  * with both queue and non-queue interrupts.  This is also used in
2784  * MSIX mode to handle the non-queue interrupts.
2785  **/
2786 static irqreturn_t i40e_intr(int irq, void *data)
2787 {
2788         struct i40e_pf *pf = (struct i40e_pf *)data;
2789         struct i40e_hw *hw = &pf->hw;
2790         irqreturn_t ret = IRQ_NONE;
2791         u32 icr0, icr0_remaining;
2792         u32 val, ena_mask;
2793
2794         icr0 = rd32(hw, I40E_PFINT_ICR0);
2795         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
2796
2797         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
2798         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
2799                 goto enable_intr;
2800
2801         /* if interrupt but no bits showing, must be SWINT */
2802         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
2803             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
2804                 pf->sw_int_count++;
2805
2806         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
2807         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
2808
2809                 /* temporarily disable queue cause for NAPI processing */
2810                 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
2811                 qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
2812                 wr32(hw, I40E_QINT_RQCTL(0), qval);
2813
2814                 qval = rd32(hw, I40E_QINT_TQCTL(0));
2815                 qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
2816                 wr32(hw, I40E_QINT_TQCTL(0), qval);
2817
2818                 if (!test_bit(__I40E_DOWN, &pf->state))
2819                         napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
2820         }
2821
2822         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
2823                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2824                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
2825         }
2826
2827         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
2828                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
2829                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
2830         }
2831
2832         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
2833                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
2834                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
2835         }
2836
2837         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
2838                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
2839                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
2840                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
2841                 val = rd32(hw, I40E_GLGEN_RSTAT);
2842                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
2843                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
2844                 if (val == I40E_RESET_CORER)
2845                         pf->corer_count++;
2846                 else if (val == I40E_RESET_GLOBR)
2847                         pf->globr_count++;
2848                 else if (val == I40E_RESET_EMPR)
2849                         pf->empr_count++;
2850         }
2851
2852         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
2853                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
2854                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
2855         }
2856
2857         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
2858                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
2859
2860                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
2861                         ena_mask &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
2862                         i40e_ptp_tx_hwtstamp(pf);
2863                         prttsyn_stat &= ~I40E_PRTTSYN_STAT_0_TXTIME_MASK;
2864                 }
2865
2866                 wr32(hw, I40E_PRTTSYN_STAT_0, prttsyn_stat);
2867         }
2868
2869         /* If a critical error is pending we have no choice but to reset the
2870          * device.
2871          * Report and mask out any remaining unexpected interrupts.
2872          */
2873         icr0_remaining = icr0 & ena_mask;
2874         if (icr0_remaining) {
2875                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
2876                          icr0_remaining);
2877                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
2878                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
2879                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK) ||
2880                     (icr0_remaining & I40E_PFINT_ICR0_MAL_DETECT_MASK)) {
2881                         dev_info(&pf->pdev->dev, "device will be reset\n");
2882                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
2883                         i40e_service_event_schedule(pf);
2884                 }
2885                 ena_mask &= ~icr0_remaining;
2886         }
2887         ret = IRQ_HANDLED;
2888
2889 enable_intr:
2890         /* re-enable interrupt causes */
2891         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
2892         if (!test_bit(__I40E_DOWN, &pf->state)) {
2893                 i40e_service_event_schedule(pf);
2894                 i40e_irq_dynamic_enable_icr0(pf);
2895         }
2896
2897         return ret;
2898 }
2899
2900 /**
2901  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
2902  * @vsi: the VSI being configured
2903  * @v_idx: vector index
2904  * @qp_idx: queue pair index
2905  **/
2906 static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
2907 {
2908         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
2909         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
2910         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
2911
2912         tx_ring->q_vector = q_vector;
2913         tx_ring->next = q_vector->tx.ring;
2914         q_vector->tx.ring = tx_ring;
2915         q_vector->tx.count++;
2916
2917         rx_ring->q_vector = q_vector;
2918         rx_ring->next = q_vector->rx.ring;
2919         q_vector->rx.ring = rx_ring;
2920         q_vector->rx.count++;
2921 }
2922
2923 /**
2924  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
2925  * @vsi: the VSI being configured
2926  *
2927  * This function maps descriptor rings to the queue-specific vectors
2928  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
2929  * one vector per queue pair, but on a constrained vector budget, we
2930  * group the queue pairs as "efficiently" as possible.
2931  **/
2932 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
2933 {
2934         int qp_remaining = vsi->num_queue_pairs;
2935         int q_vectors = vsi->num_q_vectors;
2936         int num_ringpairs;
2937         int v_start = 0;
2938         int qp_idx = 0;
2939
2940         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2941          * group them so there are multiple queues per vector.
2942          */
2943         for (; v_start < q_vectors && qp_remaining; v_start++) {
2944                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
2945
2946                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
2947
2948                 q_vector->num_ringpairs = num_ringpairs;
2949
2950                 q_vector->rx.count = 0;
2951                 q_vector->tx.count = 0;
2952                 q_vector->rx.ring = NULL;
2953                 q_vector->tx.ring = NULL;
2954
2955                 while (num_ringpairs--) {
2956                         map_vector_to_qp(vsi, v_start, qp_idx);
2957                         qp_idx++;
2958                         qp_remaining--;
2959                 }
2960         }
2961 }
2962
2963 /**
2964  * i40e_vsi_request_irq - Request IRQ from the OS
2965  * @vsi: the VSI being configured
2966  * @basename: name for the vector
2967  **/
2968 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
2969 {
2970         struct i40e_pf *pf = vsi->back;
2971         int err;
2972
2973         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2974                 err = i40e_vsi_request_irq_msix(vsi, basename);
2975         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
2976                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
2977                                   pf->misc_int_name, pf);
2978         else
2979                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
2980                                   pf->misc_int_name, pf);
2981
2982         if (err)
2983                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
2984
2985         return err;
2986 }
2987
2988 #ifdef CONFIG_NET_POLL_CONTROLLER
2989 /**
2990  * i40e_netpoll - A Polling 'interrupt'handler
2991  * @netdev: network interface device structure
2992  *
2993  * This is used by netconsole to send skbs without having to re-enable
2994  * interrupts.  It's not called while the normal interrupt routine is executing.
2995  **/
2996 static void i40e_netpoll(struct net_device *netdev)
2997 {
2998         struct i40e_netdev_priv *np = netdev_priv(netdev);
2999         struct i40e_vsi *vsi = np->vsi;
3000         struct i40e_pf *pf = vsi->back;
3001         int i;
3002
3003         /* if interface is down do nothing */
3004         if (test_bit(__I40E_DOWN, &vsi->state))
3005                 return;
3006
3007         pf->flags |= I40E_FLAG_IN_NETPOLL;
3008         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3009                 for (i = 0; i < vsi->num_q_vectors; i++)
3010                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3011         } else {
3012                 i40e_intr(pf->pdev->irq, netdev);
3013         }
3014         pf->flags &= ~I40E_FLAG_IN_NETPOLL;
3015 }
3016 #endif
3017
3018 /**
3019  * i40e_vsi_control_tx - Start or stop a VSI's rings
3020  * @vsi: the VSI being configured
3021  * @enable: start or stop the rings
3022  **/
3023 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3024 {
3025         struct i40e_pf *pf = vsi->back;
3026         struct i40e_hw *hw = &pf->hw;
3027         int i, j, pf_q;
3028         u32 tx_reg;
3029
3030         pf_q = vsi->base_queue;
3031         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3032                 j = 1000;
3033                 do {
3034                         usleep_range(1000, 2000);
3035                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3036                 } while (j-- && ((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT)
3037                                ^ (tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)) & 1);
3038
3039                 /* Skip if the queue is already in the requested state */
3040                 if (enable && (tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3041                         continue;
3042                 if (!enable && !(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3043                         continue;
3044
3045                 /* turn on/off the queue */
3046                 if (enable) {
3047                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
3048                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3049                                   I40E_QTX_ENA_QENA_STAT_MASK;
3050                 } else {
3051                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3052                 }
3053
3054                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3055
3056                 /* wait for the change to finish */
3057                 for (j = 0; j < 10; j++) {
3058                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3059                         if (enable) {
3060                                 if ((tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3061                                         break;
3062                         } else {
3063                                 if (!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3064                                         break;
3065                         }
3066
3067                         udelay(10);
3068                 }
3069                 if (j >= 10) {
3070                         dev_info(&pf->pdev->dev, "Tx ring %d %sable timeout\n",
3071                                  pf_q, (enable ? "en" : "dis"));
3072                         return -ETIMEDOUT;
3073                 }
3074         }
3075
3076         if (hw->revision_id == 0)
3077                 mdelay(50);
3078
3079         return 0;
3080 }
3081
3082 /**
3083  * i40e_vsi_control_rx - Start or stop a VSI's rings
3084  * @vsi: the VSI being configured
3085  * @enable: start or stop the rings
3086  **/
3087 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
3088 {
3089         struct i40e_pf *pf = vsi->back;
3090         struct i40e_hw *hw = &pf->hw;
3091         int i, j, pf_q;
3092         u32 rx_reg;
3093
3094         pf_q = vsi->base_queue;
3095         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3096                 j = 1000;
3097                 do {
3098                         usleep_range(1000, 2000);
3099                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3100                 } while (j-- && ((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT)
3101                                ^ (rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT)) & 1);
3102
3103                 if (enable) {
3104                         /* is STAT set ? */
3105                         if ((rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3106                                 continue;
3107                 } else {
3108                         /* is !STAT set ? */
3109                         if (!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3110                                 continue;
3111                 }
3112
3113                 /* turn on/off the queue */
3114                 if (enable)
3115                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3116                                   I40E_QRX_ENA_QENA_STAT_MASK;
3117                 else
3118                         rx_reg &= ~(I40E_QRX_ENA_QENA_REQ_MASK |
3119                                   I40E_QRX_ENA_QENA_STAT_MASK);
3120                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
3121
3122                 /* wait for the change to finish */
3123                 for (j = 0; j < 10; j++) {
3124                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3125
3126                         if (enable) {
3127                                 if ((rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3128                                         break;
3129                         } else {
3130                                 if (!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3131                                         break;
3132                         }
3133
3134                         udelay(10);
3135                 }
3136                 if (j >= 10) {
3137                         dev_info(&pf->pdev->dev, "Rx ring %d %sable timeout\n",
3138                                  pf_q, (enable ? "en" : "dis"));
3139                         return -ETIMEDOUT;
3140                 }
3141         }
3142
3143         return 0;
3144 }
3145
3146 /**
3147  * i40e_vsi_control_rings - Start or stop a VSI's rings
3148  * @vsi: the VSI being configured
3149  * @enable: start or stop the rings
3150  **/
3151 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
3152 {
3153         int ret = 0;
3154
3155         /* do rx first for enable and last for disable */
3156         if (request) {
3157                 ret = i40e_vsi_control_rx(vsi, request);
3158                 if (ret)
3159                         return ret;
3160                 ret = i40e_vsi_control_tx(vsi, request);
3161         } else {
3162                 /* Ignore return value, we need to shutdown whatever we can */
3163                 i40e_vsi_control_tx(vsi, request);
3164                 i40e_vsi_control_rx(vsi, request);
3165         }
3166
3167         return ret;
3168 }
3169
3170 /**
3171  * i40e_vsi_free_irq - Free the irq association with the OS
3172  * @vsi: the VSI being configured
3173  **/
3174 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
3175 {
3176         struct i40e_pf *pf = vsi->back;
3177         struct i40e_hw *hw = &pf->hw;
3178         int base = vsi->base_vector;
3179         u32 val, qp;
3180         int i;
3181
3182         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3183                 if (!vsi->q_vectors)
3184                         return;
3185
3186                 for (i = 0; i < vsi->num_q_vectors; i++) {
3187                         u16 vector = i + base;
3188
3189                         /* free only the irqs that were actually requested */
3190                         if (!vsi->q_vectors[i] ||
3191                             !vsi->q_vectors[i]->num_ringpairs)
3192                                 continue;
3193
3194                         /* clear the affinity_mask in the IRQ descriptor */
3195                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
3196                                               NULL);
3197                         free_irq(pf->msix_entries[vector].vector,
3198                                  vsi->q_vectors[i]);
3199
3200                         /* Tear down the interrupt queue link list
3201                          *
3202                          * We know that they come in pairs and always
3203                          * the Rx first, then the Tx.  To clear the
3204                          * link list, stick the EOL value into the
3205                          * next_q field of the registers.
3206                          */
3207                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
3208                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3209                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3210                         val |= I40E_QUEUE_END_OF_LIST
3211                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3212                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
3213
3214                         while (qp != I40E_QUEUE_END_OF_LIST) {
3215                                 u32 next;
3216
3217                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3218
3219                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3220                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3221                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3222                                          I40E_QINT_RQCTL_INTEVENT_MASK);
3223
3224                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3225                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3226
3227                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3228
3229                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3230
3231                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
3232                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
3233
3234                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3235                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3236                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3237                                          I40E_QINT_TQCTL_INTEVENT_MASK);
3238
3239                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3240                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3241
3242                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3243                                 qp = next;
3244                         }
3245                 }
3246         } else {
3247                 free_irq(pf->pdev->irq, pf);
3248
3249                 val = rd32(hw, I40E_PFINT_LNKLST0);
3250                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3251                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3252                 val |= I40E_QUEUE_END_OF_LIST
3253                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
3254                 wr32(hw, I40E_PFINT_LNKLST0, val);
3255
3256                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3257                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3258                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3259                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3260                          I40E_QINT_RQCTL_INTEVENT_MASK);
3261
3262                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3263                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3264
3265                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3266
3267                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3268
3269                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3270                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3271                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3272                          I40E_QINT_TQCTL_INTEVENT_MASK);
3273
3274                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3275                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3276
3277                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3278         }
3279 }
3280
3281 /**
3282  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
3283  * @vsi: the VSI being configured
3284  * @v_idx: Index of vector to be freed
3285  *
3286  * This function frees the memory allocated to the q_vector.  In addition if
3287  * NAPI is enabled it will delete any references to the NAPI struct prior
3288  * to freeing the q_vector.
3289  **/
3290 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
3291 {
3292         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3293         struct i40e_ring *ring;
3294
3295         if (!q_vector)
3296                 return;
3297
3298         /* disassociate q_vector from rings */
3299         i40e_for_each_ring(ring, q_vector->tx)
3300                 ring->q_vector = NULL;
3301
3302         i40e_for_each_ring(ring, q_vector->rx)
3303                 ring->q_vector = NULL;
3304
3305         /* only VSI w/ an associated netdev is set up w/ NAPI */
3306         if (vsi->netdev)
3307                 netif_napi_del(&q_vector->napi);
3308
3309         vsi->q_vectors[v_idx] = NULL;
3310
3311         kfree_rcu(q_vector, rcu);
3312 }
3313
3314 /**
3315  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
3316  * @vsi: the VSI being un-configured
3317  *
3318  * This frees the memory allocated to the q_vectors and
3319  * deletes references to the NAPI struct.
3320  **/
3321 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
3322 {
3323         int v_idx;
3324
3325         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
3326                 i40e_free_q_vector(vsi, v_idx);
3327 }
3328
3329 /**
3330  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
3331  * @pf: board private structure
3332  **/
3333 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
3334 {
3335         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
3336         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3337                 pci_disable_msix(pf->pdev);
3338                 kfree(pf->msix_entries);
3339                 pf->msix_entries = NULL;
3340         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
3341                 pci_disable_msi(pf->pdev);
3342         }
3343         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
3344 }
3345
3346 /**
3347  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
3348  * @pf: board private structure
3349  *
3350  * We go through and clear interrupt specific resources and reset the structure
3351  * to pre-load conditions
3352  **/
3353 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
3354 {
3355         int i;
3356
3357         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
3358         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
3359                 if (pf->vsi[i])
3360                         i40e_vsi_free_q_vectors(pf->vsi[i]);
3361         i40e_reset_interrupt_capability(pf);
3362 }
3363
3364 /**
3365  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
3366  * @vsi: the VSI being configured
3367  **/
3368 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
3369 {
3370         int q_idx;
3371
3372         if (!vsi->netdev)
3373                 return;
3374
3375         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3376                 napi_enable(&vsi->q_vectors[q_idx]->napi);
3377 }
3378
3379 /**
3380  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
3381  * @vsi: the VSI being configured
3382  **/
3383 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
3384 {
3385         int q_idx;
3386
3387         if (!vsi->netdev)
3388                 return;
3389
3390         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3391                 napi_disable(&vsi->q_vectors[q_idx]->napi);
3392 }
3393
3394 /**
3395  * i40e_quiesce_vsi - Pause a given VSI
3396  * @vsi: the VSI being paused
3397  **/
3398 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
3399 {
3400         if (test_bit(__I40E_DOWN, &vsi->state))
3401                 return;
3402
3403         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
3404         if (vsi->netdev && netif_running(vsi->netdev)) {
3405                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
3406         } else {
3407                 set_bit(__I40E_DOWN, &vsi->state);
3408                 i40e_down(vsi);
3409         }
3410 }
3411
3412 /**
3413  * i40e_unquiesce_vsi - Resume a given VSI
3414  * @vsi: the VSI being resumed
3415  **/
3416 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
3417 {
3418         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
3419                 return;
3420
3421         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
3422         if (vsi->netdev && netif_running(vsi->netdev))
3423                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
3424         else
3425                 i40e_up(vsi);   /* this clears the DOWN bit */
3426 }
3427
3428 /**
3429  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
3430  * @pf: the PF
3431  **/
3432 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
3433 {
3434         int v;
3435
3436         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3437                 if (pf->vsi[v])
3438                         i40e_quiesce_vsi(pf->vsi[v]);
3439         }
3440 }
3441
3442 /**
3443  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
3444  * @pf: the PF
3445  **/
3446 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
3447 {
3448         int v;
3449
3450         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3451                 if (pf->vsi[v])
3452                         i40e_unquiesce_vsi(pf->vsi[v]);
3453         }
3454 }
3455
3456 /**
3457  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
3458  * @dcbcfg: the corresponding DCBx configuration structure
3459  *
3460  * Return the number of TCs from given DCBx configuration
3461  **/
3462 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
3463 {
3464         u8 num_tc = 0;
3465         int i;
3466
3467         /* Scan the ETS Config Priority Table to find
3468          * traffic class enabled for a given priority
3469          * and use the traffic class index to get the
3470          * number of traffic classes enabled
3471          */
3472         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3473                 if (dcbcfg->etscfg.prioritytable[i] > num_tc)
3474                         num_tc = dcbcfg->etscfg.prioritytable[i];
3475         }
3476
3477         /* Traffic class index starts from zero so
3478          * increment to return the actual count
3479          */
3480         return num_tc + 1;
3481 }
3482
3483 /**
3484  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
3485  * @dcbcfg: the corresponding DCBx configuration structure
3486  *
3487  * Query the current DCB configuration and return the number of
3488  * traffic classes enabled from the given DCBX config
3489  **/
3490 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
3491 {
3492         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
3493         u8 enabled_tc = 1;
3494         u8 i;
3495
3496         for (i = 0; i < num_tc; i++)
3497                 enabled_tc |= 1 << i;
3498
3499         return enabled_tc;
3500 }
3501
3502 /**
3503  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
3504  * @pf: PF being queried
3505  *
3506  * Return number of traffic classes enabled for the given PF
3507  **/
3508 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
3509 {
3510         struct i40e_hw *hw = &pf->hw;
3511         u8 i, enabled_tc;
3512         u8 num_tc = 0;
3513         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3514
3515         /* If DCB is not enabled then always in single TC */
3516         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3517                 return 1;
3518
3519         /* MFP mode return count of enabled TCs for this PF */
3520         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3521                 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3522                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3523                         if (enabled_tc & (1 << i))
3524                                 num_tc++;
3525                 }
3526                 return num_tc;
3527         }
3528
3529         /* SFP mode will be enabled for all TCs on port */
3530         return i40e_dcb_get_num_tc(dcbcfg);
3531 }
3532
3533 /**
3534  * i40e_pf_get_default_tc - Get bitmap for first enabled TC
3535  * @pf: PF being queried
3536  *
3537  * Return a bitmap for first enabled traffic class for this PF.
3538  **/
3539 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
3540 {
3541         u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3542         u8 i = 0;
3543
3544         if (!enabled_tc)
3545                 return 0x1; /* TC0 */
3546
3547         /* Find the first enabled TC */
3548         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3549                 if (enabled_tc & (1 << i))
3550                         break;
3551         }
3552
3553         return 1 << i;
3554 }
3555
3556 /**
3557  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
3558  * @pf: PF being queried
3559  *
3560  * Return a bitmap for enabled traffic classes for this PF.
3561  **/
3562 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
3563 {
3564         /* If DCB is not enabled for this PF then just return default TC */
3565         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3566                 return i40e_pf_get_default_tc(pf);
3567
3568         /* MFP mode will have enabled TCs set by FW */
3569         if (pf->flags & I40E_FLAG_MFP_ENABLED)
3570                 return pf->hw.func_caps.enabled_tcmap;
3571
3572         /* SFP mode we want PF to be enabled for all TCs */
3573         return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
3574 }
3575
3576 /**
3577  * i40e_vsi_get_bw_info - Query VSI BW Information
3578  * @vsi: the VSI being queried
3579  *
3580  * Returns 0 on success, negative value on failure
3581  **/
3582 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
3583 {
3584         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
3585         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
3586         struct i40e_pf *pf = vsi->back;
3587         struct i40e_hw *hw = &pf->hw;
3588         i40e_status aq_ret;
3589         u32 tc_bw_max;
3590         int i;
3591
3592         /* Get the VSI level BW configuration */
3593         aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
3594         if (aq_ret) {
3595                 dev_info(&pf->pdev->dev,
3596                          "couldn't get pf vsi bw config, err %d, aq_err %d\n",
3597                          aq_ret, pf->hw.aq.asq_last_status);
3598                 return -EINVAL;
3599         }
3600
3601         /* Get the VSI level BW configuration per TC */
3602         aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
3603                                                   NULL);
3604         if (aq_ret) {
3605                 dev_info(&pf->pdev->dev,
3606                          "couldn't get pf vsi ets bw config, err %d, aq_err %d\n",
3607                          aq_ret, pf->hw.aq.asq_last_status);
3608                 return -EINVAL;
3609         }
3610
3611         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
3612                 dev_info(&pf->pdev->dev,
3613                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
3614                          bw_config.tc_valid_bits,
3615                          bw_ets_config.tc_valid_bits);
3616                 /* Still continuing */
3617         }
3618
3619         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
3620         vsi->bw_max_quanta = bw_config.max_bw;
3621         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
3622                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
3623         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3624                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
3625                 vsi->bw_ets_limit_credits[i] =
3626                                         le16_to_cpu(bw_ets_config.credits[i]);
3627                 /* 3 bits out of 4 for each TC */
3628                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
3629         }
3630
3631         return 0;
3632 }
3633
3634 /**
3635  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
3636  * @vsi: the VSI being configured
3637  * @enabled_tc: TC bitmap
3638  * @bw_credits: BW shared credits per TC
3639  *
3640  * Returns 0 on success, negative value on failure
3641  **/
3642 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
3643                                        u8 *bw_share)
3644 {
3645         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
3646         i40e_status aq_ret;
3647         int i;
3648
3649         bw_data.tc_valid_bits = enabled_tc;
3650         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3651                 bw_data.tc_bw_credits[i] = bw_share[i];
3652
3653         aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
3654                                           NULL);
3655         if (aq_ret) {
3656                 dev_info(&vsi->back->pdev->dev,
3657                          "%s: AQ command Config VSI BW allocation per TC failed = %d\n",
3658                          __func__, vsi->back->hw.aq.asq_last_status);
3659                 return -EINVAL;
3660         }
3661
3662         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3663                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
3664
3665         return 0;
3666 }
3667
3668 /**
3669  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
3670  * @vsi: the VSI being configured
3671  * @enabled_tc: TC map to be enabled
3672  *
3673  **/
3674 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3675 {
3676         struct net_device *netdev = vsi->netdev;
3677         struct i40e_pf *pf = vsi->back;
3678         struct i40e_hw *hw = &pf->hw;
3679         u8 netdev_tc = 0;
3680         int i;
3681         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3682
3683         if (!netdev)
3684                 return;
3685
3686         if (!enabled_tc) {
3687                 netdev_reset_tc(netdev);
3688                 return;
3689         }
3690
3691         /* Set up actual enabled TCs on the VSI */
3692         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
3693                 return;
3694
3695         /* set per TC queues for the VSI */
3696         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3697                 /* Only set TC queues for enabled tcs
3698                  *
3699                  * e.g. For a VSI that has TC0 and TC3 enabled the
3700                  * enabled_tc bitmap would be 0x00001001; the driver
3701                  * will set the numtc for netdev as 2 that will be
3702                  * referenced by the netdev layer as TC 0 and 1.
3703                  */
3704                 if (vsi->tc_config.enabled_tc & (1 << i))
3705                         netdev_set_tc_queue(netdev,
3706                                         vsi->tc_config.tc_info[i].netdev_tc,
3707                                         vsi->tc_config.tc_info[i].qcount,
3708                                         vsi->tc_config.tc_info[i].qoffset);
3709         }
3710
3711         /* Assign UP2TC map for the VSI */
3712         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3713                 /* Get the actual TC# for the UP */
3714                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
3715                 /* Get the mapped netdev TC# for the UP */
3716                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
3717                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
3718         }
3719 }
3720
3721 /**
3722  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
3723  * @vsi: the VSI being configured
3724  * @ctxt: the ctxt buffer returned from AQ VSI update param command
3725  **/
3726 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
3727                                       struct i40e_vsi_context *ctxt)
3728 {
3729         /* copy just the sections touched not the entire info
3730          * since not all sections are valid as returned by
3731          * update vsi params
3732          */
3733         vsi->info.mapping_flags = ctxt->info.mapping_flags;
3734         memcpy(&vsi->info.queue_mapping,
3735                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
3736         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
3737                sizeof(vsi->info.tc_mapping));
3738 }
3739
3740 /**
3741  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
3742  * @vsi: VSI to be configured
3743  * @enabled_tc: TC bitmap
3744  *
3745  * This configures a particular VSI for TCs that are mapped to the
3746  * given TC bitmap. It uses default bandwidth share for TCs across
3747  * VSIs to configure TC for a particular VSI.
3748  *
3749  * NOTE:
3750  * It is expected that the VSI queues have been quisced before calling
3751  * this function.
3752  **/
3753 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3754 {
3755         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
3756         struct i40e_vsi_context ctxt;
3757         int ret = 0;
3758         int i;
3759
3760         /* Check if enabled_tc is same as existing or new TCs */
3761         if (vsi->tc_config.enabled_tc == enabled_tc)
3762                 return ret;
3763
3764         /* Enable ETS TCs with equal BW Share for now across all VSIs */
3765         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3766                 if (enabled_tc & (1 << i))
3767                         bw_share[i] = 1;
3768         }
3769
3770         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
3771         if (ret) {
3772                 dev_info(&vsi->back->pdev->dev,
3773                          "Failed configuring TC map %d for VSI %d\n",
3774                          enabled_tc, vsi->seid);
3775                 goto out;
3776         }
3777
3778         /* Update Queue Pairs Mapping for currently enabled UPs */
3779         ctxt.seid = vsi->seid;
3780         ctxt.pf_num = vsi->back->hw.pf_id;
3781         ctxt.vf_num = 0;
3782         ctxt.uplink_seid = vsi->uplink_seid;
3783         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3784         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
3785
3786         /* Update the VSI after updating the VSI queue-mapping information */
3787         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3788         if (ret) {
3789                 dev_info(&vsi->back->pdev->dev,
3790                          "update vsi failed, aq_err=%d\n",
3791                          vsi->back->hw.aq.asq_last_status);
3792                 goto out;
3793         }
3794         /* update the local VSI info with updated queue map */
3795         i40e_vsi_update_queue_map(vsi, &ctxt);
3796         vsi->info.valid_sections = 0;
3797
3798         /* Update current VSI BW information */
3799         ret = i40e_vsi_get_bw_info(vsi);
3800         if (ret) {
3801                 dev_info(&vsi->back->pdev->dev,
3802                          "Failed updating vsi bw info, aq_err=%d\n",
3803                          vsi->back->hw.aq.asq_last_status);
3804                 goto out;
3805         }
3806
3807         /* Update the netdev TC setup */
3808         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
3809 out:
3810         return ret;
3811 }
3812
3813 /**
3814  * i40e_up_complete - Finish the last steps of bringing up a connection
3815  * @vsi: the VSI being configured
3816  **/
3817 static int i40e_up_complete(struct i40e_vsi *vsi)
3818 {
3819         struct i40e_pf *pf = vsi->back;
3820         int err;
3821
3822         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3823                 i40e_vsi_configure_msix(vsi);
3824         else
3825                 i40e_configure_msi_and_legacy(vsi);
3826
3827         /* start rings */
3828         err = i40e_vsi_control_rings(vsi, true);
3829         if (err)
3830                 return err;
3831
3832         clear_bit(__I40E_DOWN, &vsi->state);
3833         i40e_napi_enable_all(vsi);
3834         i40e_vsi_enable_irq(vsi);
3835
3836         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
3837             (vsi->netdev)) {
3838                 netdev_info(vsi->netdev, "NIC Link is Up\n");
3839                 netif_tx_start_all_queues(vsi->netdev);
3840                 netif_carrier_on(vsi->netdev);
3841         } else if (vsi->netdev) {
3842                 netdev_info(vsi->netdev, "NIC Link is Down\n");
3843         }
3844         i40e_service_event_schedule(pf);
3845
3846         return 0;
3847 }
3848
3849 /**
3850  * i40e_vsi_reinit_locked - Reset the VSI
3851  * @vsi: the VSI being configured
3852  *
3853  * Rebuild the ring structs after some configuration
3854  * has changed, e.g. MTU size.
3855  **/
3856 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
3857 {
3858         struct i40e_pf *pf = vsi->back;
3859
3860         WARN_ON(in_interrupt());
3861         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
3862                 usleep_range(1000, 2000);
3863         i40e_down(vsi);
3864
3865         /* Give a VF some time to respond to the reset.  The
3866          * two second wait is based upon the watchdog cycle in
3867          * the VF driver.
3868          */
3869         if (vsi->type == I40E_VSI_SRIOV)
3870                 msleep(2000);
3871         i40e_up(vsi);
3872         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
3873 }
3874
3875 /**
3876  * i40e_up - Bring the connection back up after being down
3877  * @vsi: the VSI being configured
3878  **/
3879 int i40e_up(struct i40e_vsi *vsi)
3880 {
3881         int err;
3882
3883         err = i40e_vsi_configure(vsi);
3884         if (!err)
3885                 err = i40e_up_complete(vsi);
3886
3887         return err;
3888 }
3889
3890 /**
3891  * i40e_down - Shutdown the connection processing
3892  * @vsi: the VSI being stopped
3893  **/
3894 void i40e_down(struct i40e_vsi *vsi)
3895 {
3896         int i;
3897
3898         /* It is assumed that the caller of this function
3899          * sets the vsi->state __I40E_DOWN bit.
3900          */
3901         if (vsi->netdev) {
3902                 netif_carrier_off(vsi->netdev);
3903                 netif_tx_disable(vsi->netdev);
3904         }
3905         i40e_vsi_disable_irq(vsi);
3906         i40e_vsi_control_rings(vsi, false);
3907         i40e_napi_disable_all(vsi);
3908
3909         for (i = 0; i < vsi->num_queue_pairs; i++) {
3910                 i40e_clean_tx_ring(vsi->tx_rings[i]);
3911                 i40e_clean_rx_ring(vsi->rx_rings[i]);
3912         }
3913 }
3914
3915 /**
3916  * i40e_setup_tc - configure multiple traffic classes
3917  * @netdev: net device to configure
3918  * @tc: number of traffic classes to enable
3919  **/
3920 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
3921 {
3922         struct i40e_netdev_priv *np = netdev_priv(netdev);
3923         struct i40e_vsi *vsi = np->vsi;
3924         struct i40e_pf *pf = vsi->back;
3925         u8 enabled_tc = 0;
3926         int ret = -EINVAL;
3927         int i;
3928
3929         /* Check if DCB enabled to continue */
3930         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
3931                 netdev_info(netdev, "DCB is not enabled for adapter\n");
3932                 goto exit;
3933         }
3934
3935         /* Check if MFP enabled */
3936         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3937                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
3938                 goto exit;
3939         }
3940
3941         /* Check whether tc count is within enabled limit */
3942         if (tc > i40e_pf_get_num_tc(pf)) {
3943                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
3944                 goto exit;
3945         }
3946
3947         /* Generate TC map for number of tc requested */
3948         for (i = 0; i < tc; i++)
3949                 enabled_tc |= (1 << i);
3950
3951         /* Requesting same TC configuration as already enabled */
3952         if (enabled_tc == vsi->tc_config.enabled_tc)
3953                 return 0;
3954
3955         /* Quiesce VSI queues */
3956         i40e_quiesce_vsi(vsi);
3957
3958         /* Configure VSI for enabled TCs */
3959         ret = i40e_vsi_config_tc(vsi, enabled_tc);
3960         if (ret) {
3961                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
3962                             vsi->seid);
3963                 goto exit;
3964         }
3965
3966         /* Unquiesce VSI */
3967         i40e_unquiesce_vsi(vsi);
3968
3969 exit:
3970         return ret;
3971 }
3972
3973 /**
3974  * i40e_open - Called when a network interface is made active
3975  * @netdev: network interface device structure
3976  *
3977  * The open entry point is called when a network interface is made
3978  * active by the system (IFF_UP).  At this point all resources needed
3979  * for transmit and receive operations are allocated, the interrupt
3980  * handler is registered with the OS, the netdev watchdog subtask is
3981  * enabled, and the stack is notified that the interface is ready.
3982  *
3983  * Returns 0 on success, negative value on failure
3984  **/
3985 static int i40e_open(struct net_device *netdev)
3986 {
3987         struct i40e_netdev_priv *np = netdev_priv(netdev);
3988         struct i40e_vsi *vsi = np->vsi;
3989         struct i40e_pf *pf = vsi->back;
3990         char int_name[IFNAMSIZ];
3991         int err;
3992
3993         /* disallow open during test */
3994         if (test_bit(__I40E_TESTING, &pf->state))
3995                 return -EBUSY;
3996
3997         netif_carrier_off(netdev);
3998
3999         /* allocate descriptors */
4000         err = i40e_vsi_setup_tx_resources(vsi);
4001         if (err)
4002                 goto err_setup_tx;
4003         err = i40e_vsi_setup_rx_resources(vsi);
4004         if (err)
4005                 goto err_setup_rx;
4006
4007         err = i40e_vsi_configure(vsi);
4008         if (err)
4009                 goto err_setup_rx;
4010
4011         snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
4012                  dev_driver_string(&pf->pdev->dev), netdev->name);
4013         err = i40e_vsi_request_irq(vsi, int_name);
4014         if (err)
4015                 goto err_setup_rx;
4016
4017         /* Notify the stack of the actual queue counts. */
4018         err = netif_set_real_num_tx_queues(netdev, vsi->num_queue_pairs);
4019         if (err)
4020                 goto err_set_queues;
4021
4022         err = netif_set_real_num_rx_queues(netdev, vsi->num_queue_pairs);
4023         if (err)
4024                 goto err_set_queues;
4025
4026         err = i40e_up_complete(vsi);
4027         if (err)
4028                 goto err_up_complete;
4029
4030 #ifdef CONFIG_I40E_VXLAN
4031         vxlan_get_rx_port(netdev);
4032 #endif
4033
4034         return 0;
4035
4036 err_up_complete:
4037         i40e_down(vsi);
4038 err_set_queues:
4039         i40e_vsi_free_irq(vsi);
4040 err_setup_rx:
4041         i40e_vsi_free_rx_resources(vsi);
4042 err_setup_tx:
4043         i40e_vsi_free_tx_resources(vsi);
4044         if (vsi == pf->vsi[pf->lan_vsi])
4045                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
4046
4047         return err;
4048 }
4049
4050 /**
4051  * i40e_close - Disables a network interface
4052  * @netdev: network interface device structure
4053  *
4054  * The close entry point is called when an interface is de-activated
4055  * by the OS.  The hardware is still under the driver's control, but
4056  * this netdev interface is disabled.
4057  *
4058  * Returns 0, this is not allowed to fail
4059  **/
4060 static int i40e_close(struct net_device *netdev)
4061 {
4062         struct i40e_netdev_priv *np = netdev_priv(netdev);
4063         struct i40e_vsi *vsi = np->vsi;
4064
4065         if (test_and_set_bit(__I40E_DOWN, &vsi->state))
4066                 return 0;
4067
4068         i40e_down(vsi);
4069         i40e_vsi_free_irq(vsi);
4070
4071         i40e_vsi_free_tx_resources(vsi);
4072         i40e_vsi_free_rx_resources(vsi);
4073
4074         return 0;
4075 }
4076
4077 /**
4078  * i40e_do_reset - Start a PF or Core Reset sequence
4079  * @pf: board private structure
4080  * @reset_flags: which reset is requested
4081  *
4082  * The essential difference in resets is that the PF Reset
4083  * doesn't clear the packet buffers, doesn't reset the PE
4084  * firmware, and doesn't bother the other PFs on the chip.
4085  **/
4086 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
4087 {
4088         u32 val;
4089
4090         WARN_ON(in_interrupt());
4091
4092         /* do the biggest reset indicated */
4093         if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) {
4094
4095                 /* Request a Global Reset
4096                  *
4097                  * This will start the chip's countdown to the actual full
4098                  * chip reset event, and a warning interrupt to be sent
4099                  * to all PFs, including the requestor.  Our handler
4100                  * for the warning interrupt will deal with the shutdown
4101                  * and recovery of the switch setup.
4102                  */
4103                 dev_info(&pf->pdev->dev, "GlobalR requested\n");
4104                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4105                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
4106                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4107
4108         } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) {
4109
4110                 /* Request a Core Reset
4111                  *
4112                  * Same as Global Reset, except does *not* include the MAC/PHY
4113                  */
4114                 dev_info(&pf->pdev->dev, "CoreR requested\n");
4115                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4116                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
4117                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4118                 i40e_flush(&pf->hw);
4119
4120         } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) {
4121
4122                 /* Request a Firmware Reset
4123                  *
4124                  * Same as Global reset, plus restarting the
4125                  * embedded firmware engine.
4126                  */
4127                 /* enable EMP Reset */
4128                 val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP);
4129                 val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK;
4130                 wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val);
4131
4132                 /* force the reset */
4133                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4134                 val |= I40E_GLGEN_RTRIG_EMPFWR_MASK;
4135                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4136                 i40e_flush(&pf->hw);
4137
4138         } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) {
4139
4140                 /* Request a PF Reset
4141                  *
4142                  * Resets only the PF-specific registers
4143                  *
4144                  * This goes directly to the tear-down and rebuild of
4145                  * the switch, since we need to do all the recovery as
4146                  * for the Core Reset.
4147                  */
4148                 dev_info(&pf->pdev->dev, "PFR requested\n");
4149                 i40e_handle_reset_warning(pf);
4150
4151         } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) {
4152                 int v;
4153
4154                 /* Find the VSI(s) that requested a re-init */
4155                 dev_info(&pf->pdev->dev,
4156                          "VSI reinit requested\n");
4157                 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4158                         struct i40e_vsi *vsi = pf->vsi[v];
4159                         if (vsi != NULL &&
4160                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
4161                                 i40e_vsi_reinit_locked(pf->vsi[v]);
4162                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
4163                         }
4164                 }
4165
4166                 /* no further action needed, so return now */
4167                 return;
4168         } else {
4169                 dev_info(&pf->pdev->dev,
4170                          "bad reset request 0x%08x\n", reset_flags);
4171                 return;
4172         }
4173 }
4174
4175 /**
4176  * i40e_do_reset_safe - Protected reset path for userland calls.
4177  * @pf: board private structure
4178  * @reset_flags: which reset is requested
4179  *
4180  **/
4181 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
4182 {
4183         rtnl_lock();
4184         i40e_do_reset(pf, reset_flags);
4185         rtnl_unlock();
4186 }
4187
4188 /**
4189  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
4190  * @pf: board private structure
4191  * @e: event info posted on ARQ
4192  *
4193  * Handler for LAN Queue Overflow Event generated by the firmware for PF
4194  * and VF queues
4195  **/
4196 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
4197                                            struct i40e_arq_event_info *e)
4198 {
4199         struct i40e_aqc_lan_overflow *data =
4200                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
4201         u32 queue = le32_to_cpu(data->prtdcb_rupto);
4202         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
4203         struct i40e_hw *hw = &pf->hw;
4204         struct i40e_vf *vf;
4205         u16 vf_id;
4206
4207         dev_info(&pf->pdev->dev, "%s: Rx Queue Number = %d QTX_CTL=0x%08x\n",
4208                  __func__, queue, qtx_ctl);
4209
4210         /* Queue belongs to VF, find the VF and issue VF reset */
4211         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
4212             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
4213                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
4214                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
4215                 vf_id -= hw->func_caps.vf_base_id;
4216                 vf = &pf->vf[vf_id];
4217                 i40e_vc_notify_vf_reset(vf);
4218                 /* Allow VF to process pending reset notification */
4219                 msleep(20);
4220                 i40e_reset_vf(vf, false);
4221         }
4222 }
4223
4224 /**
4225  * i40e_service_event_complete - Finish up the service event
4226  * @pf: board private structure
4227  **/
4228 static void i40e_service_event_complete(struct i40e_pf *pf)
4229 {
4230         BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
4231
4232         /* flush memory to make sure state is correct before next watchog */
4233         smp_mb__before_clear_bit();
4234         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
4235 }
4236
4237 /**
4238  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
4239  * @pf: board private structure
4240  **/
4241 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
4242 {
4243         if (!(pf->flags & I40E_FLAG_FDIR_REQUIRES_REINIT))
4244                 return;
4245
4246         pf->flags &= ~I40E_FLAG_FDIR_REQUIRES_REINIT;
4247
4248         /* if interface is down do nothing */
4249         if (test_bit(__I40E_DOWN, &pf->state))
4250                 return;
4251 }
4252
4253 /**
4254  * i40e_vsi_link_event - notify VSI of a link event
4255  * @vsi: vsi to be notified
4256  * @link_up: link up or down
4257  **/
4258 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
4259 {
4260         if (!vsi)
4261                 return;
4262
4263         switch (vsi->type) {
4264         case I40E_VSI_MAIN:
4265                 if (!vsi->netdev || !vsi->netdev_registered)
4266                         break;
4267
4268                 if (link_up) {
4269                         netif_carrier_on(vsi->netdev);
4270                         netif_tx_wake_all_queues(vsi->netdev);
4271                 } else {
4272                         netif_carrier_off(vsi->netdev);
4273                         netif_tx_stop_all_queues(vsi->netdev);
4274                 }
4275                 break;
4276
4277         case I40E_VSI_SRIOV:
4278                 break;
4279
4280         case I40E_VSI_VMDQ2:
4281         case I40E_VSI_CTRL:
4282         case I40E_VSI_MIRROR:
4283         default:
4284                 /* there is no notification for other VSIs */
4285                 break;
4286         }
4287 }
4288
4289 /**
4290  * i40e_veb_link_event - notify elements on the veb of a link event
4291  * @veb: veb to be notified
4292  * @link_up: link up or down
4293  **/
4294 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
4295 {
4296         struct i40e_pf *pf;
4297         int i;
4298
4299         if (!veb || !veb->pf)
4300                 return;
4301         pf = veb->pf;
4302
4303         /* depth first... */
4304         for (i = 0; i < I40E_MAX_VEB; i++)
4305                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
4306                         i40e_veb_link_event(pf->veb[i], link_up);
4307
4308         /* ... now the local VSIs */
4309         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4310                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
4311                         i40e_vsi_link_event(pf->vsi[i], link_up);
4312 }
4313
4314 /**
4315  * i40e_link_event - Update netif_carrier status
4316  * @pf: board private structure
4317  **/
4318 static void i40e_link_event(struct i40e_pf *pf)
4319 {
4320         bool new_link, old_link;
4321
4322         new_link = (pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP);
4323         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
4324
4325         if (new_link == old_link)
4326                 return;
4327
4328         if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
4329                 netdev_info(pf->vsi[pf->lan_vsi]->netdev,
4330                             "NIC Link is %s\n", (new_link ? "Up" : "Down"));
4331
4332         /* Notify the base of the switch tree connected to
4333          * the link.  Floating VEBs are not notified.
4334          */
4335         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
4336                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
4337         else
4338                 i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link);
4339
4340         if (pf->vf)
4341                 i40e_vc_notify_link_state(pf);
4342
4343         if (pf->flags & I40E_FLAG_PTP)
4344                 i40e_ptp_set_increment(pf);
4345 }
4346
4347 /**
4348  * i40e_check_hang_subtask - Check for hung queues and dropped interrupts
4349  * @pf: board private structure
4350  *
4351  * Set the per-queue flags to request a check for stuck queues in the irq
4352  * clean functions, then force interrupts to be sure the irq clean is called.
4353  **/
4354 static void i40e_check_hang_subtask(struct i40e_pf *pf)
4355 {
4356         int i, v;
4357
4358         /* If we're down or resetting, just bail */
4359         if (test_bit(__I40E_CONFIG_BUSY, &pf->state))
4360                 return;
4361
4362         /* for each VSI/netdev
4363          *     for each Tx queue
4364          *         set the check flag
4365          *     for each q_vector
4366          *         force an interrupt
4367          */
4368         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4369                 struct i40e_vsi *vsi = pf->vsi[v];
4370                 int armed = 0;
4371
4372                 if (!pf->vsi[v] ||
4373                     test_bit(__I40E_DOWN, &vsi->state) ||
4374                     (vsi->netdev && !netif_carrier_ok(vsi->netdev)))
4375                         continue;
4376
4377                 for (i = 0; i < vsi->num_queue_pairs; i++) {
4378                         set_check_for_tx_hang(vsi->tx_rings[i]);
4379                         if (test_bit(__I40E_HANG_CHECK_ARMED,
4380                                      &vsi->tx_rings[i]->state))
4381                                 armed++;
4382                 }
4383
4384                 if (armed) {
4385                         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
4386                                 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0,
4387                                      (I40E_PFINT_DYN_CTL0_INTENA_MASK |
4388                                       I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK));
4389                         } else {
4390                                 u16 vec = vsi->base_vector - 1;
4391                                 u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
4392                                            I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
4393                                 for (i = 0; i < vsi->num_q_vectors; i++, vec++)
4394                                         wr32(&vsi->back->hw,
4395                                              I40E_PFINT_DYN_CTLN(vec), val);
4396                         }
4397                         i40e_flush(&vsi->back->hw);
4398                 }
4399         }
4400 }
4401
4402 /**
4403  * i40e_watchdog_subtask - Check and bring link up
4404  * @pf: board private structure
4405  **/
4406 static void i40e_watchdog_subtask(struct i40e_pf *pf)
4407 {
4408         int i;
4409
4410         /* if interface is down do nothing */
4411         if (test_bit(__I40E_DOWN, &pf->state) ||
4412             test_bit(__I40E_CONFIG_BUSY, &pf->state))
4413                 return;
4414
4415         /* Update the stats for active netdevs so the network stack
4416          * can look at updated numbers whenever it cares to
4417          */
4418         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4419                 if (pf->vsi[i] && pf->vsi[i]->netdev)
4420                         i40e_update_stats(pf->vsi[i]);
4421
4422         /* Update the stats for the active switching components */
4423         for (i = 0; i < I40E_MAX_VEB; i++)
4424                 if (pf->veb[i])
4425                         i40e_update_veb_stats(pf->veb[i]);
4426
4427         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
4428 }
4429
4430 /**
4431  * i40e_reset_subtask - Set up for resetting the device and driver
4432  * @pf: board private structure
4433  **/
4434 static void i40e_reset_subtask(struct i40e_pf *pf)
4435 {
4436         u32 reset_flags = 0;
4437
4438         rtnl_lock();
4439         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
4440                 reset_flags |= (1 << __I40E_REINIT_REQUESTED);
4441                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
4442         }
4443         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
4444                 reset_flags |= (1 << __I40E_PF_RESET_REQUESTED);
4445                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
4446         }
4447         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
4448                 reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED);
4449                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
4450         }
4451         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
4452                 reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED);
4453                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
4454         }
4455
4456         /* If there's a recovery already waiting, it takes
4457          * precedence before starting a new reset sequence.
4458          */
4459         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
4460                 i40e_handle_reset_warning(pf);
4461                 goto unlock;
4462         }
4463
4464         /* If we're already down or resetting, just bail */
4465         if (reset_flags &&
4466             !test_bit(__I40E_DOWN, &pf->state) &&
4467             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
4468                 i40e_do_reset(pf, reset_flags);
4469
4470 unlock:
4471         rtnl_unlock();
4472 }
4473
4474 /**
4475  * i40e_handle_link_event - Handle link event
4476  * @pf: board private structure
4477  * @e: event info posted on ARQ
4478  **/
4479 static void i40e_handle_link_event(struct i40e_pf *pf,
4480                                    struct i40e_arq_event_info *e)
4481 {
4482         struct i40e_hw *hw = &pf->hw;
4483         struct i40e_aqc_get_link_status *status =
4484                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
4485         struct i40e_link_status *hw_link_info = &hw->phy.link_info;
4486
4487         /* save off old link status information */
4488         memcpy(&pf->hw.phy.link_info_old, hw_link_info,
4489                sizeof(pf->hw.phy.link_info_old));
4490
4491         /* update link status */
4492         hw_link_info->phy_type = (enum i40e_aq_phy_type)status->phy_type;
4493         hw_link_info->link_speed = (enum i40e_aq_link_speed)status->link_speed;
4494         hw_link_info->link_info = status->link_info;
4495         hw_link_info->an_info = status->an_info;
4496         hw_link_info->ext_info = status->ext_info;
4497         hw_link_info->lse_enable =
4498                 le16_to_cpu(status->command_flags) &
4499                             I40E_AQ_LSE_ENABLE;
4500
4501         /* process the event */
4502         i40e_link_event(pf);
4503
4504         /* Do a new status request to re-enable LSE reporting
4505          * and load new status information into the hw struct,
4506          * then see if the status changed while processing the
4507          * initial event.
4508          */
4509         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
4510         i40e_link_event(pf);
4511 }
4512
4513 /**
4514  * i40e_clean_adminq_subtask - Clean the AdminQ rings
4515  * @pf: board private structure
4516  **/
4517 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
4518 {
4519         struct i40e_arq_event_info event;
4520         struct i40e_hw *hw = &pf->hw;
4521         u16 pending, i = 0;
4522         i40e_status ret;
4523         u16 opcode;
4524         u32 val;
4525
4526         if (!test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state))
4527                 return;
4528
4529         event.msg_size = I40E_MAX_AQ_BUF_SIZE;
4530         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
4531         if (!event.msg_buf)
4532                 return;
4533
4534         do {
4535                 event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */
4536                 ret = i40e_clean_arq_element(hw, &event, &pending);
4537                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) {
4538                         dev_info(&pf->pdev->dev, "No ARQ event found\n");
4539                         break;
4540                 } else if (ret) {
4541                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
4542                         break;
4543                 }
4544
4545                 opcode = le16_to_cpu(event.desc.opcode);
4546                 switch (opcode) {
4547
4548                 case i40e_aqc_opc_get_link_status:
4549                         i40e_handle_link_event(pf, &event);
4550                         break;
4551                 case i40e_aqc_opc_send_msg_to_pf:
4552                         ret = i40e_vc_process_vf_msg(pf,
4553                                         le16_to_cpu(event.desc.retval),
4554                                         le32_to_cpu(event.desc.cookie_high),
4555                                         le32_to_cpu(event.desc.cookie_low),
4556                                         event.msg_buf,
4557                                         event.msg_size);
4558                         break;
4559                 case i40e_aqc_opc_lldp_update_mib:
4560                         dev_info(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
4561                         break;
4562                 case i40e_aqc_opc_event_lan_overflow:
4563                         dev_info(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
4564                         i40e_handle_lan_overflow_event(pf, &event);
4565                         break;
4566                 case i40e_aqc_opc_send_msg_to_peer:
4567                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
4568                         break;
4569                 default:
4570                         dev_info(&pf->pdev->dev,
4571                                  "ARQ Error: Unknown event 0x%04x received\n",
4572                                  opcode);
4573                         break;
4574                 }
4575         } while (pending && (i++ < pf->adminq_work_limit));
4576
4577         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
4578         /* re-enable Admin queue interrupt cause */
4579         val = rd32(hw, I40E_PFINT_ICR0_ENA);
4580         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4581         wr32(hw, I40E_PFINT_ICR0_ENA, val);
4582         i40e_flush(hw);
4583
4584         kfree(event.msg_buf);
4585 }
4586
4587 /**
4588  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
4589  * @veb: pointer to the VEB instance
4590  *
4591  * This is a recursive function that first builds the attached VSIs then
4592  * recurses in to build the next layer of VEB.  We track the connections
4593  * through our own index numbers because the seid's from the HW could
4594  * change across the reset.
4595  **/
4596 static int i40e_reconstitute_veb(struct i40e_veb *veb)
4597 {
4598         struct i40e_vsi *ctl_vsi = NULL;
4599         struct i40e_pf *pf = veb->pf;
4600         int v, veb_idx;
4601         int ret;
4602
4603         /* build VSI that owns this VEB, temporarily attached to base VEB */
4604         for (v = 0; v < pf->hw.func_caps.num_vsis && !ctl_vsi; v++) {
4605                 if (pf->vsi[v] &&
4606                     pf->vsi[v]->veb_idx == veb->idx &&
4607                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
4608                         ctl_vsi = pf->vsi[v];
4609                         break;
4610                 }
4611         }
4612         if (!ctl_vsi) {
4613                 dev_info(&pf->pdev->dev,
4614                          "missing owner VSI for veb_idx %d\n", veb->idx);
4615                 ret = -ENOENT;
4616                 goto end_reconstitute;
4617         }
4618         if (ctl_vsi != pf->vsi[pf->lan_vsi])
4619                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
4620         ret = i40e_add_vsi(ctl_vsi);
4621         if (ret) {
4622                 dev_info(&pf->pdev->dev,
4623                          "rebuild of owner VSI failed: %d\n", ret);
4624                 goto end_reconstitute;
4625         }
4626         i40e_vsi_reset_stats(ctl_vsi);
4627
4628         /* create the VEB in the switch and move the VSI onto the VEB */
4629         ret = i40e_add_veb(veb, ctl_vsi);
4630         if (ret)
4631                 goto end_reconstitute;
4632
4633         /* create the remaining VSIs attached to this VEB */
4634         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4635                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
4636                         continue;
4637
4638                 if (pf->vsi[v]->veb_idx == veb->idx) {
4639                         struct i40e_vsi *vsi = pf->vsi[v];
4640                         vsi->uplink_seid = veb->seid;
4641                         ret = i40e_add_vsi(vsi);
4642                         if (ret) {
4643                                 dev_info(&pf->pdev->dev,
4644                                          "rebuild of vsi_idx %d failed: %d\n",
4645                                          v, ret);
4646                                 goto end_reconstitute;
4647                         }
4648                         i40e_vsi_reset_stats(vsi);
4649                 }
4650         }
4651
4652         /* create any VEBs attached to this VEB - RECURSION */
4653         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
4654                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
4655                         pf->veb[veb_idx]->uplink_seid = veb->seid;
4656                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
4657                         if (ret)
4658                                 break;
4659                 }
4660         }
4661
4662 end_reconstitute:
4663         return ret;
4664 }
4665
4666 /**
4667  * i40e_get_capabilities - get info about the HW
4668  * @pf: the PF struct
4669  **/
4670 static int i40e_get_capabilities(struct i40e_pf *pf)
4671 {
4672         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
4673         u16 data_size;
4674         int buf_len;
4675         int err;
4676
4677         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
4678         do {
4679                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
4680                 if (!cap_buf)
4681                         return -ENOMEM;
4682
4683                 /* this loads the data into the hw struct for us */
4684                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
4685                                             &data_size,
4686                                             i40e_aqc_opc_list_func_capabilities,
4687                                             NULL);
4688                 /* data loaded, buffer no longer needed */
4689                 kfree(cap_buf);
4690
4691                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
4692                         /* retry with a larger buffer */
4693                         buf_len = data_size;
4694                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
4695                         dev_info(&pf->pdev->dev,
4696                                  "capability discovery failed: aq=%d\n",
4697                                  pf->hw.aq.asq_last_status);
4698                         return -ENODEV;
4699                 }
4700         } while (err);
4701
4702         if (pf->hw.revision_id == 0 && (pf->flags & I40E_FLAG_MFP_ENABLED)) {
4703                 pf->hw.func_caps.num_msix_vectors += 1;
4704                 pf->hw.func_caps.num_tx_qp =
4705                         min_t(int, pf->hw.func_caps.num_tx_qp,
4706                               I40E_MAX_NPAR_QPS);
4707         }
4708
4709         if (pf->hw.debug_mask & I40E_DEBUG_USER)
4710                 dev_info(&pf->pdev->dev,
4711                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
4712                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
4713                          pf->hw.func_caps.num_msix_vectors,
4714                          pf->hw.func_caps.num_msix_vectors_vf,
4715                          pf->hw.func_caps.fd_filters_guaranteed,
4716                          pf->hw.func_caps.fd_filters_best_effort,
4717                          pf->hw.func_caps.num_tx_qp,
4718                          pf->hw.func_caps.num_vsis);
4719
4720 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
4721                        + pf->hw.func_caps.num_vfs)
4722         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
4723                 dev_info(&pf->pdev->dev,
4724                          "got num_vsis %d, setting num_vsis to %d\n",
4725                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
4726                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
4727         }
4728
4729         return 0;
4730 }
4731
4732 /**
4733  * i40e_fdir_setup - initialize the Flow Director resources
4734  * @pf: board private structure
4735  **/
4736 static void i40e_fdir_setup(struct i40e_pf *pf)
4737 {
4738         struct i40e_vsi *vsi;
4739         bool new_vsi = false;
4740         int err, i;
4741
4742         if (!(pf->flags & (I40E_FLAG_FDIR_ENABLED |
4743                            I40E_FLAG_FDIR_ATR_ENABLED)))
4744                 return;
4745
4746         pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
4747
4748         /* find existing or make new FDIR VSI */
4749         vsi = NULL;
4750         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4751                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
4752                         vsi = pf->vsi[i];
4753         if (!vsi) {
4754                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->mac_seid, 0);
4755                 if (!vsi) {
4756                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
4757                         pf->flags &= ~I40E_FLAG_FDIR_ENABLED;
4758                         return;
4759                 }
4760                 new_vsi = true;
4761         }
4762         WARN_ON(vsi->base_queue != I40E_FDIR_RING);
4763         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_rings);
4764
4765         err = i40e_vsi_setup_tx_resources(vsi);
4766         if (!err)
4767                 err = i40e_vsi_setup_rx_resources(vsi);
4768         if (!err)
4769                 err = i40e_vsi_configure(vsi);
4770         if (!err && new_vsi) {
4771                 char int_name[IFNAMSIZ + 9];
4772                 snprintf(int_name, sizeof(int_name) - 1, "%s-fdir",
4773                          dev_driver_string(&pf->pdev->dev));
4774                 err = i40e_vsi_request_irq(vsi, int_name);
4775         }
4776         if (!err)
4777                 err = i40e_up_complete(vsi);
4778
4779         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
4780 }
4781
4782 /**
4783  * i40e_fdir_teardown - release the Flow Director resources
4784  * @pf: board private structure
4785  **/
4786 static void i40e_fdir_teardown(struct i40e_pf *pf)
4787 {
4788         int i;
4789
4790         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
4791                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
4792                         i40e_vsi_release(pf->vsi[i]);
4793                         break;
4794                 }
4795         }
4796 }
4797
4798 /**
4799  * i40e_prep_for_reset - prep for the core to reset
4800  * @pf: board private structure
4801  *
4802  * Close up the VFs and other things in prep for pf Reset.
4803   **/
4804 static int i40e_prep_for_reset(struct i40e_pf *pf)
4805 {
4806         struct i40e_hw *hw = &pf->hw;
4807         i40e_status ret;
4808         u32 v;
4809
4810         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
4811         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
4812                 return 0;
4813
4814         dev_info(&pf->pdev->dev, "Tearing down internal switch for reset\n");
4815
4816         if (i40e_check_asq_alive(hw))
4817                 i40e_vc_notify_reset(pf);
4818
4819         /* quiesce the VSIs and their queues that are not already DOWN */
4820         i40e_pf_quiesce_all_vsi(pf);
4821
4822         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4823                 if (pf->vsi[v])
4824                         pf->vsi[v]->seid = 0;
4825         }
4826
4827         i40e_shutdown_adminq(&pf->hw);
4828
4829         /* call shutdown HMC */
4830         ret = i40e_shutdown_lan_hmc(hw);
4831         if (ret) {
4832                 dev_info(&pf->pdev->dev, "shutdown_lan_hmc failed: %d\n", ret);
4833                 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
4834         }
4835         return ret;
4836 }
4837
4838 /**
4839  * i40e_reset_and_rebuild - reset and rebuild using a saved config
4840  * @pf: board private structure
4841  * @reinit: if the Main VSI needs to re-initialized.
4842  **/
4843 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
4844 {
4845         struct i40e_driver_version dv;
4846         struct i40e_hw *hw = &pf->hw;
4847         i40e_status ret;
4848         u32 v;
4849
4850         /* Now we wait for GRST to settle out.
4851          * We don't have to delete the VEBs or VSIs from the hw switch
4852          * because the reset will make them disappear.
4853          */
4854         ret = i40e_pf_reset(hw);
4855         if (ret)
4856                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
4857         pf->pfr_count++;
4858
4859         if (test_bit(__I40E_DOWN, &pf->state))
4860                 goto end_core_reset;
4861         dev_info(&pf->pdev->dev, "Rebuilding internal switch\n");
4862
4863         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
4864         ret = i40e_init_adminq(&pf->hw);
4865         if (ret) {
4866                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret);
4867                 goto end_core_reset;
4868         }
4869
4870         ret = i40e_get_capabilities(pf);
4871         if (ret) {
4872                 dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n",
4873                          ret);
4874                 goto end_core_reset;
4875         }
4876
4877         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4878                                 hw->func_caps.num_rx_qp,
4879                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
4880         if (ret) {
4881                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
4882                 goto end_core_reset;
4883         }
4884         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4885         if (ret) {
4886                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
4887                 goto end_core_reset;
4888         }
4889
4890         /* do basic switch setup */
4891         ret = i40e_setup_pf_switch(pf, reinit);
4892         if (ret)
4893                 goto end_core_reset;
4894
4895         /* Rebuild the VSIs and VEBs that existed before reset.
4896          * They are still in our local switch element arrays, so only
4897          * need to rebuild the switch model in the HW.
4898          *
4899          * If there were VEBs but the reconstitution failed, we'll try
4900          * try to recover minimal use by getting the basic PF VSI working.
4901          */
4902         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
4903                 dev_info(&pf->pdev->dev, "attempting to rebuild switch\n");
4904                 /* find the one VEB connected to the MAC, and find orphans */
4905                 for (v = 0; v < I40E_MAX_VEB; v++) {
4906                         if (!pf->veb[v])
4907                                 continue;
4908
4909                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
4910                             pf->veb[v]->uplink_seid == 0) {
4911                                 ret = i40e_reconstitute_veb(pf->veb[v]);
4912
4913                                 if (!ret)
4914                                         continue;
4915
4916                                 /* If Main VEB failed, we're in deep doodoo,
4917                                  * so give up rebuilding the switch and set up
4918                                  * for minimal rebuild of PF VSI.
4919                                  * If orphan failed, we'll report the error
4920                                  * but try to keep going.
4921                                  */
4922                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
4923                                         dev_info(&pf->pdev->dev,
4924                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
4925                                                  ret);
4926                                         pf->vsi[pf->lan_vsi]->uplink_seid
4927                                                                 = pf->mac_seid;
4928                                         break;
4929                                 } else if (pf->veb[v]->uplink_seid == 0) {
4930                                         dev_info(&pf->pdev->dev,
4931                                                  "rebuild of orphan VEB failed: %d\n",
4932                                                  ret);
4933                                 }
4934                         }
4935                 }
4936         }
4937
4938         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
4939                 dev_info(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
4940                 /* no VEB, so rebuild only the Main VSI */
4941                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
4942                 if (ret) {
4943                         dev_info(&pf->pdev->dev,
4944                                  "rebuild of Main VSI failed: %d\n", ret);
4945                         goto end_core_reset;
4946                 }
4947         }
4948
4949         /* reinit the misc interrupt */
4950         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4951                 ret = i40e_setup_misc_vector(pf);
4952
4953         /* restart the VSIs that were rebuilt and running before the reset */
4954         i40e_pf_unquiesce_all_vsi(pf);
4955
4956         /* tell the firmware that we're starting */
4957         dv.major_version = DRV_VERSION_MAJOR;
4958         dv.minor_version = DRV_VERSION_MINOR;
4959         dv.build_version = DRV_VERSION_BUILD;
4960         dv.subbuild_version = 0;
4961         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
4962
4963         dev_info(&pf->pdev->dev, "PF reset done\n");
4964
4965 end_core_reset:
4966         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
4967 }
4968
4969 /**
4970  * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild
4971  * @pf: board private structure
4972  *
4973  * Close up the VFs and other things in prep for a Core Reset,
4974  * then get ready to rebuild the world.
4975  **/
4976 static void i40e_handle_reset_warning(struct i40e_pf *pf)
4977 {
4978         i40e_status ret;
4979
4980         ret = i40e_prep_for_reset(pf);
4981         if (!ret)
4982                 i40e_reset_and_rebuild(pf, false);
4983 }
4984
4985 /**
4986  * i40e_handle_mdd_event
4987  * @pf: pointer to the pf structure
4988  *
4989  * Called from the MDD irq handler to identify possibly malicious vfs
4990  **/
4991 static void i40e_handle_mdd_event(struct i40e_pf *pf)
4992 {
4993         struct i40e_hw *hw = &pf->hw;
4994         bool mdd_detected = false;
4995         struct i40e_vf *vf;
4996         u32 reg;
4997         int i;
4998
4999         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
5000                 return;
5001
5002         /* find what triggered the MDD event */
5003         reg = rd32(hw, I40E_GL_MDET_TX);
5004         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
5005                 u8 func = (reg & I40E_GL_MDET_TX_FUNCTION_MASK)
5006                                 >> I40E_GL_MDET_TX_FUNCTION_SHIFT;
5007                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT)
5008                                 >> I40E_GL_MDET_TX_EVENT_SHIFT;
5009                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK)
5010                                 >> I40E_GL_MDET_TX_QUEUE_SHIFT;
5011                 dev_info(&pf->pdev->dev,
5012                          "Malicious Driver Detection TX event 0x%02x on q %d of function 0x%02x\n",
5013                          event, queue, func);
5014                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
5015                 mdd_detected = true;
5016         }
5017         reg = rd32(hw, I40E_GL_MDET_RX);
5018         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
5019                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK)
5020                                 >> I40E_GL_MDET_RX_FUNCTION_SHIFT;
5021                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT)
5022                                 >> I40E_GL_MDET_RX_EVENT_SHIFT;
5023                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK)
5024                                 >> I40E_GL_MDET_RX_QUEUE_SHIFT;
5025                 dev_info(&pf->pdev->dev,
5026                          "Malicious Driver Detection RX event 0x%02x on q %d of function 0x%02x\n",
5027                          event, queue, func);
5028                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
5029                 mdd_detected = true;
5030         }
5031
5032         /* see if one of the VFs needs its hand slapped */
5033         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
5034                 vf = &(pf->vf[i]);
5035                 reg = rd32(hw, I40E_VP_MDET_TX(i));
5036                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
5037                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
5038                         vf->num_mdd_events++;
5039                         dev_info(&pf->pdev->dev, "MDD TX event on VF %d\n", i);
5040                 }
5041
5042                 reg = rd32(hw, I40E_VP_MDET_RX(i));
5043                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
5044                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
5045                         vf->num_mdd_events++;
5046                         dev_info(&pf->pdev->dev, "MDD RX event on VF %d\n", i);
5047                 }
5048
5049                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
5050                         dev_info(&pf->pdev->dev,
5051                                  "Too many MDD events on VF %d, disabled\n", i);
5052                         dev_info(&pf->pdev->dev,
5053                                  "Use PF Control I/F to re-enable the VF\n");
5054                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
5055                 }
5056         }
5057
5058         /* re-enable mdd interrupt cause */
5059         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
5060         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
5061         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
5062         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
5063         i40e_flush(hw);
5064 }
5065
5066 #ifdef CONFIG_I40E_VXLAN
5067 /**
5068  * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW
5069  * @pf: board private structure
5070  **/
5071 static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf)
5072 {
5073         const int vxlan_hdr_qwords = 4;
5074         struct i40e_hw *hw = &pf->hw;
5075         i40e_status ret;
5076         u8 filter_index;
5077         __be16 port;
5078         int i;
5079
5080         if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC))
5081                 return;
5082
5083         pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC;
5084
5085         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5086                 if (pf->pending_vxlan_bitmap & (1 << i)) {
5087                         pf->pending_vxlan_bitmap &= ~(1 << i);
5088                         port = pf->vxlan_ports[i];
5089                         ret = port ?
5090                               i40e_aq_add_udp_tunnel(hw, ntohs(port),
5091                                                      vxlan_hdr_qwords,
5092                                                      I40E_AQC_TUNNEL_TYPE_VXLAN,
5093                                                      &filter_index, NULL)
5094                               : i40e_aq_del_udp_tunnel(hw, i, NULL);
5095
5096                         if (ret) {
5097                                 dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n",
5098                                          port ? "adding" : "deleting",
5099                                          ntohs(port), port ? i : i);
5100
5101                                 pf->vxlan_ports[i] = 0;
5102                         } else {
5103                                 dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n",
5104                                          port ? "Added" : "Deleted",
5105                                          ntohs(port), port ? i : filter_index);
5106                         }
5107                 }
5108         }
5109 }
5110
5111 #endif
5112 /**
5113  * i40e_service_task - Run the driver's async subtasks
5114  * @work: pointer to work_struct containing our data
5115  **/
5116 static void i40e_service_task(struct work_struct *work)
5117 {
5118         struct i40e_pf *pf = container_of(work,
5119                                           struct i40e_pf,
5120                                           service_task);
5121         unsigned long start_time = jiffies;
5122
5123         i40e_reset_subtask(pf);
5124         i40e_handle_mdd_event(pf);
5125         i40e_vc_process_vflr_event(pf);
5126         i40e_watchdog_subtask(pf);
5127         i40e_fdir_reinit_subtask(pf);
5128         i40e_check_hang_subtask(pf);
5129         i40e_sync_filters_subtask(pf);
5130 #ifdef CONFIG_I40E_VXLAN
5131         i40e_sync_vxlan_filters_subtask(pf);
5132 #endif
5133         i40e_clean_adminq_subtask(pf);
5134
5135         i40e_service_event_complete(pf);
5136
5137         /* If the tasks have taken longer than one timer cycle or there
5138          * is more work to be done, reschedule the service task now
5139          * rather than wait for the timer to tick again.
5140          */
5141         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
5142             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
5143             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
5144             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
5145                 i40e_service_event_schedule(pf);
5146 }
5147
5148 /**
5149  * i40e_service_timer - timer callback
5150  * @data: pointer to PF struct
5151  **/
5152 static void i40e_service_timer(unsigned long data)
5153 {
5154         struct i40e_pf *pf = (struct i40e_pf *)data;
5155
5156         mod_timer(&pf->service_timer,
5157                   round_jiffies(jiffies + pf->service_timer_period));
5158         i40e_service_event_schedule(pf);
5159 }
5160
5161 /**
5162  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
5163  * @vsi: the VSI being configured
5164  **/
5165 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
5166 {
5167         struct i40e_pf *pf = vsi->back;
5168
5169         switch (vsi->type) {
5170         case I40E_VSI_MAIN:
5171                 vsi->alloc_queue_pairs = pf->num_lan_qps;
5172                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5173                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5174                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5175                         vsi->num_q_vectors = pf->num_lan_msix;
5176                 else
5177                         vsi->num_q_vectors = 1;
5178
5179                 break;
5180
5181         case I40E_VSI_FDIR:
5182                 vsi->alloc_queue_pairs = 1;
5183                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
5184                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5185                 vsi->num_q_vectors = 1;
5186                 break;
5187
5188         case I40E_VSI_VMDQ2:
5189                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
5190                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5191                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5192                 vsi->num_q_vectors = pf->num_vmdq_msix;
5193                 break;
5194
5195         case I40E_VSI_SRIOV:
5196                 vsi->alloc_queue_pairs = pf->num_vf_qps;
5197                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5198                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5199                 break;
5200
5201         default:
5202                 WARN_ON(1);
5203                 return -ENODATA;
5204         }
5205
5206         return 0;
5207 }
5208
5209 /**
5210  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
5211  * @type: VSI pointer
5212  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
5213  *
5214  * On error: returns error code (negative)
5215  * On success: returns 0
5216  **/
5217 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
5218 {
5219         int size;
5220         int ret = 0;
5221
5222         /* allocate memory for both Tx and Rx ring pointers */
5223         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
5224         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
5225         if (!vsi->tx_rings)
5226                 return -ENOMEM;
5227         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
5228
5229         if (alloc_qvectors) {
5230                 /* allocate memory for q_vector pointers */
5231                 size = sizeof(struct i40e_q_vectors *) * vsi->num_q_vectors;
5232                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
5233                 if (!vsi->q_vectors) {
5234                         ret = -ENOMEM;
5235                         goto err_vectors;
5236                 }
5237         }
5238         return ret;
5239
5240 err_vectors:
5241         kfree(vsi->tx_rings);
5242         return ret;
5243 }
5244
5245 /**
5246  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
5247  * @pf: board private structure
5248  * @type: type of VSI
5249  *
5250  * On error: returns error code (negative)
5251  * On success: returns vsi index in PF (positive)
5252  **/
5253 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
5254 {
5255         int ret = -ENODEV;
5256         struct i40e_vsi *vsi;
5257         int vsi_idx;
5258         int i;
5259
5260         /* Need to protect the allocation of the VSIs at the PF level */
5261         mutex_lock(&pf->switch_mutex);
5262
5263         /* VSI list may be fragmented if VSI creation/destruction has
5264          * been happening.  We can afford to do a quick scan to look
5265          * for any free VSIs in the list.
5266          *
5267          * find next empty vsi slot, looping back around if necessary
5268          */
5269         i = pf->next_vsi;
5270         while (i < pf->hw.func_caps.num_vsis && pf->vsi[i])
5271                 i++;
5272         if (i >= pf->hw.func_caps.num_vsis) {
5273                 i = 0;
5274                 while (i < pf->next_vsi && pf->vsi[i])
5275                         i++;
5276         }
5277
5278         if (i < pf->hw.func_caps.num_vsis && !pf->vsi[i]) {
5279                 vsi_idx = i;             /* Found one! */
5280         } else {
5281                 ret = -ENODEV;
5282                 goto unlock_pf;  /* out of VSI slots! */
5283         }
5284         pf->next_vsi = ++i;
5285
5286         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
5287         if (!vsi) {
5288                 ret = -ENOMEM;
5289                 goto unlock_pf;
5290         }
5291         vsi->type = type;
5292         vsi->back = pf;
5293         set_bit(__I40E_DOWN, &vsi->state);
5294         vsi->flags = 0;
5295         vsi->idx = vsi_idx;
5296         vsi->rx_itr_setting = pf->rx_itr_default;
5297         vsi->tx_itr_setting = pf->tx_itr_default;
5298         vsi->netdev_registered = false;
5299         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
5300         INIT_LIST_HEAD(&vsi->mac_filter_list);
5301
5302         ret = i40e_set_num_rings_in_vsi(vsi);
5303         if (ret)
5304                 goto err_rings;
5305
5306         ret = i40e_vsi_alloc_arrays(vsi, true);
5307         if (ret)
5308                 goto err_rings;
5309
5310         /* Setup default MSIX irq handler for VSI */
5311         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
5312
5313         pf->vsi[vsi_idx] = vsi;
5314         ret = vsi_idx;
5315         goto unlock_pf;
5316
5317 err_rings:
5318         pf->next_vsi = i - 1;
5319         kfree(vsi);
5320 unlock_pf:
5321         mutex_unlock(&pf->switch_mutex);
5322         return ret;
5323 }
5324
5325 /**
5326  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
5327  * @type: VSI pointer
5328  * @free_qvectors: a bool to specify if q_vectors need to be freed.
5329  *
5330  * On error: returns error code (negative)
5331  * On success: returns 0
5332  **/
5333 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
5334 {
5335         /* free the ring and vector containers */
5336         if (free_qvectors) {
5337                 kfree(vsi->q_vectors);
5338                 vsi->q_vectors = NULL;
5339         }
5340         kfree(vsi->tx_rings);
5341         vsi->tx_rings = NULL;
5342         vsi->rx_rings = NULL;
5343 }
5344
5345 /**
5346  * i40e_vsi_clear - Deallocate the VSI provided
5347  * @vsi: the VSI being un-configured
5348  **/
5349 static int i40e_vsi_clear(struct i40e_vsi *vsi)
5350 {
5351         struct i40e_pf *pf;
5352
5353         if (!vsi)
5354                 return 0;
5355
5356         if (!vsi->back)
5357                 goto free_vsi;
5358         pf = vsi->back;
5359
5360         mutex_lock(&pf->switch_mutex);
5361         if (!pf->vsi[vsi->idx]) {
5362                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
5363                         vsi->idx, vsi->idx, vsi, vsi->type);
5364                 goto unlock_vsi;
5365         }
5366
5367         if (pf->vsi[vsi->idx] != vsi) {
5368                 dev_err(&pf->pdev->dev,
5369                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
5370                         pf->vsi[vsi->idx]->idx,
5371                         pf->vsi[vsi->idx],
5372                         pf->vsi[vsi->idx]->type,
5373                         vsi->idx, vsi, vsi->type);
5374                 goto unlock_vsi;
5375         }
5376
5377         /* updates the pf for this cleared vsi */
5378         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
5379         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
5380
5381         i40e_vsi_free_arrays(vsi, true);
5382
5383         pf->vsi[vsi->idx] = NULL;
5384         if (vsi->idx < pf->next_vsi)
5385                 pf->next_vsi = vsi->idx;
5386
5387 unlock_vsi:
5388         mutex_unlock(&pf->switch_mutex);
5389 free_vsi:
5390         kfree(vsi);
5391
5392         return 0;
5393 }
5394
5395 /**
5396  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
5397  * @vsi: the VSI being cleaned
5398  **/
5399 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
5400 {
5401         int i;
5402
5403         if (vsi->tx_rings && vsi->tx_rings[0]) {
5404                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
5405                         kfree_rcu(vsi->tx_rings[i], rcu);
5406                         vsi->tx_rings[i] = NULL;
5407                         vsi->rx_rings[i] = NULL;
5408                 }
5409         }
5410 }
5411
5412 /**
5413  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
5414  * @vsi: the VSI being configured
5415  **/
5416 static int i40e_alloc_rings(struct i40e_vsi *vsi)
5417 {
5418         struct i40e_pf *pf = vsi->back;
5419         int i;
5420
5421         /* Set basic values in the rings to be used later during open() */
5422         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
5423                 struct i40e_ring *tx_ring;
5424                 struct i40e_ring *rx_ring;
5425
5426                 /* allocate space for both Tx and Rx in one shot */
5427                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
5428                 if (!tx_ring)
5429                         goto err_out;
5430
5431                 tx_ring->queue_index = i;
5432                 tx_ring->reg_idx = vsi->base_queue + i;
5433                 tx_ring->ring_active = false;
5434                 tx_ring->vsi = vsi;
5435                 tx_ring->netdev = vsi->netdev;
5436                 tx_ring->dev = &pf->pdev->dev;
5437                 tx_ring->count = vsi->num_desc;
5438                 tx_ring->size = 0;
5439                 tx_ring->dcb_tc = 0;
5440                 vsi->tx_rings[i] = tx_ring;
5441
5442                 rx_ring = &tx_ring[1];
5443                 rx_ring->queue_index = i;
5444                 rx_ring->reg_idx = vsi->base_queue + i;
5445                 rx_ring->ring_active = false;
5446                 rx_ring->vsi = vsi;
5447                 rx_ring->netdev = vsi->netdev;
5448                 rx_ring->dev = &pf->pdev->dev;
5449                 rx_ring->count = vsi->num_desc;
5450                 rx_ring->size = 0;
5451                 rx_ring->dcb_tc = 0;
5452                 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
5453                         set_ring_16byte_desc_enabled(rx_ring);
5454                 else
5455                         clear_ring_16byte_desc_enabled(rx_ring);
5456                 vsi->rx_rings[i] = rx_ring;
5457         }
5458
5459         return 0;
5460
5461 err_out:
5462         i40e_vsi_clear_rings(vsi);
5463         return -ENOMEM;
5464 }
5465
5466 /**
5467  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
5468  * @pf: board private structure
5469  * @vectors: the number of MSI-X vectors to request
5470  *
5471  * Returns the number of vectors reserved, or error
5472  **/
5473 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
5474 {
5475         int err = 0;
5476
5477         pf->num_msix_entries = 0;
5478         while (vectors >= I40E_MIN_MSIX) {
5479                 err = pci_enable_msix(pf->pdev, pf->msix_entries, vectors);
5480                 if (err == 0) {
5481                         /* good to go */
5482                         pf->num_msix_entries = vectors;
5483                         break;
5484                 } else if (err < 0) {
5485                         /* total failure */
5486                         dev_info(&pf->pdev->dev,
5487                                  "MSI-X vector reservation failed: %d\n", err);
5488                         vectors = 0;
5489                         break;
5490                 } else {
5491                         /* err > 0 is the hint for retry */
5492                         dev_info(&pf->pdev->dev,
5493                                  "MSI-X vectors wanted %d, retrying with %d\n",
5494                                  vectors, err);
5495                         vectors = err;
5496                 }
5497         }
5498
5499         if (vectors > 0 && vectors < I40E_MIN_MSIX) {
5500                 dev_info(&pf->pdev->dev,
5501                          "Couldn't get enough vectors, only %d available\n",
5502                          vectors);
5503                 vectors = 0;
5504         }
5505
5506         return vectors;
5507 }
5508
5509 /**
5510  * i40e_init_msix - Setup the MSIX capability
5511  * @pf: board private structure
5512  *
5513  * Work with the OS to set up the MSIX vectors needed.
5514  *
5515  * Returns 0 on success, negative on failure
5516  **/
5517 static int i40e_init_msix(struct i40e_pf *pf)
5518 {
5519         i40e_status err = 0;
5520         struct i40e_hw *hw = &pf->hw;
5521         int v_budget, i;
5522         int vec;
5523
5524         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
5525                 return -ENODEV;
5526
5527         /* The number of vectors we'll request will be comprised of:
5528          *   - Add 1 for "other" cause for Admin Queue events, etc.
5529          *   - The number of LAN queue pairs
5530          *      - Queues being used for RSS.
5531          *              We don't need as many as max_rss_size vectors.
5532          *              use rss_size instead in the calculation since that
5533          *              is governed by number of cpus in the system.
5534          *      - assumes symmetric Tx/Rx pairing
5535          *   - The number of VMDq pairs
5536          * Once we count this up, try the request.
5537          *
5538          * If we can't get what we want, we'll simplify to nearly nothing
5539          * and try again.  If that still fails, we punt.
5540          */
5541         pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size);
5542         pf->num_vmdq_msix = pf->num_vmdq_qps;
5543         v_budget = 1 + pf->num_lan_msix;
5544         v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix);
5545         if (pf->flags & I40E_FLAG_FDIR_ENABLED)
5546                 v_budget++;
5547
5548         /* Scale down if necessary, and the rings will share vectors */
5549         v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors);
5550
5551         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
5552                                    GFP_KERNEL);
5553         if (!pf->msix_entries)
5554                 return -ENOMEM;
5555
5556         for (i = 0; i < v_budget; i++)
5557                 pf->msix_entries[i].entry = i;
5558         vec = i40e_reserve_msix_vectors(pf, v_budget);
5559         if (vec < I40E_MIN_MSIX) {
5560                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
5561                 kfree(pf->msix_entries);
5562                 pf->msix_entries = NULL;
5563                 return -ENODEV;
5564
5565         } else if (vec == I40E_MIN_MSIX) {
5566                 /* Adjust for minimal MSIX use */
5567                 dev_info(&pf->pdev->dev, "Features disabled, not enough MSIX vectors\n");
5568                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
5569                 pf->num_vmdq_vsis = 0;
5570                 pf->num_vmdq_qps = 0;
5571                 pf->num_vmdq_msix = 0;
5572                 pf->num_lan_qps = 1;
5573                 pf->num_lan_msix = 1;
5574
5575         } else if (vec != v_budget) {
5576                 /* Scale vector usage down */
5577                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
5578                 vec--;                    /* reserve the misc vector */
5579
5580                 /* partition out the remaining vectors */
5581                 switch (vec) {
5582                 case 2:
5583                         pf->num_vmdq_vsis = 1;
5584                         pf->num_lan_msix = 1;
5585                         break;
5586                 case 3:
5587                         pf->num_vmdq_vsis = 1;
5588                         pf->num_lan_msix = 2;
5589                         break;
5590                 default:
5591                         pf->num_lan_msix = min_t(int, (vec / 2),
5592                                                  pf->num_lan_qps);
5593                         pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix),
5594                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
5595                         break;
5596                 }
5597         }
5598
5599         return err;
5600 }
5601
5602 /**
5603  * i40e_alloc_q_vector - Allocate memory for a single interrupt vector
5604  * @vsi: the VSI being configured
5605  * @v_idx: index of the vector in the vsi struct
5606  *
5607  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
5608  **/
5609 static int i40e_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
5610 {
5611         struct i40e_q_vector *q_vector;
5612
5613         /* allocate q_vector */
5614         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
5615         if (!q_vector)
5616                 return -ENOMEM;
5617
5618         q_vector->vsi = vsi;
5619         q_vector->v_idx = v_idx;
5620         cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
5621         if (vsi->netdev)
5622                 netif_napi_add(vsi->netdev, &q_vector->napi,
5623                                i40e_napi_poll, vsi->work_limit);
5624
5625         q_vector->rx.latency_range = I40E_LOW_LATENCY;
5626         q_vector->tx.latency_range = I40E_LOW_LATENCY;
5627
5628         /* tie q_vector and vsi together */
5629         vsi->q_vectors[v_idx] = q_vector;
5630
5631         return 0;
5632 }
5633
5634 /**
5635  * i40e_alloc_q_vectors - Allocate memory for interrupt vectors
5636  * @vsi: the VSI being configured
5637  *
5638  * We allocate one q_vector per queue interrupt.  If allocation fails we
5639  * return -ENOMEM.
5640  **/
5641 static int i40e_alloc_q_vectors(struct i40e_vsi *vsi)
5642 {
5643         struct i40e_pf *pf = vsi->back;
5644         int v_idx, num_q_vectors;
5645         int err;
5646
5647         /* if not MSIX, give the one vector only to the LAN VSI */
5648         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5649                 num_q_vectors = vsi->num_q_vectors;
5650         else if (vsi == pf->vsi[pf->lan_vsi])
5651                 num_q_vectors = 1;
5652         else
5653                 return -EINVAL;
5654
5655         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
5656                 err = i40e_alloc_q_vector(vsi, v_idx);
5657                 if (err)
5658                         goto err_out;
5659         }
5660
5661         return 0;
5662
5663 err_out:
5664         while (v_idx--)
5665                 i40e_free_q_vector(vsi, v_idx);
5666
5667         return err;
5668 }
5669
5670 /**
5671  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
5672  * @pf: board private structure to initialize
5673  **/
5674 static void i40e_init_interrupt_scheme(struct i40e_pf *pf)
5675 {
5676         int err = 0;
5677
5678         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
5679                 err = i40e_init_msix(pf);
5680                 if (err) {
5681                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED      |
5682                                         I40E_FLAG_RSS_ENABLED      |
5683                                         I40E_FLAG_DCB_ENABLED      |
5684                                         I40E_FLAG_SRIOV_ENABLED    |
5685                                         I40E_FLAG_FDIR_ENABLED     |
5686                                         I40E_FLAG_FDIR_ATR_ENABLED |
5687                                         I40E_FLAG_VMDQ_ENABLED);
5688
5689                         /* rework the queue expectations without MSIX */
5690                         i40e_determine_queue_usage(pf);
5691                 }
5692         }
5693
5694         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
5695             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
5696                 dev_info(&pf->pdev->dev, "MSIX not available, trying MSI\n");
5697                 err = pci_enable_msi(pf->pdev);
5698                 if (err) {
5699                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err);
5700                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
5701                 }
5702         }
5703
5704         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
5705                 dev_info(&pf->pdev->dev, "MSIX and MSI not available, falling back to Legacy IRQ\n");
5706
5707         /* track first vector for misc interrupts */
5708         err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1);
5709 }
5710
5711 /**
5712  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
5713  * @pf: board private structure
5714  *
5715  * This sets up the handler for MSIX 0, which is used to manage the
5716  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
5717  * when in MSI or Legacy interrupt mode.
5718  **/
5719 static int i40e_setup_misc_vector(struct i40e_pf *pf)
5720 {
5721         struct i40e_hw *hw = &pf->hw;
5722         int err = 0;
5723
5724         /* Only request the irq if this is the first time through, and
5725          * not when we're rebuilding after a Reset
5726          */
5727         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
5728                 err = request_irq(pf->msix_entries[0].vector,
5729                                   i40e_intr, 0, pf->misc_int_name, pf);
5730                 if (err) {
5731                         dev_info(&pf->pdev->dev,
5732                                  "request_irq for msix_misc failed: %d\n", err);
5733                         return -EFAULT;
5734                 }
5735         }
5736
5737         i40e_enable_misc_int_causes(hw);
5738
5739         /* associate no queues to the misc vector */
5740         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
5741         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
5742
5743         i40e_flush(hw);
5744
5745         i40e_irq_dynamic_enable_icr0(pf);
5746
5747         return err;
5748 }
5749
5750 /**
5751  * i40e_config_rss - Prepare for RSS if used
5752  * @pf: board private structure
5753  **/
5754 static int i40e_config_rss(struct i40e_pf *pf)
5755 {
5756         /* Set of random keys generated using kernel random number generator */
5757         static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687,
5758                                 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377,
5759                                 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d,
5760                                 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be};
5761         struct i40e_hw *hw = &pf->hw;
5762         u32 lut = 0;
5763         int i, j;
5764         u64 hena;
5765
5766         /* Fill out hash function seed */
5767         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
5768                 wr32(hw, I40E_PFQF_HKEY(i), seed[i]);
5769
5770         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
5771         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
5772                 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
5773         hena |= I40E_DEFAULT_RSS_HENA;
5774         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
5775         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
5776
5777         /* Populate the LUT with max no. of queues in round robin fashion */
5778         for (i = 0, j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
5779
5780                 /* The assumption is that lan qp count will be the highest
5781                  * qp count for any PF VSI that needs RSS.
5782                  * If multiple VSIs need RSS support, all the qp counts
5783                  * for those VSIs should be a power of 2 for RSS to work.
5784                  * If LAN VSI is the only consumer for RSS then this requirement
5785                  * is not necessary.
5786                  */
5787                 if (j == pf->rss_size)
5788                         j = 0;
5789                 /* lut = 4-byte sliding window of 4 lut entries */
5790                 lut = (lut << 8) | (j &
5791                          ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
5792                 /* On i = 3, we have 4 entries in lut; write to the register */
5793                 if ((i & 3) == 3)
5794                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
5795         }
5796         i40e_flush(hw);
5797
5798         return 0;
5799 }
5800
5801 /**
5802  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
5803  * @pf: board private structure
5804  * @queue_count: the requested queue count for rss.
5805  *
5806  * returns 0 if rss is not enabled, if enabled returns the final rss queue
5807  * count which may be different from the requested queue count.
5808  **/
5809 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
5810 {
5811         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
5812                 return 0;
5813
5814         queue_count = min_t(int, queue_count, pf->rss_size_max);
5815         queue_count = rounddown_pow_of_two(queue_count);
5816
5817         if (queue_count != pf->rss_size) {
5818                 i40e_prep_for_reset(pf);
5819
5820                 pf->rss_size = queue_count;
5821
5822                 i40e_reset_and_rebuild(pf, true);
5823                 i40e_config_rss(pf);
5824         }
5825         dev_info(&pf->pdev->dev, "RSS count:  %d\n", pf->rss_size);
5826         return pf->rss_size;
5827 }
5828
5829 /**
5830  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
5831  * @pf: board private structure to initialize
5832  *
5833  * i40e_sw_init initializes the Adapter private data structure.
5834  * Fields are initialized based on PCI device information and
5835  * OS network device settings (MTU size).
5836  **/
5837 static int i40e_sw_init(struct i40e_pf *pf)
5838 {
5839         int err = 0;
5840         int size;
5841
5842         pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
5843                                 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
5844         pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG;
5845         if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
5846                 if (I40E_DEBUG_USER & debug)
5847                         pf->hw.debug_mask = debug;
5848                 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
5849                                                 I40E_DEFAULT_MSG_ENABLE);
5850         }
5851
5852         /* Set default capability flags */
5853         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
5854                     I40E_FLAG_MSI_ENABLED     |
5855                     I40E_FLAG_MSIX_ENABLED    |
5856                     I40E_FLAG_RX_1BUF_ENABLED;
5857
5858         /* Depending on PF configurations, it is possible that the RSS
5859          * maximum might end up larger than the available queues
5860          */
5861         pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width;
5862         pf->rss_size_max = min_t(int, pf->rss_size_max,
5863                                  pf->hw.func_caps.num_tx_qp);
5864         if (pf->hw.func_caps.rss) {
5865                 pf->flags |= I40E_FLAG_RSS_ENABLED;
5866                 pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus());
5867         } else {
5868                 pf->rss_size = 1;
5869         }
5870
5871         /* MFP mode enabled */
5872         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) {
5873                 pf->flags |= I40E_FLAG_MFP_ENABLED;
5874                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
5875         }
5876
5877         if (pf->hw.func_caps.dcb)
5878                 pf->num_tc_qps = I40E_DEFAULT_QUEUES_PER_TC;
5879         else
5880                 pf->num_tc_qps = 0;
5881
5882         if (pf->hw.func_caps.fd) {
5883                 /* FW/NVM is not yet fixed in this regard */
5884                 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
5885                     (pf->hw.func_caps.fd_filters_best_effort > 0)) {
5886                         pf->flags |= I40E_FLAG_FDIR_ATR_ENABLED;
5887                         dev_info(&pf->pdev->dev,
5888                                  "Flow Director ATR mode Enabled\n");
5889                         pf->flags |= I40E_FLAG_FDIR_ENABLED;
5890                         dev_info(&pf->pdev->dev,
5891                                  "Flow Director Side Band mode Enabled\n");
5892                         pf->fdir_pf_filter_count =
5893                                          pf->hw.func_caps.fd_filters_guaranteed;
5894                 }
5895         } else {
5896                 pf->fdir_pf_filter_count = 0;
5897         }
5898
5899         if (pf->hw.func_caps.vmdq) {
5900                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
5901                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
5902                 pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ;
5903         }
5904
5905 #ifdef CONFIG_PCI_IOV
5906         if (pf->hw.func_caps.num_vfs) {
5907                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
5908                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
5909                 pf->num_req_vfs = min_t(int,
5910                                         pf->hw.func_caps.num_vfs,
5911                                         I40E_MAX_VF_COUNT);
5912                 dev_info(&pf->pdev->dev,
5913                          "Number of VFs being requested for PF[%d] = %d\n",
5914                          pf->hw.pf_id, pf->num_req_vfs);
5915         }
5916 #endif /* CONFIG_PCI_IOV */
5917         pf->eeprom_version = 0xDEAD;
5918         pf->lan_veb = I40E_NO_VEB;
5919         pf->lan_vsi = I40E_NO_VSI;
5920
5921         /* set up queue assignment tracking */
5922         size = sizeof(struct i40e_lump_tracking)
5923                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
5924         pf->qp_pile = kzalloc(size, GFP_KERNEL);
5925         if (!pf->qp_pile) {
5926                 err = -ENOMEM;
5927                 goto sw_init_done;
5928         }
5929         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
5930         pf->qp_pile->search_hint = 0;
5931
5932         /* set up vector assignment tracking */
5933         size = sizeof(struct i40e_lump_tracking)
5934                 + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors);
5935         pf->irq_pile = kzalloc(size, GFP_KERNEL);
5936         if (!pf->irq_pile) {
5937                 kfree(pf->qp_pile);
5938                 err = -ENOMEM;
5939                 goto sw_init_done;
5940         }
5941         pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors;
5942         pf->irq_pile->search_hint = 0;
5943
5944         mutex_init(&pf->switch_mutex);
5945
5946 sw_init_done:
5947         return err;
5948 }
5949
5950 /**
5951  * i40e_set_features - set the netdev feature flags
5952  * @netdev: ptr to the netdev being adjusted
5953  * @features: the feature set that the stack is suggesting
5954  **/
5955 static int i40e_set_features(struct net_device *netdev,
5956                              netdev_features_t features)
5957 {
5958         struct i40e_netdev_priv *np = netdev_priv(netdev);
5959         struct i40e_vsi *vsi = np->vsi;
5960
5961         if (features & NETIF_F_HW_VLAN_CTAG_RX)
5962                 i40e_vlan_stripping_enable(vsi);
5963         else
5964                 i40e_vlan_stripping_disable(vsi);
5965
5966         return 0;
5967 }
5968
5969 #ifdef CONFIG_I40E_VXLAN
5970 /**
5971  * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port
5972  * @pf: board private structure
5973  * @port: The UDP port to look up
5974  *
5975  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
5976  **/
5977 static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port)
5978 {
5979         u8 i;
5980
5981         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5982                 if (pf->vxlan_ports[i] == port)
5983                         return i;
5984         }
5985
5986         return i;
5987 }
5988
5989 /**
5990  * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
5991  * @netdev: This physical port's netdev
5992  * @sa_family: Socket Family that VXLAN is notifying us about
5993  * @port: New UDP port number that VXLAN started listening to
5994  **/
5995 static void i40e_add_vxlan_port(struct net_device *netdev,
5996                                 sa_family_t sa_family, __be16 port)
5997 {
5998         struct i40e_netdev_priv *np = netdev_priv(netdev);
5999         struct i40e_vsi *vsi = np->vsi;
6000         struct i40e_pf *pf = vsi->back;
6001         u8 next_idx;
6002         u8 idx;
6003
6004         if (sa_family == AF_INET6)
6005                 return;
6006
6007         idx = i40e_get_vxlan_port_idx(pf, port);
6008
6009         /* Check if port already exists */
6010         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6011                 netdev_info(netdev, "Port %d already offloaded\n", ntohs(port));
6012                 return;
6013         }
6014
6015         /* Now check if there is space to add the new port */
6016         next_idx = i40e_get_vxlan_port_idx(pf, 0);
6017
6018         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6019                 netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n",
6020                             ntohs(port));
6021                 return;
6022         }
6023
6024         /* New port: add it and mark its index in the bitmap */
6025         pf->vxlan_ports[next_idx] = port;
6026         pf->pending_vxlan_bitmap |= (1 << next_idx);
6027
6028         pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6029 }
6030
6031 /**
6032  * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
6033  * @netdev: This physical port's netdev
6034  * @sa_family: Socket Family that VXLAN is notifying us about
6035  * @port: UDP port number that VXLAN stopped listening to
6036  **/
6037 static void i40e_del_vxlan_port(struct net_device *netdev,
6038                                 sa_family_t sa_family, __be16 port)
6039 {
6040         struct i40e_netdev_priv *np = netdev_priv(netdev);
6041         struct i40e_vsi *vsi = np->vsi;
6042         struct i40e_pf *pf = vsi->back;
6043         u8 idx;
6044
6045         if (sa_family == AF_INET6)
6046                 return;
6047
6048         idx = i40e_get_vxlan_port_idx(pf, port);
6049
6050         /* Check if port already exists */
6051         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6052                 /* if port exists, set it to 0 (mark for deletion)
6053                  * and make it pending
6054                  */
6055                 pf->vxlan_ports[idx] = 0;
6056
6057                 pf->pending_vxlan_bitmap |= (1 << idx);
6058
6059                 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6060         } else {
6061                 netdev_warn(netdev, "Port %d was not found, not deleting\n",
6062                             ntohs(port));
6063         }
6064 }
6065
6066 #endif
6067 static const struct net_device_ops i40e_netdev_ops = {
6068         .ndo_open               = i40e_open,
6069         .ndo_stop               = i40e_close,
6070         .ndo_start_xmit         = i40e_lan_xmit_frame,
6071         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
6072         .ndo_set_rx_mode        = i40e_set_rx_mode,
6073         .ndo_validate_addr      = eth_validate_addr,
6074         .ndo_set_mac_address    = i40e_set_mac,
6075         .ndo_change_mtu         = i40e_change_mtu,
6076         .ndo_do_ioctl           = i40e_ioctl,
6077         .ndo_tx_timeout         = i40e_tx_timeout,
6078         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
6079         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
6080 #ifdef CONFIG_NET_POLL_CONTROLLER
6081         .ndo_poll_controller    = i40e_netpoll,
6082 #endif
6083         .ndo_setup_tc           = i40e_setup_tc,
6084         .ndo_set_features       = i40e_set_features,
6085         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
6086         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
6087         .ndo_set_vf_tx_rate     = i40e_ndo_set_vf_bw,
6088         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
6089 #ifdef CONFIG_I40E_VXLAN
6090         .ndo_add_vxlan_port     = i40e_add_vxlan_port,
6091         .ndo_del_vxlan_port     = i40e_del_vxlan_port,
6092 #endif
6093 };
6094
6095 /**
6096  * i40e_config_netdev - Setup the netdev flags
6097  * @vsi: the VSI being configured
6098  *
6099  * Returns 0 on success, negative value on failure
6100  **/
6101 static int i40e_config_netdev(struct i40e_vsi *vsi)
6102 {
6103         u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
6104         struct i40e_pf *pf = vsi->back;
6105         struct i40e_hw *hw = &pf->hw;
6106         struct i40e_netdev_priv *np;
6107         struct net_device *netdev;
6108         u8 mac_addr[ETH_ALEN];
6109         int etherdev_size;
6110
6111         etherdev_size = sizeof(struct i40e_netdev_priv);
6112         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
6113         if (!netdev)
6114                 return -ENOMEM;
6115
6116         vsi->netdev = netdev;
6117         np = netdev_priv(netdev);
6118         np->vsi = vsi;
6119
6120         netdev->hw_enc_features = NETIF_F_IP_CSUM        |
6121                                   NETIF_F_GSO_UDP_TUNNEL |
6122                                   NETIF_F_TSO            |
6123                                   NETIF_F_SG;
6124
6125         netdev->features = NETIF_F_SG                  |
6126                            NETIF_F_IP_CSUM             |
6127                            NETIF_F_SCTP_CSUM           |
6128                            NETIF_F_HIGHDMA             |
6129                            NETIF_F_GSO_UDP_TUNNEL      |
6130                            NETIF_F_HW_VLAN_CTAG_TX     |
6131                            NETIF_F_HW_VLAN_CTAG_RX     |
6132                            NETIF_F_HW_VLAN_CTAG_FILTER |
6133                            NETIF_F_IPV6_CSUM           |
6134                            NETIF_F_TSO                 |
6135                            NETIF_F_TSO6                |
6136                            NETIF_F_RXCSUM              |
6137                            NETIF_F_RXHASH              |
6138                            0;
6139
6140         /* copy netdev features into list of user selectable features */
6141         netdev->hw_features |= netdev->features;
6142
6143         if (vsi->type == I40E_VSI_MAIN) {
6144                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
6145                 memcpy(mac_addr, hw->mac.perm_addr, ETH_ALEN);
6146         } else {
6147                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
6148                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
6149                          pf->vsi[pf->lan_vsi]->netdev->name);
6150                 random_ether_addr(mac_addr);
6151                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
6152         }
6153         i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
6154
6155         memcpy(netdev->dev_addr, mac_addr, ETH_ALEN);
6156         memcpy(netdev->perm_addr, mac_addr, ETH_ALEN);
6157         /* vlan gets same features (except vlan offload)
6158          * after any tweaks for specific VSI types
6159          */
6160         netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX |
6161                                                      NETIF_F_HW_VLAN_CTAG_RX |
6162                                                    NETIF_F_HW_VLAN_CTAG_FILTER);
6163         netdev->priv_flags |= IFF_UNICAST_FLT;
6164         netdev->priv_flags |= IFF_SUPP_NOFCS;
6165         /* Setup netdev TC information */
6166         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
6167
6168         netdev->netdev_ops = &i40e_netdev_ops;
6169         netdev->watchdog_timeo = 5 * HZ;
6170         i40e_set_ethtool_ops(netdev);
6171
6172         return 0;
6173 }
6174
6175 /**
6176  * i40e_vsi_delete - Delete a VSI from the switch
6177  * @vsi: the VSI being removed
6178  *
6179  * Returns 0 on success, negative value on failure
6180  **/
6181 static void i40e_vsi_delete(struct i40e_vsi *vsi)
6182 {
6183         /* remove default VSI is not allowed */
6184         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
6185                 return;
6186
6187         /* there is no HW VSI for FDIR */
6188         if (vsi->type == I40E_VSI_FDIR)
6189                 return;
6190
6191         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
6192         return;
6193 }
6194
6195 /**
6196  * i40e_add_vsi - Add a VSI to the switch
6197  * @vsi: the VSI being configured
6198  *
6199  * This initializes a VSI context depending on the VSI type to be added and
6200  * passes it down to the add_vsi aq command.
6201  **/
6202 static int i40e_add_vsi(struct i40e_vsi *vsi)
6203 {
6204         int ret = -ENODEV;
6205         struct i40e_mac_filter *f, *ftmp;
6206         struct i40e_pf *pf = vsi->back;
6207         struct i40e_hw *hw = &pf->hw;
6208         struct i40e_vsi_context ctxt;
6209         u8 enabled_tc = 0x1; /* TC0 enabled */
6210         int f_count = 0;
6211
6212         memset(&ctxt, 0, sizeof(ctxt));
6213         switch (vsi->type) {
6214         case I40E_VSI_MAIN:
6215                 /* The PF's main VSI is already setup as part of the
6216                  * device initialization, so we'll not bother with
6217                  * the add_vsi call, but we will retrieve the current
6218                  * VSI context.
6219                  */
6220                 ctxt.seid = pf->main_vsi_seid;
6221                 ctxt.pf_num = pf->hw.pf_id;
6222                 ctxt.vf_num = 0;
6223                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6224                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6225                 if (ret) {
6226                         dev_info(&pf->pdev->dev,
6227                                  "couldn't get pf vsi config, err %d, aq_err %d\n",
6228                                  ret, pf->hw.aq.asq_last_status);
6229                         return -ENOENT;
6230                 }
6231                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6232                 vsi->info.valid_sections = 0;
6233
6234                 vsi->seid = ctxt.seid;
6235                 vsi->id = ctxt.vsi_number;
6236
6237                 enabled_tc = i40e_pf_get_tc_map(pf);
6238
6239                 /* MFP mode setup queue map and update VSI */
6240                 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
6241                         memset(&ctxt, 0, sizeof(ctxt));
6242                         ctxt.seid = pf->main_vsi_seid;
6243                         ctxt.pf_num = pf->hw.pf_id;
6244                         ctxt.vf_num = 0;
6245                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
6246                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
6247                         if (ret) {
6248                                 dev_info(&pf->pdev->dev,
6249                                          "update vsi failed, aq_err=%d\n",
6250                                          pf->hw.aq.asq_last_status);
6251                                 ret = -ENOENT;
6252                                 goto err;
6253                         }
6254                         /* update the local VSI info queue map */
6255                         i40e_vsi_update_queue_map(vsi, &ctxt);
6256                         vsi->info.valid_sections = 0;
6257                 } else {
6258                         /* Default/Main VSI is only enabled for TC0
6259                          * reconfigure it to enable all TCs that are
6260                          * available on the port in SFP mode.
6261                          */
6262                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
6263                         if (ret) {
6264                                 dev_info(&pf->pdev->dev,
6265                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n",
6266                                          enabled_tc, ret,
6267                                          pf->hw.aq.asq_last_status);
6268                                 ret = -ENOENT;
6269                         }
6270                 }
6271                 break;
6272
6273         case I40E_VSI_FDIR:
6274                 /* no queue mapping or actual HW VSI needed */
6275                 vsi->info.valid_sections = 0;
6276                 vsi->seid = 0;
6277                 vsi->id = 0;
6278                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6279                 return 0;
6280                 break;
6281
6282         case I40E_VSI_VMDQ2:
6283                 ctxt.pf_num = hw->pf_id;
6284                 ctxt.vf_num = 0;
6285                 ctxt.uplink_seid = vsi->uplink_seid;
6286                 ctxt.connection_type = 0x1;     /* regular data port */
6287                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6288
6289                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6290
6291                 /* This VSI is connected to VEB so the switch_id
6292                  * should be set to zero by default.
6293                  */
6294                 ctxt.info.switch_id = 0;
6295                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
6296                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6297
6298                 /* Setup the VSI tx/rx queue map for TC0 only for now */
6299                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6300                 break;
6301
6302         case I40E_VSI_SRIOV:
6303                 ctxt.pf_num = hw->pf_id;
6304                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
6305                 ctxt.uplink_seid = vsi->uplink_seid;
6306                 ctxt.connection_type = 0x1;     /* regular data port */
6307                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
6308
6309                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6310
6311                 /* This VSI is connected to VEB so the switch_id
6312                  * should be set to zero by default.
6313                  */
6314                 ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6315
6316                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
6317                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
6318                 /* Setup the VSI tx/rx queue map for TC0 only for now */
6319                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6320                 break;
6321
6322         default:
6323                 return -ENODEV;
6324         }
6325
6326         if (vsi->type != I40E_VSI_MAIN) {
6327                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6328                 if (ret) {
6329                         dev_info(&vsi->back->pdev->dev,
6330                                  "add vsi failed, aq_err=%d\n",
6331                                  vsi->back->hw.aq.asq_last_status);
6332                         ret = -ENOENT;
6333                         goto err;
6334                 }
6335                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6336                 vsi->info.valid_sections = 0;
6337                 vsi->seid = ctxt.seid;
6338                 vsi->id = ctxt.vsi_number;
6339         }
6340
6341         /* If macvlan filters already exist, force them to get loaded */
6342         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
6343                 f->changed = true;
6344                 f_count++;
6345         }
6346         if (f_count) {
6347                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
6348                 pf->flags |= I40E_FLAG_FILTER_SYNC;
6349         }
6350
6351         /* Update VSI BW information */
6352         ret = i40e_vsi_get_bw_info(vsi);
6353         if (ret) {
6354                 dev_info(&pf->pdev->dev,
6355                          "couldn't get vsi bw info, err %d, aq_err %d\n",
6356                          ret, pf->hw.aq.asq_last_status);
6357                 /* VSI is already added so not tearing that up */
6358                 ret = 0;
6359         }
6360
6361 err:
6362         return ret;
6363 }
6364
6365 /**
6366  * i40e_vsi_release - Delete a VSI and free its resources
6367  * @vsi: the VSI being removed
6368  *
6369  * Returns 0 on success or < 0 on error
6370  **/
6371 int i40e_vsi_release(struct i40e_vsi *vsi)
6372 {
6373         struct i40e_mac_filter *f, *ftmp;
6374         struct i40e_veb *veb = NULL;
6375         struct i40e_pf *pf;
6376         u16 uplink_seid;
6377         int i, n;
6378
6379         pf = vsi->back;
6380
6381         /* release of a VEB-owner or last VSI is not allowed */
6382         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
6383                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
6384                          vsi->seid, vsi->uplink_seid);
6385                 return -ENODEV;
6386         }
6387         if (vsi == pf->vsi[pf->lan_vsi] &&
6388             !test_bit(__I40E_DOWN, &pf->state)) {
6389                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
6390                 return -ENODEV;
6391         }
6392
6393         uplink_seid = vsi->uplink_seid;
6394         if (vsi->type != I40E_VSI_SRIOV) {
6395                 if (vsi->netdev_registered) {
6396                         vsi->netdev_registered = false;
6397                         if (vsi->netdev) {
6398                                 /* results in a call to i40e_close() */
6399                                 unregister_netdev(vsi->netdev);
6400                                 free_netdev(vsi->netdev);
6401                                 vsi->netdev = NULL;
6402                         }
6403                 } else {
6404                         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
6405                                 i40e_down(vsi);
6406                         i40e_vsi_free_irq(vsi);
6407                         i40e_vsi_free_tx_resources(vsi);
6408                         i40e_vsi_free_rx_resources(vsi);
6409                 }
6410                 i40e_vsi_disable_irq(vsi);
6411         }
6412
6413         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
6414                 i40e_del_filter(vsi, f->macaddr, f->vlan,
6415                                 f->is_vf, f->is_netdev);
6416         i40e_sync_vsi_filters(vsi);
6417
6418         i40e_vsi_delete(vsi);
6419         i40e_vsi_free_q_vectors(vsi);
6420         i40e_vsi_clear_rings(vsi);
6421         i40e_vsi_clear(vsi);
6422
6423         /* If this was the last thing on the VEB, except for the
6424          * controlling VSI, remove the VEB, which puts the controlling
6425          * VSI onto the next level down in the switch.
6426          *
6427          * Well, okay, there's one more exception here: don't remove
6428          * the orphan VEBs yet.  We'll wait for an explicit remove request
6429          * from up the network stack.
6430          */
6431         for (n = 0, i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6432                 if (pf->vsi[i] &&
6433                     pf->vsi[i]->uplink_seid == uplink_seid &&
6434                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
6435                         n++;      /* count the VSIs */
6436                 }
6437         }
6438         for (i = 0; i < I40E_MAX_VEB; i++) {
6439                 if (!pf->veb[i])
6440                         continue;
6441                 if (pf->veb[i]->uplink_seid == uplink_seid)
6442                         n++;     /* count the VEBs */
6443                 if (pf->veb[i]->seid == uplink_seid)
6444                         veb = pf->veb[i];
6445         }
6446         if (n == 0 && veb && veb->uplink_seid != 0)
6447                 i40e_veb_release(veb);
6448
6449         return 0;
6450 }
6451
6452 /**
6453  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
6454  * @vsi: ptr to the VSI
6455  *
6456  * This should only be called after i40e_vsi_mem_alloc() which allocates the
6457  * corresponding SW VSI structure and initializes num_queue_pairs for the
6458  * newly allocated VSI.
6459  *
6460  * Returns 0 on success or negative on failure
6461  **/
6462 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
6463 {
6464         int ret = -ENOENT;
6465         struct i40e_pf *pf = vsi->back;
6466
6467         if (vsi->q_vectors[0]) {
6468                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
6469                          vsi->seid);
6470                 return -EEXIST;
6471         }
6472
6473         if (vsi->base_vector) {
6474                 dev_info(&pf->pdev->dev,
6475                          "VSI %d has non-zero base vector %d\n",
6476                          vsi->seid, vsi->base_vector);
6477                 return -EEXIST;
6478         }
6479
6480         ret = i40e_alloc_q_vectors(vsi);
6481         if (ret) {
6482                 dev_info(&pf->pdev->dev,
6483                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
6484                          vsi->num_q_vectors, vsi->seid, ret);
6485                 vsi->num_q_vectors = 0;
6486                 goto vector_setup_out;
6487         }
6488
6489         if (vsi->num_q_vectors)
6490                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
6491                                                  vsi->num_q_vectors, vsi->idx);
6492         if (vsi->base_vector < 0) {
6493                 dev_info(&pf->pdev->dev,
6494                          "failed to get q tracking for VSI %d, err=%d\n",
6495                          vsi->seid, vsi->base_vector);
6496                 i40e_vsi_free_q_vectors(vsi);
6497                 ret = -ENOENT;
6498                 goto vector_setup_out;
6499         }
6500
6501 vector_setup_out:
6502         return ret;
6503 }
6504
6505 /**
6506  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
6507  * @vsi: pointer to the vsi.
6508  *
6509  * This re-allocates a vsi's queue resources.
6510  *
6511  * Returns pointer to the successfully allocated and configured VSI sw struct
6512  * on success, otherwise returns NULL on failure.
6513  **/
6514 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
6515 {
6516         struct i40e_pf *pf = vsi->back;
6517         u8 enabled_tc;
6518         int ret;
6519
6520         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
6521         i40e_vsi_clear_rings(vsi);
6522
6523         i40e_vsi_free_arrays(vsi, false);
6524         i40e_set_num_rings_in_vsi(vsi);
6525         ret = i40e_vsi_alloc_arrays(vsi, false);
6526         if (ret)
6527                 goto err_vsi;
6528
6529         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
6530         if (ret < 0) {
6531                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
6532                          vsi->seid, ret);
6533                 goto err_vsi;
6534         }
6535         vsi->base_queue = ret;
6536
6537         /* Update the FW view of the VSI. Force a reset of TC and queue
6538          * layout configurations.
6539          */
6540         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
6541         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
6542         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
6543         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
6544
6545         /* assign it some queues */
6546         ret = i40e_alloc_rings(vsi);
6547         if (ret)
6548                 goto err_rings;
6549
6550         /* map all of the rings to the q_vectors */
6551         i40e_vsi_map_rings_to_vectors(vsi);
6552         return vsi;
6553
6554 err_rings:
6555         i40e_vsi_free_q_vectors(vsi);
6556         if (vsi->netdev_registered) {
6557                 vsi->netdev_registered = false;
6558                 unregister_netdev(vsi->netdev);
6559                 free_netdev(vsi->netdev);
6560                 vsi->netdev = NULL;
6561         }
6562         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
6563 err_vsi:
6564         i40e_vsi_clear(vsi);
6565         return NULL;
6566 }
6567
6568 /**
6569  * i40e_vsi_setup - Set up a VSI by a given type
6570  * @pf: board private structure
6571  * @type: VSI type
6572  * @uplink_seid: the switch element to link to
6573  * @param1: usage depends upon VSI type. For VF types, indicates VF id
6574  *
6575  * This allocates the sw VSI structure and its queue resources, then add a VSI
6576  * to the identified VEB.
6577  *
6578  * Returns pointer to the successfully allocated and configure VSI sw struct on
6579  * success, otherwise returns NULL on failure.
6580  **/
6581 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
6582                                 u16 uplink_seid, u32 param1)
6583 {
6584         struct i40e_vsi *vsi = NULL;
6585         struct i40e_veb *veb = NULL;
6586         int ret, i;
6587         int v_idx;
6588
6589         /* The requested uplink_seid must be either
6590          *     - the PF's port seid
6591          *              no VEB is needed because this is the PF
6592          *              or this is a Flow Director special case VSI
6593          *     - seid of an existing VEB
6594          *     - seid of a VSI that owns an existing VEB
6595          *     - seid of a VSI that doesn't own a VEB
6596          *              a new VEB is created and the VSI becomes the owner
6597          *     - seid of the PF VSI, which is what creates the first VEB
6598          *              this is a special case of the previous
6599          *
6600          * Find which uplink_seid we were given and create a new VEB if needed
6601          */
6602         for (i = 0; i < I40E_MAX_VEB; i++) {
6603                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
6604                         veb = pf->veb[i];
6605                         break;
6606                 }
6607         }
6608
6609         if (!veb && uplink_seid != pf->mac_seid) {
6610
6611                 for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6612                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
6613                                 vsi = pf->vsi[i];
6614                                 break;
6615                         }
6616                 }
6617                 if (!vsi) {
6618                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
6619                                  uplink_seid);
6620                         return NULL;
6621                 }
6622
6623                 if (vsi->uplink_seid == pf->mac_seid)
6624                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
6625                                              vsi->tc_config.enabled_tc);
6626                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
6627                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
6628                                              vsi->tc_config.enabled_tc);
6629
6630                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
6631                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
6632                                 veb = pf->veb[i];
6633                 }
6634                 if (!veb) {
6635                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
6636                         return NULL;
6637                 }
6638
6639                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
6640                 uplink_seid = veb->seid;
6641         }
6642
6643         /* get vsi sw struct */
6644         v_idx = i40e_vsi_mem_alloc(pf, type);
6645         if (v_idx < 0)
6646                 goto err_alloc;
6647         vsi = pf->vsi[v_idx];
6648         vsi->type = type;
6649         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
6650
6651         if (type == I40E_VSI_MAIN)
6652                 pf->lan_vsi = v_idx;
6653         else if (type == I40E_VSI_SRIOV)
6654                 vsi->vf_id = param1;
6655         /* assign it some queues */
6656         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
6657         if (ret < 0) {
6658                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
6659                          vsi->seid, ret);
6660                 goto err_vsi;
6661         }
6662         vsi->base_queue = ret;
6663
6664         /* get a VSI from the hardware */
6665         vsi->uplink_seid = uplink_seid;
6666         ret = i40e_add_vsi(vsi);
6667         if (ret)
6668                 goto err_vsi;
6669
6670         switch (vsi->type) {
6671         /* setup the netdev if needed */
6672         case I40E_VSI_MAIN:
6673         case I40E_VSI_VMDQ2:
6674                 ret = i40e_config_netdev(vsi);
6675                 if (ret)
6676                         goto err_netdev;
6677                 ret = register_netdev(vsi->netdev);
6678                 if (ret)
6679                         goto err_netdev;
6680                 vsi->netdev_registered = true;
6681                 netif_carrier_off(vsi->netdev);
6682                 /* fall through */
6683
6684         case I40E_VSI_FDIR:
6685                 /* set up vectors and rings if needed */
6686                 ret = i40e_vsi_setup_vectors(vsi);
6687                 if (ret)
6688                         goto err_msix;
6689
6690                 ret = i40e_alloc_rings(vsi);
6691                 if (ret)
6692                         goto err_rings;
6693
6694                 /* map all of the rings to the q_vectors */
6695                 i40e_vsi_map_rings_to_vectors(vsi);
6696
6697                 i40e_vsi_reset_stats(vsi);
6698                 break;
6699
6700         default:
6701                 /* no netdev or rings for the other VSI types */
6702                 break;
6703         }
6704
6705         return vsi;
6706
6707 err_rings:
6708         i40e_vsi_free_q_vectors(vsi);
6709 err_msix:
6710         if (vsi->netdev_registered) {
6711                 vsi->netdev_registered = false;
6712                 unregister_netdev(vsi->netdev);
6713                 free_netdev(vsi->netdev);
6714                 vsi->netdev = NULL;
6715         }
6716 err_netdev:
6717         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
6718 err_vsi:
6719         i40e_vsi_clear(vsi);
6720 err_alloc:
6721         return NULL;
6722 }
6723
6724 /**
6725  * i40e_veb_get_bw_info - Query VEB BW information
6726  * @veb: the veb to query
6727  *
6728  * Query the Tx scheduler BW configuration data for given VEB
6729  **/
6730 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
6731 {
6732         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
6733         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
6734         struct i40e_pf *pf = veb->pf;
6735         struct i40e_hw *hw = &pf->hw;
6736         u32 tc_bw_max;
6737         int ret = 0;
6738         int i;
6739
6740         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
6741                                                   &bw_data, NULL);
6742         if (ret) {
6743                 dev_info(&pf->pdev->dev,
6744                          "query veb bw config failed, aq_err=%d\n",
6745                          hw->aq.asq_last_status);
6746                 goto out;
6747         }
6748
6749         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
6750                                                    &ets_data, NULL);
6751         if (ret) {
6752                 dev_info(&pf->pdev->dev,
6753                          "query veb bw ets config failed, aq_err=%d\n",
6754                          hw->aq.asq_last_status);
6755                 goto out;
6756         }
6757
6758         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
6759         veb->bw_max_quanta = ets_data.tc_bw_max;
6760         veb->is_abs_credits = bw_data.absolute_credits_enable;
6761         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
6762                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
6763         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6764                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
6765                 veb->bw_tc_limit_credits[i] =
6766                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
6767                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
6768         }
6769
6770 out:
6771         return ret;
6772 }
6773
6774 /**
6775  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
6776  * @pf: board private structure
6777  *
6778  * On error: returns error code (negative)
6779  * On success: returns vsi index in PF (positive)
6780  **/
6781 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
6782 {
6783         int ret = -ENOENT;
6784         struct i40e_veb *veb;
6785         int i;
6786
6787         /* Need to protect the allocation of switch elements at the PF level */
6788         mutex_lock(&pf->switch_mutex);
6789
6790         /* VEB list may be fragmented if VEB creation/destruction has
6791          * been happening.  We can afford to do a quick scan to look
6792          * for any free slots in the list.
6793          *
6794          * find next empty veb slot, looping back around if necessary
6795          */
6796         i = 0;
6797         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
6798                 i++;
6799         if (i >= I40E_MAX_VEB) {
6800                 ret = -ENOMEM;
6801                 goto err_alloc_veb;  /* out of VEB slots! */
6802         }
6803
6804         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
6805         if (!veb) {
6806                 ret = -ENOMEM;
6807                 goto err_alloc_veb;
6808         }
6809         veb->pf = pf;
6810         veb->idx = i;
6811         veb->enabled_tc = 1;
6812
6813         pf->veb[i] = veb;
6814         ret = i;
6815 err_alloc_veb:
6816         mutex_unlock(&pf->switch_mutex);
6817         return ret;
6818 }
6819
6820 /**
6821  * i40e_switch_branch_release - Delete a branch of the switch tree
6822  * @branch: where to start deleting
6823  *
6824  * This uses recursion to find the tips of the branch to be
6825  * removed, deleting until we get back to and can delete this VEB.
6826  **/
6827 static void i40e_switch_branch_release(struct i40e_veb *branch)
6828 {
6829         struct i40e_pf *pf = branch->pf;
6830         u16 branch_seid = branch->seid;
6831         u16 veb_idx = branch->idx;
6832         int i;
6833
6834         /* release any VEBs on this VEB - RECURSION */
6835         for (i = 0; i < I40E_MAX_VEB; i++) {
6836                 if (!pf->veb[i])
6837                         continue;
6838                 if (pf->veb[i]->uplink_seid == branch->seid)
6839                         i40e_switch_branch_release(pf->veb[i]);
6840         }
6841
6842         /* Release the VSIs on this VEB, but not the owner VSI.
6843          *
6844          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
6845          *       the VEB itself, so don't use (*branch) after this loop.
6846          */
6847         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6848                 if (!pf->vsi[i])
6849                         continue;
6850                 if (pf->vsi[i]->uplink_seid == branch_seid &&
6851                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
6852                         i40e_vsi_release(pf->vsi[i]);
6853                 }
6854         }
6855
6856         /* There's one corner case where the VEB might not have been
6857          * removed, so double check it here and remove it if needed.
6858          * This case happens if the veb was created from the debugfs
6859          * commands and no VSIs were added to it.
6860          */
6861         if (pf->veb[veb_idx])
6862                 i40e_veb_release(pf->veb[veb_idx]);
6863 }
6864
6865 /**
6866  * i40e_veb_clear - remove veb struct
6867  * @veb: the veb to remove
6868  **/
6869 static void i40e_veb_clear(struct i40e_veb *veb)
6870 {
6871         if (!veb)
6872                 return;
6873
6874         if (veb->pf) {
6875                 struct i40e_pf *pf = veb->pf;
6876
6877                 mutex_lock(&pf->switch_mutex);
6878                 if (pf->veb[veb->idx] == veb)
6879                         pf->veb[veb->idx] = NULL;
6880                 mutex_unlock(&pf->switch_mutex);
6881         }
6882
6883         kfree(veb);
6884 }
6885
6886 /**
6887  * i40e_veb_release - Delete a VEB and free its resources
6888  * @veb: the VEB being removed
6889  **/
6890 void i40e_veb_release(struct i40e_veb *veb)
6891 {
6892         struct i40e_vsi *vsi = NULL;
6893         struct i40e_pf *pf;
6894         int i, n = 0;
6895
6896         pf = veb->pf;
6897
6898         /* find the remaining VSI and check for extras */
6899         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6900                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
6901                         n++;
6902                         vsi = pf->vsi[i];
6903                 }
6904         }
6905         if (n != 1) {
6906                 dev_info(&pf->pdev->dev,
6907                          "can't remove VEB %d with %d VSIs left\n",
6908                          veb->seid, n);
6909                 return;
6910         }
6911
6912         /* move the remaining VSI to uplink veb */
6913         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
6914         if (veb->uplink_seid) {
6915                 vsi->uplink_seid = veb->uplink_seid;
6916                 if (veb->uplink_seid == pf->mac_seid)
6917                         vsi->veb_idx = I40E_NO_VEB;
6918                 else
6919                         vsi->veb_idx = veb->veb_idx;
6920         } else {
6921                 /* floating VEB */
6922                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6923                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
6924         }
6925
6926         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
6927         i40e_veb_clear(veb);
6928
6929         return;
6930 }
6931
6932 /**
6933  * i40e_add_veb - create the VEB in the switch
6934  * @veb: the VEB to be instantiated
6935  * @vsi: the controlling VSI
6936  **/
6937 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
6938 {
6939         bool is_default = false;
6940         bool is_cloud = false;
6941         int ret;
6942
6943         /* get a VEB from the hardware */
6944         ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid,
6945                               veb->enabled_tc, is_default,
6946                               is_cloud, &veb->seid, NULL);
6947         if (ret) {
6948                 dev_info(&veb->pf->pdev->dev,
6949                          "couldn't add VEB, err %d, aq_err %d\n",
6950                          ret, veb->pf->hw.aq.asq_last_status);
6951                 return -EPERM;
6952         }
6953
6954         /* get statistics counter */
6955         ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL,
6956                                          &veb->stats_idx, NULL, NULL, NULL);
6957         if (ret) {
6958                 dev_info(&veb->pf->pdev->dev,
6959                          "couldn't get VEB statistics idx, err %d, aq_err %d\n",
6960                          ret, veb->pf->hw.aq.asq_last_status);
6961                 return -EPERM;
6962         }
6963         ret = i40e_veb_get_bw_info(veb);
6964         if (ret) {
6965                 dev_info(&veb->pf->pdev->dev,
6966                          "couldn't get VEB bw info, err %d, aq_err %d\n",
6967                          ret, veb->pf->hw.aq.asq_last_status);
6968                 i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL);
6969                 return -ENOENT;
6970         }
6971
6972         vsi->uplink_seid = veb->seid;
6973         vsi->veb_idx = veb->idx;
6974         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
6975
6976         return 0;
6977 }
6978
6979 /**
6980  * i40e_veb_setup - Set up a VEB
6981  * @pf: board private structure
6982  * @flags: VEB setup flags
6983  * @uplink_seid: the switch element to link to
6984  * @vsi_seid: the initial VSI seid
6985  * @enabled_tc: Enabled TC bit-map
6986  *
6987  * This allocates the sw VEB structure and links it into the switch
6988  * It is possible and legal for this to be a duplicate of an already
6989  * existing VEB.  It is also possible for both uplink and vsi seids
6990  * to be zero, in order to create a floating VEB.
6991  *
6992  * Returns pointer to the successfully allocated VEB sw struct on
6993  * success, otherwise returns NULL on failure.
6994  **/
6995 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
6996                                 u16 uplink_seid, u16 vsi_seid,
6997                                 u8 enabled_tc)
6998 {
6999         struct i40e_veb *veb, *uplink_veb = NULL;
7000         int vsi_idx, veb_idx;
7001         int ret;
7002
7003         /* if one seid is 0, the other must be 0 to create a floating relay */
7004         if ((uplink_seid == 0 || vsi_seid == 0) &&
7005             (uplink_seid + vsi_seid != 0)) {
7006                 dev_info(&pf->pdev->dev,
7007                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
7008                          uplink_seid, vsi_seid);
7009                 return NULL;
7010         }
7011
7012         /* make sure there is such a vsi and uplink */
7013         for (vsi_idx = 0; vsi_idx < pf->hw.func_caps.num_vsis; vsi_idx++)
7014                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
7015                         break;
7016         if (vsi_idx >= pf->hw.func_caps.num_vsis && vsi_seid != 0) {
7017                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
7018                          vsi_seid);
7019                 return NULL;
7020         }
7021
7022         if (uplink_seid && uplink_seid != pf->mac_seid) {
7023                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
7024                         if (pf->veb[veb_idx] &&
7025                             pf->veb[veb_idx]->seid == uplink_seid) {
7026                                 uplink_veb = pf->veb[veb_idx];
7027                                 break;
7028                         }
7029                 }
7030                 if (!uplink_veb) {
7031                         dev_info(&pf->pdev->dev,
7032                                  "uplink seid %d not found\n", uplink_seid);
7033                         return NULL;
7034                 }
7035         }
7036
7037         /* get veb sw struct */
7038         veb_idx = i40e_veb_mem_alloc(pf);
7039         if (veb_idx < 0)
7040                 goto err_alloc;
7041         veb = pf->veb[veb_idx];
7042         veb->flags = flags;
7043         veb->uplink_seid = uplink_seid;
7044         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
7045         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
7046
7047         /* create the VEB in the switch */
7048         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
7049         if (ret)
7050                 goto err_veb;
7051
7052         return veb;
7053
7054 err_veb:
7055         i40e_veb_clear(veb);
7056 err_alloc:
7057         return NULL;
7058 }
7059
7060 /**
7061  * i40e_setup_pf_switch_element - set pf vars based on switch type
7062  * @pf: board private structure
7063  * @ele: element we are building info from
7064  * @num_reported: total number of elements
7065  * @printconfig: should we print the contents
7066  *
7067  * helper function to assist in extracting a few useful SEID values.
7068  **/
7069 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
7070                                 struct i40e_aqc_switch_config_element_resp *ele,
7071                                 u16 num_reported, bool printconfig)
7072 {
7073         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
7074         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
7075         u8 element_type = ele->element_type;
7076         u16 seid = le16_to_cpu(ele->seid);
7077
7078         if (printconfig)
7079                 dev_info(&pf->pdev->dev,
7080                          "type=%d seid=%d uplink=%d downlink=%d\n",
7081                          element_type, seid, uplink_seid, downlink_seid);
7082
7083         switch (element_type) {
7084         case I40E_SWITCH_ELEMENT_TYPE_MAC:
7085                 pf->mac_seid = seid;
7086                 break;
7087         case I40E_SWITCH_ELEMENT_TYPE_VEB:
7088                 /* Main VEB? */
7089                 if (uplink_seid != pf->mac_seid)
7090                         break;
7091                 if (pf->lan_veb == I40E_NO_VEB) {
7092                         int v;
7093
7094                         /* find existing or else empty VEB */
7095                         for (v = 0; v < I40E_MAX_VEB; v++) {
7096                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
7097                                         pf->lan_veb = v;
7098                                         break;
7099                                 }
7100                         }
7101                         if (pf->lan_veb == I40E_NO_VEB) {
7102                                 v = i40e_veb_mem_alloc(pf);
7103                                 if (v < 0)
7104                                         break;
7105                                 pf->lan_veb = v;
7106                         }
7107                 }
7108
7109                 pf->veb[pf->lan_veb]->seid = seid;
7110                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
7111                 pf->veb[pf->lan_veb]->pf = pf;
7112                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
7113                 break;
7114         case I40E_SWITCH_ELEMENT_TYPE_VSI:
7115                 if (num_reported != 1)
7116                         break;
7117                 /* This is immediately after a reset so we can assume this is
7118                  * the PF's VSI
7119                  */
7120                 pf->mac_seid = uplink_seid;
7121                 pf->pf_seid = downlink_seid;
7122                 pf->main_vsi_seid = seid;
7123                 if (printconfig)
7124                         dev_info(&pf->pdev->dev,
7125                                  "pf_seid=%d main_vsi_seid=%d\n",
7126                                  pf->pf_seid, pf->main_vsi_seid);
7127                 break;
7128         case I40E_SWITCH_ELEMENT_TYPE_PF:
7129         case I40E_SWITCH_ELEMENT_TYPE_VF:
7130         case I40E_SWITCH_ELEMENT_TYPE_EMP:
7131         case I40E_SWITCH_ELEMENT_TYPE_BMC:
7132         case I40E_SWITCH_ELEMENT_TYPE_PE:
7133         case I40E_SWITCH_ELEMENT_TYPE_PA:
7134                 /* ignore these for now */
7135                 break;
7136         default:
7137                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
7138                          element_type, seid);
7139                 break;
7140         }
7141 }
7142
7143 /**
7144  * i40e_fetch_switch_configuration - Get switch config from firmware
7145  * @pf: board private structure
7146  * @printconfig: should we print the contents
7147  *
7148  * Get the current switch configuration from the device and
7149  * extract a few useful SEID values.
7150  **/
7151 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
7152 {
7153         struct i40e_aqc_get_switch_config_resp *sw_config;
7154         u16 next_seid = 0;
7155         int ret = 0;
7156         u8 *aq_buf;
7157         int i;
7158
7159         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
7160         if (!aq_buf)
7161                 return -ENOMEM;
7162
7163         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
7164         do {
7165                 u16 num_reported, num_total;
7166
7167                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
7168                                                 I40E_AQ_LARGE_BUF,
7169                                                 &next_seid, NULL);
7170                 if (ret) {
7171                         dev_info(&pf->pdev->dev,
7172                                  "get switch config failed %d aq_err=%x\n",
7173                                  ret, pf->hw.aq.asq_last_status);
7174                         kfree(aq_buf);
7175                         return -ENOENT;
7176                 }
7177
7178                 num_reported = le16_to_cpu(sw_config->header.num_reported);
7179                 num_total = le16_to_cpu(sw_config->header.num_total);
7180
7181                 if (printconfig)
7182                         dev_info(&pf->pdev->dev,
7183                                  "header: %d reported %d total\n",
7184                                  num_reported, num_total);
7185
7186                 if (num_reported) {
7187                         int sz = sizeof(*sw_config) * num_reported;
7188
7189                         kfree(pf->sw_config);
7190                         pf->sw_config = kzalloc(sz, GFP_KERNEL);
7191                         if (pf->sw_config)
7192                                 memcpy(pf->sw_config, sw_config, sz);
7193                 }
7194
7195                 for (i = 0; i < num_reported; i++) {
7196                         struct i40e_aqc_switch_config_element_resp *ele =
7197                                 &sw_config->element[i];
7198
7199                         i40e_setup_pf_switch_element(pf, ele, num_reported,
7200                                                      printconfig);
7201                 }
7202         } while (next_seid != 0);
7203
7204         kfree(aq_buf);
7205         return ret;
7206 }
7207
7208 /**
7209  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
7210  * @pf: board private structure
7211  * @reinit: if the Main VSI needs to re-initialized.
7212  *
7213  * Returns 0 on success, negative value on failure
7214  **/
7215 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
7216 {
7217         u32 rxfc = 0, txfc = 0, rxfc_reg;
7218         int ret;
7219
7220         /* find out what's out there already */
7221         ret = i40e_fetch_switch_configuration(pf, false);
7222         if (ret) {
7223                 dev_info(&pf->pdev->dev,
7224                          "couldn't fetch switch config, err %d, aq_err %d\n",
7225                          ret, pf->hw.aq.asq_last_status);
7226                 return ret;
7227         }
7228         i40e_pf_reset_stats(pf);
7229
7230         /* fdir VSI must happen first to be sure it gets queue 0, but only
7231          * if there is enough room for the fdir VSI
7232          */
7233         if (pf->num_lan_qps > 1)
7234                 i40e_fdir_setup(pf);
7235
7236         /* first time setup */
7237         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
7238                 struct i40e_vsi *vsi = NULL;
7239                 u16 uplink_seid;
7240
7241                 /* Set up the PF VSI associated with the PF's main VSI
7242                  * that is already in the HW switch
7243                  */
7244                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
7245                         uplink_seid = pf->veb[pf->lan_veb]->seid;
7246                 else
7247                         uplink_seid = pf->mac_seid;
7248                 if (pf->lan_vsi == I40E_NO_VSI)
7249                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
7250                 else if (reinit)
7251                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
7252                 if (!vsi) {
7253                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
7254                         i40e_fdir_teardown(pf);
7255                         return -EAGAIN;
7256                 }
7257         } else {
7258                 /* force a reset of TC and queue layout configurations */
7259                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
7260                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
7261                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
7262                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
7263         }
7264         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
7265
7266         /* Setup static PF queue filter control settings */
7267         ret = i40e_setup_pf_filter_control(pf);
7268         if (ret) {
7269                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
7270                          ret);
7271                 /* Failure here should not stop continuing other steps */
7272         }
7273
7274         /* enable RSS in the HW, even for only one queue, as the stack can use
7275          * the hash
7276          */
7277         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
7278                 i40e_config_rss(pf);
7279
7280         /* fill in link information and enable LSE reporting */
7281         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
7282         i40e_link_event(pf);
7283
7284         /* Initialize user-specific link properties */
7285         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
7286                                   I40E_AQ_AN_COMPLETED) ? true : false);
7287         /* requested_mode is set in probe or by ethtool */
7288         if (!pf->fc_autoneg_status)
7289                 goto no_autoneg;
7290
7291         if ((pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) &&
7292             (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX))
7293                 pf->hw.fc.current_mode = I40E_FC_FULL;
7294         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
7295                 pf->hw.fc.current_mode = I40E_FC_TX_PAUSE;
7296         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
7297                 pf->hw.fc.current_mode = I40E_FC_RX_PAUSE;
7298         else
7299                 pf->hw.fc.current_mode = I40E_FC_NONE;
7300
7301         /* sync the flow control settings with the auto-neg values */
7302         switch (pf->hw.fc.current_mode) {
7303         case I40E_FC_FULL:
7304                 txfc = 1;
7305                 rxfc = 1;
7306                 break;
7307         case I40E_FC_TX_PAUSE:
7308                 txfc = 1;
7309                 rxfc = 0;
7310                 break;
7311         case I40E_FC_RX_PAUSE:
7312                 txfc = 0;
7313                 rxfc = 1;
7314                 break;
7315         case I40E_FC_NONE:
7316         case I40E_FC_DEFAULT:
7317                 txfc = 0;
7318                 rxfc = 0;
7319                 break;
7320         case I40E_FC_PFC:
7321                 /* TBD */
7322                 break;
7323         /* no default case, we have to handle all possibilities here */
7324         }
7325
7326         wr32(&pf->hw, I40E_PRTDCB_FCCFG, txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
7327
7328         rxfc_reg = rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
7329                    ~I40E_PRTDCB_MFLCN_RFCE_MASK;
7330         rxfc_reg |= (rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT);
7331
7332         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rxfc_reg);
7333
7334         goto fc_complete;
7335
7336 no_autoneg:
7337         /* disable L2 flow control, user can turn it on if they wish */
7338         wr32(&pf->hw, I40E_PRTDCB_FCCFG, 0);
7339         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
7340                                          ~I40E_PRTDCB_MFLCN_RFCE_MASK);
7341
7342 fc_complete:
7343         i40e_ptp_init(pf);
7344
7345         return ret;
7346 }
7347
7348 /**
7349  * i40e_set_rss_size - helper to set rss_size
7350  * @pf: board private structure
7351  * @queues_left: how many queues
7352  */
7353 static u16 i40e_set_rss_size(struct i40e_pf *pf, int queues_left)
7354 {
7355         int num_tc0;
7356
7357         num_tc0 = min_t(int, queues_left, pf->rss_size_max);
7358         num_tc0 = min_t(int, num_tc0, num_online_cpus());
7359         num_tc0 = rounddown_pow_of_two(num_tc0);
7360
7361         return num_tc0;
7362 }
7363
7364 /**
7365  * i40e_determine_queue_usage - Work out queue distribution
7366  * @pf: board private structure
7367  **/
7368 static void i40e_determine_queue_usage(struct i40e_pf *pf)
7369 {
7370         int accum_tc_size;
7371         int queues_left;
7372
7373         pf->num_lan_qps = 0;
7374         pf->num_tc_qps = rounddown_pow_of_two(pf->num_tc_qps);
7375         accum_tc_size = (I40E_MAX_TRAFFIC_CLASS - 1) * pf->num_tc_qps;
7376
7377         /* Find the max queues to be put into basic use.  We'll always be
7378          * using TC0, whether or not DCB is running, and TC0 will get the
7379          * big RSS set.
7380          */
7381         queues_left = pf->hw.func_caps.num_tx_qp;
7382
7383         if   (!(pf->flags & I40E_FLAG_MSIX_ENABLED) ||
7384                 !(pf->flags & (I40E_FLAG_RSS_ENABLED |
7385                 I40E_FLAG_FDIR_ENABLED | I40E_FLAG_DCB_ENABLED)) ||
7386                 (queues_left == 1)) {
7387
7388                 /* one qp for PF, no queues for anything else */
7389                 queues_left = 0;
7390                 pf->rss_size = pf->num_lan_qps = 1;
7391
7392                 /* make sure all the fancies are disabled */
7393                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED       |
7394                                 I40E_FLAG_FDIR_ENABLED     |
7395                                 I40E_FLAG_FDIR_ATR_ENABLED |
7396                                 I40E_FLAG_DCB_ENABLED      |
7397                                 I40E_FLAG_SRIOV_ENABLED    |
7398                                 I40E_FLAG_VMDQ_ENABLED);
7399
7400         } else if (pf->flags & I40E_FLAG_RSS_ENABLED      &&
7401                    !(pf->flags & I40E_FLAG_FDIR_ENABLED)  &&
7402                    !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7403
7404                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7405
7406                 queues_left -= pf->rss_size;
7407                 pf->num_lan_qps = pf->rss_size_max;
7408
7409         } else if (pf->flags & I40E_FLAG_RSS_ENABLED      &&
7410                    !(pf->flags & I40E_FLAG_FDIR_ENABLED)  &&
7411                    (pf->flags & I40E_FLAG_DCB_ENABLED)) {
7412
7413                 /* save num_tc_qps queues for TCs 1 thru 7 and the rest
7414                  * are set up for RSS in TC0
7415                  */
7416                 queues_left -= accum_tc_size;
7417
7418                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7419
7420                 queues_left -= pf->rss_size;
7421                 if (queues_left < 0) {
7422                         dev_info(&pf->pdev->dev, "not enough queues for DCB\n");
7423                         return;
7424                 }
7425
7426                 pf->num_lan_qps = pf->rss_size_max + accum_tc_size;
7427
7428         } else if (pf->flags & I40E_FLAG_RSS_ENABLED   &&
7429                   (pf->flags & I40E_FLAG_FDIR_ENABLED) &&
7430                   !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7431
7432                 queues_left -= 1; /* save 1 queue for FD */
7433
7434                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7435
7436                 queues_left -= pf->rss_size;
7437                 if (queues_left < 0) {
7438                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director\n");
7439                         return;
7440                 }
7441
7442                 pf->num_lan_qps = pf->rss_size_max;
7443
7444         } else if (pf->flags & I40E_FLAG_RSS_ENABLED   &&
7445                   (pf->flags & I40E_FLAG_FDIR_ENABLED) &&
7446                   (pf->flags & I40E_FLAG_DCB_ENABLED)) {
7447
7448                 /* save 1 queue for TCs 1 thru 7,
7449                  * 1 queue for flow director,
7450                  * and the rest are set up for RSS in TC0
7451                  */
7452                 queues_left -= 1;
7453                 queues_left -= accum_tc_size;
7454
7455                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7456                 queues_left -= pf->rss_size;
7457                 if (queues_left < 0) {
7458                         dev_info(&pf->pdev->dev, "not enough queues for DCB and Flow Director\n");
7459                         return;
7460                 }
7461
7462                 pf->num_lan_qps = pf->rss_size_max + accum_tc_size;
7463
7464         } else {
7465                 dev_info(&pf->pdev->dev,
7466                          "Invalid configuration, flags=0x%08llx\n", pf->flags);
7467                 return;
7468         }
7469
7470         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
7471             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
7472                 pf->num_req_vfs = min_t(int, pf->num_req_vfs, (queues_left /
7473                                                                pf->num_vf_qps));
7474                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
7475         }
7476
7477         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
7478             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
7479                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
7480                                           (queues_left / pf->num_vmdq_qps));
7481                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
7482         }
7483
7484         pf->queues_left = queues_left;
7485         return;
7486 }
7487
7488 /**
7489  * i40e_setup_pf_filter_control - Setup PF static filter control
7490  * @pf: PF to be setup
7491  *
7492  * i40e_setup_pf_filter_control sets up a pf's initial filter control
7493  * settings. If PE/FCoE are enabled then it will also set the per PF
7494  * based filter sizes required for them. It also enables Flow director,
7495  * ethertype and macvlan type filter settings for the pf.
7496  *
7497  * Returns 0 on success, negative on failure
7498  **/
7499 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
7500 {
7501         struct i40e_filter_control_settings *settings = &pf->filter_settings;
7502
7503         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
7504
7505         /* Flow Director is enabled */
7506         if (pf->flags & (I40E_FLAG_FDIR_ENABLED | I40E_FLAG_FDIR_ATR_ENABLED))
7507                 settings->enable_fdir = true;
7508
7509         /* Ethtype and MACVLAN filters enabled for PF */
7510         settings->enable_ethtype = true;
7511         settings->enable_macvlan = true;
7512
7513         if (i40e_set_filter_control(&pf->hw, settings))
7514                 return -ENOENT;
7515
7516         return 0;
7517 }
7518
7519 /**
7520  * i40e_probe - Device initialization routine
7521  * @pdev: PCI device information struct
7522  * @ent: entry in i40e_pci_tbl
7523  *
7524  * i40e_probe initializes a pf identified by a pci_dev structure.
7525  * The OS initialization, configuring of the pf private structure,
7526  * and a hardware reset occur.
7527  *
7528  * Returns 0 on success, negative on failure
7529  **/
7530 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
7531 {
7532         struct i40e_driver_version dv;
7533         struct i40e_pf *pf;
7534         struct i40e_hw *hw;
7535         static u16 pfs_found;
7536         u16 link_status;
7537         int err = 0;
7538         u32 len;
7539
7540         err = pci_enable_device_mem(pdev);
7541         if (err)
7542                 return err;
7543
7544         /* set up for high or low dma */
7545         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
7546                 /* coherent mask for the same size will always succeed if
7547                  * dma_set_mask does
7548                  */
7549                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
7550         } else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) {
7551                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
7552         } else {
7553                 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
7554                 err = -EIO;
7555                 goto err_dma;
7556         }
7557
7558         /* set up pci connections */
7559         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
7560                                            IORESOURCE_MEM), i40e_driver_name);
7561         if (err) {
7562                 dev_info(&pdev->dev,
7563                          "pci_request_selected_regions failed %d\n", err);
7564                 goto err_pci_reg;
7565         }
7566
7567         pci_enable_pcie_error_reporting(pdev);
7568         pci_set_master(pdev);
7569
7570         /* Now that we have a PCI connection, we need to do the
7571          * low level device setup.  This is primarily setting up
7572          * the Admin Queue structures and then querying for the
7573          * device's current profile information.
7574          */
7575         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
7576         if (!pf) {
7577                 err = -ENOMEM;
7578                 goto err_pf_alloc;
7579         }
7580         pf->next_vsi = 0;
7581         pf->pdev = pdev;
7582         set_bit(__I40E_DOWN, &pf->state);
7583
7584         hw = &pf->hw;
7585         hw->back = pf;
7586         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
7587                               pci_resource_len(pdev, 0));
7588         if (!hw->hw_addr) {
7589                 err = -EIO;
7590                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
7591                          (unsigned int)pci_resource_start(pdev, 0),
7592                          (unsigned int)pci_resource_len(pdev, 0), err);
7593                 goto err_ioremap;
7594         }
7595         hw->vendor_id = pdev->vendor;
7596         hw->device_id = pdev->device;
7597         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
7598         hw->subsystem_vendor_id = pdev->subsystem_vendor;
7599         hw->subsystem_device_id = pdev->subsystem_device;
7600         hw->bus.device = PCI_SLOT(pdev->devfn);
7601         hw->bus.func = PCI_FUNC(pdev->devfn);
7602         pf->instance = pfs_found;
7603
7604         /* do a special CORER for clearing PXE mode once at init */
7605         if (hw->revision_id == 0 &&
7606             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
7607                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
7608                 i40e_flush(hw);
7609                 msleep(200);
7610                 pf->corer_count++;
7611
7612                 i40e_clear_pxe_mode(hw);
7613         }
7614
7615         /* Reset here to make sure all is clean and to define PF 'n' */
7616         err = i40e_pf_reset(hw);
7617         if (err) {
7618                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
7619                 goto err_pf_reset;
7620         }
7621         pf->pfr_count++;
7622
7623         hw->aq.num_arq_entries = I40E_AQ_LEN;
7624         hw->aq.num_asq_entries = I40E_AQ_LEN;
7625         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
7626         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
7627         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
7628         snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1,
7629                  "%s-pf%d:misc",
7630                  dev_driver_string(&pf->pdev->dev), pf->hw.pf_id);
7631
7632         err = i40e_init_shared_code(hw);
7633         if (err) {
7634                 dev_info(&pdev->dev, "init_shared_code failed: %d\n", err);
7635                 goto err_pf_reset;
7636         }
7637
7638         /* set up a default setting for link flow control */
7639         pf->hw.fc.requested_mode = I40E_FC_NONE;
7640
7641         err = i40e_init_adminq(hw);
7642         dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw));
7643         if (((hw->nvm.version & I40E_NVM_VERSION_HI_MASK)
7644                  >> I40E_NVM_VERSION_HI_SHIFT) != I40E_CURRENT_NVM_VERSION_HI) {
7645                 dev_info(&pdev->dev,
7646                          "warning: NVM version not supported, supported version: %02x.%02x\n",
7647                          I40E_CURRENT_NVM_VERSION_HI,
7648                          I40E_CURRENT_NVM_VERSION_LO);
7649         }
7650         if (err) {
7651                 dev_info(&pdev->dev,
7652                          "init_adminq failed: %d expecting API %02x.%02x\n",
7653                          err,
7654                          I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR);
7655                 goto err_pf_reset;
7656         }
7657
7658         i40e_clear_pxe_mode(hw);
7659         err = i40e_get_capabilities(pf);
7660         if (err)
7661                 goto err_adminq_setup;
7662
7663         err = i40e_sw_init(pf);
7664         if (err) {
7665                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
7666                 goto err_sw_init;
7667         }
7668
7669         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7670                                 hw->func_caps.num_rx_qp,
7671                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
7672         if (err) {
7673                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
7674                 goto err_init_lan_hmc;
7675         }
7676
7677         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7678         if (err) {
7679                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
7680                 err = -ENOENT;
7681                 goto err_configure_lan_hmc;
7682         }
7683
7684         i40e_get_mac_addr(hw, hw->mac.addr);
7685         if (!is_valid_ether_addr(hw->mac.addr)) {
7686                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
7687                 err = -EIO;
7688                 goto err_mac_addr;
7689         }
7690         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
7691         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
7692
7693         pci_set_drvdata(pdev, pf);
7694         pci_save_state(pdev);
7695
7696         /* set up periodic task facility */
7697         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
7698         pf->service_timer_period = HZ;
7699
7700         INIT_WORK(&pf->service_task, i40e_service_task);
7701         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
7702         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
7703         pf->link_check_timeout = jiffies;
7704
7705         /* WoL defaults to disabled */
7706         pf->wol_en = false;
7707         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
7708
7709         /* set up the main switch operations */
7710         i40e_determine_queue_usage(pf);
7711         i40e_init_interrupt_scheme(pf);
7712
7713         /* Set up the *vsi struct based on the number of VSIs in the HW,
7714          * and set up our local tracking of the MAIN PF vsi.
7715          */
7716         len = sizeof(struct i40e_vsi *) * pf->hw.func_caps.num_vsis;
7717         pf->vsi = kzalloc(len, GFP_KERNEL);
7718         if (!pf->vsi) {
7719                 err = -ENOMEM;
7720                 goto err_switch_setup;
7721         }
7722
7723         err = i40e_setup_pf_switch(pf, false);
7724         if (err) {
7725                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
7726                 goto err_vsis;
7727         }
7728
7729         /* The main driver is (mostly) up and happy. We need to set this state
7730          * before setting up the misc vector or we get a race and the vector
7731          * ends up disabled forever.
7732          */
7733         clear_bit(__I40E_DOWN, &pf->state);
7734
7735         /* In case of MSIX we are going to setup the misc vector right here
7736          * to handle admin queue events etc. In case of legacy and MSI
7737          * the misc functionality and queue processing is combined in
7738          * the same vector and that gets setup at open.
7739          */
7740         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7741                 err = i40e_setup_misc_vector(pf);
7742                 if (err) {
7743                         dev_info(&pdev->dev,
7744                                  "setup of misc vector failed: %d\n", err);
7745                         goto err_vsis;
7746                 }
7747         }
7748
7749         /* prep for VF support */
7750         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
7751             (pf->flags & I40E_FLAG_MSIX_ENABLED)) {
7752                 u32 val;
7753
7754                 /* disable link interrupts for VFs */
7755                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
7756                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
7757                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
7758                 i40e_flush(hw);
7759         }
7760
7761         pfs_found++;
7762
7763         i40e_dbg_pf_init(pf);
7764
7765         /* tell the firmware that we're starting */
7766         dv.major_version = DRV_VERSION_MAJOR;
7767         dv.minor_version = DRV_VERSION_MINOR;
7768         dv.build_version = DRV_VERSION_BUILD;
7769         dv.subbuild_version = 0;
7770         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7771
7772         /* since everything's happy, start the service_task timer */
7773         mod_timer(&pf->service_timer,
7774                   round_jiffies(jiffies + pf->service_timer_period));
7775
7776         /* Get the negotiated link width and speed from PCI config space */
7777         pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status);
7778
7779         i40e_set_pci_config_data(hw, link_status);
7780
7781         dev_info(&pdev->dev, "PCI Express: %s %s\n",
7782                 (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" :
7783                  hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" :
7784                  hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" :
7785                  "Unknown"),
7786                 (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" :
7787                  hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" :
7788                  hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" :
7789                  hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" :
7790                  "Unknown"));
7791
7792         if (hw->bus.width < i40e_bus_width_pcie_x8 ||
7793             hw->bus.speed < i40e_bus_speed_8000) {
7794                 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
7795                 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
7796         }
7797
7798         return 0;
7799
7800         /* Unwind what we've done if something failed in the setup */
7801 err_vsis:
7802         set_bit(__I40E_DOWN, &pf->state);
7803         i40e_clear_interrupt_scheme(pf);
7804         kfree(pf->vsi);
7805 err_switch_setup:
7806         i40e_reset_interrupt_capability(pf);
7807         del_timer_sync(&pf->service_timer);
7808 err_mac_addr:
7809 err_configure_lan_hmc:
7810         (void)i40e_shutdown_lan_hmc(hw);
7811 err_init_lan_hmc:
7812         kfree(pf->qp_pile);
7813         kfree(pf->irq_pile);
7814 err_sw_init:
7815 err_adminq_setup:
7816         (void)i40e_shutdown_adminq(hw);
7817 err_pf_reset:
7818         iounmap(hw->hw_addr);
7819 err_ioremap:
7820         kfree(pf);
7821 err_pf_alloc:
7822         pci_disable_pcie_error_reporting(pdev);
7823         pci_release_selected_regions(pdev,
7824                                      pci_select_bars(pdev, IORESOURCE_MEM));
7825 err_pci_reg:
7826 err_dma:
7827         pci_disable_device(pdev);
7828         return err;
7829 }
7830
7831 /**
7832  * i40e_remove - Device removal routine
7833  * @pdev: PCI device information struct
7834  *
7835  * i40e_remove is called by the PCI subsystem to alert the driver
7836  * that is should release a PCI device.  This could be caused by a
7837  * Hot-Plug event, or because the driver is going to be removed from
7838  * memory.
7839  **/
7840 static void i40e_remove(struct pci_dev *pdev)
7841 {
7842         struct i40e_pf *pf = pci_get_drvdata(pdev);
7843         i40e_status ret_code;
7844         u32 reg;
7845         int i;
7846
7847         i40e_dbg_pf_exit(pf);
7848
7849         i40e_ptp_stop(pf);
7850
7851         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
7852                 i40e_free_vfs(pf);
7853                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
7854         }
7855
7856         /* no more scheduling of any task */
7857         set_bit(__I40E_DOWN, &pf->state);
7858         del_timer_sync(&pf->service_timer);
7859         cancel_work_sync(&pf->service_task);
7860
7861         i40e_fdir_teardown(pf);
7862
7863         /* If there is a switch structure or any orphans, remove them.
7864          * This will leave only the PF's VSI remaining.
7865          */
7866         for (i = 0; i < I40E_MAX_VEB; i++) {
7867                 if (!pf->veb[i])
7868                         continue;
7869
7870                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
7871                     pf->veb[i]->uplink_seid == 0)
7872                         i40e_switch_branch_release(pf->veb[i]);
7873         }
7874
7875         /* Now we can shutdown the PF's VSI, just before we kill
7876          * adminq and hmc.
7877          */
7878         if (pf->vsi[pf->lan_vsi])
7879                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
7880
7881         i40e_stop_misc_vector(pf);
7882         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7883                 synchronize_irq(pf->msix_entries[0].vector);
7884                 free_irq(pf->msix_entries[0].vector, pf);
7885         }
7886
7887         /* shutdown and destroy the HMC */
7888         ret_code = i40e_shutdown_lan_hmc(&pf->hw);
7889         if (ret_code)
7890                 dev_warn(&pdev->dev,
7891                          "Failed to destroy the HMC resources: %d\n", ret_code);
7892
7893         /* shutdown the adminq */
7894         ret_code = i40e_shutdown_adminq(&pf->hw);
7895         if (ret_code)
7896                 dev_warn(&pdev->dev,
7897                          "Failed to destroy the Admin Queue resources: %d\n",
7898                          ret_code);
7899
7900         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
7901         i40e_clear_interrupt_scheme(pf);
7902         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7903                 if (pf->vsi[i]) {
7904                         i40e_vsi_clear_rings(pf->vsi[i]);
7905                         i40e_vsi_clear(pf->vsi[i]);
7906                         pf->vsi[i] = NULL;
7907                 }
7908         }
7909
7910         for (i = 0; i < I40E_MAX_VEB; i++) {
7911                 kfree(pf->veb[i]);
7912                 pf->veb[i] = NULL;
7913         }
7914
7915         kfree(pf->qp_pile);
7916         kfree(pf->irq_pile);
7917         kfree(pf->sw_config);
7918         kfree(pf->vsi);
7919
7920         /* force a PF reset to clean anything leftover */
7921         reg = rd32(&pf->hw, I40E_PFGEN_CTRL);
7922         wr32(&pf->hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
7923         i40e_flush(&pf->hw);
7924
7925         iounmap(pf->hw.hw_addr);
7926         kfree(pf);
7927         pci_release_selected_regions(pdev,
7928                                      pci_select_bars(pdev, IORESOURCE_MEM));
7929
7930         pci_disable_pcie_error_reporting(pdev);
7931         pci_disable_device(pdev);
7932 }
7933
7934 /**
7935  * i40e_pci_error_detected - warning that something funky happened in PCI land
7936  * @pdev: PCI device information struct
7937  *
7938  * Called to warn that something happened and the error handling steps
7939  * are in progress.  Allows the driver to quiesce things, be ready for
7940  * remediation.
7941  **/
7942 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
7943                                                 enum pci_channel_state error)
7944 {
7945         struct i40e_pf *pf = pci_get_drvdata(pdev);
7946
7947         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
7948
7949         /* shutdown all operations */
7950         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
7951                 rtnl_lock();
7952                 i40e_prep_for_reset(pf);
7953                 rtnl_unlock();
7954         }
7955
7956         /* Request a slot reset */
7957         return PCI_ERS_RESULT_NEED_RESET;
7958 }
7959
7960 /**
7961  * i40e_pci_error_slot_reset - a PCI slot reset just happened
7962  * @pdev: PCI device information struct
7963  *
7964  * Called to find if the driver can work with the device now that
7965  * the pci slot has been reset.  If a basic connection seems good
7966  * (registers are readable and have sane content) then return a
7967  * happy little PCI_ERS_RESULT_xxx.
7968  **/
7969 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
7970 {
7971         struct i40e_pf *pf = pci_get_drvdata(pdev);
7972         pci_ers_result_t result;
7973         int err;
7974         u32 reg;
7975
7976         dev_info(&pdev->dev, "%s\n", __func__);
7977         if (pci_enable_device_mem(pdev)) {
7978                 dev_info(&pdev->dev,
7979                          "Cannot re-enable PCI device after reset.\n");
7980                 result = PCI_ERS_RESULT_DISCONNECT;
7981         } else {
7982                 pci_set_master(pdev);
7983                 pci_restore_state(pdev);
7984                 pci_save_state(pdev);
7985                 pci_wake_from_d3(pdev, false);
7986
7987                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
7988                 if (reg == 0)
7989                         result = PCI_ERS_RESULT_RECOVERED;
7990                 else
7991                         result = PCI_ERS_RESULT_DISCONNECT;
7992         }
7993
7994         err = pci_cleanup_aer_uncorrect_error_status(pdev);
7995         if (err) {
7996                 dev_info(&pdev->dev,
7997                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
7998                          err);
7999                 /* non-fatal, continue */
8000         }
8001
8002         return result;
8003 }
8004
8005 /**
8006  * i40e_pci_error_resume - restart operations after PCI error recovery
8007  * @pdev: PCI device information struct
8008  *
8009  * Called to allow the driver to bring things back up after PCI error
8010  * and/or reset recovery has finished.
8011  **/
8012 static void i40e_pci_error_resume(struct pci_dev *pdev)
8013 {
8014         struct i40e_pf *pf = pci_get_drvdata(pdev);
8015
8016         dev_info(&pdev->dev, "%s\n", __func__);
8017         if (test_bit(__I40E_SUSPENDED, &pf->state))
8018                 return;
8019
8020         rtnl_lock();
8021         i40e_handle_reset_warning(pf);
8022         rtnl_lock();
8023 }
8024
8025 /**
8026  * i40e_shutdown - PCI callback for shutting down
8027  * @pdev: PCI device information struct
8028  **/
8029 static void i40e_shutdown(struct pci_dev *pdev)
8030 {
8031         struct i40e_pf *pf = pci_get_drvdata(pdev);
8032         struct i40e_hw *hw = &pf->hw;
8033
8034         set_bit(__I40E_SUSPENDED, &pf->state);
8035         set_bit(__I40E_DOWN, &pf->state);
8036         rtnl_lock();
8037         i40e_prep_for_reset(pf);
8038         rtnl_unlock();
8039
8040         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8041         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8042
8043         if (system_state == SYSTEM_POWER_OFF) {
8044                 pci_wake_from_d3(pdev, pf->wol_en);
8045                 pci_set_power_state(pdev, PCI_D3hot);
8046         }
8047 }
8048
8049 #ifdef CONFIG_PM
8050 /**
8051  * i40e_suspend - PCI callback for moving to D3
8052  * @pdev: PCI device information struct
8053  **/
8054 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
8055 {
8056         struct i40e_pf *pf = pci_get_drvdata(pdev);
8057         struct i40e_hw *hw = &pf->hw;
8058
8059         set_bit(__I40E_SUSPENDED, &pf->state);
8060         set_bit(__I40E_DOWN, &pf->state);
8061         rtnl_lock();
8062         i40e_prep_for_reset(pf);
8063         rtnl_unlock();
8064
8065         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8066         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8067
8068         pci_wake_from_d3(pdev, pf->wol_en);
8069         pci_set_power_state(pdev, PCI_D3hot);
8070
8071         return 0;
8072 }
8073
8074 /**
8075  * i40e_resume - PCI callback for waking up from D3
8076  * @pdev: PCI device information struct
8077  **/
8078 static int i40e_resume(struct pci_dev *pdev)
8079 {
8080         struct i40e_pf *pf = pci_get_drvdata(pdev);
8081         u32 err;
8082
8083         pci_set_power_state(pdev, PCI_D0);
8084         pci_restore_state(pdev);
8085         /* pci_restore_state() clears dev->state_saves, so
8086          * call pci_save_state() again to restore it.
8087          */
8088         pci_save_state(pdev);
8089
8090         err = pci_enable_device_mem(pdev);
8091         if (err) {
8092                 dev_err(&pdev->dev,
8093                         "%s: Cannot enable PCI device from suspend\n",
8094                         __func__);
8095                 return err;
8096         }
8097         pci_set_master(pdev);
8098
8099         /* no wakeup events while running */
8100         pci_wake_from_d3(pdev, false);
8101
8102         /* handling the reset will rebuild the device state */
8103         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
8104                 clear_bit(__I40E_DOWN, &pf->state);
8105                 rtnl_lock();
8106                 i40e_reset_and_rebuild(pf, false);
8107                 rtnl_unlock();
8108         }
8109
8110         return 0;
8111 }
8112
8113 #endif
8114 static const struct pci_error_handlers i40e_err_handler = {
8115         .error_detected = i40e_pci_error_detected,
8116         .slot_reset = i40e_pci_error_slot_reset,
8117         .resume = i40e_pci_error_resume,
8118 };
8119
8120 static struct pci_driver i40e_driver = {
8121         .name     = i40e_driver_name,
8122         .id_table = i40e_pci_tbl,
8123         .probe    = i40e_probe,
8124         .remove   = i40e_remove,
8125 #ifdef CONFIG_PM
8126         .suspend  = i40e_suspend,
8127         .resume   = i40e_resume,
8128 #endif
8129         .shutdown = i40e_shutdown,
8130         .err_handler = &i40e_err_handler,
8131         .sriov_configure = i40e_pci_sriov_configure,
8132 };
8133
8134 /**
8135  * i40e_init_module - Driver registration routine
8136  *
8137  * i40e_init_module is the first routine called when the driver is
8138  * loaded. All it does is register with the PCI subsystem.
8139  **/
8140 static int __init i40e_init_module(void)
8141 {
8142         pr_info("%s: %s - version %s\n", i40e_driver_name,
8143                 i40e_driver_string, i40e_driver_version_str);
8144         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
8145         i40e_dbg_init();
8146         return pci_register_driver(&i40e_driver);
8147 }
8148 module_init(i40e_init_module);
8149
8150 /**
8151  * i40e_exit_module - Driver exit cleanup routine
8152  *
8153  * i40e_exit_module is called just before the driver is removed
8154  * from memory.
8155  **/
8156 static void __exit i40e_exit_module(void)
8157 {
8158         pci_unregister_driver(&i40e_driver);
8159         i40e_dbg_exit();
8160 }
8161 module_exit(i40e_exit_module);